import tempfile
from functools import partial
from typing import Any, Dict, List

import pytest
import pytorch_lightning as pl
import torch
from torch import nn
from torch.optim import Adam
from torch.utils.data import DataLoader, Dataset

from oml.const import (
    EMBEDDINGS_KEY,
    INPUT_TENSORS_KEY,
    IS_GALLERY_KEY,
    IS_QUERY_KEY,
    LABELS_KEY,
)
from oml.datasets.triplet import TItem, tri_collate
from oml.lightning.callbacks.metric import MetricValCallback
from oml.losses.triplet import TripletLossPlain, TripletLossWithMiner
from oml.metrics.embeddings import EmbeddingMetrics
from oml.metrics.triplets import AccuracyOnTriplets
from oml.samplers.balance import BalanceSampler

class DummyTripletDataset(Dataset):
    def __init__(self, num_triplets: int, im_size: int):
        self.num_triplets = num_triplets
        self.im_size = im_size

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        input_tensors = torch.rand((3, 3, self.im_size, self.im_size))
        return {INPUT_TENSORS_KEY: input_tensors}

    def __len__(self) -> int:
        return self.num_triplets

class DummyRetrievalDataset(Dataset):
    def __init__(self, labels: List[int], im_size: int):
        self.labels = labels
        self.im_size = im_size

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        input_tensors = torch.rand((3, self.im_size, self.im_size))
        label = torch.tensor(self.labels[idx]).long()
        return {INPUT_TENSORS_KEY: input_tensors, LABELS_KEY: label, IS_QUERY_KEY: True, IS_GALLERY_KEY: True}

    def __len__(self) -> int:
        return len(self.labels)

class DummyCommonModule(pl.LightningModule):
    def __init__(self, im_size: int):
        super().__init__()
        self.model = nn.Sequential(
            nn.AvgPool2d(kernel_size=(im_size, im_size)), nn.Flatten(start_dim=1), nn.Linear(3, 5), nn.Linear(5, 5)
        )

    def configure_optimizers(self) -> torch.optim.Optimizer:
        return Adam(self.model.parameters(), lr=1e-4)

    def validation_step(self, batch: TItem, batch_idx: int, *_: Any) -> Dict[str, Any]:
        embeddings = self.model(batch[INPUT_TENSORS_KEY])
        return {**batch, **{EMBEDDINGS_KEY: embeddings.detach().cpu()}}

class DummyTripletModule(DummyCommonModule):
    def __init__(self, im_size: int):
        super().__init__(im_size=im_size)
        self.criterion = TripletLossPlain(margin=None)

    def training_step(self, batch_multidataloader: List[TItem], batch_idx: int) -> torch.Tensor:
        embeddings = torch.cat([self.model(batch[INPUT_TENSORS_KEY]) for batch in batch_multidataloader])
        loss = self.criterion(embeddings)
        return loss

class DummyExtractorModule(DummyCommonModule):
    def __init__(self, im_size: int):
        super().__init__(im_size=im_size)
        self.criterion = TripletLossWithMiner(margin=None, need_logs=True)

    def training_step(self, batch_multidataloader: List[TItem], batch_idx: int) -> torch.Tensor:
        embeddings = torch.cat([self.model(batch[INPUT_TENSORS_KEY]) for batch in batch_multidataloader])
        labels = torch.cat([batch[LABELS_KEY] for batch in batch_multidataloader])
        loss = self.criterion(embeddings, labels)
        return loss

def create_triplet_dataloader(num_samples: int, im_size: int, num_workers: int) -> DataLoader:
    dataset = DummyTripletDataset(num_triplets=num_samples, im_size=im_size)
    dataloader = DataLoader(
        dataset=dataset, batch_size=num_samples // 2, num_workers=num_workers, collate_fn=tri_collate
    )
    return dataloader

def create_retrieval_dataloader(
    num_samples: int, im_size: int, n_labels: int, n_instances: int, num_workers: int
) -> DataLoader:
    assert num_samples % (n_labels * n_instances) == 0

    labels = [idx // n_instances for idx in range(num_samples)]

    dataset = DummyRetrievalDataset(labels=labels, im_size=im_size)

    sampler_retrieval = BalanceSampler(labels=labels, n_labels=n_labels, n_instances=n_instances)
    train_retrieval_loader = DataLoader(
        dataset=dataset,
        batch_sampler=sampler_retrieval,
        num_workers=num_workers,
    )
    return train_retrieval_loader

def create_triplet_callback(loader_idx: int, samples_in_getitem: int) -> MetricValCallback:
    metric = AccuracyOnTriplets(embeddings_key=EMBEDDINGS_KEY)
    metric_callback = MetricValCallback(metric=metric, loader_idx=loader_idx, samples_in_getitem=samples_in_getitem)
    return metric_callback

def create_retrieval_callback(loader_idx: int, samples_in_getitem: int) -> MetricValCallback:
    metric = EmbeddingMetrics(
        embeddings_key=EMBEDDINGS_KEY,
        labels_key=LABELS_KEY,
        is_query_key=IS_QUERY_KEY,
        is_gallery_key=IS_GALLERY_KEY,
    )
    metric_callback = MetricValCallback(metric=metric, loader_idx=loader_idx, samples_in_getitem=samples_in_getitem)
    return metric_callback

@pytest.mark.parametrize(
    "samples_in_getitem, is_error_expected, pipeline",
    [
        (1, True, "triplet"),
        (3, False, "triplet"),
        (5, True, "triplet"),
        (1, False, "retrieval"),
        (2, True, "retrieval"),
    ],
)
@pytest.mark.parametrize("num_dataloaders", [1, 2])
def test_lightning(
    samples_in_getitem: int, is_error_expected: bool, num_dataloaders: int, pipeline: str, num_workers: int
) -> None:
    num_samples = 12
    im_size = 6
    n_labels = 2
    n_instances = 3

    if pipeline == "triplet":
        create_dataloader = create_triplet_dataloader
        lightning_module = DummyTripletModule(im_size=im_size)
        create_callback = create_triplet_callback
    elif pipeline == "retrieval":
        create_dataloader = partial(create_retrieval_dataloader, n_labels=n_labels, n_instances=n_instances)
        lightning_module = DummyExtractorModule(im_size=im_size)
        create_callback = create_retrieval_callback
    else:
        raise ValueError

    train_dataloaders = [
        create_dataloader(num_samples=num_samples, im_size=im_size, num_workers=num_workers)
        for _ in range(num_dataloaders)
    ]
    val_dataloaders = [
        create_dataloader(num_samples=num_samples, im_size=im_size, num_workers=num_workers)
        for _ in range(num_dataloaders)
    ]
    callbacks = [create_callback(loader_idx=k, samples_in_getitem=samples_in_getitem) for k in range(num_dataloaders)]

    trainer = pl.Trainer(
        default_root_dir=tempfile.gettempdir(),
        max_epochs=2,
        enable_progress_bar=False,
        num_nodes=1,
        gpus=None,
        replace_sampler_ddp=False,
        callbacks=callbacks,
        num_sanity_val_steps=0,
    )

    if is_error_expected:
        with pytest.raises(ValueError, match=callbacks[0].metric.__class__.__name__):
            trainer.fit(model=lightning_module, train_dataloaders=train_dataloaders, val_dataloaders=val_dataloaders)
            trainer.validate(model=lightning_module, dataloaders=val_dataloaders)
    else:
        trainer.fit(model=lightning_module, train_dataloaders=train_dataloaders, val_dataloaders=val_dataloaders)
        trainer.validate(model=lightning_module, dataloaders=val_dataloaders)


# -*- coding: utf-8 -*-
import datetime
import os
import time

import pkg_resources

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
build_date = datetime.datetime.utcfromtimestamp(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))

extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx'
    ]

intersphinx_mapping = {
    'https://docs.python.org/3/': None,
}

# General
source_suffix = '.rst'
master_doc = 'index'
project = 'mortar_sqlalchemy'
copyright = '2015-%s Chris Withers' % build_date.year
version = release = pkg_resources.get_distribution(project).version
exclude_trees = ['_build']
pygments_style = 'sphinx'

# Options for HTML output
html_theme = 'furo'
htmlhelp_basename = project+'doc'

# Options for LaTeX output
exclude_patterns = ['**/furo.js.LICENSE.txt']

autodoc_member_order = 'bysource'


import torch
from torch.utils.data import DataLoader
import numpy as np
import argparse

import dataset as dataset

from utils.config import cfg, cfg_from_list

from trainer import Trainer

def set_seed():
    # set random seed, default is 0
    # cudnn
    torch.backends.cudnn.deterministic = True
    # random
    seed = cfg.GAN.SEED
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    np.random.seed(seed)

def arg_parse(parser):
    # parse args
    parser.add_argument('--summary', type=bool, default=False, help='if summarize model -> True')

    args = parser.parse_args()

    return args

# def fix_config(args):
#     if args.ngf is not None:
#         cfg_from_list(['GAN.GF_DIM', args.ngf])
#     if args.ndf is not None:
#         cfg_from_list(['GAN.DF_DIM', args.ndf])
#     if args.ncf is not None:
#         cfg_from_list(['GAN.CONDITION_DIM', args.ncf])

def get_dataloader(mode='train', batch_size=8, resplit=False):
    shapenet = dataset.ShapeNetDataset(mode=mode, resplit=resplit)
    return DataLoader(shapenet, batch_size, shuffle=True)

if __name__ == '__main__':
    # argparse
    parser = argparse.ArgumentParser()
    args = arg_parse(parser)
#     fix_config(args)
    # device selection
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    set_seed()
    train_dataloader = get_dataloader(mode='train', batch_size=cfg.TRAIN.BATCH_SIZE)

    t2s_trainer = Trainer(train_dataloader, device)
    t2s_trainer.train(summary=args.summary)

#     arg = arg_parse()


'''
# This code compares the scattering intensities calculated with the DDA (ADDA code)
# for the scattering of two different Bessel beams (option_1 and option_2) by a sphere.
'''

import os, re, math
import matplotlib.pyplot as plt
from matplotlib import rc

# path to adda executable
#adda_exec = "../../win64/adda.exe"
adda_exec = os.path.abspath(__file__ + "/../../../src/seq/adda")

# define here different parameters for 2 options (see ADDA manual)
run_options = [' -beam besselLE  2 15',  # option 1
               ' -beam besselLM  2 15']  # option 2

# =============================================================================
# Bessel beams in ADDA:
#
#   besselCS <order> <angle>
#       -Bessel beam with circularly symmetric energy density.
#   besselCSp <order> <angle>
#       -Alternative Bessel beam with circularly symmetric energy density.
#   besselM <order> <angle> <ReMex> <ReMey> <ReMmx> <ReMmy> [<ImMex> <ImMey> <ImMmx> <ImMmy>]
#       -Generalized Bessel beam. The beam is defined by 2x2 matrix M:
#       (Mex, Mey, Mmx, Mmy). Real parts of these four elements are obligatory,
#       while imaginary parts are optional (zero, by default).
#   besselLE <order> <angle>
#       -Bessel beam with linearly polarized electric field.
#   besselLM <order> <angle>
#       -Bessel beam with linearly polarized magnetic field.
#   besselTEL <order> <angle>
#       -Linear component of the TE Bessel beam.
#   besselTML <order> <angle>
#       -Linear component of the TM Bessel beam.
#
#   Order is integer (of any sign) and the half-cone angle (in degrees)
#   is measured from the z-axis.
# =============================================================================

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# data generation (run of ADDA code)
def adda_run(mode): # option 1 or 2

    # cmd line generation (see ADDA manual)

    # common parameters for 2 options
    cmdline = adda_exec
    cmdline += ' -grid 16' # particle discretization
    cmdline += ' -sym enf' # do not simulate second polarization
    cmdline += ' -ntheta 180' # number of scattering angles
    cmdline += ' -store_beam' # save incident field
    cmdline += ' -dir option_' + str(mode) # save path
    cmdline += run_options[mode-1]
    print(cmdline)

    os.system(cmdline)

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# data extraction
def extractData(mode):
    path = 'option_' + str(mode)
    files = os.listdir(path)
    with open(path + '/' + str(files[files.index('mueller')])) as f:
        fileM = f.read()
    f.close()
    dat = re.split('[\n \t]',fileM)
    theta = dat[17:-1:17]
    s11 = dat[18:-1:17]
    s12 = dat[19:-1:17]

    with open(path + '/' + str(files[files.index('IncBeam-Y')])) as f:
        fileF = f.read()
    f.close()
    dat = re.split('[\n \t]',fileF)
    xd = dat[10:-1:10]
    yd = dat[11:-1:10]
    zd = dat[12:-1:10]
    ed = dat[13:-1:10]

    theta = [float(th) for th in theta]
    s11 = [float(s) for s in s11]
    s12 = [float(s) for s in s12]
    i_per = [x-y for x, y in zip(s11, s12)]
    i_par = [x+y for x, y in zip(s11, s12)]
    xd = [float(i) for i in xd]
    yd = [float(i) for i in yd]
    zd = [float(i) for i in zd]
    ed = [float(i) for i in ed]
    minz = min([abs(i) for i in zd])
    zslice = [i for i in range(len(zd)) if zd[i]==minz]
    xd = [xd[i] for i in zslice]
    yd = [yd[i] for i in zslice]
    ed = [ed[i] for i in zslice]

    return theta,i_per,i_par,xd,yd,ed,minz

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# data visualisation

# plots of scattering intensities
def plotData(xv1,yv1,xv2,yv2,flag):
    rc('font',**{'family':'sans-serif','sans-serif':['Arial']})
    plt.plot(xv1, yv1, label = 'option 1', color = (0.5, 0.7, 0.9))
    plt.plot(xv2, yv2, label = 'option 2', color = 'm',linestyle='dashed')
    plt.minorticks_on()
    plt.tick_params(which='major',right=True, top=True)
    plt.tick_params(which='minor',right=True, top=True)
    plt.xlabel(r'Scattering angle $\theta$, deg')
    if flag == 1:
        plt.ylabel(r'$I_{\perp}$ (H plane)')
    if flag == 2:
        plt.ylabel(r'$I_{\parallel}$ (E plane)')
    plt.legend()
    plt.xlim(0, 180)
    plt.yscale('log')

    maxint = max(yv2)
    dev = list(map(lambda x, y: math.fabs((x-y)/maxint*100), yv1,yv2))
    dev2 = list(map(lambda x, y: (math.fabs((x-y)/maxint))**2, yv1,yv2))

    print('\nDeviations ' + str(flag))
    print('Max relative error, %:')
    print(max(dev))
    print('Average relative error,%:')
    print(sum(dev)/len(dev))
    print('RMS, %:')
    print(math.sqrt(sum(dev2)/len(xv2)*100))

# Visualisation of the amplitude of the incident electric field almost in the middle
# of the particle (the nearest to the center xy-plane of dipoles is used, its z-coordinate is shown on the plot)
def plotField(xd,yd,ed,z0,mode):
    ax.set_title(r'OPTION '+str(mode)+':\n '+run_options[mode-1]+'\nIntensity profile of $|E_{inc}|^2$\n(z = '+str(round(z0,2))+')');
    ax.scatter(xd, yd, ed, c=ed, cmap='viridis', linewidth=0.5);
    #ax.plot_trisurf(xd, yd, ed,cmap='viridis', edgecolor='none');
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
    ax.w_yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
    ax.w_zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

adda_run(1)
adda_run(2)

theta_1,iper_1,ipar_1,xd_1,yd_1,ed_1,z0_1 = extractData(1) # extraction of ADDA results for option 1
theta_2,iper_2,ipar_2,xd_2,yd_2,ed_2,z0_2 = extractData(2) # extraction of ADDA results for option 2

# data visualisation

fig = plt.figure()
# Visualisation of the amplitude of the incident electric field for option 1
ax = fig.add_subplot(221,projection='3d')
plotField(xd_1,yd_1,ed_1,z0_1,1)
# Visualisation of the amplitude of the incident electric field for option 2
ax = fig.add_subplot(222,projection='3d')
plotField(xd_2,yd_2,ed_2,z0_2,2)
# Perpendicular scattering intensity (1)
ax = fig.add_subplot(223)
plotData(theta_1,iper_1,theta_2,iper_2,1)
# Parallel scattering intensity (2)
ax = fig.add_subplot(224)
plotData(theta_1,ipar_1,theta_2,ipar_2,2)

plt.tight_layout()

# data save
os.makedirs('saved', exist_ok=True)

plt.savefig('saved/results.pdf', bbox_inches='tight')

plt.show()


# 图片预处理
# 输入参数：输入图片文件夹路径（具体文件夹结构见原文档）、输出图片文件夹路径
# 函数无直接输出

import facenet
import align.align_dataset_mtcnn
import sys
import os
import argparse
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'

def main(args):
    if os.path.exists(args.in_dir):
        if __name__ == '__main__':
            align.align_dataset_mtcnn.main(align.align_dataset_mtcnn.parse_arguments([args.in_dir,args.out_dir,'--image_size','160','--margin','20','--random_order']))
        '''
        out = facenet.get_image_paths(args.in_dir)
        for imgs in out:
            imgs_path = facenet.get_image_paths(imgs)
            path=args.out_dir+imgs[len(args.in_dir):]
            if not os.path.exists(path):  # 如果路径不存在
                os.makedirs(path)
            for img in imgs_path:
                name=args.out_dir+img[len(args.in_dir):]
                print(name)
                final.Align.main(final.Align.parse_arguments(["-in_dir", img,"-out_dir",name]))
        '''

    else:
        print("no such path!")

def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument('-in_dir', type=str , default='../in')

    parser.add_argument('-out_dir', type=str , default='../out')

    return parser.parse_args(argv)

if __name__ == '__main__':
    # 这是一个从外部输入参数的代码。
    main(parse_arguments(sys.argv[1:]))


import json

from pymongo import MongoClient

from date import current_date

# Load 'config.json' file
conf = json.load(
    open("config.json", "r+")
)

# MongoDB Init
client = MongoClient(conf["mongodb"]["connection-string"])
db = client.MyEshop

affiliates = db.Affiliate

# Create 'Affiliate' class
# This will handle creating users and auth
class Affiliate():
    def __init__(self, campaign):
        self.campaign = campaign

        self.query = affiliates.find_one({"campaign": str(campaign)})

        if self.query != None:
            self.visits = self.query["visits"]

    # Create new campaign
    def create_campaign(self):

        try:
            if self.query == None:
                affiliates.insert_one({
                    "campaign": str(self.campaign),
                    "visits": 0
                })

            return True

        except:
            return False

    # Add campaign visit
    def add_visit(self):
        try:
            if self.query != None:
                current_visits = int(self.visits)

                affiliates.update_one({
                    "campaign": str(self.campaign)
                },
                {
                    "$set": {
                        "visits": str(current_visits + 1)
                    }
                })

                return True

            else:
                return False

        except:
            return False

    # Delete campaign
    def delete_affiliate(self):
        try:
            if self.query != None:
                affiliates.delete_one({
                    "campaign": str(self.campaign)
                })

                return True

            else:
                return False

        except:
            return False


from django.urls import include, path
from rest_framework.routers import DefaultRouter
from users.views import UserViewSet, create_user, get_jwt_token

from .views import (CategoryViewSet, CommentViewSet, GenreViewSet,
                    ReviewViewSet, TitleViewSet)

router = DefaultRouter(trailing_slash=True)
router.register('categories', CategoryViewSet, basename='categories')
router.register('genres', GenreViewSet, basename='genres')
router.register('titles', TitleViewSet, basename='titles')
router.register(r'titles/(?P<title_id>\d+)/reviews',
                ReviewViewSet, basename='reviews')
router.register(r'titles/(?P<title_id>\d+)/reviews/(?P<review_id>\d+)'
                r'/comments', CommentViewSet, basename='comments')
router.register('users', UserViewSet, basename='users')

urlpatterns = [
    path('v1/', include(router.urls)),
    path('v1/auth/signup/', create_user, name='signup'),
    path('v1/auth/token/', get_jwt_token, name='token'),
]


# -*- coding: UTF-8 -*-

import pymysql    #导入pymysql模块 这里面模块名称可以小写

class Sqlbase(object):

	def __init__(self):
		# 打开数据库连接
		self.db = pymysql.connect("localhost", "root", "123456", "idata")
		# 使用cursor()方法获取操作游标
		self.cursor = self.db.cursor()

	def select(self,sql):
		try:
			# 执行sql语句
			result=self.cursor.execute(sql)
			# 提交到数据库执行
			self.db.commit()
			return result
		except:
			# 如果发生错误则回滚
			self.db.rollback()
		finally:
			# 关闭数据库连接
			self.db.close()

	def execute(self,sql):
		try:
			# 执行sql语句
			result=self.cursor.execute(sql)
			# 提交到数据库执行
			self.db.commit()
		except:
			# 如果发生错误则回滚
			self.db.rollback()

		# 关闭数据库连接
		self.db.close()

if __name__ == '__main__':
	sqlbase=Sqlbase()
	# SQL 插入语句
	sql = """INSERT INTO tb_test(test)
		         VALUES ('xx')"""
	sqlbase.execute(sql)


import openai
import telebot

from bot.constant_messages import empty_voice_message, api_key_not_set_message, api_key_expired_message
from bot.constants import bot
from bot.error_handlers import on_api_telegram_exception_429
from db.db import insert_message, insert_user_if_not_exists, insert_group_if_not_exists
from open_ai.chatgpt.reply import get_chatgpt_reply
from open_ai.common import is_openai_api_key_set
from open_ai.whisper.whisper import transcribe_voice

def reply_to_voice_prompt(message: telebot.types.Message):
    insert_user_if_not_exists(message)
    if message.chat.type == 'group' or message.chat.type == 'supergroup':
        insert_group_if_not_exists(message)
    if is_openai_api_key_set(message.from_user.id):
        transcribed_voice = transcribe_voice(message)
        if not transcribed_voice:
            bot.reply_to(message=message, text=empty_voice_message)
        else:
            try:
                reply = get_chatgpt_reply(
                    content=transcribed_voice,
                    user_id=message.from_user.id,
                    full_name=message.from_user.full_name,
                    username=message.from_user.username
                )
                insert_message(message=message, reply=reply, transcribed_voice=transcribed_voice)
                bot.reply_to(message=message, text=reply)
            except telebot.apihelper.ApiTelegramException:
                on_api_telegram_exception_429(reply_to_voice_prompt, message)
            except openai.error.RateLimitError:
                bot.reply_to(message, api_key_expired_message)
    else:
        bot.reply_to(message, api_key_not_set_message('OpenAI'))

def register_chat_voice():
    bot.register_message_handler(
        reply_to_voice_prompt,
        content_types=['voice'],
        chat_types=['private']
    )


"""Custom loader for loading a chapter."""

import os.path
from django.db import transaction
from utils.TranslatableModelLoader import TranslatableModelLoader
from utils.errors.MissingRequiredFieldError import MissingRequiredFieldError
from utils.language_utils import get_default_language
from utils.check_required_files import check_interactives
from chapters.models import Chapter

class ChaptersLoader(TranslatableModelLoader):
    """Custom loader for loading chapters."""

    extra_converter_templates_directory = "chapter"

    def __init__(self, factory, chapter_number, **kwargs):
        """Create the loader for loading a Chapter.

        Args:
            factory (LoaderFactory): Object for creating other loaders.
        """
        super().__init__(**kwargs)
        self.factory = factory
        self.chapter_slug = self.content_path
        self.chapter_number = chapter_number

    @transaction.atomic
    def load(self):
        """Load the content for a chapter.

        Raises:
            MissingRequiredFieldError: When a config (yaml) file is missing a required
                field.
        """
        chapter_structure = self.load_yaml_file(self.structure_file_path)

        sections = chapter_structure.get("sections", None)
        if sections is None:
            raise MissingRequiredFieldError(
                self.structure_file_path,
                ["sections"],
                "Chapter"
            )

        chapter_translations = self.get_blank_translation_dictionary()

        introduction_filename = "{}.md".format(self.chapter_slug)
        introduction_translations = self.get_markdown_translations(introduction_filename)
        for language, content in introduction_translations.items():
            chapter_translations[language]["introduction"] = content.html_string
            chapter_translations[language]["name"] = content.title

        chapter_icon = chapter_structure.get("icon", None)
        if chapter_icon is None:
            raise MissingRequiredFieldError(
                self.structure_file_path,
                ["icon"],
                "Chapter"
            )
        else:
            # TODO: Check icon exists here before path modification
            # Remove directory and extension as svg templatetag automatically adds these
            chapter_icon = chapter_icon[4:-4]

        video = chapter_structure.get("video", "")
        if video != "" and "vimeo" not in video:
            raise ValueError("Video must be a Vimeo video.")

        # Create or update chapter object and save to the db
        chapter, created = Chapter.objects.update_or_create(
            number=self.chapter_number,
            defaults={
                'slug': self.chapter_slug,
                'icon': chapter_icon,
                'video': video
            }
        )

        self.populate_translations(chapter, chapter_translations)
        self.mark_translation_availability(chapter, required_fields=["name", "introduction"])
        chapter.save()

        if created:
            term = 'Created'
        else:
            term = 'Updated'
        self.log(f'{term} chapter: {chapter.name}')

        check_interactives(
            introduction_translations[get_default_language()].required_files["interactives"],
            self.structure_file_path,
            chapter,
        )

        # Load chapter sections
        content_path, structure_filename = os.path.split(sections)
        self.factory.create_chapter_section_loader(
            chapter,
            base_path=self.base_path,
            content_path=os.path.join(self.content_path, content_path),
            structure_filename=structure_filename
        ).load()


#  Copyright (c) Microsoft Corporation.
#  Licensed under the MIT license.
import subprocess as sp
import os
import argparse
import glob

CODE_ROOT = "./"
PYTHON_EXE = "python"

def eval(prediction_path, output_path, data_type):
    os.chdir(CODE_ROOT)
    ref_path = f'{CODE_ROOT}/data/test.refs.txt'
    key_path = f'{CODE_ROOT}/data/keys.2k.txt'
    cmd = ['dstc.py',
           prediction_path,
           '--refs',
           ref_path,
           '--keys',
           key_path,
           '--clean']
    cmd = " ".join(cmd) #% {"CODE_ROOT": CODE_ROOT}
    print(cmd)

    ret = sp.run([PYTHON_EXE] + cmd.split(" "), stdout=sp.PIPE, stderr=sp.STDOUT)

    with open(output_path, "w", encoding="utf-8") as out_f:
        out_f.write(ret.stdout.decode("utf-8"))

parser = argparse.ArgumentParser(description="""
Given `input_dir`, create eval directory under it.
for each *.predicted.txt under `input_dir`, evaluate it, push the evaluation result to files under eval directory.
The file name is *.eval.txt
""")
parser.add_argument("--input_dir", type=str, default="./")
parser.add_argument("--data_type", type=str, default="test", help="could be 'valid' or 'test' ")

args = parser.parse_args()
assert args.data_type in ("valid", "test")

output_dir = os.path.join(args.input_dir, "eval")
if not os.path.exists(output_dir):
    os.makedirs(output_dir)

resp_dir = os.path.join(args.input_dir, "resp")
if not os.path.exists(resp_dir):
    os.makedirs(resp_dir)

for prediction_path in glob.glob(os.path.join(args.input_dir, "*.resp.txt")):
    resp_path = os.path.join(resp_dir , os.path.basename(prediction_path))
    cmd = ['less',
           prediction_path,
           '|grep -v "^Val"',
           '|sed \'/^$/d\'',
           '>',
           resp_path]
    cmd = " ".join(cmd)
    print(cmd)
    ret = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.STDOUT)
    output = ret.communicate()[0]
    print(output)
    output_name = os.path.basename(prediction_path).replace(".resp.txt", ".eval.txt")
    output_path = os.path.join(output_dir, output_name)
    eval(resp_path, output_path, args.data_type)


from pythonosc import udp_client

import time

class Light:
  def __init__(self,host=None,port=None):
    self.host = host or "127.0.0.1"
    self.port = port or 11000
    self.client = udp_client.SimpleUDPClient(self.host, self.port)

  def goToSeq(self,seqName,time=0):
    print("go to ",seqName, "in ",time, self.host,self.port)
    self.sendOSC('/sequencePlayer/goToStateNamed',[seqName,time])

  def sendOSC(self,addr,args):
    try:
      self.client.send_message(addr, args)
    except:
      print ("no client connected")

if __name__=="__main__":
  l = Light()
  i=100
  delay = 1
  while i>0:

    l.goToSeq("1",delay)
    time.sleep(delay)
    l.goToSeq("base",delay)
    time.sleep(delay)
    i-=1


## @package gdayf.persistence.persistencehandler
# Define all objects, functions and structures related to physically store information on persistence system
#  on an unified way

'''
 * This file is part of the gDayF AutoML Core Framework project
 * distribution (https://github.com/e2its/gdayf-core).
 * Copyright (c) 2016-2019 Jose Luis Sanchez del Coso <e2its.es@gmail.com>.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 3.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 ** Written by Jose L. Sanchez <e2its.es@gmail.com>, 2016-2019
'''

from json import dump, dumps, load, loads
from collections import OrderedDict
import codecs
from os import path as ospath, chmod, remove
from os import path
from shutil import rmtree, copy2
from pathlib import Path
from gdayf.common.utils import hash_key
from gdayf.conf.loadconfig import LoadConfig
from gdayf.common.utils import get_model_fw
from gdayf.common.storagemetadata import StorageMetadata
import gzip
import mimetypes
from hdfs import InsecureClient as Client, HdfsError
from pymongo import MongoClient
from pymongo.errors import *
#from pymongo import *
from copy import deepcopy
import bson
from bson.codec_options import CodecOptions

## Class to manage trasient information between all persistence options and models on an unified way
class PersistenceHandler(object):
    ## Class Constructor
    # @param e_c context pointer
    def __init__(self, e_c):
        self._ec = e_c
        self._config = self._ec.config.get_config()['storage']
        self._persistence = self._ec.config.get_config()['persistence']

    ## Method used to store a file on one persistence system ['localfs', ' hdfs']
    # using mmap structure to manage multi-persistence features
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param filename file full path string
    # @return global_op state (0 success) (n number of errors)
    def store_file(self, storage_json, filename):
        global_op = 0

        '''try:
            file = open(filename, 'rb')
            mmap_ = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ)
        except IOError:
            return 1'''

        for each_storage_type in storage_json:
            if each_storage_type['type'] == 'localfs':
                result, each_storage_type['hash_value'] = self._store_file_to_localfs(each_storage_type, filename)
                global_op += result
            elif each_storage_type['type'] == 'hdfs':
                result, each_storage_type['hash_value'] = self._store_file_to_hdfs(each_storage_type, filename)
                global_op += result
        '''mmap_.close()'''
        return global_op

    ## Protected method used to store a file on ['localfs']
    # using mmap structure to manage multi-persistence features
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param filename file to store
    # @returns status (0,1) (hash_key)
    def _store_file_to_hdfs(self, storage_json, filename):
        try:
            client = Client(url=self._config['hdfs']['url'])
        except HdfsError as hexecution_error:
            print(repr(hexecution_error))
            return 1, None
        except IOError as iexecution_error:
            print(repr(iexecution_error))
            return 1, None
        except OSError as oexecution_error:
            print(repr(oexecution_error))
            return 1, None
        try:
            self._mkdir_hdfs(path=path.dirname(storage_json['value']),
                             grants=self._config['grants'],
                             client=client)
            #client.write(storage_json['value'], data=mmap_, encoding='utf-8', overwrite=True)
            client.upload(storage_json['value'], filename, overwrite=True)

            '''with client.write(storage_json['value'],overwrite=True) as wfile:
                mmap_.seek(0)
                iterator = 0
                while iterator < mmap_.size():
                    wfile.write(mmap_.read())
                    wfile.flush()
                    iterator += 1
                wfile.close()'''

        except HdfsError as hexecution_error:
            print(repr(hexecution_error))
            return 1, None
        except IOError as iexecution_error:
            print(repr(iexecution_error))
            return 1, None
        except OSError as oexecution_error:
            print(repr(oexecution_error))
            return 1, None
        finally:
            del client

        return 0, None

    ## Protected method used to store a file on ['hdfs']
    #Not implemented yet !!
    # using mmap structure to manage multi-persistence features
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param mmap mmap structure containing the file to store
    # @returns status (0,1) (hash_key)
    def _store_file_to_localfs(self, storage_json, filename):
        if not ospath.exists(path=storage_json['value']):
            try:
                self._mkdir_localfs(path=path.dirname(storage_json['value']), grants=int(self._config['grants'], 8))
                copy2(filename, storage_json['value'])
                '''with open(storage_json['value'], 'wb') as wfile:
                    mmap_.seek(0)
                    iterator = 0
                    while iterator < mmap_.size():
                        wfile.write(mmap_.read())
                        wfile.flush()
                        iterator += 1
                    wfile.close()'''
                chmod(storage_json['value'], int(self._config['grants'], 8))
            except IOError:
                return 1, None

        return 0, hash_key(hash_type=storage_json['hash_type'], filename=storage_json['value'])

    ## Method used to remove a file on one persistence system ['localfs',' hdfs']
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param ar_metadata model_structure
    # @return global_op state (0 success) (n number of errors)
    def remove_file(self, load_path):
        global_op = 0

        storage_metadata = StorageMetadata(self._ec)
        for each_storage_type in load_path:
            if each_storage_type['type'] == 'localfs':
                result, storage = self._remove_file_to_localfs(each_storage_type)
                if storage is not None:
                    storage_metadata.append(storage)
                global_op += result
            elif each_storage_type['type'] == 'hdfs':
                result, storage = self._remove_file_to_hdfs(each_storage_type)
                if storage is not None:
                    storage_metadata.append(storage)
                global_op += result
        return global_op, storage_metadata.copy()

    ## Method used to remove a file on one persistence system ['hdfs']
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @return global_op state (0 success) (n number of errors)
    def _remove_file_to_hdfs(self,storage_json):
        path = storage_json['value']
        url_beginning = path.find('//') + 2
        url_ending = path.find('/', url_beginning)
        path = path[url_ending:]

        try:
            client = Client(url=self._config['hdfs']['url'])
        except HdfsError as hexecution_error:
            print(repr(hexecution_error))
            return 1, None
        except IOError as iexecution_error:
            print(repr(iexecution_error))
            return 1, None
        except OSError as oexecution_error:
            print(repr(oexecution_error))
            return 1, None
        try:
            if client.delete(hdfs_path=path, recursive=True):
                return 0, None
            else:
                return 1, storage_json

        except HdfsError as hexecution_error:
            print(repr(hexecution_error))
            return 1, storage_json
        except IOError as iexecution_error:
            print(repr(iexecution_error))
            return 1, storage_json
        except OSError as oexecution_error:
            print(repr(oexecution_error))
            return 1, storage_json
        finally:
            del client

    ## Method used to remove a file on one persistence system ['localfs']
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @return boolean: op_success, path value [None if deleted or not exists]
    @staticmethod
    def _remove_file_to_localfs(storage_json):
        if not ospath.exists(path=storage_json['value']):
            return 0, None
        else:
            try:
                if ospath.isdir(storage_json['value']):
                    rmtree(storage_json['value'])
                else:
                    remove(storage_json['value'])
                return 0, None
            except OSError:
                return 1, storage_json

    ## Method used to store a json on all persistence system ['localfs', ' hdfs', ' mongoDB']
    # oriented to store full Analysis_results json but useful on whole json
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param ar_json file ArMetadata Class or OrderedDict() compatible object
    # @param other file OrderedDict() compatible object to be stored
    # @return global_op status (0 success) (n number of errors)
    def store_json(self, storage_json, ar_json, other=None):
        '''assert isinstance(storage_json, StorageMetadata)'''
        global_op = 0
        for each_storage_type in storage_json:
            if each_storage_type['type'] == 'localfs':
                if other is None:
                    global_op += self._store_json_to_localfs(each_storage_type, ar_json)
                else:
                    global_op += self._store_json_to_localfs(each_storage_type, other)
            elif each_storage_type['type'] == 'hdfs':
                if other is None:
                    global_op += self._store_json_to_hdfs(each_storage_type, ar_json)
                else:
                    global_op += self._store_json_to_hdfs(each_storage_type, other)
            elif each_storage_type['type'] == 'mongoDB':
                if other is None:
                    global_op += self._store_json_to_mongoDB(each_storage_type, ar_json)
                else:
                    global_op += self._store_other_to_mongoDB(each_storage_type, other)
        return global_op

    ## Protected method used to store a json on ['localfs'] persistence system
    # oriented to store full Analysis_results json but useful on whole json
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param ar_json file ArMetadata Class or OrderedDict() compatible object
    # @param prediction file OrderedDict() compatible object
    # @return global_op status (0 success) (1 error)
    def _store_json_to_localfs(self, storage_json, ar_json):
        compress = self._persistence['compress_json']
        #if not ospath.exists(storage_json['value']):
        try:
            self._mkdir_localfs(path=path.dirname(storage_json['value']), grants=int(self._config['grants'], 8))
            if compress:
                file = gzip.GzipFile(storage_json['value'], 'w')
                json_str = dumps(ar_json, indent=4)
                json_bytes = json_str.encode('utf-8')
                file.write(json_bytes)
            else:
                # file = open(storage_json['value'], 'w')
                file = codecs.open(storage_json['value'], 'w', encoding='utf-8')
                dump(ar_json, file, indent=4, ensure_ascii=False)
            file.close()
            chmod(storage_json['value'], int(self._config['grants'], 8))
            return 0
        except IOError as iexecution_error:
            print(repr(iexecution_error))
            return 1

    ## Method used to store a json on ['hdfs'] persistence system
    # oriented to store full Analysis_results json but useful on whole json
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param ar_json file ArMetadata Class or OrderedDict() compatible object
    # @param client Cliente HDFS
    # @param prediction file OrderedDict() compatible object
    # @return operation status (0 success) (1 error)
    def _store_json_to_hdfs(self, storage_json, ar_json, client=None):
        remove_client = False
        if client is None:
            client = Client(url=self._config['hdfs']['url'])
            remove_client = True
        compress = self._persistence['compress_json']
        #if not ospath.exists(storage_json['value']):
        try:
            self._mkdir_hdfs(path=path.dirname(storage_json['value']),
                             grants=self._config['grants'],
                             client=client)
            if compress:
                json_str = dumps(ar_json, indent=4)
                json_bytes = json_str.encode('utf-8')
                client.write(storage_json['value'],
                             data=gzip.compress(json_bytes),
                             overwrite=True)
            else:
                client.write(storage_json['value'], data=dumps(ar_json, indent=4, ensure_ascii=False),
                             encoding='utf-8', overwrite=True)
            return 0
        except HdfsError as hexecution_error:
            print(repr(hexecution_error))
            return 1
        except IOError as iexecution_error:
            print(repr(iexecution_error))
            return 1
        except OSError as oexecution_error:
            print(repr(oexecution_error))
            return 1
        finally:
            if remove_client:
                del client

    ## Protected method used to store an ar_json on ['mongoDB'] persistence system
    # oriented to store full Analysis_results json but useful on whole json
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param ar_json file ArMetadata Class or OrderedDict() compatible object
    # @param client Cliente MongoClient()
    # @return operation status (0 success) (1 error)
    def _store_json_to_mongoDB(self, storage_json, ar_json, client=None):
        remove_client = False
        if client is None or not isinstance(client(MongoClient)):
            try:
                client = MongoClient(host=self._config['mongoDB']['url'],
                                     port=int(self._config['mongoDB']['port']),
                                     document_class=OrderedDict)
                remove_client = True
            except ConnectionFailure as cexecution_error:
                print(repr(cexecution_error))
                return 1
        try:
            description = Path(storage_json['value']).parts
            db = client[self._config['mongoDB']['value']]
            collection = db[description[1]]
            model_id = ar_json['model_parameters'][get_model_fw(ar_json)]['parameters']['model_id']['value']
            filter_cond = "model_parameters." + get_model_fw(ar_json) + ".parameters.model_id.value"
            cond = [{filter_cond: model_id}, {"type": ar_json['type']},
                    {"model_id": ar_json['model_id']},  {"timestamp": ar_json['timestamp']}]
            query = {"$and": cond}

            count = collection.find(query).count()
            new_ar_json = OrderedDict(ar_json)
            if count == 1:
                collection.delete_one(query)
                collection.insert(new_ar_json, check_keys=False)
                return 0
            elif count == 0:
                collection.insert(new_ar_json, check_keys=False)
                return 0
            else:
                print("Trace: Duplicate Model %s" % model_id)
                return 1
        finally:
            if remove_client:
                client.close()

    ## Protected method used to store a dayf compatible_json on ['mongoDB'] persistence system
    # oriented to store full Analysis_results json but useful on whole json
    # @param self object pointer
    # @param storage_json (list of storagemetadata objects or OrderedDict() compatible objects)
    # @param other file  dayf compatible_json OrderedDict() object
    # @param client Cliente MongoClient()
    # @return operation status (0 success) (1 error)
    def _store_other_to_mongoDB(self, storage_json, other, client=None):

        remove_client = False
        if client is None or not isinstance(client(MongoClient)):
            try:
                client = MongoClient(host=self._config['mongoDB']['url'],
                                     port=int(self._config['mongoDB']['port']),
                                     document_class=OrderedDict)
                remove_client = True
            except ConnectionFailure as cexecution_error:
                print(repr(cexecution_error))
                return 1
        try:
            db = client[self._config['mongoDB']['value']]
            collection = db[self._ec.get_id_user() + '_' + storage_json['value']]
            new_ar_json = OrderedDict(other)
            try:
                collection.insert(new_ar_json, check_keys=False)
                return 0
            except Exception as execution_error:
                print(repr(execution_error))
                return 1
        finally:
            if remove_client:
                client.close()

    ## Method used to recover an experiment as [ar_metadata]
    # oriented to store full Analysis_results json but useful on whole json
    # @param self object pointer
    # @param client Cliente MongoClient()
    # @return [ArMetadata]
    def recover_experiment_mongoDB(self, client=None):
        execution_list = list()
        remove_client = False
        if client is None or not isinstance(client(MongoClient)):
            try:
                client = MongoClient(host=self._config['mongoDB']['url'],
                                     port=int(self._config['mongoDB']['port']),
                                     document_class=OrderedDict)
                remove_client = True
            except ConnectionFailure as cexecution_error:
                print(repr(cexecution_error))
                return execution_list
        try:
            db = client[self._config['mongoDB']['value']]
            collection = db[self._ec.get_id_user()]
            query = {"$and": [{"model_id": self._ec.get_id_analysis()}, {"type": "train"}]}
            for element in collection.find(query):
                execution_list.append(element)
            for element in execution_list:
                element.pop('_id')
            #print(execution_list)
        except PyMongoError as pexecution_error:
            print(repr(pexecution_error))
        finally:
            if remove_client:
                client.close()
            return deepcopy(execution_list)

    ## Method used to check and make directory os similar path structures
    # on all persistence system ['localfs', ' hdfs', ' mongoDB'] over agnostic way
    # @param self object pointer
    # @param type ['localfs', ' hdfs', ' mongoDB']
    # @param path directory or persistence structure to be created
    # @param grants on a 0o#### format (octalpython format)
    # @return operation status (0 success) (1 error)
    def mkdir(self, type, path, grants):
        if type == 'localfs':
            return self._mkdir_localfs(path=path, grants=int(grants, 8))
        elif type == 'hdfs':
            return self._mkdir_hdfs(path=path, grants=grants)
        elif type == 'mongoDB':
            return self._mkdir_mongoDB(path=path, grants=grants)

    ## Static protected  method used to check and make directory
    # on ['localfs']
    # @param self object pointer
    # @param path directory or persistence structure to be created
    # @param grants on a 0o#### format (octalpython format)
    # @return operation status (0 success) (1 error)
    @staticmethod
    def _mkdir_localfs(path, grants):
        try:
            Path(path).mkdir(mode=grants, parents=True, exist_ok=True)
            return 0
        except IOError:
            return 1

    ## Static protected method used to check and make directory
    # on ['hdfs']
    # Not implemented yet!
    # @param self object pointer
    # @param path directory or persistence structure to be created
    # @param grants on a 0o#### format (octalpython format)
    # @param client Cliente HDFS
    # @return operation status (0 success) (1 error)
    def _mkdir_hdfs(self, path, grants, client=None):
        remove_client = False
        if client is None:
            client = Client(url=self._config['hdfs']['url'])
            remove_client = True
        try:
            if client.status(hdfs_path=path, strict=False) is None:
                client.makedirs(hdfs_path=path, permission=grants)
            return 0
        except HdfsError as hexecution_error:
            print(repr(hexecution_error))
            return 1
        except IOError as iexecution_error:
            print(repr(iexecution_error))
            return 1
        except OSError as oexecution_error:
            print(repr(oexecution_error))
            return 1
        finally:
            if remove_client:
                del client

    ## Static protected method used to check and make directory
    # on ['mongoDB']
    # Not necessary throught pymongo!
    # @param self object pointer
    # @param path directory or persistence structure to be created
    # @param grants on a 0o#### format (octalpython format)
    # @return operation status (0 success) (1 error)
    @staticmethod
    def _mkdir_mongoDB(path, grants):
        try:
            return 0
        except IOError:
            return 1

    ## Method base to get an ArMetadata Structure from file
    # @param self object pointer
    # @param path FilePath for localfs or hdfs and /user_id/workflow_id/analysis_id/model_id for MongoDB
    # @return operation status (0 success /1 error, ArMetadata/None)
    def get_ar_from_engine(self, path):
        found = False
        for storage in ['localfs', 'hdfs', 'mongoDB']:
            if storage == 'localfs' and not found:
                if ospath.exists(path):
                    _, type = mimetypes.guess_type(path)
                    if type == 'gzip':
                        file = gzip.GzipFile(filename=path, mode='r')
                        json_bytes = file.read()
                        json_str = json_bytes.decode('utf-8')
                        ar_metadata = loads(json_str, object_hook=OrderedDict)
                    else:
                        #Bug pending
                        file = open(path, 'r')
                        ar_metadata = load(file, object_hook=OrderedDict)
                    file.close()
                    return 0, ar_metadata
            elif storage == 'hdfs' and not found:
                url = self._config[storage]['url']
                client = Client(url=url)
                remove_client = True
                try:
                    if client.status(hdfs_path=path, strict=False) is not None:
                        _, type = mimetypes.guess_type(path)
                        if type == 'gzip':
                            with client.read(path) as file_hdfs:
                                file = gzip.GzipFile(fileobj=file_hdfs)
                                json_bytes = file.read()
                                json_str = json_bytes.decode('utf-8')
                                ar_metadata = loads(json_str, object_hook=OrderedDict)
                                file.close()
                        else:
                            with client.read(path) as file_hdfs:
                                json_bytes = file_hdfs.read()
                                json_str = json_bytes.decode('utf-8')
                                ar_metadata = loads(json_str, object_hook=OrderedDict)
                        return 0, ar_metadata
                except HdfsError as hexecution_error:
                    print(repr(hexecution_error))
                    return 1, repr(hexecution_error)
                except IOError as iexecution_error:
                    print(repr(iexecution_error))
                    return 1, repr(iexecution_error)
                except OSError as oexecution_error:
                    print(repr(oexecution_error))
                    return 1, repr(oexecution_error)
                finally:
                    if remove_client:
                        del client
            elif storage == 'mongoDB' and not found:
                try:
                    client = MongoClient(host=self._config['mongoDB']['url'],
                                         port=int(self._config['mongoDB']['port']),
                                         document_class=OrderedDict)
                    remove_client = True
                except ConnectionFailure as cexecution_error:
                    print(repr(cexecution_error))
                    return 1, repr(cexecution_error)
                try:
                    db = client[self._config['mongoDB']['value']]
                    description = Path(path).parts
                    if description[1] is not None \
                            and description[2] is not None \
                            and description[3] is not None \
                            and description[4] is not None \
                            and description[1] in db.collection_names():

                        collection = db[description[1]]
                        query1 = {"$and": [{"model_id": description[3]},
                                           {"workflow_id": description[2]},
                                           {'type': 'train'}]
                                  }
                        for element in collection.find(query1):
                            # Return the first element found
                            if element['model_parameters'][get_model_fw(element)]['parameters']['model_id']['value'] \
                                    == description[4]:
                                element.pop('_id')
                                print(element)
                                return 0, element
                        return 1, None

                    else:
                        return 1, None
                except PyMongoError as pexecution_error:
                    print(repr(pexecution_error))
                    return 1, repr(pexecution_error)
                finally:
                    if remove_client:
                        client.close()

        return 1, None


"""A collection of classes intended to represent one MuseScore file.

The classes build on each other hierarchically. The class hierarchy:

.. code ::

    MscoreFile
        MscoreXmlTree
            MscoreStyleInterface
            MscoreLyricsInterface
"""

import fnmatch
import os
import shutil
import string
import tempfile
import zipfile
from typing import List, Optional

import lxml
import lxml.etree  # Needed for type hints

from mscxyz.utils import mscore, re_open

def list_scores(
    path: str, extension: str = "both", glob: Optional[str] = None
) -> List[str]:
    """List all scores in path.

    :param path: The path so search for score files.
    :param extension: Possible values: “both”, “mscz” or “mscx”.
    :param glob: A glob string, see fnmatch
    """
    if not glob:
        if extension == "both":
            glob = "*.msc[xz]"
        elif extension in ("mscx", "mscz"):
            glob = "*.{}".format(extension)
        else:
            raise ValueError(
                "Possible values for the argument “extension” "
                "are: “both”, “mscx”, “mscz”"
            )
    if os.path.isfile(path):
        if fnmatch.fnmatch(path, glob):
            return [path]
        else:
            return []
    out: List[str] = []
    for root, _, scores in os.walk(path):
        for score in scores:
            if fnmatch.fnmatch(score, glob):
                scores_path = os.path.join(root, score)
                out.append(scores_path)
    out.sort()
    return out

def list_zero_alphabet() -> List[str]:
    """Build a list: 0, a, b, c etc."""
    score_dirs = ["0"]
    for char in string.ascii_lowercase:
        score_dirs.append(char)
    return score_dirs

###############################################################################
# Class hierarchy level 1
###############################################################################

class MscoreFile(object):
    """This class holds basic file properties of the MuseScore score file.

    :param relpath: The relative (or absolute) path of a MuseScore
        file.
    """

    errors: List[Exception]
    """A list to store errors."""

    relpath: str
    """The relative path of the score file, for example:
    ``files_mscore2/simple.mscx``.
    """

    abspath: str
    """The absolute path of the score file, for example:
    ``/home/jf/test/files_mscore2/simple.mscx``."""

    extension: str
    """The extension (``mscx`` or ``mscz``) of the score file, for
    example: ``mscx``."""

    relpath_backup: str

    dirname: str
    """The name of the containing directory of the MuseScore file, for
    example: ``files_mscore2``."""

    filename: str
    """The filename of the MuseScore file, for example:
    ``simple.mscx``."""

    basename: str
    """The basename of the score file, for example: ``simple``."""

    loadpath: str
    """The load path of the score file"""

    def __init__(self, relpath: str):
        self.errors = []
        self.relpath = relpath
        self.abspath = os.path.abspath(relpath)
        self.extension = relpath.split(".")[-1].lower()
        self.relpath_backup = relpath.replace(
            "." + self.extension, "_bak." + self.extension
        )
        self.dirname = os.path.dirname(relpath)
        self.filename = os.path.basename(relpath)
        self.basename = self.filename.replace(".mscx", "")

        if self.extension == "mscz":
            self.loadpath = self._unzip(self.abspath)
        else:
            self.loadpath = self.abspath

    @staticmethod
    def _unzip(abspath: str) -> str:
        tmp_zipdir = tempfile.mkdtemp()
        zip_ref = zipfile.ZipFile(abspath, "r")
        zip_ref.extractall(tmp_zipdir)
        zip_ref.close()
        con = os.path.join(tmp_zipdir, "META-INF", "container.xml")
        container_info = lxml.etree.parse(con)
        mscx = container_info.xpath("string(/container/rootfiles/rootfile/@full-path)")
        return os.path.join(tmp_zipdir, str(mscx))

    def backup(self):
        """Make a copy of the MuseScore file."""
        shutil.copy2(self.relpath, self.relpath_backup)

    def export(self, extension: str = "pdf"):
        """Export the score to the specifed file type.

        :param extension: The extension (default: pdf)
        """
        score = self.relpath
        mscore(
            ["--export-to", score.replace("." + self.extension, "." + extension), score]
        )

###############################################################################
# Class hierarchy level 2
###############################################################################

class MscoreXmlTree(MscoreFile):
    """XML tree manipulation

    :param relpath: The relative (or absolute) path of a MuseScore file.
    """

    def __init__(self, relpath: str):
        super(MscoreXmlTree, self).__init__(relpath)
        try:
            self.xml_tree = lxml.etree.parse(self.loadpath)
        except lxml.etree.XMLSyntaxError as e:
            self.errors.append(e)
        else:
            self.xml_root = self.xml_tree.getroot()
            musescore = self.xml_tree.xpath("/museScore")
            version = musescore[0].get("version")
            self.version_major = int(version.split(".")[0])
            """The major MuseScore version, for example 2 or 3"""
            self.version = float(version)
            """The MuseScore version, for example 2.03 or 3.01"""

    def add_sub_element(self, root_tag, tag, text: str):
        tag = lxml.etree.SubElement(root_tag, tag)
        tag.text = text

    def strip_tags(self, *tag_names: str):
        """Delete / strip some tag names."""
        lxml.etree.strip_tags(self.xml_tree, tag_names)

    def remove_tags_by_xpath(self, *xpath_strings: str):
        """Remove tags by xpath strings.

        :param xpath_strings: A xpath string.

        .. code:: Python

            tree.remove_tags_by_xpath(
                '/museScore/Score/Style', '//LayoutBreak', '//StemDirection'
            )

        """
        for xpath_string in xpath_strings:
            for rm in self.xml_tree.xpath(xpath_string):
                rm.getparent().remove(rm)

    def merge_style(self, styles: str):
        """Merge styles into the XML tree.

        :param styles: The path of the style file or a string containing
          the XML style markup.

        ``styles`` may not contain surrounding ``<Style>`` tags. This input is
        valid:

        .. code :: XML

            <TextStyle>
              <halign>center</halign>
              <valign>bottom</valign>
              <xoffset>0</xoffset>
              <yoffset>-1</yoffset>
              <offsetType>spatium</offsetType>
              <name>Form Section</name>
              <family>Alegreya Sans</family>
              <size>12</size>
              <bold>1</bold>
              <italic>1</italic>
              <sizeIsSpatiumDependent>1</sizeIsSpatiumDependent>
              <frameWidthS>0.1</frameWidthS>
              <paddingWidthS>0.2</paddingWidthS>
              <frameRound>0</frameRound>
              <frameColor r="0" g="0" b="0" a="255"/>
              </TextStyle>

        This input is invalid:

        .. code :: XML

            <?xml version="1.0"?>
            <museScore version="2.06">
              <Style>
                <TextStyle>
                  <halign>center</halign>
                  <valign>bottom</valign>
                  <xoffset>0</xoffset>
                  <yoffset>-1</yoffset>
                  <offsetType>spatium</offsetType>
                  <name>Form Section</name>
                  <family>Alegreya Sans</family>
                  <size>12</size>
                  <bold>1</bold>
                  <italic>1</italic>
                  <sizeIsSpatiumDependent>1</sizeIsSpatiumDependent>
                  <frameWidthS>0.1</frameWidthS>
                  <paddingWidthS>0.2</paddingWidthS>
                  <frameRound>0</frameRound>
                  <frameColor r="0" g="0" b="0" a="255"/>
                  </TextStyle>
                </Style>
              </museScore>

        """
        if os.path.exists(styles):
            style = lxml.etree.parse(styles).getroot()
        else:
            # <?xml ... tag without encoding to avoid this error:
            # ValueError: Unicode strings with encoding declaration are
            # not supported. Please use bytes input or XML fragments without
            # declaration.
            pre = '<?xml version="1.0"?><museScore version="2.06"><Style>'
            post = "</Style></museScore>"
            style = lxml.etree.XML(pre + styles + post)

        for score in self.xml_tree.xpath("/museScore/Score"):
            score.insert(0, style[0])

    def clean(self):
        """Remove the style, the layout breaks, the stem directions and the
        ``font``, ``b``, ``i``, ``pos``, ``offset`` tags"""
        self.remove_tags_by_xpath(
            "/museScore/Score/Style", "//LayoutBreak", "//StemDirection"
        )
        self.strip_tags("font", "b", "i", "pos", "offset")

    def save(self, new_name: str = "", mscore: bool = False):
        """Save the MuseScore file.

        :param new_name: Save the MuseScore file under a new name.
        :param mscore: Save the MuseScore file by opening it with the
          MuseScore executable and save it there.
        """
        if new_name:
            filename = new_name
        elif self.extension == "mscz":
            filename = self.loadpath
        else:
            filename = self.relpath
        if not self.errors:
            # To get the same xml tag structure as the original score file
            # has.
            for xpath in (
                "//LayerTag",
                "//metaTag",
                "//font",
                "//i",
                "//evenFooterL",
                "//evenFooterC",
                "//evenFooterR",
                "//oddFooterL",
                "//oddFooterC",
                "//oddFooterR",
                "//chord/name",
                "//chord/render",
                "//StaffText/text",
                "//Jump/continueAt",
            ):

                for tag in self.xml_tree.xpath(xpath):
                    if not tag.text:
                        tag.text = ""

            score = open(filename, "w")
            score.write('<?xml version="1.0" encoding="UTF-8"?>\n')
            score.write(
                lxml.etree.tostring(self.xml_root, encoding="UTF-8").decode("utf-8")
            )
            score.write("\n")
            score.close()

            if self.extension == "mscz":
                # Need some tmp directory cleanup for working with mscz files
                tmpdir = os.path.dirname(filename)
                zip_ref = zipfile.ZipFile(self.abspath, "w")
                for root, _, files in os.walk(tmpdir):
                    for file in files:
                        arcname = os.path.join(root.replace(tmpdir, ""), file)
                        zip_ref.write(os.path.join(root, file), arcname)

            if mscore:
                re_open(filename)

###############################################################################
# Class hierarchy level 3
###############################################################################

class MscoreStyleInterface(MscoreXmlTree):
    """
    Interface specialized for the style manipulation.

    :param relpath: The relative (or absolute) path of a MuseScore file.
    """

    def __init__(self, relpath: str):
        super(MscoreStyleInterface, self).__init__(relpath)
        styles = self.xml_tree.xpath("/museScore/Score/Style")
        if styles:
            self.style = styles[0]
            """The ``/museScore/Score/Style`` element object, see
            https://lxml.de/tutorial.html#the-element-class
            """
        else:
            self.style = self._create_parent_style()

    def _create_parent_style(self):
        score = self.xml_tree.xpath("/museScore/Score")
        return lxml.etree.SubElement(score[0], "Style")

    def _create(self, tag: str) -> lxml.etree.Element:
        """
        :param tag: Nested tags are supported, for example ``TextStyle/halign``
        """
        tags = tag.split("/")
        parent = self.style
        for tag in tags:
            element = parent.find(tag)
            if element is None:
                parent = lxml.etree.SubElement(parent, tag)
            else:
                parent = element
        return parent

    def get_element(
        self, element_path: str, create: bool = False
    ) -> lxml.etree.Element:
        """
        Get a lxml element which is parent to the ``Style`` tag.

        :param element_path: see
          http://lxml.de/tutorial.html#elementpath
        :param create: Create the element if not present in the parent
          ``Style`` tag.

        Example code:

        .. code:: Python

            # Set attributes on a maybe non-existent style tag.
            # <measureNumberOffset x="0.5" y="-2"/>
            test = MscoreStyleInterface('text.mscx')
            element = test.get_element('measureNumberOffset', create=True)
            element.attrib['x'] = '0.5'
            element.attrib['y'] = '-2'
            test.save()
        """
        element = self.style.find(element_path)
        if element is None and create:
            element = self._create(element_path)
        return element

    def get_value(self, element_path: str) -> str:
        """
        Get the value (text) of a style tag.

        :param element_path: see
          http://lxml.de/tutorial.html#elementpath
        """
        element = self.get_element(element_path)
        return element.text

    def set_attributes(self, element_path: str, attributes: dict) -> lxml.etree.Element:
        """Set attributes on a style child tag.

        :param element_path: see
          http://lxml.de/tutorial.html#elementpath
        """
        element = self.get_element(element_path, create=True)
        for name, value in attributes.items():
            element.attrib[name] = str(value)
        return element

    def set_value(self, element_path: str, value: str):
        """
        :param element_path: see
          http://lxml.de/tutorial.html#elementpath
        """
        element = self.style.find(element_path)
        if element is None:
            element = self._create(element_path)
        element.text = str(value)

    def _get_text_style_element(self, name: str) -> lxml.etree.Element:
        if self.version_major != 2:
            raise ValueError(
                "This operation is only allowed for MuseScore 2 score files"
            )
        xpath = '//TextStyle/name[contains(., "{}")]'.format(name)
        child = self.xml_tree.xpath(xpath)
        if child:
            return child[0].getparent()
        else:
            el_text_style = lxml.etree.SubElement(self.style, "TextStyle")
            el_name = lxml.etree.SubElement(el_text_style, "name")
            el_name.text = name
            return el_text_style

    def get_text_style(self, name: str) -> dict:
        """Get text styles. Only MuseScore2!

        :param name: The name of the text style.
        """
        text_style = self._get_text_style_element(name)
        out = {}
        for child in text_style.iterchildren():
            out[child.tag] = child.text
        return out

    def set_text_style(self, name: str, values: dict):
        """Set text styles. Only MuseScore2!

        :param name: The name of the text style.
        :param values: A dictionary. The keys are the tag names, values are
          the text values of the child tags, for example
          ``{size: 14, bold: 1}``.
        """
        text_style = self._get_text_style_element(name)
        for element_name, value in values.items():
            el = text_style.find(element_name)
            if el is None:
                el = lxml.etree.SubElement(text_style, element_name)
            el.text = str(value)


import glob
import json
import os
from urllib.request import urlretrieve
import re

BASE_URL = 'http://projects.bobbelderbos.com/pcc/omdb/'
MOVIES = ('bladerunner2049 fightclub glengary '
          'horrible-bosses terminator').split()
TMP = '/tmp'
# TMP = os.getcwd()

# print(TMP)

# little bit of prework (yes working on pip installables ...)
for movie in MOVIES:
    fname = f'{movie}.json'
    remote = os.path.join(BASE_URL, fname)
    local = os.path.join(TMP, fname)
    urlretrieve(remote, local)

files = glob.glob(os.path.join(TMP, '*json'))

def get_movie_data(files=files):

    movies = []

    for file in files:

        with open(file) as f:
            movie_dict = json.load(f)
            movies.append(movie_dict)

    return movies

def get_single_comedy(movies):
    return [m['Title'] for m in movies if 'comedy' in m['Genre'].lower()][0]

def get_movie_most_nominations(movies):

    nominations = []

    award_list = [(m['Title'], m['Awards']) for m in movies]

    nom_pattern = re.compile(r'([0-9]+)*\snominations')

    for a in award_list:
        nom_num = nom_pattern.search(a[1])
        nominations.append((a[0], int(nom_num.group(1))))

    return max(nominations, key=lambda x: x[1])[0]

def get_movie_longest_runtime(movies):

    runtimes = []

    runtime_list = [(m['Title'], m['Runtime']) for m in movies]

    rt_pattern = re.compile(r'([0-9]+)*\smin')

    for rt in runtime_list:
        rt_num = rt_pattern.search(rt[1])
        runtimes.append((rt[0], (int(rt_num.group(1)))))

    return max(runtimes, key=lambda x: x[1])[0]

# def main():
#     movies = get_movie_data()

#     assert len(movies) == 5
#     assert all(type(m) == dict for m in movies)

#     assert get_single_comedy(movies) == 'Horrible Bosses'
#     assert get_movie_most_nominations(movies) == 'Fight Club'

#     assert get_movie_longest_runtime(movies) == 'Blade Runner 2049'

# if __name__ == '__main__':
#     main()


from four import *
import time
import pygame
from pygame.locals import QUIT,MOUSEBUTTONDOWN
import threading

pygame.init()

width = 500
height = 500
SCREEN = (width, height)

screen = pygame.display.set_mode(SCREEN)

mediumFont = pygame.font.Font("OpenSans-Regular.ttf", 28)
largeFont = pygame.font.Font("OpenSans-Regular.ttf", 40)
moveFont = pygame.font.Font("OpenSans-Regular.ttf", 60)

black = (0, 0, 0)
white = (255, 255, 255)
red = (255,50,50)
yellow = (255,255,50)

board = initial_state()

def draw(board):
    board = transpose(board)
    for j in range(6):
        for i in range(7):
            if board[j][i] == 'X':
                pygame.draw.circle(screen, red, (100+50*i, 175+50*j), 20)
            elif board[j][i] == 'O':
                pygame.draw.circle(screen, yellow, (100+50*i, 175+50*j), 20)

continue_code = True
user = None
comp = False

while continue_code:
    for event in pygame.event.get():
        if event.type == QUIT:
            continue_code = False

    screen.fill((100,100,100))

    if user is None:
        # Draw title
        title = largeFont.render("CONNECT 4", True, white)
        titleRect = title.get_rect()
        titleRect.center = ((width / 2), 50)
        screen.blit(title, titleRect)

        # Draw buttons
        playXButton = pygame.Rect((width / 8), (height / 2), width / 4, 50)
        playX = mediumFont.render("Player 1", True, black)
        playXRect = playX.get_rect()
        playXRect.center = playXButton.center
        pygame.draw.rect(screen, white, playXButton)
        screen.blit(playX, playXRect)

        playOButton = pygame.Rect(5 * (width / 8), (height / 2), width / 4, 50)
        playO = mediumFont.render("Player 2", True, black)
        playORect = playO.get_rect()
        playORect.center = playOButton.center
        pygame.draw.rect(screen, white, playOButton)
        screen.blit(playO, playORect)

        # Check if button is clicked
        click, _, _ = pygame.mouse.get_pressed()
        if click == 1:
            mouse = pygame.mouse.get_pos()
            if playXButton.collidepoint(mouse):
                time.sleep(0.2)
                user = True
            elif playOButton.collidepoint(mouse):
                time.sleep(0.2)
                user = False

    else:
        pygame.draw.rect(screen, (50,50,255), pygame.Rect(75,150,350,300), border_radius=20)
        for i in range(7):
            for j in range(6):
                pygame.draw.circle(screen, (100,100,100), (100+50*i, 175+50*j), 20)
        draw(board)

        Player = player(board)

        if terminal(board):
            Winner = winner(board)
            if Winner is None:
                title = f"Game Over: Tie."
            elif Winner == user:
                title = f"You win!."
            else:
                title = f"I win"
        elif user == Player:
            title = f"Your turn"
        else:
            title = f"Computer thinking..."
        title = largeFont.render(title, True, white)
        titleRect = title.get_rect()
        titleRect.center = ((width / 2), 30)
        screen.blit(title, titleRect)

        if terminal(board):
            againButton = pygame.Rect(width / 3, 100, width / 3, 50)
            again = mediumFont.render("Play Again", True, black)
            againRect = again.get_rect()
            againRect.center = againButton.center
            pygame.draw.rect(screen, white, againButton)
            screen.blit(again, againRect)
            click, _, _ = pygame.mouse.get_pressed()
            if click == 1:
                mouse = pygame.mouse.get_pos()
                if againButton.collidepoint(mouse):
                    time.sleep(0.2)
                    user = None
                    board = initial_state()
        click, _, _ = pygame.mouse.get_pressed()
        if click == 1 and user == Player and not terminal(board):
            x,y = pygame.mouse.get_pos()
            X = (x-75)//50
            board = result(board, X)

        elif user != Player and not terminal(board):
            if comp:
                time.sleep(0.5)
                move = minimax(board)
                board = result(board, move)
                comp = False
            else:
                comp = True

    pygame.display.flip()

pygame.quit()


import pandas as pd
import numpy as np
import sys
import os
import utils

def process_parameters():
    if len(sys.argv) < 2:
        print("Please specify name for the output")
        sys.exit(1)

    if len(sys.argv) < 3:
        print("Please specify name for the output folder")
        sys.exit(1)

    print("Analysis invasion : ")
    file_csv = sys.argv[1]
    print("- Output data file : %s" % file_csv)
    physiboss_final_net_folders = sys.argv[2:]
    print("- Final net folders:")
    for folder in physiboss_final_net_folders:
        print("  - %s" % str(folder))

    return physiboss_final_net_folders, file_csv

def main(physiboss_final_net_folders, file_csv):
    singles = []
    clusters = []
    ratios = []
    cells_in_cluster = []

    for replicate in physiboss_final_net_folders:

        single = []
        cluster = []
        ratio = []
        cell_in_cluster = []

        path = os.path.join(replicate, "final_net.csv")
        net_df = pd.read_csv(path)
        utils.split_col(net_df)

        I = utils.create_graph(net_df, " neighID")
        comp_x = utils.count_component(I)[0]
        comp_y = utils.count_component(I)[1]
        comp_z = utils.count_cell_in_cluster(I)[0]
        comp_w = utils.count_cell_in_cluster(I)[1]
        single.append(comp_x)
        cluster.append(comp_y)
        ratio.append(comp_z)
        cell_in_cluster.append(comp_w)

        singles.append(single)
        clusters.append(cluster)
        ratios.append(ratio)
        cells_in_cluster.append(cell_in_cluster)

    avg_singles = []
    avg_clusters = []
    avg_ratios = []
    avg_cells_in_cluster = []
    std_singles = []
    std_clusters = []
    std_ratios = []
    std_cells_in_cluster = []
    for i, _ in enumerate(singles[0]):
        avg_singles.append(
            np.mean([single[i] for single in singles if i < len(single)])
        )
        avg_clusters.append(
            np.mean([cluster[i] for cluster in clusters if i < len(cluster)])
        )
        avg_ratios.append(np.mean([ratio[i] for ratio in ratios if i < len(ratio)]))
        avg_cells_in_cluster.append(
            np.mean(
                [
                    cell_in_cluster[i]
                    for cell_in_cluster in cells_in_cluster
                    if i < len(cell_in_cluster)
                ]
            )
        )
        std_singles.append(np.std([single[i] for single in singles if i < len(single)]))
        std_clusters.append(
            np.std([cluster[i] for cluster in clusters if i < len(cluster)])
        )
        std_ratios.append(np.std([ratio[i] for ratio in ratios if i < len(ratio)]))
        std_cells_in_cluster.append(
            np.std(
                [
                    cell_in_cluster[i]
                    for cell_in_cluster in cells_in_cluster
                    if i < len(cell_in_cluster)
                ]
            )
        )

    data = {
        "single": avg_singles,
        "cluster": avg_clusters,
        "ratio": avg_ratios,
        "cells_in_cluster": avg_cells_in_cluster,
        "error_single": std_singles,
        "error_cluster": std_clusters,
        "error_ratio": std_ratios,
        "error_cells_in_cluster": std_cells_in_cluster,
    }

    df = pd.DataFrame(data)

    df.to_csv(file_csv, index=True, header=True)

    print("Done")

if __name__ == "__main__":
    physiboss_final_net_folders, file_csv = process_parameters()
    main(physiboss_final_net_folders, file_csv)


import numpy as np
import utils
from utils import get_radius_of_equal_distance, spherical_to_cartesian
from abc import ABC
from antenna_element import AntennaElement

class AntennaArray(ABC):
    """
    Abstract base class for all antenna array classes
    """
    def __init__(self, arrayElements, ax_size, arrow_size):
        """
        This ABC requires the list of antenna elements, axis size for antenna
        element plot by [-ax_size, ax_size] and the arrow size for overlaying
        quiver plot.

        Parameters
        ----------
        arrayElements : list(AntennaElement)
            list of antenna element objects belonging to this antenna array.
        ax_size : floating point
            axis size for antenna element plot by [-ax_size, ax_size].
        arrow_size : floating point
            arrow
            size for overlaying quiver plot.

        Returns
        -------
        None.

        """
        self.arrayElements = arrayElements
        self.ax_size = ax_size
        self.arrow_size = arrow_size

class LinearArray(AntennaArray):
    """
    Class that creates and stores attributes of a linear array formation
    """
    def __init__(self, antennaPattern, wavelength, elementDistanceFactor=0.5,
                 numElements=10, xOffset=None):
        """
        Create linear antenna array.

        Parameters
        ----------
        antennaPattern : AntennaPattern

        wavelength : floating point
            lambda of the system.
        elementDistanceFactor : floating poing, optional
            Defines the distance of the antenna element.
            elementDistance = wavelength*elementDistanceFactor.
            The default is 0.5.
        numElements : integer, optional
            number of elements for the linear array. The default is 10.
        xOffset : floating point, optional
            x-position offset of the linear array position. The default is None.

        Returns
        -------
        None.

        """
        self.antennaPattern = antennaPattern
        self.wavelength = wavelength
        self.elementDistance = wavelength*elementDistanceFactor
        self.numElements = numElements
        if not xOffset:
            self.xOffset = -numElements*self.elementDistance/2
        else:
            self.xOffset = xOffset

        arrayElements = self.generateArray()

        ax_size = 1.2*numElements*self.elementDistance
        arrow_size = 0.5*self.elementDistance

        super().__init__(arrayElements, ax_size, arrow_size)

    def generateArray(self):
        arrayElements = []

        # thetaOrientVec = utils.spherical_to_cartesian(1, np.pi/4, np.pi/4)
        thetaOrientVec = np.array([0, 0, 1])
        phiOrientVec = np.array([1, 0, 0])
        for n in range(self.numElements):
            xPos = (self.elementDistance*n)+self.xOffset
            positionVector = np.array([xPos, 0, 0])
            arrayElements.append(AntennaElement(self.antennaPattern, positionVector, thetaOrientVec, phiOrientVec))

        return arrayElements

class CircularArray(AntennaArray):
    """
    Class that creates and stores attributes of a circular array formation
    """
    def __init__(self, antennaPattern, wavelength, elementDisctanceFactor=0.5,
                 numElements=10, circularAng=2*np.pi,
                 elementPoleAng=np.pi/2, circularAzimuthOffset=0):
        """
        Create circular array.

        Parameters
        ----------
        antennaPattern : AntennaPattern
            Object that defines the antenna radiation pattern of the antenna element used in this array.
        wavelength : floating point
            lambda of the system.
        elementDisctanceFactor : floating point, optional
            Defines the distance of the antenna element.
            elementDistance = wavelength*elementDistanceFactor.
            The default is 0.5.
        numElements : integer, optional
            number of elements for the circular array. The default is 10.
        circularAng : floating poing, optional
            Angle of the arc. The default is 2*np.pi (whole circle).
        elementPoleAng : floating poing, optional
            Pole angle orientation of the antenna elements.
            The default is np.pi/2 (pointing outwards).
        circularAzimuthOffset : floating poing, optional
            Azimuth angle offset where the first antenna element shall be positioned.
            The default is 0.

        Returns
        -------
        None.

        """
        self.antennaPattern = antennaPattern
        self.wavelength = wavelength
        self.elementDistance = wavelength*elementDisctanceFactor
        self.numElements = numElements
        self.circularAng = circularAng
        self.elementRadius = get_radius_of_equal_distance(self.elementDistance,
                                                          numElements,
                                                          ang=circularAng)
        self.elementPoleAng = elementPoleAng
        self.circularAzimuthOffset = circularAzimuthOffset

        ax_size = 1.2*2*self.elementRadius
        arrow_size = 0.5*(2*np.pi*self.elementRadius/numElements)

        arrayElements = self.generateArray()

        super().__init__(arrayElements, ax_size, arrow_size)

    def generateArray(self):
        angularDistance = 2*np.pi/self.numElements

        arrayElements = []
        for n in range(self.numElements):
            elementAzimuth = n*angularDistance + self.circularAzimuthOffset
            xPos = np.cos(elementAzimuth)*self.elementRadius
            yPos = np.sin(elementAzimuth)*self.elementRadius
            positionVector = np.array([xPos, yPos, 0])
            thetaOrientVec = spherical_to_cartesian(1, self.elementPoleAng, elementAzimuth)
            phiOrientVec = spherical_to_cartesian(1, np.pi/2, elementAzimuth+np.pi/2)
            arrayElements.append(AntennaElement(self.antennaPattern, positionVector, thetaOrientVec, phiOrientVec))

        return arrayElements

class LinearCircularArray(AntennaArray):
    """
    Class that creates and stores attributes of a cylindrical array formation
    """
    def __init__(self, antennaPattern, wavelength,
                 linearDistanceFactor=0.5, circularDistanceFactor=0.5,
                 numLinearElements=4, numCircularElements=10,
                 circularAng=2*np.pi):
        """
        Create cylindrical array.
        The cylindrical array consists of multiple linear arrays arranged in a circle.

        Parameters
        ----------
        antennaPattern : AntennaPattern
            Object that defines the antenna radiation pattern of the antenna element used in this array.
        wavelength : floating poing
            lambda of the system.
        linearDistanceFactor : floating point, optional
            Defines the distance of the linear array antenna element.
            linearElementDistance = wavelength*linearDistanceFactor.
            The default is 0.5.
        circularDistanceFactor : floating point, optional
            Defines the distance of the circular array antenna element.
            circularElementDistance = wavelength*circularDistanceFactor.
            The default is 0.5.
        numLinearElements : int, optional
            number of elements for the linear array. The default is 10.
        numCircularElements : TYPE, optional
            number of elements for the circular array. The default is 10.
        circularAng : TYPE, optional
            Angle of the circle arc. The default is 2*np.pi (whole circle).

        Returns
        -------
        None.

        """
        self.antennaPattern = antennaPattern
        self.wavelength = wavelength
        self.linearElementDistance = wavelength*linearDistanceFactor
        self.circularElementDistance = wavelength*circularDistanceFactor
        self.numLinearElements = numLinearElements
        self.numCircularElements = numCircularElements
        self.circularAng = circularAng
        self.circularRadius = get_radius_of_equal_distance(self.circularElementDistance,
                                                           self.numCircularElements,
                                                           ang=circularAng)

        ax_size = 1.2 * max(self.numLinearElements*self.linearElementDistance,
                            2*self.circularRadius)
        arrow_size = 0.5 * min(self.linearElementDistance,
                               2*np.pi*self.circularRadius/self.numCircularElements)

        arrayElements = self.generateArray()

        super().__init__(arrayElements, ax_size, arrow_size)

    def generateArray(self):
        angularDistance = 2*np.pi/self.numCircularElements
        linearElementPositions = self.linearElementDistance*np.array(range(self.numLinearElements))
        linearArrayLength = np.max(linearElementPositions)
        linearElementCenteredPositions = linearElementPositions - linearArrayLength/2

        arrayElements = []
        for n in range(self.numCircularElements):
            elementAzimuth = n*angularDistance
            xPos = np.cos(elementAzimuth)*self.circularRadius
            yPos = np.sin(elementAzimuth)*self.circularRadius
            for m in range(self.numLinearElements):
                zPos = linearElementCenteredPositions[m]
                positionVector = np.array([xPos, yPos, zPos])
                # elements are facing outwards
                thetaOrientVec = utils.spherical_to_cartesian(1, np.pi/2, elementAzimuth)
                phiOrientVec = utils.spherical_to_cartesian(1, np.pi/2, elementAzimuth+np.pi/2)
                arrayElements.append(AntennaElement(self.antennaPattern, positionVector, thetaOrientVec, phiOrientVec))

        return arrayElements

class BowCircularArray(AntennaArray):
    """
    Class that creates and stores attributes of a bow circular array formation
    """
    def __init__(self, antennaPattern, wavelength,
                 bowDistanceFactor=0.5, circularDistanceFactor=0.5,
                 numBowElements=4, numCircularElements=10,
                 bowAng=np.pi/2, circularAng=2*np.pi):
        """
        Create bow-circular array.
        Bow-circular array consists of multiple bow arrays (antennas placed along
        an arc) arranged in a circle.

        Parameters
        ----------
        antennaPattern : AntennaPattern
            Object that defines the antenna radiation pattern of the antenna element used in this array.
        wavelength : floating point
            lambda of the system.
        bowDistanceFactor : floating point, optional
            Defines the distance of the bow array antenna elements.
            bowElementDistance = wavelength*bowDistanceFactor.
            The default is 0.5.
        circularDistanceFactor : floating point, optional
            Defines the distance of the circular array antenna element.
            circularElementDistance = wavelength*circularDistanceFactor.
            The default is 0.5.
        numBowElements : integer, optional
            number of elements for the bow array. The default is 10.
        numCircularElements : integer, optional
            number of elements for the circular array. The default is 10.
        bowAng : TYPE, optional
            Angle of the bow arc. The default is np.pi/2 (quarter circle).
        circularAng : TYPE, optional
            Angle of the circle arc. The default is 2*np.pi (whole circle).

        Returns
        -------
        None.

        """
        self.antennaPattern = antennaPattern
        self.wavelength = wavelength
        self.bowElementDistance = wavelength*bowDistanceFactor
        self.circularElementDistance = wavelength*circularDistanceFactor
        self.numBowElements = numBowElements
        self.numCircularElements = numCircularElements
        self.bowAng = bowAng
        self.bowRadius = get_radius_of_equal_distance(self.bowElementDistance,
                                                      numBowElements,
                                                      ang=self.bowAng)
        self.circularAng = circularAng
        if numCircularElements == 1:
            self.circularRadius = 0.1
        else:
            self.circularRadius = get_radius_of_equal_distance(self.circularElementDistance,
                                                           numCircularElements,
                                                           ang=self.circularAng)

        ax_size = 1.2 * 2*self.circularRadius
        arrow_size_circ = 0.5 * 2*np.pi*self.circularRadius/numCircularElements
        arrow_size_bow = 0.5 * 2*np.pi*self.bowRadius/numBowElements
        arrow_size = min(arrow_size_circ, arrow_size_bow)

        arrayElements = self.generateArray()

        super().__init__(arrayElements, ax_size, arrow_size)

    def generateArray(self):
        circAngularDistance = self.circularAng/self.numCircularElements

        # generate positions of a arm of bow
        numBowAngSlices = self.numBowElements-1
        bowAngularDistance = self.bowAng/numBowAngSlices  # angle between bow elements
        bowElementElevations = -self.bowAng/2 + np.array(range(self.numBowElements))*bowAngularDistance
        xPosVec = self.bowRadius * np.cos(bowElementElevations) + (self.circularRadius - self.bowRadius)
        zPosVec = self.bowRadius * np.sin(bowElementElevations)
        elementPoleAngles = np.pi/2 - bowElementElevations

        arrayElements = []
        for n in range(self.numCircularElements):
            elementAzimuth = n*circAngularDistance
            rotMatZ = np.array([[np.cos(elementAzimuth), -np.sin(elementAzimuth), 0],
                                [np.sin(elementAzimuth), np.cos(elementAzimuth), 0],
                                [0, 0, 1]])
            for m in range(self.numBowElements):
                refPosVec = np.array([[xPosVec[m]], [0], [zPosVec[m]]])
                rotPosVec = np.squeeze(np.matmul(rotMatZ, refPosVec))
                thetaOrientVec = utils.spherical_to_cartesian(1,
                                                              elementPoleAngles[m],
                                                              elementAzimuth)
                phiOrientVec = utils.spherical_to_cartesian(1,
                                                            np.pi/2,
                                                            elementAzimuth+np.pi/2)
                arrayElements.append(AntennaElement(self.antennaPattern, rotPosVec, thetaOrientVec, phiOrientVec))

        return arrayElements