from django.db import models, connection
from django.db.models import Q, Max, Case, Value, When, Exists, OuterRef, \
    UniqueConstraint, Subquery
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.core.exceptions import FieldError, ValidationError
from django.template import Template, loader
from django.utils.functional import cached_property
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.translation import gettext_lazy as _
from django.utils import timezone
from django.urls import reverse
from polymorphic.models import PolymorphicModel
from polymorphic.managers import PolymorphicManager
import uuid
from itertools import groupby
from pathlib import Path
from datetime import timedelta
import os

from ..stock import StockWidget
from ..filetype import FileType
from ..utils import create_model, remove_p, send_email, submission_link, \
    thumbnail_path, MarkdownFormatter
from .ranked import RankedModel, UnderscoredRankedModel
from .automatic import AutoSlugModel

markdown = MarkdownFormatter()

class ProgramManager(models.Manager):
    def get_by_natural_key(self, slug):
        return self.get(slug=slug)

class Program(AutoSlugModel):
    class Meta:
        ordering = ['created']

    sites = models.ManyToManyField('Site', blank=True, related_name='programs',
                                   related_query_name='program')
    name = models.CharField(max_length=64)
    slug = models.SlugField(max_length=30, unique=True, allow_unicode=True,
                            verbose_name='identifier')
    db_slug = models.SlugField(max_length=30, unique=True, allow_unicode=True,
                               editable=False)
    description = models.CharField(max_length=250, blank=True)
    options = models.JSONField(default=dict, blank=True)
    hidden = models.BooleanField(default=False)
    created = models.DateTimeField(auto_now_add=True)

    objects = ProgramManager()

    def __str__(self):
        return self.name

    def natural_key(self):
        return (self.slug,)

    def validate_unique(self, exclude=None):
        super().validate_unique(exclude=exclude)
        if not self._state.adding: return

        if Program.objects.filter(db_slug=self.slug.replace('-', '')).exists():
            msg = 'Identifier (with hyphens removed) must be unique.'
            raise ValidationError(msg)

    def visible_forms(self):
        pub = self.forms.exclude(status=Form.Status.DRAFT)
        return pub.exclude(options__hidden__isnull=False)

    def home_url(self):
        if 'home_url' in self.options: return self.options['home_url']
        return None

class FormManager(models.Manager):
    def get_by_natural_key(self, program_slug, slug):
        return self.get(program__slug=program_slug, slug=slug)

class Form(AutoSlugModel):
    class Meta:
        constraints = [
            UniqueConstraint(fields=['program', 'slug'], name='unique_slug'),
            UniqueConstraint(fields=['program', 'db_slug'],
                             name='unique_db_slug')
        ]

    class Status(models.TextChoices):
        DRAFT = 'draft', _('unpublished')
        DISABLED = 'disabled', _('submissions disabled')
        ENABLED = 'enabled', _('published/enabled')
        COMPLETED = 'completed', _('completed')

    class Validation(models.TextChoices):
        # currently, validation to create a submission is always by email
        EMAIL = 'email', _('email address')

    program = models.ForeignKey(Program, models.CASCADE,
                                related_name='forms', related_query_name='form')
    name = models.CharField(max_length=64)
    slug = models.SlugField(max_length=30, allow_unicode=True,
                            verbose_name='identifier')
    db_slug = models.SlugField(max_length=30, allow_unicode=True,
                               editable=False)
    status = models.CharField(max_length=16, default=Status.DRAFT,
                              choices=Status.choices)
    options = models.JSONField(default=dict, blank=True)
    created = models.DateTimeField(auto_now_add=True)
    # this is also the published date, for enabled and completed forms
    modified = models.DateTimeField(default=timezone.now, editable=False)
    completed = models.DateTimeField(null=True, blank=True, editable=False)
    validation_type = models.CharField(max_length=16, editable=False,
                                       default=Validation.EMAIL,
                                       choices=Validation.choices)
    objects = FormManager()

    def __str__(self):
        return self.name

    def natural_key(self):
        return self.program.natural_key() + (self.slug,)
    natural_key.dependencies = ['formative.program']

    def validate_unique(self, exclude=None):
        super().validate_unique(exclude=exclude)
        if not self._state.adding: return

        if Form.objects.filter(program=self.program,
                               db_slug=self.slug.replace('-', '')).exists():
            msg = 'Identifier (with hyphens removed) must be unique ' \
                  'within this program.'
            raise ValidationError(msg)

    @cached_property
    def model(self):
        if self.status == self.Status.DRAFT: return None

        fields = []
        for block in self.blocks.exclude(page=0, _rank__gt=1):
            fields += block.fields()

        name = self.program.db_slug + '_' + self.db_slug
        class Meta:
            verbose_name = self.slug + ' submission'
            verbose_name_plural = self.slug + ' submissions'
        return create_model(name, fields, program=self.program.db_slug,
                            base_class=Submission, meta=Meta)

    @cached_property
    def item_model(self):
        if self.status == self.Status.DRAFT: return None

        collections = self.collections()
        if not collections: return None

        names = []
        for c in collections:
            for field in c.collection_fields():
                if field not in names: names.append(field)

        fields = [
            # the first column links submission items to the submission
            ('_submission', models.ForeignKey(self.model, models.CASCADE,
                                              related_name='_items',
                                              related_query_name='_item'))
        ]

        field_blocks = { b.name: b for b in self.collection_field_blocks() }
        for n in names:
            # look for a CustomBlock with the same name on page 0 (hidden)
            if n in field_blocks: block = field_blocks[n]
            # otherwise, use the default text CustomBlock
            else: block = CustomBlock.text_create()
            fields.append((n, block.field()))

        name = self.program.db_slug + '_' + self.db_slug + '_i'
        class Meta:
            constraints = [
                UniqueConstraint(
                    fields=['_submission', '_collection', '_block', '_rank'],
                    name=self.program.db_slug+'_'+self.db_slug+'_u'
                )
            ]
            verbose_name = self.slug + ' item'
            verbose_name_plural = self.slug + ' items'
        return create_model(name, fields, program=self.program.db_slug,
                            base_class=SubmissionItem, meta=Meta)

    def cache_dirty(self):
        version = cache.get('models_version')
        if version is None: cache.set('models_version', 1, timeout=None)
        else: cache.incr('models_version')

    def publish_model(self, model, admin=None):
        from ..signals import all_forms_publish

        with connection.schema_editor() as editor:
            editor.create_model(model)
        ctype = ContentType(app_label=model._meta.app_label,
                            model=model.__name__)
        ctype.save()
        ContentType.objects.clear_cache()

        all_forms_publish.send(self, content_type=ctype)
        self.cache_dirty()

    def unpublish_model(self, model):
        from ..signals import all_forms_unpublish

        self.cache_dirty()
        ctype = ContentType.objects.get_for_model(model)
        all_forms_unpublish.send(self, content_type=ctype)

        ctype.delete()
        ContentType.objects.clear_cache()
        with connection.schema_editor() as editor:
            editor.delete_model(model)

    def publish(self):
        if self.status != self.Status.DRAFT: return

        self.status = self.Status.ENABLED
        if 'model' in self.__dict__: del self.model
        if 'item_model' in self.__dict__: del self.item_model

        from ..admin import SubmissionAdmin, SubmissionItemAdmin
        self.publish_model(self.model, admin=SubmissionAdmin)
        if self.item_model:
            self.publish_model(self.item_model, admin=SubmissionItemAdmin)

        self.modified = timezone.now()
        self.save()

    def unpublish(self):
        if self.status == self.Status.DRAFT: return

        rec_type = SubmissionRecord.RecordType.SUBMISSION
        recs = SubmissionRecord.objects.filter(program=self.program,
                                               form=self.slug, type=rec_type)
        recs.update(deleted=True)

        self.unpublish_model(self.model)
        if self.item_model: self.unpublish_model(self.item_model)

        self.status = self.Status.DRAFT
        if 'model' in self.__dict__: del self.model
        if 'item_model' in self.__dict__: del self.item_model

        self.modified, self.completed = timezone.now(), None
        self.save()

    def get_available_plugins(self):
        from ..plugins import get_available_plugins
        return get_available_plugins(self)

    def get_plugins(self):
        if 'plugins' in self.options: return self.options['plugins']
        return []

    def add_plugins(self, plugins):
        available = self.get_available_plugins()

        enabled = self.get_plugins()
        enable = [ p for p in plugins if p in available and p not in enabled ]
        for plugin in enable:
            if hasattr(available[plugin].app, 'installed'):
                getattr(available[plugin].app, 'installed')(self)

        if 'plugins' not in self.options: self.options['plugins'] = []
        self.options['plugins'] += plugins

    def remove_plugins(self, plugins):
        available = self.get_available_plugins()

        enabled = self.get_plugins()
        for plugin in [ p for p in plugins if p in enabled ]:
            if hasattr(available[plugin].app, 'uninstalled'):
                getattr(available[plugin].app, 'uninstalled')(self)

        new_plugins = [ p for p in enabled if p not in plugins ]
        self.options['plugins'] = new_plugins

    def default_text_label_style(self):
        if 'default_text_label_style' in self.options:
            return self.options['default_text_label_style']
        return FormLabel.LabelStyle.WIDGET

    def num_pages(self):
        return self.blocks.aggregate(Max('page'))['page__max']

    def custom_blocks(self):
        return CustomBlock.objects.filter(form=self).non_polymorphic()

    def submission_blocks(self):
        blocks = self.blocks.not_instance_of(CollectionBlock)
        return blocks.exclude(page=0, _rank__gt=0)

    def collection_field_blocks(self):
        return self.custom_blocks().filter(page=0, _rank__gt=1)

    def collections(self, name=None):
        blocks = CollectionBlock.objects.filter(form=self)
        if name: return blocks.filter(name=name).non_polymorphic()
        return blocks.non_polymorphic()

    def validation_block(self):
        return self.blocks.get(page=0, _rank=1)

    def visible_blocks(self, page=None, skip=None):
        query = self.blocks.all()
        if skip: query = query.exclude(id__in=skip)
        if page and page > 0: return query.filter(page=page)
        else: return query.exclude(page=0, _rank__gt=0)
        return query.filter(page__gt=0)

    def visible_items(self, submission, page=None, skip=None):
        if not self.item_model: return []

        query = self.item_model.objects.filter(_submission=submission)
        if skip: query = query.exclude(_block__in=skip)
        if page and page > 0:
            block_ids = Subquery(self.blocks.filter(page=page).values('pk'))
            query = query.filter(_block__in=block_ids)

        query = query.exclude(_file='', _filesize__gt=0) # upload in progress
        return query.order_by('_collection', '_block', '_rank')

    def field_labels(self):
        labels = {}
        for label in self.labels.all():
            key, target = label.path, labels
            if '.' in label.path:
                base, key = label.path.split('.')
                if key[-1] == '_': base, key = base + '_', key[:-1]

                if base not in labels: labels[base] = {}
                target = labels[base]
            if key not in target: target[key] = {}
            target[key][label.style] = label

        return labels

    def label_class(self):
        return FormLabel

    def status_message(self):
        if self.status == self.Status.DRAFT:
            return 'NA'
        elif self.status == self.Status.DISABLED:
            if 'disabled_message' in self.options:
                return self.options['disabled_message']
            return _('Not yet open for submissions')
        elif self.status == self.Status.COMPLETED:
            if 'completed_message' in self.options:
                return self.options['completed_message']
            return _('Closed')
        if 'enabled_message' in self.options:
            return self.options['enabled_message']
        return _('Open for submissions')

    def hidden(self):
        return self.status != self.Status.ENABLED or 'hidden' in self.options

    def access_enable(self):
        if 'access_enable' in self.options: return self.options['access_enable']
        return None

    def timed_completion(self):
        if 'timed_completion' in self.options:
            return self.options['timed_completion']
        return None

    def complete_submit_time(self):
        if 'complete_submit_time' in self.options:
            return self.options['complete_submit_time']
        return 5 # minutes

    def extra_time(self):
        extra = self.complete_submit_time()
        if timezone.now() - self.completed <= timedelta(minutes=extra):
            return True
        return False

    def review_pre(self, prefix=''):
        name = prefix + 'review_pre'
        if name in self.options:
            return mark_safe(markdown.convert(self.options[name]))
        return ''

    def review_post(self):
        if 'review_post' in self.options:
            return mark_safe(markdown.convert(self.options['review_post']))
        return ''

    def submit_submission(self, submission):
        submission._submit()

        rec, created = SubmissionRecord.objects.get_or_create(
            program=self.program, form=self.slug, submission=submission._id,
            type=SubmissionRecord.RecordType.SUBMISSION
        )
        rec.text = submission._email
        rec.save()

        if self.item_model:
            dir = os.path.join(settings.MEDIA_ROOT, str(submission._id))
            if os.path.isdir(dir):
                Path(os.path.join(dir, 'submitted')).touch()

    def submitted_review_pre(self):
        return self.review_pre(prefix='submitted_')

    def review_after_submit(self):
        return 'no_review_after_submit' not in self.options

    def submit_button_label(self):
        if 'submit_button_label' in self.options:
            return self.options['submit_button_label']
        return 'submit'

    def thanks(self):
        if 'thanks' in self.options:
            return mark_safe(markdown.convert(self.options['thanks']))

    def emails(self):
        if 'emails' in self.options: return self.options['emails']
        return {}

    def email_names(self):
        names = list(self.emails())
        for name in ('confirmation', 'continue'):
            if name not in names: names.insert(0, name)
        return names

    def load_email_templates(self, n):
        subject = loader.get_template('formative/emails/' + n + '_subject.html')
        content = loader.get_template('formative/emails/' + n + '.html')
        return subject, content

    def email_templates(self):
        emails = self.emails()
        for name in ('continue', 'confirmation'):
            if name in emails: continue
            subject, content = self.load_email_templates(name)

            emails[name] = {'content': content.template.source,
                            'subject': subject.template.source.rstrip('\n')}
        return emails

class FormLabelManager(models.Manager):
    def get_by_natural_key(self, program_slug, form_slug, path, style):
        return self.get(form__program__slug=program_slug, form__slug=form_slug,
                        path=path, style=style)

class FormLabel(models.Model):
    class Meta:
        constraints = [
            UniqueConstraint(fields=['form', 'path', 'style'],
                             name='unique_path_style')
        ]

    class LabelStyle(models.TextChoices):
        WIDGET = 'widget', _('widget label')
        VERTICAL = 'vertical', _('vertical label')
        HORIZONTAL = 'horizontal', _('horizontal label')

    form = models.ForeignKey('Form', models.CASCADE, related_name='labels',
                             related_query_name='label')
    path = models.CharField(max_length=128)
    text = models.CharField(max_length=1000)
    style = models.CharField(max_length=16, choices=LabelStyle.choices,
                             default=LabelStyle.WIDGET)

    objects = FormLabelManager()

    def __str__(self):
        return self.path

    def natural_key(self):
        return self.form.natural_key() + (self.path, self.style)
    natural_key.dependencies = ['formative.form']

    def display(self, inline=False):
        s = markdown.convert(self.text)
        if inline: return mark_safe(remove_p(s))
        return mark_safe(s)

    def display_inline(self):
        return self.display(inline=True)

class FormDependencyManager(models.Manager):
    def get_by_natural_key(self, program_slug, form_slug, name, value):
        return self.get(block__form__program__slug=program_slug,
                        block__form__slug=form_slug,
                        block__name=name, value=value)

class FormDependency(models.Model):
    class Meta:
        verbose_name = 'dependency value'
        verbose_name_plural = 'dependency values'
        constraints = [
            UniqueConstraint(fields=['block', 'value'], name='unique_blockval')
        ]

    block = models.ForeignKey('FormBlock', models.CASCADE,
                              related_name='dependencies',
                              related_query_name='dependency')
    value = models.CharField(max_length=64, blank=True)

    objects = FormDependencyManager()

    def __str__(self):
        if self.block.dependence:
            return f'{self.block.dependence.name}="{self.value}"'
        return f'?="{self.value}"'

    def natural_key(self):
        return self.block.natural_key() + (self.value,)
    natural_key.dependencies = ['formative.formblock', 'formative.customblock',
                                'formative.collectionblock']

class FormBlockManager(PolymorphicManager):
    def get_by_natural_key(self, program_slug, form_slug, name):
        return self.non_polymorphic().get(form__program__slug=program_slug,
                                          form__slug=form_slug, name=name)

class FormBlock(PolymorphicModel, RankedModel):
    class Meta(PolymorphicModel.Meta, RankedModel.Meta):
        constraints = [
            UniqueConstraint(fields=['form', 'page', '_rank'],
                             name='unique_rank'),
        ]
        ordering = ['form', 'page', '_rank']

    form = models.ForeignKey(Form, models.CASCADE,
                             related_name='blocks', related_query_name='block')
    name = models.SlugField(max_length=32, verbose_name='identifier',
                            allow_unicode=True)
    options = models.JSONField(default=dict, blank=True)
    page = models.PositiveIntegerField(default=1)
    dependence = models.ForeignKey('FormBlock', models.CASCADE,
                                   null=True, blank=True,
                                   related_name='dependents',
                                   related_query_name='dependent')
    negate_dependencies = models.BooleanField(default=False,
                                              verbose_name='negate dependency')
    objects = FormBlockManager()

    def __str__(self):
        return self.name

    def natural_key(self):
        return self.form.natural_key() + (self.name,)
    natural_key.dependencies = ['formative.form']

    def rank_group(self):
        return FormBlock.objects.filter(form=self.form, page=self.page)

    def validate_unique(self, exclude=None):
        super().validate_unique(exclude=exclude)
        if not self._state.adding: return

        if self.name == 'email' and self.page:
            raise ValidationError('There is already a block called "email."')

        # name of a collection block identifies its "bucket", not its field(s)
        # in this case, it's not required to be unique
        if self.block_type() == 'collection': return

        qs = FormBlock.objects.filter(form_id=self.form_id)
        if self.page: qs = qs.filter(page__gt=0)
        else: qs = qs.filter(page=0)
        if qs.filter(name=self.name).exists():
            msg = 'Identifiers for stock and custom blocks must be unique.'
            raise ValidationError(msg)

    def block_type(self):
        if type(self) == CustomBlock: return 'custom'
        if type(self) == CollectionBlock: return 'collection'
        return 'stock'

    def stock_type(self):
        if 'type' not in self.options:
            raise FieldError('untyped stock widget')

        return StockWidget.by_type(self.options['type'])

    @cached_property
    def stock(self):
        return self.stock_type()(self.name, **self.options)

    def fields(self):
        return self.stock.fields()

    def enabled_blocks(self, value, page=None):
        # blocks on the given page that depend on self, and enabled given value
        query = self.form.blocks.filter(dependence_id=self.id)
        if page: query = query.filter(page=page)

        if type(value) == bool: value = value and 'yes' or 'no' # TODO: numeric
        if value is None: value = ''

        val = FormDependency.objects.filter(block_id=OuterRef('id'),
                                            value=Value(value))
        cond = Case(
            When(negate_dependencies=False, then=Exists(val)),
            When(negate_dependencies=True, then=~Exists(val))
        )
        query = query.annotate(en=cond).filter(en=True)
        return query.values_list('id', flat=True)

    def min_allowed_page(self):
        if not self.page: return 0
        min_page = 1

        if self.dependence: min_page = self.dependence.page + 1
        return min_page

    def max_allowed_page(self, last_page=None):
        if not self.page: return 0
        if last_page is None:
            last_page = self.form.blocks.aggregate(p=Max('page'))['p'] or 1
        max_page = last_page

        for block in self.dependents.all():
            if block.page - 1 < max_page: max_page = block.page - 1
        return max_page

    def show_in_review(self):
        return 'no_review' not in self.options

class CustomBlock(FormBlock):
    class Meta:
        db_table = 'formative_formcustomblock'

    CHOICE_VAL_MAXLEN = 64
    DEFAULT_TEXT_MAXLEN = 1000
    MAX_TEXT_MAXLEN = 65535

    class InputType(models.TextChoices):
        TEXT = 'text', _('text')
        NUMERIC = 'num', _('numeric')
        CHOICE = 'choice', _('multiple choice')
        BOOLEAN = 'bool', _('true/false choice')

    block = models.OneToOneField(FormBlock, on_delete=models.CASCADE,
                                 parent_link=True, primary_key=True)
    type = models.CharField(max_length=16, choices=InputType.choices,
                            default=InputType.TEXT)
    required = models.BooleanField(default=False)
    num_lines = models.PositiveIntegerField(default=1)
    min_chars = models.PositiveIntegerField(null=True, blank=True)
    max_chars = models.PositiveIntegerField(null=True, blank=True)
    min_words = models.PositiveIntegerField(null=True, blank=True)
    max_words = models.PositiveIntegerField(null=True, blank=True)

    @classmethod
    def text_create(cls, *args, **kwargs):
        if 'max_chars' not in kwargs:
            kwargs['max_chars'] = cls.DEFAULT_TEXT_MAXLEN
        return cls(*args, **kwargs, type=cls.InputType.TEXT)

    def choices(self, include_empty=False):
        if include_empty:
            if 'choices' in self.options: return self.options['choices']
            return []
        if 'choices' not in self.options or not self.options['choices']:
            raise FieldError('choices not defined')

        return self.options['choices']

    def field(self):
        # fields are NULL when we haven't yet reached the page, or if their
        # block had a dependency that wasn't met. non-required fields may
        # have a different way to record that no input was made, usually ''
        if self.type == self.InputType.TEXT:
            blank, max_chars = False, self.max_chars

            if not self.min_chars and not self.min_words: blank = True
            if not self.max_chars or self.max_chars > self.MAX_TEXT_MAXLEN:
                max_chars = self.MAX_TEXT_MAXLEN

            if self.num_lines > 1 or max_chars > self.DEFAULT_TEXT_MAXLEN:
                return models.TextField(null=True, max_length=max_chars,
                                        blank=blank)
            return models.CharField(null=True, max_length=self.max_chars,
                                    blank=blank)

        elif self.type == self.InputType.NUMERIC:
            return models.IntegerField(null=True, blank=(not self.required))

        elif self.type == self.InputType.CHOICE:
            return models.CharField(null=True, blank=(not self.required),
                                    max_length=self.CHOICE_VAL_MAXLEN,
                                    choices=[(c, c) for c in self.choices()])

        elif self.type == self.InputType.BOOLEAN:
            return models.BooleanField(null=True)

    def fields(self):
        return [(self.name, self.field())]

    def form_field(self, model_field, **kwargs):
        if self.type == self.InputType.TEXT:
            return model_field.formfield(min_length=self.min_chars, **kwargs)

        elif self.type == self.InputType.NUMERIC:
            return model_field.formfield(min_value=self.numeric_min(),
                                         max_value=self.numeric_max(), **kwargs)

        # or use the ModelForm factory's default:
        return model_field.formfield(**kwargs)

    def clean_field(self, data, field):
        # currently, all are handled from validators set up on the form
        return data

    def numeric_min(self):
        if 'numeric_min' in self.options: return self.options['numeric_min']
        return None

    def numeric_max(self):
        if 'numeric_max' in self.options: return self.options['numeric_max']
        return None

    def default_value(self):
        if self.type == self.InputType.TEXT: return None

        if 'default_value' in self.options: return self.options['default_value']
        return None

    def conditional_value(self, value):
        if self.type in (self.InputType.TEXT, self.InputType.NUMERIC):
            # in this case, condition is whether the field was filled out
            # (and is non-zero for numeric)
            return bool(value)

        return value

    def span(self, media=None):
        width = 6
        if self.max_chars and self.max_chars > 50: width = 8
        if self.num_lines > 1: width = 8
        if self.num_lines > 4: width = 10
        if self.type in (self.InputType.CHOICE, self.InputType.BOOLEAN):
            width = 8
        elif self.type == self.InputType.NUMERIC: width = 2

        if self.type == self.InputType.NUMERIC:
            if 'span_phone' in self.options:
                return min(self.options['span_phone'], 4)

        if 'span_tablet' in self.options:
            if not media: return min(width, self.options['span_tablet'], 4)
        elif not media: return min(width, 4)

        if media == 'tablet' and 'span_tablet' in self.options:
            return self.options['span_tablet']
        if media == 'desktop' and 'span_desktop' in self.options:
            return self.options['span_desktop']

        return width

    def tablet_span(self): return self.span(media='tablet')
    def desktop_span(self): return self.span(media='desktop')

class CollectionBlock(FormBlock):
    class Meta:
        db_table = 'formative_formcollectionblock'

    FIXED_CHOICE_VAL_MAXLEN = 100

    class AlignType(models.TextChoices):
        TABULAR = 'tabular', _('tabular')
        STACKED = 'stacked', _('stacked')

    block = models.OneToOneField(FormBlock, on_delete=models.CASCADE,
                                 parent_link=True, primary_key=True)
    fixed = models.BooleanField(default=False)
    min_items = models.PositiveIntegerField(null=True, blank=True) # null if
    max_items = models.PositiveIntegerField(null=True, blank=True) # fixed
    has_file = models.BooleanField(default=False)
    file_optional = models.BooleanField(default=False)
    # we don't need these references indexed or validated here, so no SlugField
    name1 = models.CharField(max_length=32, default='', blank=True)
    name2 = models.CharField(max_length=32, default='', blank=True)
    name3 = models.CharField(max_length=32, default='', blank=True)
    align_type = models.CharField(max_length=16, choices=AlignType.choices,
                                  default=AlignType.TABULAR)

    def fields(self):
        return []

    def collection_fields(self):
        fields = []
        if self.name1: fields.append(self.name1)
        if self.name2: fields.append(self.name2)
        if self.name3: fields.append(self.name3)

        return fields

    def tabular(self):
        return self.align_type == self.AlignType.TABULAR

    def max_filesize(self):
        if 'max_filesize' in self.options: return self.options['max_filesize']
        return None # TODO: overall default max

    def allowed_filetypes(self):
        if 'file_types' in self.options:
            return self.options['file_types']

        return None # allow any file extension

    def allowed_extensions(self):
        types = self.allowed_filetypes()
        if not types: return None

        extensions = []
        for filetype in types:
            extensions += FileType.by_type(filetype)().allowed_extensions()
        return extensions

    def autoinit_filename(self):
        if 'autoinit_filename' in self.options: return True
        return False

    def fixed_choices(self):
        if 'choices' not in self.options:
            msg = 'choices must be provided for a fixed collection block'
            raise FieldError(msg)

        return self.options['choices']

    def num_choices(self):
        return len(self.fixed_choices())

    def file_limits(self):
        if 'file_limits' in self.options: return self.options['file_limits']
        return {}

    def process_options(self, filetype):
        if 'file_processing' not in self.options: return {}
        if filetype in self.options['file_processing']:
            return self.options['file_processing'][filetype]
        return {}

    def span(self, media=None):
        width = 10
        if media == 'tablet': width = 8

        if not media: return 4

        if media == 'tablet' and 'span_tablet' in self.options:
            return max(4, self.options['span_tablet'])
        if media == 'desktop' and 'span_desktop' in self.options:
            return max(4, self.options['span_desktop'])

        return width

    def tablet_span(self): return self.span(media='tablet')
    def desktop_span(self): return self.span(media='desktop')

    def total_colspan(self):
         fields = self.collection_fields()
         if fields: return len(fields)
         return 1

    def horizontal_width(self, field):
        total = self.total_colspan()
        if 'wide' in self.options:
            total += len(self.options['wide'])
            if field in self.options['wide']: return 200.0 / total
        return 100.0 / total

    def collection_fields_as_blocks(self):
        class ColFieldBlock:
            def __init__(self, name, width):
                self.name, self.width = name, width

        fields = self.collection_fields()
        return [ ColFieldBlock(n, self.horizontal_width(n)) for n in fields ]

    def items_sortable(self):
        return 'unsortable' not in self.options

    def button_text(self):
        if 'button_text' in self.options: return self.options['button_text']

        if self.has_file and not self.file_optional:
            if self.max_items and self.max_items > 1: return _('add files')
            return _('add file')
        return _('add item')

class SubmissionRecord(models.Model):
    class Meta:
        constraints = [
            UniqueConstraint(fields=['submission', 'type'],
                             name='unique_submission_record_type')
        ]

    class RecordType(models.TextChoices):
        FILES = 'files', _('uploaded files')
        SUBMISSION = 'submission', _('form submission')

    program = models.ForeignKey(Program, models.SET_NULL, null=True, blank=True)
    form = models.SlugField(max_length=64, allow_unicode=True)
    submission = models.UUIDField(editable=False)
    type = models.CharField(max_length=32)
    recorded = models.DateTimeField(auto_now=True, verbose_name='recorded at')
    text = models.TextField(blank=True)
    number = models.PositiveBigIntegerField(null=True, blank=True)
    deleted = models.BooleanField(default=False)

# abstract classes, used as templates for the dynamic models:

class Submission(models.Model):
    class Meta:
        abstract = True

    _id = models.UUIDField(primary_key=True, default=uuid.uuid4,
                           editable=False)
    # valid up to page N:
    _valid = models.PositiveIntegerField(default=0, editable=False)
    # an array of N block id arrays, those skipped for form dependency not met:
    _skipped = models.JSONField(default=list, blank=True, editable=False)
    _created = models.DateTimeField(auto_now_add=True)
    _modified = models.DateTimeField(auto_now=True)
    _submitted = models.DateTimeField(null=True, blank=True)

    @classmethod
    def _get_form(cls):
        program_slug = cls._meta.program_slug
        slug = cls._meta.model_name[len(program_slug)+1:]
        return Form.objects.get(program__db_slug=program_slug, db_slug=slug)

    def __str__(self):
        if hasattr(self, '_email'): return self._email
        return str(self._id)

    def _get_absolute_url(self):
        form = self._get_form()
        args = {'program_slug': form.program.slug, 'form_slug': form.slug,
                'sid': self._id}
        return reverse('submission', kwargs=args)

    def _update_context(self, form, context):
        context['review_link'] = submission_link(self, form, rest='review')

        for block in form.visible_blocks():
            if block.block_type() == 'custom':
                context[block.name] = getattr(self, block.name)
            elif block.block_type() == 'stock':
                class Obj: pass
                obj = Obj()
                obj.__dict__ = { n: getattr(self, block.stock.field_name(n))
                                 for n in block.stock.widget_names() }
                if len(obj.__dict__) > 1: context[block.name] = obj
                else: context[block.name] = next(iter(obj.__dict__.values()))

    def _send_email(self, form, name, **kwargs):
        form_emails = form.emails()
        if name in form_emails:
            subject = Template(form_emails[name]['subject'])
            template = Template(form_emails[name]['content'])
        else: subject, template = form.load_email_templates(name)

        context = {
            'submission': self, 'form': form,
            'submission_link': submission_link(self, form)
        }
        if self._submitted: self._update_context(form, context)

        return send_email(template=template, to=self._email,
                          subject=subject, context=context, **kwargs)

    def _submit(self):
        self._submitted = timezone.now()
        self.save()

    def _collections(self, queryset=None, form=None):
        if not form: form = self._get_form()
        if not queryset: queryset = self._items.all()

        # form's order also orders blocks' items with the same collection name
        block = FormBlock.objects.filter(form=form, pk=OuterRef('_block'))
        queryset = queryset.annotate(page=Subquery(block.values('page')))
        queryset = queryset.annotate(block_rank=Subquery(block.values('_rank')))
        items = queryset.order_by('_collection', 'page', 'block_rank', '_rank')
        collections = groupby(items, key=lambda item: item._collection)
        return { k: list(items) for k, items in collections }

def file_path(instance, filename):
    return os.path.join(str(instance._submission_id), filename)

class SubmissionItem(UnderscoredRankedModel):
    class Meta:
        abstract = True

    _id = models.BigAutoField(primary_key=True, editable=False)
    # see Form.item_model() for _submission = models.ForeignKey(Submission)

    # the item's collection name == the name of the CollectionBlock
    _collection = models.CharField(max_length=32)

    # id of the collection block this item came from, as some may have same name
    _block = models.PositiveBigIntegerField()

    _file = models.FileField(upload_to=file_path, max_length=172, blank=True)
    _filesize = models.PositiveBigIntegerField(default=0)
    _filemeta = models.JSONField(default=dict, blank=True)
    _error = models.BooleanField(default=False)
    _message = models.CharField(max_length=100, default='', blank=True)

    @classmethod
    def _filename_maxlen(cls):
        # use 37 for directory uuid, 8 for possible alt name, 7 for order prefix
        return cls._meta.get_field('_file').max_length - 37 - 8 - 7

    @classmethod
    def _message_maxlen(cls):
        return cls._meta.get_field('_message').max_length

    def _rank_group(self):
        return self.__class__.objects.filter(_submission=self._submission,
                                             _collection=self._collection,
                                             _block=self._block)

    def _file_name(self):
        if not self._file: return None
        return self._file.name[self._file.name.index('/')+1:]

    def _file_type(self):
        if 'type' in self._filemeta: return self._filemeta['type']
        return ''

    def _artifact_url(self, name='thumbnail'):
        if not self._file: return None
        type = self._file_type()
        if not type: return None
        if name != 'thumbnail' or type not in ('image', 'video'):
            filetype = FileType.by_type(type)()
            return filetype.artifact_url(name, self._file.url)

        if type == 'image': return thumbnail_path(self._file.url)
        elif type == 'video': return thumbnail_path(self._file.url, ext='jpg')
        return None

{'all_forms_publish': 'signals.all_forms_publish', 'all_forms_unpublish': 'signals.all_forms_unpublish', 'SubmissionAdmin': 'admin.SubmissionAdmin', 'SubmissionItemAdmin': 'admin.SubmissionItemAdmin', 'get_available_plugins': 'plugins.get_available_plugins'}.objects.get


from itertools import product
from random import choice
from time import sleep
from os import system
from math import floor
from colorama import Back, Fore, Style

################################################################################
# Simulation of Conway's Game of Life. The goal here was to write this with a
# small amount of code as a proof-of-concept that could be run in the terminal.
#
# If you'd like to tinker with the rules, see the conditionals defined in the
# `advance/1` function. For other parameters, like the board size and refresh
# rate, refer to the while-loop defined at the bottom of this file.
################################################################################

def init_board(n, init_alive_percentage):
    """Initialize a board of size `n` by `n`. Supply a percentage,
    `init_alive_percentage`, representing the number of cells in the board that
    should be alive from the start."""
    alive_count = floor(n * init_alive_percentage)
    distribution = [True] * alive_count + [False] * (n - alive_count)
    return [[choice(distribution) for _ in range(n)] for _ in range(n)]

def neighbors(coord, board):
    """Return the neighbors for a given `coord` on a `board`."""
    n = len(board)
    row, col = coord
    return [
        board[(row + row_d) % n][(col + col_d) % n]
        for row_d, col_d in product([-1, 0, 1], [-1, 0, 1])
        if (row_d, col_d) != (0, 0)
    ]

def advance(board):
    """Advance the state of the `board` from T[n] to T[n+1]."""
    n = len(board)
    new_board = [[False for _ in range(n)] for _ in range(n)]
    for row in range(n):
        for col in range(n):
            alive_count = len([x for x in neighbors((row, col), board) if x])
            # Loneliness
            if alive_count == 0:
                new_board[row][col] = False
            # Status Quo
            elif alive_count == 1:
                new_board[row][col] = board[row][col]
            # Cooperation
            elif alive_count == 2:
                new_board[row][col] = True
            # Resource starvation
            elif alive_count >= 3:
                new_board[row][col] = False
    return new_board

def print_board(board):
    """Print the game `board` in a human-readable way."""
    result = ''
    for row in board:
        for col in row:
            if col:
                result += Back.GREEN + '1 ' + Style.RESET_ALL
            else:
                result += Back.RED + '0 ' + Style.RESET_ALL
        result += '\n'
    print(result)

board = init_board(100, 0.50)
while True:
    system('clear')
    print_board(board)
    sleep(0.15)
    board = advance(board)


from __future__ import print_function
import argparse
import torch
import torch.utils.data
from torch import optim
from torch import nn
from torch.utils.data import DataLoader
from gensim.models import KeyedVectors

import os
import numpy as np
from collections import OrderedDict
from multiprocessing import cpu_count

from ptb import PTB
from model import SentenceVAE

parser = argparse.ArgumentParser(description='Sentence VAE Example')

parser.add_argument('--seed', type=int, default=1, metavar='S',
                    help='random seed (default: 1)')
parser.add_argument('--data_dir', type=str, default='data')
parser.add_argument('--create_data', action='store_true')

parser.add_argument('--epochs', type=int, default=10)
parser.add_argument('--batch_size', type=int, default=32)

parser.add_argument('--k', type=float, default=0.0025)
parser.add_argument('--x0', type=int, default=2500)
parser.add_argument('--log_interval', type=int, default=50)
parser.add_argument('--save_model_path', type=str, default='model')
parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='enables CUDA training')

args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()

# torch.manual_seed(args.seed)
device = torch.device("cuda" if args.cuda else "cpu")

splits = ['train', 'valid']
datasets = OrderedDict()

for split in splits:
    datasets[split] = PTB(
        data_dir=args.data_dir,
        split=split,
        create_data=args.create_data,
        max_sequence_length=60
    )

# vocab_size = datasets['train'].vocab_size
sos_idx = datasets['train'].sos_idx
eos_idx = datasets['train'].eos_idx
pad_idx = datasets['train'].pad_idx

embedding = KeyedVectors.load('model/pretrained_embedding')
if args.cuda:
    weights = torch.FloatTensor(embedding.syn0).cuda()
else:
    weights = torch.FloatTensor(embedding.syn0)

model = SentenceVAE(weights.size(0), sos_idx, eos_idx, pad_idx, training=True).to(device)

def init_weights(m):
    if type(m) == torch.nn.Linear:
        torch.nn.init.xavier_uniform(m.weight)
        m.bias.data.fill_(0)

model.apply(init_weights)
model.emb = nn.Embedding.from_pretrained(weights)
optimizer = optim.Adam(filter(lambda p: p.requires_grad,model.parameters()),
                       lr=1e-4, weight_decay=1e-3)

def kl_anneal_function(step):
    k = args.k
    x0 = args.x0
    weight = float(1 / (1 + np.exp(-k * (step - x0))))
    return weight

criterion = torch.nn.NLLLoss(size_average=False, ignore_index=pad_idx)

def loss_function(reconx, x, mu, logvar, step):
    # print(torch.argmax(torch.exp(reconx), dim=-1)[0][:50])
    x = x.view(-1).long()
    reconx = reconx.view(-1, reconx.size(2))
    NLL_loss = criterion(reconx, x)
    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
    beta = kl_anneal_function(step)
    loss = NLL_loss + beta * KLD
    return loss, NLL_loss, KLD, beta

def train(epoch, step):
    model.train()

    data_loader = DataLoader(
        dataset=datasets['train'],
        batch_size=args.batch_size,
        shuffle=True,
        num_workers=cpu_count(),
        pin_memory=torch.cuda.is_available()
    )

    train_loss = 0
    for batch_idx, batch in enumerate(data_loader):
        batch_size = batch['input'].size(0)
        if torch.cuda.is_available():
            batch['input'] = batch['input'].cuda()
            batch['target'] = batch['target'].cuda()
        logp, mu, logvar, z = model(batch['input'])
        loss, NLL_loss, KL_loss, KL_weight = loss_function(logp, batch['target'],
                                                           mu, logvar, step)

        loss = loss / batch_size
        if step == 10:
            checkpoint_path = os.path.join(args.save_model_path,
                                           "model_epoch_%i" % (epoch))
            torch.save(model.state_dict(), checkpoint_path)
            print("Model saved at %s" % checkpoint_path)

        optimizer.zero_grad()
        loss.backward()
        train_loss += loss.item()
        torch.nn.utils.clip_grad_norm(model.parameters(), 5)
        optimizer.step()
        step += 1

        if batch_idx % args.log_interval == 0:
            print('Train Epoch {} [{}/{}] Loss {:.2f} | NLL {:.2f}'
                  ' | KL {:.2f} | Beta {:.3f}'.format(epoch,
                   batch_idx * batch_size, len(data_loader.dataset),
                   loss.item(), NLL_loss.item() / batch_size,
                   KL_loss.item() / batch_size, KL_weight))

    print('====> Epoch: {} Average loss: {:.4f} steps: {}'.format(
          epoch, train_loss * args.batch_size / len(data_loader.dataset), step))

    checkpoint_path = os.path.join(args.save_model_path, "model_epoch_%i" % (epoch))
    torch.save(model.state_dict(), checkpoint_path)
    print("Model saved at %s" % checkpoint_path)
    return step

def test(step):
    model.eval()
    test_loss = 0

    data_loader = DataLoader(
        dataset=datasets['valid'],
        batch_size=args.batch_size,
        shuffle=False,
        num_workers=cpu_count(),
        pin_memory=torch.cuda.is_available()
    )

    with torch.no_grad():
        for batch_index, batch in enumerate(data_loader):
            if torch.cuda.is_available():
                batch['input'] = batch['input'].cuda()
                batch['target'] = batch['target'].cuda()
            logp, mu, logvar, z = model(batch['input'])
            loss, _, _, _ = loss_function(logp, batch['target'],
                                          mu, logvar, step)
            test_loss += loss.item()

    test_loss /= len(data_loader.dataset)
    print('====> Test set loss: {:.4f}'.format(test_loss))

step = 0
for epoch in range(1, args.epochs + 1):
    step = train(epoch, step)
    test(step)


import os
import math
import numpy as np
import shutil
import cv2
from colormath.color_conversions import *
from colormath.color_objects import *
from sklearn.manifold import TSNE
import json
from similarity_measurer import SimilarityMeasurer
from color_palette import ColorPalette
from geo_sorter_helper import *
from OrderSorter import OrderSorter
from palette_to_image import save

if __name__ == '__main__':
    # 随机得到展示的数据
    # numbers = 50
    # k_list = [5, 10, 15, 20, 25, 30]
    # jitters = [0, 5, 10, 15]
    # for k in k_list:
    #     replaces = [0, math.ceil(k / 10)]
    #     target_dir = 'Datasets/WedData/k{}'.format(k)
    #     if not os.path.exists(target_dir):
    #         os.makedirs(target_dir)
    #     for i in range(numbers):
    #         j = jitters[np.random.randint(0, len(jitters))]
    #         r = replaces[np.random.randint(0, 2)]
    #         sub_dir = 'Datasets/LHSP/LHSP-k{}-jitter{}-replacement{}/retrieval-palettes-images'.format(k, j, r)
    #         image_list = os.listdir(sub_dir)
    #         src_image = image_list[np.random.randint(0, len(image_list))]
    #         dst_image = ('k{}-p{}' + os.path.splitext(src_image)[-1]).format(k, i)
    #         print(os.path.join(sub_dir, src_image))
    #         shutil.copyfile(os.path.join(sub_dir, src_image), os.path.join(target_dir, dst_image))

    k_list = [x for x in range(5, 16, 1)]
    # tsne_results = []
    json_path = 'Datasets/WedData/'
    # 存每个palette的颜色
    palette_color = {}  # 未排序
    palette_color_sorted = {}  # 排序
    if not os.path.exists(json_path):
        os.makedirs(json_path)
    all_palettes = []
    all_names = []
    for k in k_list:
        print('k=', k)
        base_dir = 'Datasets/WedData/images/k{}'.format(k)
        save_dir = 'Datasets/WedData/images-sorted/k{}'.format(k)
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)
        palettes = []
        # palettes_sorted = []
        lab_lists = []
        names = []
        # cnt = 6  # 每个长度计算6个就行
        for img in os.listdir(base_dir):
            name = img[:img.index('.')]
            names.append(name)
            all_names.append(name)
            palette_path = os.path.join(base_dir, img)
            image = cv2.imread(palette_path)
            image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            # print(image_rgb.shape)
            palette = []
            lab_list = []
            for i in range(0, image_rgb.shape[1], image_rgb.shape[0]):
                r, g, b = image_rgb[0][i]
                rgb = sRGBColor(r, g, b, True)
                lab = convert_color(rgb, LabColor)
                palette.append(rgb.get_rgb_hex())
                lab_list.extend(list(lab.get_value_tuple()))

            # 对于长度不够的进行扩充 在计算总体时
            # for i in range(k, 15):
            #     lab_list.extend([100, 0, 0])

            palettes.append(palette)
            lab_lists.append(lab_list)  #
            palette_color[name] = palette
            all_palettes.append(lab_list)  #

            # 排序
            sorter = OrderSorter(palette=palette, name='_')
            sorter.calculate_distance_matrix()
            step = 1
            sorter.discrete_sampling(step=step)
            best_points = sorter.sort()  # 采样点取值
            sorter.solve(best_points)  # 排序
            palette_sorted = [palette[x] for x in sorter.samples_sorted]
            palette_color_sorted[name] = palette_sorted
            save(palette_sorted, name + '-sorted', image_dir=save_dir, repeat=True,
                 color_block_width=100)
            # palettes_sorted.append(palette_sorted)

            # cnt -= 1
            # if cnt <= 0:
            #     break

        # ========================分长度计算 TSNE 保存json
        # data = np.array(lab_lists)
        # tsne = TSNE(n_components=2)
        # result = tsne.fit_transform(data.reshape(data.shape[0], -1))
        # # tsne_results.append(result)
        # json_text = {}
        # for i in range(len(palettes)):
        #     json_text[names[i]] = {'pointX': result[i][0].item(), 'pointY': result[i][1].item()}
        # print(json_text)
        # json_data = json.dumps(json_text, indent=4)
        #
        # file = os.path.join(json_path, 'k{}.json'.format(k))
        # with open(file, 'w') as f:
        #     f.write(json_data)

        # ======================================= 计算相似度
        # num = len(palettes)
        # similarity_list = {}
        # for i in range(num):
        #     sub_list = []
        #     pa = ColorPalette(auto_fetched=False, colors=palettes[i])
        #     for j in range(num):
        #         pb = ColorPalette(auto_fetched=False, colors=palettes[j])
        #         similarity_measurer = SimilarityMeasurer(pa, pb, LabDistanceMode.CIEDE2000, improved=True)
        #         sim, _ = similarity_measurer.measure()
        #         sub_list.append({"id": names[j], "similarity": sim["DynamicClosestColorWarping"]})
        #     similarity_list[names[i]] = sub_list
        #     json_path = 'Datasets/WedData/similarity'
        #     if not os.path.exists(json_path):
        #         os.makedirs(json_path)
        #     json_data = json.dumps(similarity_list, indent=4)
        #     file = os.path.join(json_path, 'k{}.json'.format(k))
        #     with open(file, 'w') as f:
        #         f.write(json_data)

    # =========== 保存palette 的颜色信息
    json_data = json.dumps(palette_color_sorted, indent=4)
    file = os.path.join(json_path, 'palette_color_sorted.json')
    with open(file, 'w') as f:
        f.write(json_data)

    # ========================合并所有长度计算 TSNE 保存json
    # print(all_palettes)
    # all_data = np.array(all_palettes)
    # print(all_data.shape)
    # all_tsne = TSNE(n_components=2)
    # result = all_tsne.fit_transform(all_data.reshape(all_data.shape[0], -1))
    # # tsne_results.append(result)
    # json_text = {}
    # for i in range(len(all_palettes)):
    #     json_text[all_names[i]] = {'pointX': result[i][0].item(), 'pointY': result[i][1].item()}
    # print(json_text)
    # json_data = json.dumps(json_text, indent=4)
    #
    # file = os.path.join(json_path, 'all_palettes.json')
    # with open(file, 'w') as f:
    #     f.write(json_data)


import scipy.io
import pylab
import numpy
from math import *
import scipy.misc
import scipy.sparse
import sys
import time
import matplotlib.ticker
#import HCIL_model # make sure this model accounts for alignments etc
from matplotlib import pyplot as plt
import socket
import time
from InitializationPY import *
from EKF_MAT2PY import mat2py, recvall

# G = numpy.concatenate([numpy.concatenate([numpy.real(G1),
# 	numpy.imag(G1)], axis=1), numpy.concatenate([numpy.real(G2), numpy.imag(G2)],
# 	axis=1)], axis=0).T # real matrix [Re(G1),Im(G1); Re(G2), Im(G2)], for dark hole area

# Only use G2 for control right now
G = numpy.concatenate([numpy.concatenate([numpy.real(G2), numpy.imag(G2)],
	axis=1)], axis=0).T
# G = numpy.concatenate([numpy.real(G2), numpy.imag(G2)],
# 	axis=1).T
print(G.shape)

## DM setting after creating the dark hole

u0 = command #numpy.concatenate([DM1command, DM2command])

## Correction on top of u0
u = u0*0.0

## Drift covariances (might be a bit off, but the filter is not very sensitive to this)
Qlarge = G.dot(numpy.eye(len(u))).dot(G.T)* drift_std**2 # this needs to be number of actuators available, not of full control

## The Extended Kalman Filter class which stores its state
class Filter:
    def __init__(self, index, x0 = numpy.zeros(2)):
        self.index = index #pixel index
        self.x_hat = numpy.array(x0) #electric field (state) estimate (real 2x1 vector with real and imag parts)

        # self.Q = drift_stats["covs"][self.index] #process noise covariance
        self.Q = Qlarge[2*index:2*index+2, 2*index:2*index+2]
        self.P = 16*self.Q #initial state covariance

    def advance(self, Gu, I):
        ## "predic" step
        E_hat = complex(self.x_hat[0] + Gu[self.index], self.x_hat[1] + Gu[self.index+G.shape[0]//2]) #closed loop electric field estimate (open loop field + Jacobian * controls)
        z_hat = abs(E_hat)**2 + i_inco_avg#*scale # intensity estimate
        z = I[self.index] # intensity measurement

        H = [2*Gu[self.index] + 2*self.x_hat[0], 2*Gu[self.index+G.shape[0]//2] + 2*self.x_hat[1]] #observation equation linearization

        ## Kalman gain:
        S = self.P.dot(H).dot(H) + (dark_curr)**2
        K = self.P.dot(H)/S

        ## "update" step
        self.x_hat = self.x_hat + K.dot(z - z_hat) #closed loop electric field estimate
        self.P = self.P - numpy.outer(K,H).dot(self.P) #electric field covariance

        ## "advance" step
        self.P = self.P + self.Q #covariance increases due to druft
        self.x_hat = self.x_hat*1.0 #mean remains the same (the drift is zero mean)

        return E_hat

## Computing EFC gain
M = G.T.dot(G)
EFC_gain = numpy.linalg.pinv(M + alpha*numpy.eye(len(M))).dot(G.T) #EFC function in runEFC

## Sample random wavefront drift and dithering
numpy.random.seed(0)
u_drift = numpy.zeros(n_act_drift)
u_drifts = []
dus = []

for j in range(T):
    ## Random walk drift
    u_drifts.append(u_drift) #change drift
    u_drift = u_drift + drift_std*(numpy.random.random(u_drift.shape)-0.5) # in nm?

    ## Zero mean dithering
    dus.append(numpy.random.normal(0, sigma_u, u.shape))

## Initialize filters
#I0, E0_init = mat2py(DM1command, u0 ,len(DM1command),G1.shape[1])
fs = [Filter(index,[numpy.real(E0_init[index]), numpy.imag(E0_init[index])]) for index in range(G.shape[0]//2)]

## Run closed loop
for j in range(T):

	cont_power = j>4
	drift_power = j>4

	I_OL, E_OL = mat2py(DM1command + drift_power*u_drifts[j], u0 ,len(DM1command),G1.shape[1])
	I, E = mat2py(DM1command + drift_power*u_drifts[j], u0 + u,len(DM1command),G1.shape[1])

	E_OL_split  = numpy.zeros(2*len(E_OL))
	E_OL_split[0:n_px] = numpy.real(E_OL)
	E_OL_split[n_px:2*n_px] = numpy.imag(E_OL)

	E_split  = numpy.zeros(2*len(E))
	E_split[0:n_px]  = numpy.real(E)
	E_split[n_px:2*n_px]= numpy.imag(E)
	# plt.plot(u0+u, label="DM2 EFC+Dither+DH")
	# plt.plot(u0, label="DM2 DH command")
	# plt.plot(u, label="DM2 EFC+Dither")
	# plt.plot(DM1command, label="DM1 DH command")
	# plt.plot(u_drifts[j], label="DM1 Drift")
	# plt.legend()
	# plt.show()
	# print("Iteration %d: avg closed loop intensity %.01f, avg open loop intensity %.01f"%(j, numpy.linalg.norm(E)**2/scale, numpy.linalg.norm(E0)**2/scale))

	# print("Iteration %d: avg closed loop intensity %.01f"%(j, numpy.linalg.norm(I)/scale))

	# I = measurement(numpy.abs(E)**2) # get from getImage in lab, dark hole area only
	I2D_temp = numpy.zeros(99*83)
	# print('red',len(I2D_temp))
	I2D_temp[pixel_indices] = I
	I2D = I2D_temp.reshape(99,83)
	# print(I2D.shape)

	Gu = G.dot(u) #precompute the difference between closed and open loop fields
	Iperf = numpy.abs(E)**2
	E_hat_hat = []
	for f in fs:
			f.advance(Gu,I) #advance the filters
			# E_hat_hat.append(f.advance(Gu,Iperf+dark_curr))

    # Get field estimate
	x_hats = numpy.array([f.x_hat for f in fs])
	E_hat = numpy.concatenate([x_hats[:,0], x_hats[:,1]])

	# x_hat_hat = numpy.array(E_hat_hat)
	# print(len(E_hat_hat))
	# x_hathat = numpy.concatenate([x_hat_hat, x_hat_hat[:,1]])

	#print("mean estimated intensity")
	# E_hat2 = numpy.concatenate([numpy.real(E0), numpy.imag(E0)]) #for debugging only

	# print(numpy.linalg.norm(E_hat-E_hat2)/numpy.linalg.norm(E_hat2))

	## Record iteration data
	I_arr.append(I)
	I_OL_arr.append(I_OL)
	u_arr.append(u)
	E_hat_arr.append(E_hat)
	E_arr.append(E)
	E_OL_arr.append(E_OL)

	print("Iteration: ",j, "\navg closed loop intensity: ", numpy.mean(I),
		"\navg open loop intensity: ", numpy.mean(I_OL), "\nmean estimated OL intensity: ",
		numpy.mean(E_hat[0:n_px]**2+ E_hat[n_px:2*n_px]**2))

	# fig = plt.figure()
    # self.ax1 = fig.add_subplot(1, 2, 1)
	deltaE = G2.T.dot(u)
	deltaE_split = numpy.zeros(2*len(deltaE))
	deltaE_split[::2] = numpy.real(deltaE)
	deltaE_split[1::2] = numpy.imag(deltaE)

	Gu_comb = Gu[0:n_px] + Gu[n_px:2*n_px]*complex(0,1)

	# plt.plot(-Gu,label='Split -Gu')
	# # plt.plot(-Gu_comb,label='-Gu combined')
	# plt.plot(E_OL_split-E_split,lw=3,alpha = 0.5,label='Split OL-CL')
	# plt.plot(-G2.T.dot(u),label='-G2u')
	# plt.plot(E_OL-E,lw=2,alpha = 0.3,label='OL-CL')
	# plt.plot(E_hat,label = 'Ehat')
	# plt.plot(E_OL_split,lw=2,alpha = 0.5,label = "E_OL")

	# plt.plot(E,label = 'E')
	# plt.plot(E_hat_hat,lw=2,alpha = 0.5,label = "E_hat_hat")
	# plt.legend()
	# plt.title("Iteration %d"%(j))
	# plt.show()

    # Compute next command (EFC + dithering)

	u = dus[j] - cont_power*EFC_gain.dot(E_hat)
	# u = -EFC_gain.dot(E0) + dus[j]
	# ADD A SEND DM COMMAND

	#
	# print("Iteration %d: avg closed loop intensity %.01f, \
	# 	avg open loop intensity %.01f"%(j, numpy.mean(I), \
	# 	numpy.mean(I_OL)))

print("saving data")
to_save = ["I_arr","I_OL_arr","E_hat_arr","u_arr", "dus", "u_drifts","pixel_indices","i_inco_avg","dark_curr"]
scipy.io.savemat(fname, dict((s, numpy.array(eval(s))) for s in to_save))
print("data saved")


#!/usr/bin/python
# -*- coding: utf-8 -*
from fabric.api import *
from fabric.context_managers import *
from fabric.contrib.console import confirm
from fabric.contrib.files import *
from fabric.contrib.project import rsync_project
import fabric.operations
import time,os
import logging
import base64
from getpass import getpass
import json
import sys

# 定义一些常量
## 本地软件目录
env.local_softdir="/opt/software/"
## 远端软件目录
env.remote_softdir="/opt/software/"
## 远端家目录
env.remote_dir="/opt/machtalk/"

############## CDH

@task
@roles("cdh")
def cdh_pufile():
    # 上传文件
    run("mkdir -p /opt/cloudera/parcel-repo/")
    run("mkdir -p /opt/cdh-cm/share/cmf/lib/")
    put(env.local_softdir + "cloudera-manager-el6-cm5.5.0_x86_64.tar.gz", "/opt/")
    put(env.local_softdir + "solrcloud.tar.gz", "/opt/")
    put(env.local_softdir + "CDH-5.5.0-1.cdh5.5.0.p0.8-el6.parcel", "/opt/cloudera/parcel-repo/")
    put(env.local_softdir + "CDH-5.5.0-1.cdh5.5.0.p0.8-el6.parcel.sha", "/opt/cloudera/parcel-repo/")
    put(env.local_softdir + "manifest.json", "/opt/cloudera/parcel-repo/")
    put(env.local_softdir + "mysql-connector-java-5.1.38-bin.jar", "/opt/cdh-cm/share/cmf/lib/")

@task
@roles("cdh")
def cdh_deploy():
    with cd("/opt/"):
        # 判断目录是否存在，如果存在就退出
        run(""" [ -e "/opt/cm" ] && exit 1 || echo '开始部署CDH！' """)

        # 根据变量获取
        ip = env.host
        ipListNumber = json.loads(bo.servers).index(ip) + 1
        serverName = "cdhslave%s" % (ipListNumber)
        if ipListNumber == 1:
            serverName = "cdhmanager"

        # 设置主机名
        sudo("""
    cat << 'EOF' > /etc/sysconfig/network
    NETWORKING=yes
    HOSTNAME=%s
    EOF
    hostname %s
                    """ % (serverName, serverName))

        # 设置hosts
        conf_hosts = ""
        itemNumber = 0
        for item in json.loads(bo.servers):
            if itemNumber == 0:
                conf_hosts += """
    %s cdhmanager""" % (item)
            else:
                conf_hosts += """
    %s cdhslave%s""" % (item, itemNumber + 1)
            itemNumber += 1
        sudo("""
                        sed -i "/cdhslave/d" /etc/hosts
                        sed -i "/cdhmanager/d" /etc/hosts
                        service network restart
                        echo '%s' >> /etc/hosts
                    """ % (conf_hosts))

        # 上传文件
        # put(Const.SOURCE_DIR + "cloudera-manager-el6-cm5.5.0_x86_64.tar.gz", "/opt/")
        run("tar -xzf /opt/cloudera-manager-el6-cm5.5.0_x86_64.tar.gz -C /opt/")
        run("ln -s /opt/cm-5.5.0 /opt/cdh-cm")
        # put(Const.SOURCE_DIR + "solrcloud.tar.gz", "/opt/")
        run("tar -xzf ./solrcloud.tar.gz")
        # put(Const.SOURCE_DIR + "CDH-5.5.0-1.cdh5.5.0.p0.8-el6.parcel", "/opt/cloudera/parcel-repo/")
        #put(Const.SOURCE_DIR + "CDH-5.5.0-1.cdh5.5.0.p0.8-el6.parcel.sha", "/opt/cloudera/parcel-repo/")
        #put(Const.SOURCE_DIR + "manifest.json", "/opt/cloudera/parcel-repo/")
        #put(Const.SOURCE_DIR + "mysql-connector-java-5.1.38-bin.jar", "/opt/cdh-cm/share/cmf/lib/")

        # 修改主机名 为 cdhmanassger
        run("""
        hostname cdhmanager
        sed -i 's/HOSTNAME=.*/HOSTNAME=cdhmanager/g' /etc/sysconfig/network
        sed -i '/cdhmanager/d' /etc/hosts
        echo "%s  cdhmanager" >> /etc/hosts
        service network restart
        """%env.host)

        # 添加cloudera-scm账户
        run("useradd --system --home=/opt/cdh-cm/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment 'Cloudera SCM User' cloudera-scm | echo 'account alreay exists'")

        # cdhmanager服务器ssh key
        run('''
            ssh-keygen -t rsa -C 'cdh' -P '' -f  ~/.ssh/id_rsa
            cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
            chmod 600 ~/.ssh/authorized_keys
            ''')

        # 只有cdhamnager需要执行下面的命令
        if serverName == "cdhmanager":
            # 配置修改
            ## agent
            run(
                "sed -i 's/server_host=localhost/server_host=cdhmanager/g' /opt/cdh-cm/etc/cloudera-scm-agent/config.ini")
            # 数据库配置
            conf_cdhserver = """
    com.cloudera.cmf.db.type=mysql
    com.cloudera.cmf.db.host=%s
    com.cloudera.cmf.db.name=%s
    com.cloudera.cmf.db.user=%s
    com.cloudera.cmf.db.password=%s
                """ % (info['services']['cdh']['dbip'], "cm", "scm", "scm")
            run(""" echo '%s' > /opt/cm-5.5.0/etc/cloudera-scm-server/db.properties """ % conf_cdhserver)

            # 数据库建库
            # 需要变量dbip dbname dbpass
            ## 确保mysql客户端存在
            run(""" yum install -y mysql || echo 'mysql客户端已经安装!'""")
            ## for database amon
            run(
                """ mysql -h %s -u%s -p%s -e "create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;" """ % (
                info['services']['cdh']['dbip'], info['services']['cdh']['dbusrname'], info['services']['cdh']['dbpwd']))
            run(
                """ mysql -h %s -u%s -p%s -e "grant all privileges on amon.* to 'amon'@'%s' identified by 'amon'; flush privileges;" """ % (
                    info['services']['cdh']['dbip'], info['services']['cdh']['dbusrname'],
                    info['services']['cdh']['dbpwd'], "%"))
            run(
                """ mysql -h %s -u%s -p%s -e "grant all privileges on amon.* to 'amon'@'%s' identified by 'amon'; flush privileges;" """ % (
                    info['services']['cdh']['dbip'], info['services']['cdh']['dbusrname'],
                    info['services']['cdh']['dbpwd'], "localhost"))
            ##  for database cm
            run(""" mysql -h %s -u%s -p%s -e "create database cm DEFAULT CHARSET utf8 COLLATE utf8_general_ci;" """ % (
                info['services']['cdh']['dbip'], info['services']['cdh']['dbusrname'], info['services']['cdh']['dbpwd']))
            run(
                """ mysql -h %s -u%s -p%s -e "grant all privileges on cm.* to 'scm'@'%s' identified by 'scm'; flush privileges;" """ % (
                    info['services']['cdh']['dbip'], info['services']['cdh']['dbusrname'],
                    info['services']['cdh']['dbpwd'], "%"))
            run(
                """ mysql -h %s -u%s -p%s -e "grant all privileges on cm.* to 'scm'@'%s' identified by 'scm'; flush privileges;" """ % (
                    info['services']['cdh']['dbip'], info['services']['cdh']['dbusrname'],
                    info['services']['cdh']['dbpwd'], "localhost"))

            # 数据库初始化
            run('/opt/cdh-cm/share/cmf/schema/scm_prepare_database.sh mysql cm --scm-host ' + ip + ' scm scm scm')

            # 启动server
            run("/opt/cdh-cm/etc/init.d/cloudera-scm-server start")

        # 启动agent
        run("""
                sed -i 's:server_host=.*:server_host=cdhmanager:g' /opt/cm-5.5.0/etc/cloudera-scm-agent/config.ini
                /opt/cm-5.5.0/etc/init.d/cloudera-scm-agent restart
            """)

    ''' 伟大的注释
    # cdh - 3 - 数据库初始化
    @task
    @roles('cdh')
    def cdh3():
      ip = run("/sbin/ifconfig | grep '10\.\|192\.168\.' | head -n 1 | awk -F\: '{print $2}' | awk '{print $1}'")
      dbip=ip
      # for database amon
      run(""" mysql -e "create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;" """)
      run(""" mysql -e "grant all privileges on amon.* to 'amon'@'%s' identified by 'amon'; flush privileges;" """%("%"))
      run(""" mysql -e "grant all privileges on amon.* to 'amon'@'%s' identified by 'amon'; flush privileges;" """%("localhost"))
      #A for database cm
      run(""" mysql -e "create database cm DEFAULT CHARSET utf8 COLLATE utf8_general_ci;" """)
      run(""" mysql -e "grant all privileges on cm.* to 'scm'@'%s' identified by 'scm'; flush privileges;" """%("%"))
      run(""" mysql -e "grant all privileges on cm.* to 'scm'@'%s' identified by 'scm'; flush privileges;" """%("localhost"))
      # for database hive
      #run(""" mysql -e "create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;" """)
      #run(""" mysql -e "grant all privileges on hive.* to 'hive'@'%s' identified by 'hive'; flush privileges;" """%("%"))
      #run(""" mysql -e "grant all privileges on hive.* to 'hive'@'%s' identified by 'hive'; flush privileges;" """%("localhost"))
      # for database hue
      #run(""" mysql -e "create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;" """)
      #run(""" mysql -e "grant all privileges on hue.* to 'hue'@'%s' identified by 'hue'; flush privileges;" """%("%"))
      #run(""" mysql -e "grant all privileges on hue.* to 'hue'@'%s' identified by 'hue'; flush privileges;" """%("localhost"))
      # for a special user
      #run(""" mysql -e "grant all privileges on *.* to 'machtalk'@'%s' identified by 'machmydb'; flush privileges;" """%("%"))
      #run(""" mysql -e "grant all privileges on *.* to 'machtalk'@'%s' identified by 'machmydb'; flush privileges;" """%("localhost"))
      #run(""" mysql -e "grant all privileges on *.* to 'machtalk'@'%s' identified by 'machmydb'; flush privileges;" """%("cdhmanager"))
      #scm back is clouder_manager_server ip
      run('/opt/cdh-cm/share/cmf/schema/scm_prepare_database.sh mysql cm --scm-host '+ip+' scm scm scm')

    # cdh - 4 - 启动服务
    @task
    @roles('cdh')
    def cdh4():
      run("/opt/cdh-cm/etc/init.d/cloudera-scm-server start")
      run("/opt/cdh-cm/etc/init.d/cloudera-scm-agent start")
    '''

    '''
    清空当前cloudera
    cd /opt ; rm -rf cloud*
    cd /opt ; rm -rf cm*
    cd /opt ; rm -rf solr*
    cd /opt ; rm -rf cdh-cm
    '''

    ''' 最后配置
    su hdfs
    hadoop fs -setrep 2 /

    '''

@task
@roles("cdh")
def cdh_clean():
    pass


# imports
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from TaxiFareModel.encoders import TimeFeaturesEncoder, DistanceTransformer
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from TaxiFareModel.utils import compute_rmse
import mlflow, memoized_property
from mlflow.tracking import MlflowClient

class Trainer():
    def __init__(self, X, y):
        """
            X: pandas DataFrame
            y: pandas Series
        """
        self.pipeline = None
        self.X = X
        self.y = y
        self.experiment_name = "[#871] [Berlin] [KiKar31] pipe v1"

    def set_pipeline(self):
        """defines the pipeline as a class attribute"""
        dist_pipe = Pipeline([
            ('dist_trans', DistanceTransformer()),
            ('stdscaler', StandardScaler())
        ])
        time_pipe = Pipeline([
            ('time_enc', TimeFeaturesEncoder('pickup_datetime')),
            ('ohe', OneHotEncoder(handle_unknown='ignore'))
        ])
        preproc_pipe = ColumnTransformer([
            ('distance', dist_pipe, ["pickup_latitude", "pickup_longitude", 'dropoff_latitude', 'dropoff_longitude']),
            ('time', time_pipe, ['pickup_datetime'])
        ], remainder="drop")
        pipe = Pipeline([
            ('preproc', preproc_pipe),
            ('linear_model', LinearRegression())
        ])
        self.pipeline = pipe

    def run(self):
        """set and train the pipeline"""
        return self.pipeline.fit(self.X, self.y)

    def evaluate(self, X_test, y_test):
        """evaluates the pipeline on df_test and return the RMSE"""
        y_pred = self.pipeline.predict(X_test)
        return compute_rmse(y_pred, y_test)

    @memoized_property
    def mlflow_client(self):
        mlflow.set_tracking_uri("https://mlflow.lewagon.ai/")
        return MlflowClient()

    @memoized_property
    def mlflow_experiment_id(self):
        try:
            return self.mlflow_client.create_experiment(self.experiment_name)
        except BaseException:
            return self.mlflow_client.get_experiment_by_name(self.experiment_name).experiment_id

    @memoized_property
    def mlflow_run(self):
        return self.mlflow_client.create_run(self.mlflow_experiment_id)

    def mlflow_log_param(self, key, value):
        self.mlflow_client.log_param(self.mlflow_run.info.run_id, key, value)

    def mlflow_log_metric(self, key, value):
        self.mlflow_client.log_metric(self.mlflow_run.info.run_id, key, value)

if __name__ == "__main__":
    # get data
    # clean data
    # set X and y
    # hold out
    # train
    # evaluate
    print('TODO')


from data_loader import *
from plotter import *
from utils import *
import netCDF4 as nc
import argparse
import pandas as pd
from tkinter import *

# TODO: Add gui....eventually
# import customtkinter
#
# customtkinter.set_appearance_mode('system')
# root = customtkinter.CTk()
# root.geometry('300x400')
# button = customtkinter.CTkButton(master=root, text='Hello World!!!')
# button.place(relx=0.5, rely=0.5, anchor=CENTER)
# root.mainloop()

# TODO: add style checks
# TODO: add option to wget data
# TODO: plot sc orbit locations

# def process_spacecraft_data(g17_file, g18_file, gk2a_file, g17_deg,
# g18_deg, gk2a_deg, save_path):
def process_spacecraft_data(g17_file=None, g18_file=None, gk2a_file=None,
                            g17_deg=None, g18_deg=None, gk2a_deg=None,
                            save_path=None):
    # For multiple s/c, one day is typical, unless use aggregate_nc_file to
    # look at multiple days at a time.
    # Plots mag inclination angle
    noonmidnighttimes_dict = {}
    goes_time_fromnc = None
    goes17_bgse_stacked = goes18_bgse_stacked = gk2a_bgse_stacked = None

    if g17_file:
        goes17coloc_dataset = nc.Dataset(g17_file)
        goes17_bgse_stacked = process_goes_dataset(
            goes17coloc_dataset['b_gse'])
        goes_time_fromnc = goes_epoch_to_datetime(
            goes17coloc_dataset['time'][:])
        goes17_VDH = gse_to_vdh(goes17_bgse_stacked, goes_time_fromnc)

    if g18_file:
        goes18coloc_dataset = nc.Dataset(g18_file)
        goes18_bgse_stacked = process_goes_dataset(
            goes18coloc_dataset['b_gse'])
        goes_time_fromnc = goes_epoch_to_datetime(
            goes18coloc_dataset['time'][:])
        goes18_VDH = gse_to_vdh(goes18_bgse_stacked, goes_time_fromnc)

    if gk2a_file:
        gk2a_dataset = nc.Dataset(gk2a_file)
        gk2a_bgse_stacked = stack_gk2a_data(gk2a_dataset)
        gk2a_VDH = gse_to_vdh(gk2a_bgse_stacked, goes_time_fromnc)

    date_str = get_date_str_from_goesTime(goes_time_fromnc)

    # Used to plot 'noon' and 'midnight' times (optional arg)
    noonmidnighttimes_dict = {}
    gk2a_noon, gk2a_midnight, g17_noon, g17_midnight, g18_noon, g18_midnight \
        = [
              None] * 6

    if gk2a_deg:
        gk2a_time_diff = calculate_time_difference(float(gk2a_deg), 'E')
        gk2a_noon, gk2a_midnight = find_noon_and_midnight_time(gk2a_time_diff,
                                                               date_str,
                                                               gk2a=True)
        noonmidnighttimes_dict['gk2a'] = {'noon': gk2a_noon,
                                          'midnight': gk2a_midnight}

    if g17_deg:
        g17_time_diff = calculate_time_difference(float(g17_deg))
        g17_noon, g17_midnight = find_noon_and_midnight_time(g17_time_diff,
                                                             date_str)
        noonmidnighttimes_dict['g17'] = {'noon': g17_noon,
                                         'midnight': g17_midnight}
    if g18_deg:
        g18_time_diff = calculate_time_difference(float(g18_deg))
        g18_noon, g18_midnight = find_noon_and_midnight_time(g18_time_diff,
                                                             date_str)
        noonmidnighttimes_dict['g18'] = {'noon': g18_noon,
                                         'midnight': g18_midnight}

    # Plot B field in GSE coords:
    plot_BGSE_fromdata_ontop(goes_time_fromnc, goes17_bgse_stacked,
                             goes18_bgse_stacked, 'G17', 'G18', 'SOSMAG',
                             gk2a_bgse_stacked, date_str, save_path,
                             noonmidnighttimes_dict)

    # Plot mag incl (theta) over time:
    plot_magnetic_inclination_over_time_3sc(date_str, goes_time_fromnc,
                                            goes17_VDH, goes18_VDH, gk2a_VDH,
                                            save_path, noonmidnighttimes_dict)

    # print(gk2a_noon, gk2a_midnight, g17_noon, g17_midnight, g18_noon,
    # g18_midnight)
    # print(noonmidnighttimes_dict)

# def analyze_pickle_data_statstics(data1, data2):
#     # Perform statistical analysis on entire directories of pickle data

def main():
    # picklefile = 'Z:/Data/GK2A/model_outputs/202208/sosmag_modout_OMNI2022
    # -08-04.pickle'
    # with open(picklefile, 'rb') as file:
    #     data = pickle.load(file)
    # print(data.keys())

    parser = argparse.ArgumentParser()

    # all spacecraft data is optional, and at least one is required.
    group = parser.add_argument_group('Spacecraft Data')
    group.add_argument("--g17-file", help="File path for GOES-17 mag data")
    group.add_argument("--g18-file", help="File path for GOES-18 mag data")
    group.add_argument("--gk2a-file", help="File path for GK2A SOSMAG data")

    # Optional arguments
    parser.add_argument("--save-path", default=None,
                        help="Save path for the figure \n(optional)")

    parser.add_argument("--g17-deg", default=None,
                        help="GOES17 s/c longitude in degrees (WEST), "
                             "ex. 105\n(optional)")
    parser.add_argument("--g18-deg", default=None,
                        help="GOES18 s/c longitude in degrees (WEST), "
                             "ex. 137.0\n(optional)")
    parser.add_argument("--gk2a-deg", default=None,
                        help="GK2A s/c longitude in degrees (EAST), "
                             "ex. 128.2\n(optional)")

    args = parser.parse_args()

    # Needed to initialize:
    # goes17_bgse_stacked, goes18_bgse_stacked, gk2a_bgse_stacked, gk2a_VDH, \
    #     goes17_VDH, goes17_VDH, save_path = [None] * 7

    # Check if at least one spacecraft data is provided
    if not any([args.g17_file, args.g18_file, args.gk2a_file]):
        parser.error('At least one spacecraft data file must be provided.')

    process_spacecraft_data(g17_file=args.g17_file, g18_file=args.g18_file,
                            gk2a_file=args.gk2a_file,
                            g17_deg=args.g17_deg, g18_deg=args.g18_deg,
                            gk2a_deg=args.gk2a_deg,
                            save_path=args.save_path)

if __name__ == "__main__":
    main()


import socket
import sys
import logging

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

# Create a TCP/IP socket
sock = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
# Connect the socket to the port where the server is listening
server_address = ('localhost', 5000)
logging.info ('connecting to {} port {}'.format (*server_address))
sock.connect (server_address)
try:
    # Send data
    message = b'00010sinitasipe'
    logging.info ('sending {!r}'.format (message))
    sock.sendall (message)
    while True:
        # Look for the response
        logging.info ("Waiting for register transaction")
        amount_received = 0
        amount_expected = int(sock.recv(5))
        while amount_received < amount_expected:
            data = sock.recv (amount_expected - amount_received)
            amount_received += len (data)
            logging.info('received {!r}'.format(data))
            logging.info ("Visualización de Asistencia por Personal ...")
            data = data.decode().split()
            try:
                opcion = data[1]
                PersonalRut = data[2]
                Fecha = data[3]
                #FechaDesde = data[4]
                #FechaHasta = data[5]

                largo = len(PersonalRut+Fecha+opcion) + 8
                message = '000{}datas {} {} {}'.format(largo,opcion,PersonalRut,Fecha).encode()
                logging.info ('sending to bbdd {!r}'.format (message))
                sock.sendall(message)
                algo = sock.recv(4096).decode()
                if algo:
                    if 'rut' in algo:
                        message = '00011asiperut'.encode()
                    elif 'fecha' in algo:
                        message = '00011asipefecha'.encode()
                    else:
                        message = '00010asipeexito {}'.format(algo).encode()
                    logging.info ('sending {!r}'.format (message))
                    sock.send(message)
            except:
                pass
            logging.info('-------------------------------')

finally:
    logging.info ('closing socket')
    sock.close ()


import os
import glob
import logging
import numbers
import numpy as np
import pandas as pd
from ast import literal_eval
from django.conf import settings

from ..utils.custom_decorator import where_exception
from ..data_preprocess.preprocess_base import PreprocessorBase

logger = logging.getLogger("collect_log_helper")

def _error_return_dict(error_type, error_msg):
    """
    Return common error dictionary type

        Parameters:
        -----------
             error_type (str) : type of error (eg. '4102')
             error_msg (str) : detail message of the error

        Returns:
        --------
             (dict) : common error dictionary
    """
    return dict(error_type=error_type, error_msg=error_msg)

class BatchTestResult(PreprocessorBase):
    def __init__(self, batch_service, test_data_path):
        self.batch_manager_id = batch_service['BATCH_SERVICE_SEQUENCE_PK']
        self.model_summary = batch_service['MODEL_SUMMARY']
        self.model_command = batch_service['MODEL_COMMAND']
        self.pdata_summary = batch_service['PREPROCESSED_DATA_SUMMARY']
        self.model_sandbox_pk = batch_service['MODEL_SANDBOX_SEQUENCE_FK1']
        self.trans_sandbox_pk = batch_service['PREPROCESSED_DATA_SANDBOX_SEQUENCE_FK2']

        self.nfs_dir = settings.ANALYTICS_MANAGER_NFS # /ANALYTICS_MANAGER_NFS/batchServer
        self.test_data_path = test_data_path
        self.nfs_batch_info_dir = os.path.join(self.nfs_dir, f'batchService_{self.batch_manager_id}')
        self.nfs_model_path = os.path.join(self.nfs_batch_info_dir, f'M_{self.model_sandbox_pk}.pickle')
        self.nfs_trans_path = glob.glob(
            os.path.join(self.nfs_batch_info_dir, f'T_{self.trans_sandbox_pk}_*.pickle'))

    # 모델학습에서 사용한 데이터와 테스트 데이터이 컬럼이 일치하는지 확인하는 함수
    @staticmethod
    def _check_train_columns(data_set, train_summary, target_data):
        test_data_columns = list(data_set.columns.values)
        test_data_columns.remove(target_data)
        test_data_columns.sort()
        train_data_summary = literal_eval(train_summary)
        train_data_columns = train_data_summary["model_train_columns"]
        train_data_columns.sort()

        if test_data_columns == train_data_columns:
            return True
        else:
            return False

    # Train Data 와 동일한 변환기로 Test Data 에 전처리를 수행하는 함수
    def _test_data_transformer(self, data_set, pdata_summary):
        test_data_columns = list(data_set.columns.values)
        train_pdata_summary = literal_eval(pdata_summary)  # str => list

        # 학습된 데이터의 전처리 정보를 읽어서 차례대로 동일하게 수행하는 코드
        for preprocess_info_dict in train_pdata_summary:
            field_name = preprocess_info_dict["field_name"]
            func_name = preprocess_info_dict["function_name"]
            file_name = preprocess_info_dict["file_name"]
            logger.info(f"[모델 배치] {func_name} applied to {field_name}")

            if field_name not in test_data_columns:
                return False
            else:
                if func_name == "DropColumns":
                    data_set = super()._drop_columns(data_set, field_name)
                else:
                    transformer = super()._load_pickle(
                        base_path=self.nfs_batch_info_dir, file_name=file_name
                    )
                    changed_field = transformer.transform(
                        data_set[field_name].values.reshape(-1, 1)
                    )
                    changed_field = super()._to_array(changed_field)

                    # transform 된 데이터와 원본 데이터 통합(NEW) - preprocess_helper.py 참고
                    if len(changed_field.shape) == 2 and changed_field.shape[1] == 1:
                        if func_name == "Normalizer":
                            logger.warning("Not working in this version!!!")
                        else:
                            data_set[field_name] = changed_field
                    elif len(changed_field.shape) == 1:  # LabelEncoder
                        data_set[field_name] = changed_field
                    else:
                        col_name = super()._new_columns(
                            field_name=field_name, after_fitted=changed_field
                        )
                        new_columns = pd.DataFrame(changed_field, columns=col_name)
                        data_set = pd.concat(
                            [data_set, new_columns], axis=1, sort=False
                        )
                        data_set = data_set.drop(field_name, axis=1)
        return data_set

    # 배치 서비스 요청에 대한 요청 파라미터 검사하는 함수
    def check_request_batch_path(self):
        check_list = [self.test_data_path, self.nfs_model_path, self.nfs_trans_path[0]]
        for check_path in check_list:
            logger.info(f"경로 확인 중... [{check_path}]")
            if not os.path.isfile(check_path):
                logger.error(f"{check_path} 경로가 존재하지 않습니다")
                return dict(error_type="4004", error_msg=check_path)
        return True

    # 예측값 또는 스코어를 출력하는 함수
    def get_batch_test_result(self):
        try:
            # 테스트 데이터 로드
            if self.test_data_path.endswith(".csv"):
                test_data = pd.read_csv(self.test_data_path)
            elif self.test_data_path.endswith(".json"):
                test_data = pd.read_json(
                    self.test_data_path, lines=True, encoding="utf-8"
                )
            logger.info(f"[모델 배치] Batch ID [{self.batch_manager_id}] Data Load!")

            # 테스트 데이터 전처리
            pdata_test = self._test_data_transformer(
                data_set=test_data, pdata_summary=self.pdata_summary
            )

            if isinstance(pdata_test, bool):  # 오류 발생시 False 반환
                logger.error(
                    f"[모델 배치 err1] Batch ID [{self.batch_manager_id}] Check Columns Name"
                )
                return _error_return_dict("4022", "Data is not suitable for the model")
            target = literal_eval(self.model_command)["train_parameters"]["y"]
            is_same_columns = self._check_train_columns(
                data_set=pdata_test,
                train_summary=self.model_summary,
                target_data=target,
            )

            if not is_same_columns:
                logger.error(
                    f"[모델 배치 err2] Batch ID [{self.batch_manager_id}] Check Columns Name"
                )
                return _error_return_dict("4022", "Data is not suitable for the model")
            # 모델 로드
            model_load = super()._load_pickle(
                base_path=self.nfs_batch_info_dir,
                file_name="M_{}.pickle".format(self.model_sandbox_pk),
            )
            logger.info(f"[모델 배치] Batch ID [{self.batch_manager_id}] Model Load!")

            # 모델 테스트 결과9
            X_ = super()._drop_columns(pdata_test, target)
            y_ = np.array(pdata_test[target]).reshape(-1, 1)
            score_ = model_load.score(X=X_, y=y_)
            predict_ = model_load.predict(X=X_)
            logger.info(
                f"[모델 배치] Batch ID [{self.batch_manager_id}] Predict Result Return!"
            )

            if isinstance(predict_[0], numbers.Integral):
                result_response = {"score": "%.3f" % score_, "predict": predict_}
                return result_response
            else:
                result_response = ["%.3f" % elem for elem in predict_]
                result_response = {"score": "%.3f" % score_, "predict": result_response}
                return result_response
        except Exception as e:
            where_exception(error_msg=e)


from openpyxl import load_workbook

class doExcel():

    def __init__(self,file_path,sheet_name):
        self.file_path=file_path
        self.sheet_name=sheet_name
        self.wb=load_workbook(self.file_path)
        self.sh=self.wb[self.sheet_name]
        #获取当前sheet最大的行数
        self.row_max=self.sh.max_row
        # 获取当前sheet最大的列数
        self.column_max=self.sh.max_column

    def get_sheet(self):
        sheet_list=[]
        for i in range(2,self.row_max+1):
            sheet_row_list={}
            sheet_row_list["interface_name"]=self.sh.cell(i,1).value
            sheet_row_list["interface_url"]=self.sh.cell(i,2).value
            sheet_row_list["interface_method"]=self.sh.cell(i,3).value
            sheet_row_list["interface_body"]=self.sh.cell(i,4).value
            sheet_row_list["interface_expectedOutcome"]=self.sh.cell(i,5).value
            sheet_row_list["interface_actualResults"]=self.sh.cell(i,6).value
            sheet_list.append(sheet_row_list)
        return sheet_list

    def get_cell(self,row,column):
        return self.sh.cell(row,column).value

    def set_cell(self,row,column,cellvalue):
        self.sh.cell(row=row, column=column).value = cellvalue
        self.wb.save(self.file_path)

file_path="C:\\Users\\admin\\Desktop\\test_api.xlsx"
sheet_name="Sheet1"
do=doExcel(file_path,sheet_name)
do.set_cell(10,10,"")
print(do.get_cell(10,10))


import os
import json
from flask import Flask, request, send_file, jsonify
from picamera2 import Picamera2, Preview
import time
from PIL import Image

picam2 = Picamera2()
camera_config = picam2.create_still_configuration(main={"size": (256, 256)})
picam2.set_controls({"ExposureTime": 5000})
picam2.configure(camera_config)
# picam2.start_preview(Preview.DRM)
picam2.start()
time.sleep(2)
# picam2.capture_file("test.jpg")

app = Flask(__name__)
counter = 0
# Simulated raw_image (you should replace this with your own captured image)
# For demonstration purposes, we'll assume it's already a PIL Image.
# raw_image = Image.new("RGB", (640, 480), "white")

@app.route('/')
def index():
    return 'Camera Server is running!'

@app.route('/capture', methods=['GET'])
def capture():
    try:
        raw_image = picam2.capture_image("main")
        # Flip the image horizontally
        flipped_image = raw_image.transpose(Image.FLIP_TOP_BOTTOM)

        # Create a unique filename based on the current timestamp
        filename = f"photo_{int(time.time())}.jpeg"

        # Save the flipped image as a JPEG file
        flipped_image.save(filename, "JPEG")

        # Send the image to the client
        return send_file(filename, mimetype='image/jpeg')

    except Exception as e:
        return str(e)

@app.route('/store_inference', methods=['POST'])
def store_inference():
    global counter

    data = request.get_json()
    print(data)

    infer_text = data.get('infer_text', '')
    infer_img = data.get('infer_img', '')

    # Increment the counter
    counter += 1

    # Create the JSON payload
    inference_json = {
        'infer_text': infer_text,
        'infer_img': infer_img
    }

    # Save the JSON to a file
    filename = f'inference_{counter}.json'
    file_path = os.path.join('./inferences', filename)  # Replace with your desired directory
    with open(file_path, 'w') as f:
        json.dump(inference_json, f)

    # Respond with a success message
    return jsonify({'message': f'Inference stored successfully. Saved as {filename}'})

@app.route('/get_inference', methods=['GET'])
def get_inference():
    index = request.args.get('index', type=int)

    # Generate the filename based on the index
    filename = f'inference_{index}.json'
    file_path = os.path.join("./inferences", filename)

    if os.path.exists(file_path):
        # Load the JSON from the specified file
        with open(file_path, 'r') as f:
            inference_data = json.load(f)
        return jsonify(inference_data)
    else:
        return jsonify({'error': f'Inference file for index {index} not found.'}), 404

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8000)