"""
.. module::  contacts.models
   :synopsis:  Contact models module.

Contact models module.  A primary design decision was to be inspired by key
abstractions from rfc6350 "vCard Format Specification".  In addition to design
reuse, the objective is to be able to import and export data in compliance
with rfc6350.

This lead to a more complex business object model.  The intention is
to hide the complexity  in the client application.

VCard import/export per VCard spec version issues have to be handled
by the import/export process logic.  There may be cases were
an import->export process will not yield an exact "equal" VCard instance.

One of the design principles adhered to was allowing the grouping, tagging,
or organization of instances using the definition of an associated 'Type' class
(i.e. Phone->PhoneType, Email->EmailType)
"""
#
# Note: Adding db table name naming convention control to allow flexible
#    integration of the application in target db environments.
#    Have not found a way to have a class decorator simplify the
#    implementation with Django 1.9.  A db table is named using the following
#    substrings, each separated by an '_':
#    a) Site label (i.e. 'sl')
#    b) Application name (i.e. contacts)
#    c) Class name in lower case with underscores (i.e. contact_name)
#    Based on the above, for a class named ContactName, the db table name
#    would be:
#        'sl_contacts_contact_name'
#

# @TODO: add missing rfc6350 fields (i.e. Contact Key, audio, Media
#        see https://github.com/areski/django-audiofield)
# @TODO: For image, review approach to have bundled
#        thumbnail/compressed version
# @TODO: check implementation of related contact when uri is used,not contact
#        defined in model
# @TODO: contact accesss requires quite a few joins, look at optimization
# @TODO: str methods
# @TODO: review on_delete=CASCADE for reference data types
#        in many-2-many relationships
# @TODO: review manager add/remove method implementation; too many methods
from __future__ import absolute_import

# @TODO: review class field layout
# @TODO: review each of the types, determine which should be optional
import logging
from inflection import humanize, pluralize, underscore

from django.conf import settings
import django.contrib.auth.models
from django.db.models import Model
from django.db.models import CASCADE
from django.utils.translation import ugettext_lazy as _

from guardian.models import UserObjectPermissionBase
from guardian.models import GroupObjectPermissionBase

from python_core_utils.core import class_name
from django_core_utils import fields

from django_core_models.core.models import Annotation, Category
from django_core_models.demographics.models import Gender
from django_core_models.images.models import ImageReference
from django_core_models.locations.models import (Address, AddressType,
                                                 GeographicLocation,
                                                 GeographicLocationType,
                                                 Language, LanguageType,
                                                 Timezone, TimezoneType)
from django_core_models.organizations.models import (Organization,
                                                     OrganizationUnit, Role,
                                                     Title)
from django_core_models.social_media.models import (Email, EmailType,
                                                    FormattedName, Group,
                                                    InstantMessaging,
                                                    InstantMessagingType,
                                                    LogoType, Name, Nickname,
                                                    NicknameType, Phone,
                                                    PhoneType, PhotoType, Url,
                                                    UrlType)

from django_core_utils.models import (NamedModel, PrioritizedModel,
                                      VersionedModelManager,
                                      db_table, related_name_base)

from . import validation

logger = logging.getLogger(__name__)
_app_label = 'contacts'

def create_fields(instance, **kwargs):
    """Return dict of fields to be used as create params."""
    site = kwargs.pop("site", instance.site)
    creation_user = kwargs.pop("creation_user", instance.creation_user)
    effective_user = kwargs.pop("effective_user", instance.effective_user)
    update_user = kwargs.pop("update_user", instance.update_user)
    params = dict(
        creation_user=creation_user, effective_user=effective_user,
        site=site, update_user=update_user)
    params.update(kwargs)
    return params

def delete_association(association_class, **kwargs):
    """Remove an association.

    Requires deletion when using 'through'.
    """

    try:
        instance = association_class.objects.get(**kwargs)
    except association_class.DoesNotExist:
        logger.exception(
            "%s instance with kwargs %s not found",
            class_name(association_class), kwargs)
        raise

    return instance.delete()

def create_association(association_class, **kwargs):
    """Create association instance."""
    return association_class.objects.create(**kwargs)

_contact_type = "ContactType"
_contact_type_vebose = humanize(underscore(_contact_type))

class ContactType(NamedModel):
    """Contact type model class.

    Allows the organization of contacts into logical types.
    Values may include individual, organization, unknown.
    """
    # @TODO: is this duplicate of category?
    class Meta(NamedModel.Meta):
        """Model meta class declaration."""
        app_label = _app_label
        db_table = db_table(_app_label, _contact_type)
        verbose_name = _(_contact_type_vebose)
        verbose_name_plural = _(pluralize(_contact_type_vebose))

_related_contact_type = "ContactRelationshipType"
_related_contact_type_verbose = humanize(underscore(_related_contact_type))

class ContactRelationshipType(NamedModel):
    """Related contact type model class.

    Allows classification of contact relationships.
    Sample values may include "unknown",
    "acquaintance", "parent", "child", "co-worker",  "friend"
    """
    class Meta(NamedModel.Meta):
        app_label = _app_label
        db_table = db_table(_app_label, _related_contact_type)
        verbose_name = _(_related_contact_type_verbose)
        verbose_name_plural = _(pluralize(_related_contact_type_verbose))

class ContactsModel(PrioritizedModel):
    """Base contacts class."""
    class Meta(PrioritizedModel.Meta):
        """Model meta class declaration."""
        app_label = _app_label
        abstract = True

class ContactManager(VersionedModelManager):
    """Contact manager class."""

    def address_add(self, contact, address, **kwargs):
        """Add contact and address association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactAddress, contact=contact,
            address=address, **params)

    def address_remove(self, contact, address, **kwargs):
        """Remove contact and address association."""
        return delete_association(
            ContactAddress, contact=contact, address=address)

    def annotation_add(self, contact, annotation, **kwargs):
        """Add contact and annotation association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactAnnotation, contact=contact,
            annotation=annotation, **params)

    def annotation_remove(self, contact, annotation, **kwargs):
        """Remove contact and annotation association."""
        return delete_association(
            ContactAnnotation, contact=contact, annotation=annotation)

    def category_add(self, contact, category, **kwargs):
        """Add contact and categry association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactCategory, contact=contact,
            category=category, **params)

    def category_remove(self, contact, category):
        """Remove contact and category association."""
        return delete_association(
            ContactCategory, contact=contact, category=category)

    def email_add(self, contact, email, **kwargs):
        """Add contact and email association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactEmail, contact=contact,
            email=email, **params)

    def email_remove(self, contact, email):
        """Remove contact and email association."""
        return delete_association(
            ContactEmail, contact=contact, email=email)

    def formatted_name_add(self, contact, name, **kwargs):
        """Add contact and formatted name association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactFormattedName, contact=contact,
            name=name, **params)

    def formatted_name_remove(self, contact, name):
        """Remove contact and formatted name association."""
        return delete_association(
            ContactFormattedName, contact=contact, name=name)

    def geographic_location_add(self, contact, geographic_location, **kwargs):
        """Add contact and geographic location association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactGeographicLocation, contact=contact,
            geographic_location=geographic_location, **params)

    def geographic_location_remove(self, contact, geographic_location):
        """Remove contact and geographic location association."""
        return delete_association(
            ContactGeographicLocation, contact=contact,
            geographic_location=geographic_location)

    def group_add(self, contact, group, **kwargs):
        """Add contact and group association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactGroup, contact=contact,
            group=group, **params)

    def group_remove(self, contact, group):
        """Remove contact and group association."""
        return delete_association(
            ContactGroup, contact=contact, group=group)

    def instant_messaging_add(self, contact, instant_messaging, **kwargs):
        """Add contact and instant messaging association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactInstantMessaging, contact=contact,
            instant_messaging=instant_messaging, **params)

    def instant_messaging_remove(self, contact, instant_messaging):
        """Remove contact and instant_messaging association."""
        return delete_association(
            ContactInstantMessaging, contact=contact,
            instant_messaging=instant_messaging)

    def language_add(self, contact, language, **kwargs):
        """Add contact and language association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactLanguage, contact=contact,
            language=language, **params)

    def language_remove(self, contact, language):
        """Remove contact and language association."""
        return delete_association(
            ContactLanguage, contact=contact,
            language=language)

    def logo_add(self, contact, image_reference, **kwargs):
        """Add contact and logo  association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactLogo, contact=contact,
            image_reference=image_reference, **params)

    def logo_remove(self, contact, image_reference):
        """Remove contact and logo association."""
        return delete_association(
            ContactLogo, contact=contact,
            image_reference=image_reference)

    def name_add(self, contact, name, **kwargs):
        """Add contact and name association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactName, contact=contact,
            name=name, **params)

    def name_remove(self, contact, name):
        """Remove contact and name association."""
        return delete_association(ContactName, contact=contact, name=name)

    def nickname_add(self, contact, name, **kwargs):
        """Add contact and nickname association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactNickname, contact=contact,
            name=name, **params)

    def nickname_remove(self, contact, name):
        """Remove contact and nickname association."""
        return delete_association(ContactNickname, contact=contact, name=name)

    def organization_add(self, contact, organization, **kwargs):
        """Add contact and organization association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactOrganization, contact=contact,
            organization=organization, **params)

    def organization_remove(self, contact, organization):
        """Remove contact and organization association."""
        return delete_association(
            ContactOrganization, contact=contact, organization=organization)

    def phone_add(self, contact, phone, **kwargs):
        """Add contact and phone association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactPhone, contact=contact,
            phone=phone, **params)

    def phone_remove(self, contact, phone):
        """Remove contact and phone association."""
        return delete_association(
            ContactPhone, contact=contact, phone=phone)

    def photo_add(self, contact, image_reference, **kwargs):
        """Add contact and photo association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactPhoto, contact=contact,
            image_reference=image_reference, **params)

    def photo_remove(self, contact, image_reference):
        """Remove contact and photo association."""
        return delete_association(
            ContactPhoto, contact=contact, image_reference=image_reference)

    def related_contact_add(self, from_contact, to_contact, **kwargs):
        """Add contact and photo association."""
        params = create_fields(from_contact, **kwargs)
        return create_association(
            RelatedContact, from_contact=from_contact,
            to_contact=to_contact, **params)

    def related_contact_remove(self, from_contact, to_contact):
        """Remove contact and photo association."""
        return delete_association(
            RelatedContact, from_contact=from_contact, to_contact=to_contact)

    def role_add(self, contact, role, **kwargs):
        """Add contact and role association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactRole, contact=contact,
            role=role, **params)

    def role_remove(self, contact, role):
        """Remove contact and role association."""
        return delete_association(
            ContactRole, contact=contact, role=role)

    def timezone_add(self, contact, timezone, **kwargs):
        """Add contact and timezone association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactTimezone, contact=contact,
            timezone=timezone, **params)

    def timezone_remove(self, contact, timezone):
        """Remove contact and timezone association."""
        return delete_association(
            ContactTimezone, contact=contact, timezone=timezone)

    def title_add(self, contact, title, **kwargs):
        """Add contact and title association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactTitle, contact=contact,
            title=title, **params)

    def title_remove(self, contact, title):
        """Remove contact and title association."""
        return delete_association(
            ContactTitle, contact=contact, title=title)

    def url_add(self, contact, url, **kwargs):
        """Add contact and url association."""
        params = create_fields(contact, **kwargs)
        return create_association(
            ContactUrl, contact=contact,
            url=url, **params)

    def url_remove(self, contact, url):
        """Remove contact and url association."""
        return delete_association(
            ContactUrl, contact=contact, url=url)

_contact = "Contact"
_contact_verbose = humanize(underscore(_contact))

class Contact(ContactsModel):
    """Contact model class.

    Capture contact attributes.  Designed to allow having multiple contact
    instances per person, organization, etc.

    Either name or formatted name have to be set on an instance.
    """
    formatted_name = fields.foreign_key_field(
        FormattedName, blank=True, null=True)
    name = fields.foreign_key_field(Name, blank=True, null=True)

    # 'simple fields'
    anniversary = fields.date_field(blank=True, null=True)
    birth_date = fields.date_field(blank=True, null=True)
    contact_type = fields.foreign_key_field(ContactType, blank=True, null=True)
    gender = fields.foreign_key_field(Gender, blank=True, null=True)

    # many-2-many fields
    addresses = fields.many_to_many_field(Address, through="ContactAddress")
    annotations = fields.many_to_many_field(Annotation,
                                            through="ContactAnnotation")
    categories = fields.many_to_many_field(Category,
                                           through="ContactCategory")
    emails = fields.many_to_many_field(Email,
                                       through="ContactEmail")
    formatted_names = fields.many_to_many_field(FormattedName,
                                                through="ContactFormattedName")
    geographic_locations = fields.many_to_many_field(
        GeographicLocation, through="ContactGeographicLocation")
    groups = fields.many_to_many_field(Group,
                                       through="ContactGroup")
    instant_messaging = fields.many_to_many_field(
        InstantMessaging, through="ContactInstantMessaging")
    languages = fields.many_to_many_field(Language, through="ContactLanguage")
    logos = fields.many_to_many_field(
        ImageReference, through="ContactLogo",
        related_name=related_name_base + "contact_logo")
    names = fields.many_to_many_field(Name, through="ContactName")
    nicknames = fields.many_to_many_field(Nickname,
                                          through="ContactNickname")
    organizations = fields.many_to_many_field(Organization,
                                              through="ContactOrganization")
    phones = fields.many_to_many_field(Phone,
                                       through="ContactPhone")
    photos = fields.many_to_many_field(
        ImageReference, through="ContactPhoto",
        related_name=related_name_base + "contact_photo")
    related_contacts = fields.many_to_many_field(
        "self", through="RelatedContact", symmetrical=False)
    roles = fields.many_to_many_field(Role, through="ContactRole")
    timezones = fields.many_to_many_field(Timezone, through="ContactTimezone")
    titles = fields.many_to_many_field(Title, through="ContactTitle")
    urls = fields.many_to_many_field(Url, through="ContactUrl")

    objects = ContactManager()

    class Meta(ContactsModel.Meta):
        db_table = db_table(_app_label, _contact)
        verbose_name = _(_contact_verbose)
        verbose_name_plural = _(pluralize(_contact_verbose))
        permissions = (
            ("read_contact", "Can read contacts"),
            ("write_contact", "Can write contacts"),
        )

    def clean(self):
        super(Contact, self).clean()
        validation.contact_validation(self)

    @property
    def display_name(self):
        return self.name if self.name else self.formatted_name

    def __str__(self):
        return str(self.display_name)

class ContactAssociation(ContactsModel):
    """Base class for contact association."""
    class Meta(ContactsModel.Meta):
        abstract = True

_contact_address = "ContactAddress"
_contact_address_verbose = humanize(underscore(_contact_address))

class ContactAddress(ContactAssociation):
    """Contact address model class.

    Capture the attributes of contact address(s).
    A contact may be associated with 0 or more addresses as follows:
    Contact(1)  -------> ContactAddress(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    address = fields.foreign_key_field(Address, on_delete=CASCADE)
    address_type = fields.foreign_key_field(AddressType, null=True, blank=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_address)
        verbose_name = _(_contact_address_verbose)
        verbose_name_plural = _(pluralize(_contact_address_verbose))
        unique_together = ("contact", "address", "address_type")

_contact_annotation = "ContactAnnotation"
_contact_annotation_verbose = humanize(underscore(_contact_annotation))

class ContactAnnotation(ContactAssociation):
    """Contact annotation model class.

    Capture contact annotation/notes attributes.  A contact may be associated
    with 0 or more  annotations:
        Contact(1) ------> Annotation(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    annotation = fields.foreign_key_field(Annotation, on_delete=CASCADE)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_annotation)
        verbose_name = _(_contact_annotation_verbose)
        verbose_name_plural = _(pluralize(_contact_annotation_verbose))
        unique_together = ("contact", "annotation")

_contact_category = "ContactCategory"
_contact_category_verbose = humanize(underscore(_contact_category))

class ContactCategory(ContactAssociation):
    """Contact category model class.

    Capture contact category attributes.  A contact may be associated
    with 0 or more contact categories:
        Contact(1) ------> Category(0..*)
    """
    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_category)
        verbose_name = _(_contact_category_verbose)
        verbose_name_plural = _(pluralize(_contact_category_verbose))
        unique_together = ("contact", "category")

    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    category = fields.foreign_key_field(Category, on_delete=CASCADE)

_contact_email = "ContactEmail"
_contact_email_verbose = humanize(underscore(_contact_email))

class ContactEmail(ContactAssociation):
    """Contact email address model class.

    Captures contact email address(s).  Contact may be associated with
    0 or more email addresses:
        Contact (1) ------> Email(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    email = fields.foreign_key_field(Email, on_delete=CASCADE)
    email_type = fields.foreign_key_field(EmailType, null=True, blank=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_email)
        verbose_name = _(_contact_email_verbose)
        verbose_name_plural = _(pluralize(_contact_email_verbose))
        unique_together = ("contact", "email", "email_type", )

_formatted_name = "FormattedName"
_formatted_name_verbose = humanize(underscore(_formatted_name))

class ContactFormattedName(ContactAssociation):
    """Contact formatted name model class.

    Specifies the formatted contact name.
    A Contact must be associated with at least one FormatedName instance:
        Contact(1) ------> FormattedName(1..*)
    """

    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    name = fields.foreign_key_field(FormattedName, on_delete=CASCADE)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _formatted_name)
        verbose_name = _(_formatted_name_verbose)
        verbose_name_plural = _(pluralize(_formatted_name_verbose))
        unique_together = ('contact', 'name')

    def clean(self):
        super(ContactFormattedName, self).clean()
        validation.contact_formatted_name_validation(self)

_contact_geographic_location = "ContactGeographicLocation"
_contact_geographic_location_verbose = humanize(
    underscore(_contact_geographic_location))

class ContactGeographicLocation(ContactAssociation):
    """
    Contact geographic location model class.

    Capture contact geographic location(s) information.  A contact may be
    associated with 0 or more geographic locations:
        Contact ------> GeographicLocation(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    geographic_location = fields.foreign_key_field(GeographicLocation,
                                                   on_delete=CASCADE)
    geographic_location_type = fields.foreign_key_field(
        GeographicLocationType, blank=True, null=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_geographic_location)
        verbose_name = _(_contact_geographic_location_verbose)
        verbose_name_plural = _(
            pluralize(_contact_geographic_location_verbose))
        unique_together = ("contact",
                           "geographic_location",
                           "geographic_location_type")

_contact_group = "ContactGroup"
_contact_group_verbose = humanize(underscore(_contact_group))

class ContactGroup(ContactAssociation):
    """ContactGroup model class.

    Capture contact group attributes.  A contact may be associated
    with 0 or more groups :
        Contact(1) ------> Group(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    group = fields.foreign_key_field(Group, on_delete=CASCADE)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_group)
        verbose_name = _(_contact_group_verbose)
        verbose_name_plural = _(pluralize(_contact_group_verbose))
        unique_together = ("contact", "group")

class ContactImage(ContactAssociation):
    """Contact image abstract model base class.

    Capture contact  photograph information.
    A contact may be associated with 0 or more images:
        Contact(1) -------> Image(0..*).

    An image may be associated with one or more contacts.
    """
    contact = fields.foreign_key_field(
        Contact, on_delete=CASCADE,
        related_name="%(app_label)s_%(class)s_related_contact")
    image_reference = fields.foreign_key_field(ImageReference,
                                               on_delete=CASCADE)

    class Meta(ContactAssociation.Meta):
        abstract = True

_contact_instant_messaging = "ContactInstantMessaging"
_contact_instant_messaging_verbose = humanize(
    underscore(_contact_instant_messaging))

class ContactInstantMessaging(ContactAssociation):
    """Contact instant messaging model class.

    Capture the URI(s) for contact instant messaging.
    A contact may be associated with  0 or more instance messaging uri's:
        Contact(1) -------> InstantMessaging(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    instant_messaging = fields.foreign_key_field(InstantMessaging,
                                                 on_delete=CASCADE)
    instant_messaging_type = fields.foreign_key_field(InstantMessagingType,
                                                      blank=True, null=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_instant_messaging)
        verbose_name = _(_contact_instant_messaging_verbose)
        verbose_name_plural = _(pluralize(_contact_instant_messaging_verbose))
        unique_together = ("contact",
                           "instant_messaging",
                           "instant_messaging_type", )

_contact_language = "ContactLanguage"
_contact_language_verbose = humanize(underscore(_contact_language))

class ContactLanguage(ContactAssociation):
    """Contact language model class.

    Specify the language(s) that may be used for communicating with
    a contact.  A contact may be associated 0 or more languages:
        Contact(1) -------> ContactLanguage(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    language = fields.foreign_key_field(Language, on_delete=CASCADE)
    language_type = fields.foreign_key_field(
        LanguageType, blank=True, null=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_language)
        verbose_name = _(_contact_language_verbose)
        verbose_name_plural = _(pluralize(_contact_language_verbose))
        unique_together = ("contact", "language", "language_type")

_contact_logo = "ContactLogo"
_contact_logo_verbose = humanize(underscore(_contact_logo))

class ContactLogo(ContactImage):
    """Contact logo class.

    Capture contact logo information.
    A contact may be associated with 0 or more logos:
        Contact(1) -------> Logo(0..*).
    """
    logo_type = fields.foreign_key_field(LogoType, null=True, blank=True)

    class Meta(ContactImage.Meta):
        db_table = db_table(_app_label, _contact_logo)
        verbose_name = _(_contact_logo_verbose)
        verbose_name_plural = _(pluralize(_contact_logo_verbose))
        unique_together = ("contact", "image_reference", "logo_type")

_contact_name = "ContactName"
_contact_name_verbose = humanize(underscore(_contact_name))

class ContactName(ContactAssociation):
    """Contact name model class.

    Defines the contact name components.
    While the RFC specifies that a contact may be associated with at most one
    Name instance, this implementation supports many such instances:
        Contact(1) --------> Name(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    name = fields.foreign_key_field(Name, on_delete=CASCADE)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_name)
        verbose_name = _(_contact_name_verbose)
        verbose_name_plural = _(pluralize(_contact_name_verbose))
        unique_together = ('contact', 'name')

    def clean(self):
        super(ContactName, self).clean()
        validation.contact_name_validation(self)

_contact_nickname = "ContactNickname"
_contact_nickname_verbose = humanize(underscore(_contact_nickname))

class ContactNickname(ContactAssociation):
    """Contact nickname model class.

    Allow a association of contact with a nickname.
    A Contact may be associated with 0 or more Nickname instances:
        Contact(1) ------> Nickname(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    name = fields.foreign_key_field(Nickname, on_delete=CASCADE)
    nickname_type = fields.foreign_key_field(
        NicknameType, null=True, blank=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_nickname)
        verbose_name = _(_contact_nickname_verbose)
        verbose_name_plural = _(pluralize(_contact_nickname_verbose))
        unique_together = ("contact", "name", "nickname_type")

_contact_organization = "ContactOrganization"
_contact_organization_verbose = humanize(underscore(_contact_organization))

class ContactOrganization(ContactAssociation):
    """Contact organization model class.
    A contact may be associated
    with 0 or more organizations:
        Contact(1) ------> Organization(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    organization = fields.foreign_key_field(Organization, on_delete=CASCADE)
    unit = fields.foreign_key_field(OrganizationUnit, blank=True, null=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_organization)
        verbose_name = _(_contact_organization_verbose)
        verbose_name_plural = _(pluralize(_contact_organization_verbose))
        unique_together = ("contact", "organization")

_contact_phone = "ContactPhone"
_contact_phone_verbose = humanize(underscore(_contact_phone))

class ContactPhone(ContactAssociation):
    """Contact phone model class.

    Capture contact telephone(s) attributes.
    A contact may be associated with 0 or more  telephones:
        Contact(1) ------> Phone(0:*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    phone = fields.foreign_key_field(Phone, on_delete=CASCADE)
    phone_type = fields.foreign_key_field(PhoneType, blank=True, null=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_phone)
        verbose_name = _(_contact_phone_verbose)
        verbose_name_plural = _(pluralize(_contact_phone_verbose))
        unique_together = ("contact", "phone", "phone_type")

_contact_photo = "ContactPhoto"
_contact_photo_verbose = humanize(underscore(_contact_photo))

class ContactPhoto(ContactImage):
    """Contact photo class.

    Capture contact photo information.
    A contact may be associated with 0 or more photos:
        Contact(1) -------> Photo(0..*).
    """

    photo_type = fields.foreign_key_field(PhotoType, null=True, blank=True)

    class Meta(ContactImage.Meta):
        db_table = db_table(_app_label, _contact_photo)
        verbose_name = _(_contact_photo_verbose)
        verbose_name_plural = _(pluralize(_contact_photo_verbose))
        unique_together = ("contact", "image_reference", "photo_type")

_contact_role = "ContactRole"
_contact_role_verbose = humanize(underscore(_contact_role))

class ContactRole(ContactAssociation):
    """
    Contract role model class.

    Within the context of an organization, capture contact role attributes.
    A contact may be associated with multiple roles:
        Contact(1) -----> Role(0:*)

    The organization for which the role is in effect is optional.
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    role = fields.foreign_key_field(Role, on_delete=CASCADE)
    organization = fields.foreign_key_field(Organization,
                                            blank=True, null=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_role)
        verbose_name = _(_contact_role_verbose)
        verbose_name_plural = _(pluralize(_contact_role_verbose))
        unique_together = ("contact", "role", "organization")

_contact_timezone = "ContactTimezone"
_contact_timezone_verbose = humanize(underscore(_contact_timezone))

class ContactTimezone(ContactAssociation):
    """
    Contact time zone model class.

    Capture contact time zone(s) information.  A contact may be
    associated with 0 or more time zone:
        Contact ------> Timezone(0..*)
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    timezone = fields.foreign_key_field(Timezone, on_delete=CASCADE)
    timezone_type = fields.foreign_key_field(
        TimezoneType, blank=True, null=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_timezone)
        verbose_name = _(_contact_timezone_verbose)
        verbose_name_plural = _(pluralize(_contact_timezone_verbose))
        unique_together = (
            "contact", "timezone", "timezone_type")

_contact_title = "ContactTitle"
_contact_title_verbose = humanize(underscore(_contact_title))

class ContactTitle(ContactAssociation):
    """
    Contact title model class.

    Within the context of an organization, capture contact title attributes.
    A contact may be associated with multiple titles:
        Contact(1) -----> Title(0:*)

    The organization for which the title is in effect is optional.
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    title = fields.foreign_key_field(Title, on_delete=CASCADE)
    organization = fields.foreign_key_field(Organization,
                                            blank=True, null=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_title)
        verbose_name = _(_contact_title_verbose)
        verbose_name_plural = _(pluralize(_contact_title_verbose))
        unique_together = ("contact", "title", "organization")

_contact_url = "ContactUrl"
_contact_url_verbose = humanize(underscore(_contact_url))

class ContactUrl(ContactAssociation):
    """ContactUrl model class.

    A contact may be associated with 0:* urls.  These may include
    social media urls including Linkedln, Facebook, Twitter.
    """
    contact = fields.foreign_key_field(Contact, on_delete=CASCADE)
    url = fields.foreign_key_field(Url, on_delete=CASCADE)
    url_type = fields.foreign_key_field(UrlType, null=True, blank=True)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _contact_url)
        verbose_name = _(_contact_url_verbose)
        verbose_name_plural = _(pluralize(_contact_url_verbose))
        unique_together = ("contact", "url", "url_type",)

_related_contact = "RelatedContact"
_related_contact_verbose = humanize(underscore(_related_contact))

class RelatedContact(ContactAssociation):
    """Related contact model class.

    Capture related contact attributes.  A contact may be associated
    with 0 or more contact instances:
        Contact(1) ------> Contact(0..*)
    """
    from_contact = fields.foreign_key_field(
        Contact,
        on_delete=CASCADE,
        related_name="%(app_label)s_%(class)s_related_from_contact")
    to_contact = fields.foreign_key_field(
        Contact,
        on_delete=CASCADE,
        related_name="%(app_label)s_%(class)s_related_to_contact")
    contact_relationship_type = fields.foreign_key_field(
        ContactRelationshipType)

    class Meta(ContactAssociation.Meta):
        db_table = db_table(_app_label, _related_contact)
        verbose_name = _(_related_contact_verbose)
        verbose_name_plural = _(pluralize(_related_contact_verbose))
        unique_together = ("from_contact", "to_contact",
                           "contact_relationship_type")

    def clean(self):
        super(RelatedContact, self).clean()
        validation.related_contact_validation(self)

PERMISSION_ADD = "contacts.add_contact"
PERMISSION_CHANGE = "contacts.change_contact"
PERMISSION_DELETE = "contacts.delete_contact"
PERMISSION_READ = "contacts.read_contact"
PERMISSION_WRITE = "contacts.write_contact"
PERMISSIONS_CONTACT_OBJECT = (PERMISSION_READ, PERMISSION_WRITE)
PERMISSIONS_CONTACT_FUNCTIONAL = (PERMISSION_ADD,
                                  PERMISSION_CHANGE,
                                  PERMISSION_DELETE)

# As per guardian documentation, defining these derived classes
# with physical foreign key definitions results in improved performance.
# The on_delete parameter is defined such that on contact instance deletion
# the associated object permissions are deleted as well.

class ContactObjectPermission(UserObjectPermissionBase):
    """User contact permission class."""
    content_object = fields.foreign_key_field(Contact, on_delete=CASCADE)

class ContactGroupObjectPermission(GroupObjectPermissionBase):
    """Group contact permission class."""
    content_object = fields.foreign_key_field(Contact, on_delete=CASCADE)

_contacts_user_profile = "UserProfile"
_contacts_user_profile_verbose = humanize(underscore(_contacts_user_profile))
_groups_read = "groups_read"
_groups_write = "groups_write"
_users_read = "users_read"
_users_write = "users_write"

def _field_helper(to_class, relation_type):
    return fields.many_to_many_field(
        to_class=to_class,
        db_table="{}_{}".format(
            db_table(_app_label, _contacts_user_profile), relation_type),
        related_name=related_name_base + relation_type)

class UserProfile(Model):
    """Contacts application user profile model class.

    Each user has an associated ContactsUserProfile instance which defines
    which users and groups are granted read/write access permissions
    for contacts created by the user.
    """
    user = fields.one_to_one_field(settings.AUTH_USER_MODEL,
                                   on_delete=CASCADE)
    groups_read = _field_helper(
        django.contrib.auth.models.Group, _groups_read)
    groups_write = _field_helper(
        django.contrib.auth.models.Group, _groups_write)
    users_read = _field_helper(
        django.contrib.auth.models.User, _users_read)
    users_write = _field_helper(
        django.contrib.auth.models.User, _users_write)

    class Meta(ContactsModel.Meta):
        db_table = db_table(_app_label, _contacts_user_profile)
        verbose_name = _(_contacts_user_profile_verbose)
        verbose_name_plural = _(pluralize(_contacts_user_profile_verbose))


#!/usr/bin/env python

import requests
import json
import re
from datetime import datetime
from config import dpla_api_key, data_dir

today = datetime.now()

# create request for each disaster, using pagination
def getdata(disasters):
    for subject in disasters:
        p = 1  # start with first page
        size = 500 # results size

        data = requestdata(subject, p, size) # first call to get count
        createfile(data['docs'], subject, p) # write first call

        pages = data['count'] / size  # get number of pages

        for i in range(pages): # iterate through pages
            p = i + 2  # skips over page "0" (doesn't exist)
                       # and "1" (already got it); get modulo
            data = requestdata(subject, p, size)
            createfile(data['docs'], subject, p)

# make request to dpla
def requestdata(subject, p, size):
    query = {'api_key':dpla_api_key, 'sourceResource.subject.name':subject,
            'sourceResource.type':'image', 'page_size':size, 'page':p,
            'fields':'id,isShownAt,object,provider,sourceResource'}
    r = requests.get('http://api.dp.la/v2/items?', params=query)
    return r.json()

# create time-stamped and page-stamped files
def createfile(data, subject, p):
    e = re.compile('\W.*') # regex to simplify search statements that have
                           # punctuation or boolean into mongo-acceptable form
    subject = e.sub('', subject)
    filename = data_dir + '{0}-{1}-{2}-dpla.json'.format(subject,today.isoformat(), p)
    with open(filename, 'w') as f:
        f.write(json.dumps(data))

disasters = ['earthquakes','hurricanes','floods','forest+AND+fires']

getdata(disasters)


import unittest

from mock import MagicMock

from bounded_contexts.central_console.domain_services import control_node_repository
from bounded_contexts.central_console.domain_services import deployment_info_repository
from bounded_contexts.central_console.domain_services import license_repository
from bounded_contexts.central_console.domain_services import abstract_sensor_repository
from bounded_contexts.central_console.models import deployment_info

class TestDeploymentInfoRepository(unittest.TestCase):

    def setUp(self):
        self.deployment_info_constructor = MagicMock(spec=deployment_info.DeploymentInfo)
        self.control_node_repo = MagicMock(spec=control_node_repository.ControlNodeRepository)
        self.license_repo = MagicMock(spec=license_repository.LicenseRepository)
        self.sensor_repo = MagicMock(spec=abstract_sensor_repository.AbstractSensorRepository)

        self.deployment_info_repo = deployment_info_repository.DeploymentInfoRepository(
            self.deployment_info_constructor,
            self.control_node_repo,
            self.license_repo,
            self.sensor_repo
        )

    def test_get_deployment_info(self):
        self.deployment_info_constructor.return_value = expected_info = MagicMock()
        self.control_node_repo.get_control_node.return_value = expected_control_node = MagicMock()
        self.license_repo.get_license.return_value = expected_license = MagicMock()
        self.sensor_repo.get_sensors.return_value = expected_sensors = [MagicMock(), MagicMock()]

        actual_info = self.deployment_info_repo.get_deployment_info()

        self.deployment_info_constructor.assert_called_once_with(
            expected_control_node,
            expected_license,
            expected_sensors
        )
        self.assertEqual(actual_info, expected_info)

if __name__ == '__main__':
    unittest.main()


from django.dispatch import receiver
from .models import Response
from django.db.models.signals import post_save
from django.core.mail import EmailMultiAlternatives

@receiver(post_save, sender=Response)
def response_created(instance,  created, **kwargs):
    if not created:
        return

    email = instance.post.author.email

    subject = f'Новый отклик'

    text_content = (
        f'{instance.post.author.username}, вам отклик от {instance.author} на объявление "{instance.get_absolute_url()}"!'
    )

    html_content = (
        f'{instance.post.author.username}, вам отклик от {instance.author} на объявление!'
    )

    msg = EmailMultiAlternatives(subject, text_content, None, [email])
    msg.attach_alternative(html_content, "text/html")
    msg.send()

@receiver(post_save, sender=Response)
def response_accept(instance, **Kwargs):
    if not instance.status:
        return

    email = instance.author.email

    subject = f'Ваш отклик приняли!'

    text_content = (
        f'{instance.author.username}, ваш отклик к "{instance.post.title}" приняли!'
    )

    html_content = (
        f'{instance.author.username}, ваш отклик к {instance.post.title} приняли!'
    )

    msg = EmailMultiAlternatives(subject, text_content, None, [email])
    msg.attach_alternative(html_content, "text/html")
    msg.send()


import discord

from discord.ext import commands

class Testing(commands.Cog):
	""" Testing cog for trying out experimental code. """
	def __init__(self, bot):
		self._bot = bot

	@commands.is_owner()
	@commands.command(hidden=True)
	async def poke_testing(self, ctx):
		""" Utility function to check testing cog is active. """
		command_names = " | ".join(c.name for c in self.get_commands())
		await ctx.send(f"`Testing cog active. Test commands = {command_names}`")

	@commands.is_owner()
	@commands.command(hidden=True)
	async def raise_error(self, ctx, *, args):
		""" Used to raise specific errors for the purpose of testing the error handler. """
		class DummyResponse:
			def __init__(self):
				self.status = None
				self.reason = None

		if args == "a":
			raise discord.Forbidden(DummyResponse(), "")
		elif args == "b":
			raise discord.NotFound(DummyResponse(), "")
		elif args == "c":
			raise discord.errors.NotFound(DummyResponse(), "")
		elif args == "d":
			raise discord.errors.Forbidden(DummyResponse(), "")

	@commands.is_owner()
	@commands.command(hidden=True)
	async def test_emojis(self, ctx):
		""" Check if certain emojis are compatible with discord. """
		await ctx.send("\U0001F3B2")
		await ctx.send("\U0000274E")

def setup(bot):
	bot.add_cog(Testing(bot))


import pandas as pd
import glob
from pathlib import Path
from fpdf import FPDF

filepaths = glob.glob("Invoices/*.xlsx")
for filepath in filepaths:
    pdf = FPDF(orientation="P", unit="mm", format="A4")
    pdf.add_page()
    filename = Path(filepath).stem
    invoice_nr = filename.split("-")[0]
    invoice_date = filename.split("-")[1]
    pdf.set_font(family="Times", style="B", size=16)
    pdf.cell(w=50, h=8, txt=f"Invoice nr.{invoice_nr}", align="L")
    pdf.ln()
    pdf.cell(w=50, h=8, txt=f"Date:{invoice_date}", align="L", ln=1)
    df = pd.read_excel(filepath, sheet_name="Sheet 1")
    column = list(df.columns)
    column = [item.replace("_", " ").title() for item in column]
    pdf.set_font(family="Times", size=10, style="B")
    # pdf.set_text_color(80, 80, 80)
    pdf.cell(w=30, h=8, txt=column[0], border=1)
    pdf.cell(w=65, h=8, txt=column[1], border=1)
    pdf.cell(w=33, h=8, txt=column[2], border=1)
    pdf.cell(w=30, h=8, txt=column[3], border=1)
    pdf.cell(w=25, h=8, txt=column[4], border=1, ln=1)
    for index, row in df.iterrows():
        pdf.set_font(family="Times", size=10)
        pdf.set_text_color(80, 80, 80)
        pdf.cell(w=30, h=8, txt=str(row["product_id"]), border=1)
        pdf.cell(w=65, h=8, txt=str(row["product_name"]), border=1)
        pdf.cell(w=33, h=8, txt=str(row["amount_purchased"]), border=1)
        pdf.cell(w=30, h=8, txt=str(row["price_per_unit"]), border=1)
        pdf.cell(w=25, h=8, txt=str(row["total_price"]), border=1, ln=1)
    total_amount = df["total_price"].sum()
    pdf.set_font(family="Times", size=10)
    pdf.set_text_color(80, 80, 80)
    pdf.cell(w=30, h=8, txt="", border=1)
    pdf.cell(w=65, h=8, txt="", border=1)
    pdf.cell(w=33, h=8, txt="", border=1)
    pdf.cell(w=30, h=8, txt="", border=1)
    pdf.cell(w=25, h=8, txt=str(total_amount), border=1, ln=1)

    pdf.set_font(family="Times", size=10, style="B")
    pdf.cell(w=30, h=10, txt=f"The total price is INR {total_amount}.", ln=1)

    pdf.set_font(family="Times", size=13, style="B")
    pdf.cell(w=25, h=8, txt="Python How")
    pdf.image("pythonhow.png", w=10)

    pdf.output(f"PDFs/{filename}.pdf")


import logging
import os
import psutil
import redis
import signal
import socket
import sys
import threading

from config import REDIS_URL

class HunterAgent(threading.Thread):
    def __init__(self, redis_url):
        threading.Thread.__init__(self)
        self.agent_hostname = socket.gethostname()
        self.agent_pid = os.getpid()
        self.agent_name = '{h.agent_hostname}@{h.agent_pid}'.format(h=self)

        self.logger = logging.getLogger(name=self.agent_name)

        self.redis_url = redis_url
        self.redis = redis.StrictRedis.from_url(self.redis_url)
        self.pubsub = self.redis.pubsub()
        self.pubsub.subscribe([
            'hunter'
        ])

        self.logger.debug("Created HunterAgent")

        self.commands = {
            "Q_CPUPERCENT": self.cpu_percent
        }

    def run(self):
        self.redis.publish('hunter', 'AGENT_BEGIN {h.agent_name}'.format(h=self))
        self.redis.sadd("HUNTER_CLIENTS", self.agent_name)
        for item in self.pubsub.listen():
            self.logger.debug("Received: {}".format(item))
            if item['type'] == "message":
                data = item['data'].decode('utf-8')
                try:
                    command, args = data.split(" ", 1)
                except ValueError:
                    command = data
                    args = None

                if command == "SHUTDOWN" and args == self.agent_name:
                    self.shutdown()
                    break

                if command in self.commands:
                    self.logger.debug("Dispatching Command: {}".format(command))
                    self.commands[command](command, args)

    def shutdown(self):
        self.logger.debug("Shutting Down")
        self.pubsub.unsubscribe()
        self.redis.publish('hunter', 'AGENT_END {h.agent_name}'.format(h=self))
        self.redis.srem("HUNTER_CLIENTS", self.agent_name)

    def _reply(self, question, answer):
        self.redis.publish('hunter', "RESPONSE {} {h.agent_name} {}".format(question, answer, h=self))

    def cpu_percent(self, cmd, args):
        self._reply(cmd, psutil.cpu_percent(interval=0.5))

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    # data = psutil.cpu_percent(interval=1)

    # print(data)

    agent = HunterAgent(REDIS_URL)

    def ctrl_c_handler(signal, frame):
        agent.shutdown()
        sys.exit(0)

    signal.signal(signal.SIGINT, ctrl_c_handler)
    agent.run()


from price_stream import read_stream, get_market_hours
from datetime import date, datetime
from strategies.strategyv0 import MyStrategy
from postprocess_times import post_process_csv

import asyncio
import boto3
import csv
import secret_keys
import sys

class MarketStreamer:
    def __init__(self):
        filename = date.today().strftime("%m_%d_%y") + ".csv"
        self.f_meta = open('META_' + filename, 'w')
        self.f_aapl = open('AAPL_' + filename, 'w')
        self.f_qqq = open('QQQ_' + filename, 'w')
        self.writer_qqq = csv.writer(self.f_qqq)
        self.writer_meta = csv.writer(self.f_meta)
        self.writer_aapl = csv.writer(self.f_aapl)

        self.trader = MyStrategy(5000, 120.34, 1000)

    def trading_equities_handler(self, message):
        time = datetime.utcfromtimestamp(message["timestamp"]/1000)
        sys.stdout.flush()
        if "content" in message:
            for content in message["content"]:
                symbol = content["key"]
                bid_price = content["BID_PRICE"] if "BID_PRICE" in content else None
                ask_price = content["ASK_PRICE"] if "ASK_PRICE" in content else None
                total_volume = content["TOTAL_VOLUME"] if "TOTAL_VOLUME" in content else None
                open_price = content["OPEN_PRICE"] if "OPEN_PRICE" in content else None
                quote_time = content["QUOTE_TIME"] if "QUOTE_TIME" in content else None
                ask_size = content["ASK_SIZE"] if "ASK_SIZE" in content else None
                bid_size = content["BID_SIZE"] if "BID_SIZE" in content else None

                if symbol == 'META':
                    self.writer_meta.writerow([time, bid_price, ask_price, total_volume, open_price, quote_time, ask_size, bid_size])
                    self.f_meta.flush()

                    if bid_price != None or ask_price != None:
                        self.trader.should_make_trade(bid_price, ask_price, time)

                if symbol == 'AAPL':
                    self.writer_aapl.writerow([time, bid_price, ask_price, total_volume, open_price, ask_size, bid_size, quote_time])
                    self.f_aapl.flush()

                if symbol == 'QQQ':
                    self.writer_qqq.writerow([time, bid_price, ask_price, total_volume, open_price, ask_size, bid_size, quote_time])
                    self.f_qqq.flush()

    def stream_data(self):
        print("market hours: ", get_market_hours())
        sys.stdout.flush()
        asyncio.run(read_stream(self.trading_equities_handler, get_market_hours()))

        self.f_meta.close()
        self.f_aapl.close()
        self.f_qqq.close()

        s3 = boto3.resource(
            service_name='s3',
            region_name='us-east-2',
            aws_access_key_id=secret_keys.access_key,
            aws_secret_access_key=secret_keys.secret_key
        )

        post_process_csv(self.f_meta.name)
        post_process_csv(self.f_aapl.name)
        post_process_csv(self.f_qqq.name)

        s3.Bucket('equities-streaming-data').upload_file(Filename=self.f_meta.name, Key=self.f_meta.name)
        s3.Bucket('equities-streaming-data').upload_file(Filename=self.f_aapl.name, Key=self.f_aapl.name)
        s3.Bucket('equities-streaming-data').upload_file(Filename=self.f_qqq.name, Key=self.f_qqq.name)

        print("bucket_uploaded")
        sys.stdout.flush()

if __name__ == "__main__":
    streamer = MarketStreamer()
    streamer.stream_data()


from typing import List, Tuple

import cv2
import numpy as np

from src.util import load_yaml

cfg = load_yaml()

def four_point_transform(image, pts):
    max_x, max_y = np.max(pts[:, 0]).astype(np.int32), np.max(pts[:, 1]).astype(np.int32)

    dst = np.array([
        [0, 0],
        [image.shape[1] - 1, 0],
        [image.shape[1] - 1, image.shape[0] - 1],
        [0, image.shape[0] - 1]], dtype="float32")

    # 透視変換
    M = cv2.getPerspectiveTransform(dst, pts)
    warped = cv2.warpPerspective(image, M, (max_x, max_y))

    return warped

def generate_gaussian() -> np.ndarray:
    """正規分布の生成

    Returns:
        np.ndarray: 正規分布
    """

    spread = cfg['gaussian_spread']
    sigma = cfg['gaussian_sigma']

    extent = int(spread * sigma)
    center = spread * sigma / 2
    gaussian_heatmap = np.zeros([extent, extent], dtype=np.float32)

    for i_ in range(extent):
        for j_ in range(extent):
            gaussian_heatmap[i_, j_] = 1 / 2 / np.pi / (sigma ** 2) * np.exp(
                -1 / 2 * ((i_ - center - 0.5) ** 2 + (j_ - center - 0.5) ** 2) / (sigma ** 2))

    gaussian_heatmap = (gaussian_heatmap / np.max(gaussian_heatmap) * 255).astype(np.uint8)
    return gaussian_heatmap

def add_character(image, bbox):
    """文字のヒートマップを取得するための処理
    """
    # 左上の座標を整数値に変換
    top_left = np.array([np.min(bbox[:, 0]), np.min(bbox[:, 1])]).astype(np.int32)

    # bboxの中に0以下が含まれていない、bboxのx座標が画像の幅を超えてないか、bboxのy座標が画像の高さを超えてないか
    if np.any(bbox < 0) or np.any(bbox[:, 0] > image.shape[1]) or np.any(bbox[:, 1] > image.shape[0]):
        return image

    # 左上の座標を(0, 0)に移動
    bbox -= top_left[None, :]

    transformed = four_point_transform(generate_gaussian().copy(), bbox.astype(np.float32))

    start_row = max(top_left[1], 0) - top_left[1]
    start_col = max(top_left[0], 0) - top_left[0]
    end_row = min(top_left[1] + transformed.shape[0], image.shape[0])
    end_col = min(top_left[0] + transformed.shape[1], image.shape[1])

    image[max(top_left[1], 0):end_row, max(top_left[0], 0):end_col] += transformed[start_row:end_row - top_left[1],
                                                                                start_col:end_col - top_left[0]]

    return image

def add_affinity(image, bbox_1, bbox_2):
    '''文字間のヒートマップを取得する処理'''
    center_1, center_2 = np.mean(bbox_1, axis=0), np.mean(bbox_2, axis=0)
    tl = np.mean([bbox_1[0], bbox_1[1], center_1], axis=0)
    bl = np.mean([bbox_1[2], bbox_1[3], center_1], axis=0)
    tr = np.mean([bbox_2[0], bbox_2[1], center_2], axis=0)
    br = np.mean([bbox_2[2], bbox_2[3], center_2], axis=0)

    affinity = np.array([tl, tr, br, bl])

    return add_character(image, affinity)

def generate_region_score(image_size: Tuple[int, int, int], character_bbox: np.ndarray) \
        -> Tuple[np.ndarray, np.ndarray]:
    """Region Score Mapの生成

    Args:
        image_size (Tuple[int, int, int]): 画像サイズ
        character_bbox (np.ndarray): 文字のバウディングボックス
                                     shape (文字数, bboxの頂点(左下から反時計回りの順番), 座標)

    Returns:
        Tuple[np.ndarray, np.ndarray, np.ndarray]: Region Score, 正規化されたRegion Score
    """
    height, width, _ = image_size

    # ヒートマップの基を生成
    region_score = np.zeros([height, width], dtype=np.uint8)

    # 文字単位でRegionScoreを生成
    for i in range(character_bbox.shape[0]):
        region_score = add_character(region_score, character_bbox[i])

    return region_score.copy(), region_score.copy() / 255

def generate_affinity_score(image_size: Tuple[int, int, int], character_bbox: np.ndarray, text: List[List[str]])\
        -> Tuple[np.ndarray, np.ndarray]:
    """Affinity Scoreの生成

    Args:
        image_size (Tuple[int, int, int]): 画像サイズ
        character_bbox (np.ndarray): 文字のバウディングボックス
                                     shape (文字数, bboxの頂点(左下から反時計回りの順番), 座標)
        text (List[List[str]]): 埋め込まれている文字列の配列

    Returns:
        Tuple[np.ndarray, np.ndarray, np.ndarray]: Affinity Score, 正規化されたAffinity Score
    """
    height, width, _ = image_size

    affinity_score = np.zeros([height, width], dtype=np.uint8)

    total_letters = 0

    # 単語単位でAffinityScoreを生成
    for word in text:
        for _ in range(len(word) - 1):
            affinity_score = add_affinity(affinity_score,
                                          character_bbox[total_letters].copy(),
                                          character_bbox[total_letters + 1].copy())
            total_letters += 1
        total_letters += 1

    return affinity_score.copy(), affinity_score.copy() / 255


#!/usr/bin/env python
"""
	Visualize sequence features
"""

import dnaplotlib as dpl
import matplotlib.pyplot as plt
from matplotlib import gridspec

# Required for drawing shapes
from matplotlib.patches import Polygon
from matplotlib.lines import Line2D
from matplotlib.patheffects import Stroke
import matplotlib.patches as patches

__author__  = 'Thomas Gorochowski <tom@chofski.co.uk>, Voigt Lab, MIT'
__license__ = 'MIT'
__version__ = '1.0'

##############################################################################
# HELPER FUNCTIONS
# Could be integrated into DNAplotlib in the future. You can create renderer
# functions for each type of annoation and these are then easily used by
# DNAplotlib. I created a very simple one for what we discussed - these can
# be customised further by the opts dictionary in the later section to capture
# metadata such as strength, etc.
##############################################################################

# Function for writing a sequence to axis
def draw_sequence (ax, start_pos, y_offset, seq):
	# For each letter position in center of range
	for c_idx, c in enumerate(seq):
		ax.annotate(c,(start_pos+c_idx+0.5, y_offset), ha='center', va='bottom', fontsize=8)

# Custom renderer for promoter -35 to -10 site (this is used by DNAplotlib to render the sites in a design)
def promoter_region (ax, type, num, start, end, prev_end, scale, linewidth, opts):
	# Default parameters - these can be added to, but we usually use this style (probably should simplify in future)
	y_offset = 0.0
	color_35 = (0.5,0.5,0.5)
	color_10 = (0.5,0.5,0.5)
	color_connector = (0,0,0)
	linewidth_connector = 2.0
	len_35 = 4
	len_10 = 2
	y_extent = 4.0
	# Update default parameters if provided
	if opts != None:
		if 'y_extent' in list(opts.keys()):
			y_extent = opts['y_extent']
		if 'y_offset' in list(opts.keys()):
			y_offset = opts['y_offset']
		if 'linewidth' in list(opts.keys()):
			linewidth = opts['linewidth']
		if 'color_35' in list(opts.keys()):
			color_35 = opts['color_35']
		if 'color_10' in list(opts.keys()):
			color_10 = opts['color_10']
		if 'color_connector' in list(opts.keys()):
			color_connector = opts['color_connector']
		if 'linewidth_connector' in list(opts.keys()):
			linewidth_connector = opts['linewidth_connector']
		if 'len_35' in list(opts.keys()):
			len_35 = opts['len_35']
		if 'len_10' in list(opts.keys()):
			len_10 = opts['len_10']
	# Check direction (we don't use at moment)
	fwd = True
	if start > end:
		fwd = False
	# Draw the -35 site (from start to start + length of -35 site)
	p35 = Polygon([(start, y_offset),
		           (start, y_offset+y_extent),
		           (start+len_35,y_offset+y_extent),
		           (start+len_35,y_offset)],
		            edgecolor=(0,0,0), facecolor=color_35, linewidth=linewidth, zorder=11,
		            path_effects=[Stroke(joinstyle="miter")])
	ax.add_patch(p35)
	# Draw the -10 site (from end-length of -10 site to end)
	p10 = Polygon([(end-len_10, y_offset),
		           (end-len_10, y_offset+y_extent),
		           (end,y_offset+y_extent),
		           (end,y_offset)],
		            edgecolor=(0,0,0), facecolor=color_10, linewidth=linewidth, zorder=11,
		            path_effects=[Stroke(joinstyle="miter")])
	ax.add_patch(p10)
	l1 = Line2D([start+len_35, end-len_10],
                [y_offset+(y_extent/2.0), y_offset+(y_extent/2.0)], linewidth=linewidth_connector,
                color=color_connector, zorder=10)
	ax.add_line(l1)

	# Add a label if needed
	if opts != None and 'label' in list(opts.keys()):
		if final_start > final_end:
			dpl.write_label(ax, opts['label'], final_end+((final_start-final_end)/2.0), opts=opts)
		else:
			dpl.write_label(ax, opts['label'], final_start+((final_end-final_start)/2.0), opts=opts)
	# Return the final start and end positions to the DNA renderer
	return start, end

##############################################################################
# CREATES THE FIGURE
##############################################################################

# Create the figure
fig = plt.figure(figsize=(3.0,1.5))
gs = gridspec.GridSpec(1, 1)
ax_dna = plt.subplot(gs[0])

# This is the main sequence
seq1 = 'AATTCCGAGCGCGAGCTTTGCGAGTGA'
draw_sequence(ax_dna, 0, 0, seq1)

# Another sequence to show you can overlay them if needed with an offset
seq2 = 'TTGCGAGTGA'
draw_sequence(ax_dna, 8, 5, seq2)

# Create the DNAplotlib renderer and map of part types to renderering functions
dr = dpl.DNARenderer()
part_renderers = {'PromoterRegion': promoter_region}

# Create the sites to draw
r1 = {'type':'PromoterRegion', 'name':'region1', 'start': 2, 'end': 13, 'fwd':True, 'opts':{'y_offset':10, 'len_35':1, 'len_10':1, 'color_35':(0,0.5,0.2), 'color_10':(0.9,0.2,0.7)}}
r2 = {'type':'PromoterRegion', 'name':'region2', 'start': 15, 'end': 22, 'fwd':True, 'opts':{'y_offset':20, 'color_connector':(1,0,0), 'linewidth_connector':4.0}}
r3 = {'type':'PromoterRegion', 'name':'region3', 'start': 5, 'end': 24, 'fwd':True, 'opts':{'y_offset':30}}

# We don't use a design, so just annotate an axis
dr.annotate(ax_dna, part_renderers, r1)
dr.annotate(ax_dna, part_renderers, r2)
dr.annotate(ax_dna, part_renderers, r3)

# Set up bounds of the axis
ax_dna.set_xlim([-0.5, len(seq1)+0.5])
ax_dna.set_ylim([0,40])
ax_dna.set_xticks([])
ax_dna.set_yticks([])
ax_dna.axis('off')

# Update subplot spacing
plt.subplots_adjust(left=0.01, right=0.99, top=0.99, bottom=0.01)

# Save the figure
fig.savefig('sequence_features.pdf', transparent=True)
fig.savefig('sequence_features.png', dpi=300)

# Clear the plotting cache
plt.close('all')


####################################################################################
## AUTOR: CAIO FALCO (DADOS COM CAIO)
## DATA: 12/09/2022
## DESCRICAO: DAG QUE REALIZA CARGA DE ARQUIVOS CSV NO BIGQUERY, GERA UMA TABELA FILTRADA E MOVE OS ARQUIVOS FONTES.
####################################################################################

from datetime import datetime, timedelta
import airflow
from airflow import DAG
from airflow.operators import bash_operator
import airflow.providers.google.cloud.operators.bigquery
from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator
from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator

from airflow.providers.google.cloud.operators.bigquery import (
    BigQueryExecuteQueryOperator,
)

default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'start_date': datetime(2021, 1, 20),
    'email': ['airflow@example.com'],
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 0,
    'schedule_interval': '30 20 * * *'
}

with DAG(
    dag_id='gcp_dag',
    default_args=default_args,
    schedule_interval=None,
    tags=['gcp','dados-com-caio','big-query'],
) as dag:

	t1_load_csv_bq = GCSToBigQueryOperator(
    task_id='loading_csv_from_gcs_to_bq',
    bucket='meu-bucket-gcs',
    source_objects='v15/landing_zone/*.csv',
	field_delimiter=',',
	skip_leading_rows=1,
    destination_project_dataset_table=f"meu_dataset.tb_raw_ny_bike_trips_airflow",
    schema_fields=[
        {'name': 'tripduration', 'type': 'INTEGER', 'mode': 'NULLABLE'},
        {'name': 'starttime', 'type': 'DATETIME', 'mode': 'NULLABLE'},
		{'name': 'stoptime', 'type': 'DATETIME', 'mode': 'NULLABLE'},
        {'name': 'start_station_id', 'type': 'INTEGER', 'mode': 'NULLABLE'},
		{'name': 'start_station_name', 'type': 'STRING', 'mode': 'NULLABLE'},
        {'name': 'start_station_latitude', 'type': 'FLOAT', 'mode': 'NULLABLE'},
		{'name': 'start_station_longitude', 'type': 'FLOAT', 'mode': 'NULLABLE'},
        {'name': 'end_station_id', 'type': 'INTEGER', 'mode': 'NULLABLE'},
		{'name': 'end_station_name', 'type': 'STRING', 'mode': 'NULLABLE'},
		{'name': 'end_station_latitude', 'type': 'FLOAT', 'mode': 'NULLABLE'},
        {'name': 'end_station_longitude', 'type': 'FLOAT', 'mode': 'NULLABLE'},
		{'name': 'usertype', 'type': 'STRING', 'mode': 'NULLABLE'},
		{'name': 'birth_year', 'type': 'INTEGER', 'mode': 'NULLABLE'},
		{'name': 'gender', 'type': 'STRING', 'mode': 'NULLABLE'},
    ],
    write_disposition='WRITE_TRUNCATE',
    dag=dag,
)

	t2_creating_table_from_query = BigQueryExecuteQueryOperator(
    task_id="querying_and_creating_bq_table",
    sql="""
    SELECT
      *
    FROM
      `projeto-gcp-356114.meu_dataset.tb_raw_ny_bike_trips_airflow`
	WHERE STARTTIME BETWEEN '2017-01-01T00:00:00' AND '2017-06-30T23:59:59' AND
	GENDER = 'female' AND
	birth_year > 1999
    """,
    destination_dataset_table=f"projeto-gcp-356114.meu_dataset.tb_filtered_ny_bike_trips_airflow",
    write_disposition="WRITE_TRUNCATE",
    gcp_conn_id="cnx-composer-bigquery",
    use_legacy_sql=False,
)

	t3_move_src_files_to_bkp = GCSToGCSOperator(
    task_id="move_src_file_to_bkp",
    source_bucket='meu-bucket-gcs',
    source_object='v15/landing_zone/*.csv',
    destination_bucket='meu-bucket-gcs',
    destination_object='v15/bkp/',
    move_object=True,
)

	t1_load_csv_bq >> t2_creating_table_from_query >> t3_move_src_files_to_bkp


# -*- coding:utf8 -*-
import cv2
from pylab import *
import numpy as np
import matplotlib.pyplot as plt
import pywt
#本程序依赖matplotlib, numpy, scipy,opencv-python等python库

#给图像添加椒盐噪声及高斯噪声
def add_noise(image, method, intensity):        #image为图像，method为噪音类型，intensit为噪音强度
    if method == "pepper_salt":
        #使用随机函数生成盐噪声
        salt_noise = np.random.randint(0, 256, image.shape)
        salt_noise = np.where(salt_noise < (intensity / 200.0 * 256), 255, 0)
        salt_noise = salt_noise.astype(float32)
        #使用随机函数生成椒噪声
        pepper_noise = np.random.randint(0, 256, image.shape)
        pepper_noise = np.where(pepper_noise < (intensity / 200.0 * 256), -255, 0)
        pepper_noise = pepper_noise.astype(float32)
        #将椒盐噪声添加到图像中去
        image = image + salt_noise + pepper_noise
        image = np.where(image < 0, 0, np.where(image > 255, 255, image))
    elif method == "gauss":
        #使用随机函数生成高斯白噪声，均值为0，方差为噪音强度
        gauss_noise = np.random.normal(0, intensity, image.shape)
        gauss_noise = gauss_noise.astype(float32)
        #将高斯白噪声添加到图像中去
        image = image + gauss_noise
        image = np.where(image < 0, 0, np.where(image > 255, 255, image))
    else:
        raise ("Type Error")
    return image

#读取图像并转换成灰度图
image = cv2.imread("../image/church.jpg")
image = cv2.resize(image, (1080, 1080))
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image = image.astype(np.float32)

#生成椒盐噪音强度为25%的噪音图像并进行小波变换
pep_salt_image = add_noise(image, "pepper_salt", 25)
cv2.imwrite("./pepper-salt noise image.jpg", pep_salt_image)
coeffs = pywt.dwt2(pep_salt_image, 'haar')
cA, (cH, cV, cD) = coeffs           # 其中cA为图像的LL系数，cH为LH系数，cV为HL系数以及cD为HH系数
HH_salt_image = cD.astype(np.int64)

#统计HH子带小波系数的分布情况
amp_range = 400
Pcoeff_salt_image = zeros((2*amp_range, 1))
for i in HH_salt_image:
    for j in i:
        Pcoeff_salt_image[j+amp_range-1, 0] += 1

#生成高斯白噪音强度为25的噪音图像并进行小波变换
gauss_image = add_noise(image, "gauss", 25)
cv2.imwrite("./gauss noise image.jpg", gauss_image)
coeffs = pywt.dwt2(gauss_image, 'haar')
cA, (cH, cV, cD) = coeffs           # 其中cA为图像的LL系数，cH为LH系数，cV为HL系数以及cD为HH系数
HH_gauss_image = cD.astype(np.int64)
#统计HH子带小波系数的分布情况
Pcoeff_gauss_image = zeros((2*amp_range, 1))
for i in HH_gauss_image:
    for j in i:
        Pcoeff_gauss_image[j+amp_range-1, 0] += 1

#画出椒盐噪音图像下噪音图像的HH子带小波系数不同幅值系数出现的概率分布图并保存
x = np.arange(-amp_range, amp_range, 1)
plt.plot(x, Pcoeff_salt_image, color="k")
plt.xlabel("Coe")
plt.ylabel("Pcoe")
plt.savefig("HH coefficient distribution by pepper-salt noise.jpg")
plt.clf()

#画出高斯噪音图像下噪音图像的HH子带小波系数不同幅值系数出现的概率分布图并保存
plt.plot(x, Pcoeff_gauss_image, color="k")
plt.xlabel("Coe")
plt.ylabel("Pcoe")
plt.savefig("HH coefficient distribution by gauss noise.jpg")
plt.clf()


import os
import json

from aubio import source, pitch
from google.cloud import texttospeech
from google.oauth2 import service_account
import ffmpeg
import numpy as np
from scipy.cluster.hierarchy import linkage

from utils import path_in_medialib, NoteTools

class GoogleSpeaker:
    FILENAME_MAX_CHARS = 40
    FILENAME_EXTENSION = 'mp3'
    DEFAULT_CREDENTIAL_FILEPATH = \
        "/Users/elio/projects/writersup/credentials.json"
    default_credentials = None

    def __init__(self):
        credential_filepath = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
        if not credential_filepath or not os.path.exists(credential_filepath):
            self.default_credentials = \
                service_account.Credentials.from_service_account_file(
                    self.DEFAULT_CREDENTIAL_FILEPATH)
        self.client = texttospeech.TextToSpeechClient() \
            if not self.default_credentials \
            else texttospeech.TextToSpeechClient(
                    credentials=self.default_credentials
                )

    @staticmethod
    def mp3_to_wav(mp3_path, wav_path):
        try:
            mp3 = ffmpeg.input(mp3_path)
            outdict = {
                'acodec': 'pcm_u8',
            }
            ffmpeg.output(mp3, wav_path, **outdict).run(
                capture_stdout=True,
                capture_stderr=True
            )
        except ffmpeg.Error as e:
            print('stdout:', e.stdout.decode('utf8'))
            print('stderr:', e.stderr.decode('utf8'))
            raise e

    def filename_from_text(self, text, language_code, speak_rate):
        out = text.replace(" ", "_").lower()
        out = out[:self.FILENAME_MAX_CHARS] \
            if self.FILENAME_MAX_CHARS < len(out) else out
        out = f"{out}_{language_code}_{str(speak_rate).replace('.','_')}"
        return out

    def speak(
        self,
        text,
        rate=0.8,
        language='en-US',
        filename=None,
        voice_name='en-US-Wavenet-D'
    ):
        """
        Create the audio file from text.
        """
        if not filename:
            filename = self.filename_from_text(text, rate, language)
        filepath = path_in_medialib(filename)
        filepath_wav = f'{filepath}.wav'
        filepath_mp3 = f'{filepath}.mp3'
        if os.path.exists(filepath_wav):
            return filepath_wav
        synthesis_input = texttospeech.SynthesisInput(text=text)
        voice = texttospeech.VoiceSelectionParams(
            language_code=language,
            name=voice_name
        )
        # Select the type of audio file you want returned
        audio_config = texttospeech.AudioConfig(
            audio_encoding=texttospeech.AudioEncoding.MP3,
            speaking_rate=rate
        )
        # voice parameters and audio file type
        response = self.client.synthesize_speech(
            input=synthesis_input, voice=voice, audio_config=audio_config
        )
        # The response's audio_content is binary.
        with open(filepath_mp3, "wb") as out:
            out.write(response.audio_content)
        self.mp3_to_wav(filepath_mp3, filepath_wav)
        return filepath_wav

class AudioAnalyst:
    debug = True
    hop = 512  # downsample # hop size
    downsample = 1
    win_s = 4096  # downsample # fft size
    tolerance = 0.8
    silence = -40
    clean = False
    clean_tollerance = 0.35
    clean_value = -1.0
    analysis = None

    def __init__(
        self,
        filename,
        title,
        unit='midi',
        method='yin',
        samplerate=12800,
        hop=512,
        win_s=4096
    ):
        """
        Default values  128000 / 512 = 25 sample per second.
        """
        self.filename = filename
        self.title = title
        self.unit = unit
        self.hop = hop
        self.win_s = win_s
        # Support to frequency not completed and never used
        self.is_midi = self.unit == 'midi'
        self.method = method
        self.samplerate = samplerate
        self.s = source(filename, self.samplerate, self.hop)
        self.pitch_o = pitch(
            self.method, self.win_s, self.hop, self.samplerate
        )
        self.pitch_o.set_unit(self.unit)
        self.pitch_o.set_silence(self.silence)
        self.pitch_o.set_tolerance(self.tolerance)

    def plot_samples(self, plot):
        times = [t * self.hop for t in range(len(self.samples))]
        plot.plot(times, self.samples, '.-')
        plot.axis(ymin=0.0, ymax=1.1 * self.max_y)

    def plot_histogram(self, plot):
        plot.hist(self.histogram)

    def save_file(self, filepath):
        with open(filepath) as json_file:
            json.dump(self.analysis, json_file)

    def cluster(self):
        """
        Create clusters using hierarchical clustering
        """
        zeros = np.zeros(len(self.samples))
        points = np.column_stack(
            (zeros.astype(np.double), self.samples.astype(np.double))
        )
        self.linkage = linkage(points, 'ward')
        return self.linkage

    def set_analysis(self, samples):
        """
        Set samples and compute stats
        """
        self.samples = samples
        self.max_x = self.samples.size
        self.min_y = np.min(self.samples)
        self.max_y = np.max(self.samples)
        self.histogram = np.histogram(
            self.samples,
            bins=int(self.max_y)
        )
        if not self.is_midi:
            self.max_note, self.min_note = NoteTools.freq_to_note(
                self.max_y), \
                NoteTools.freq_to_note(self.min_y)
        else:
            self.max_note, self.min_note = NoteTools.midi_to_note(
                self.max_y), \
                NoteTools.midi_to_note(self.min_y)

        self.analysis = {
                'samples': self.samples.tolist(),
                'tolerance': self.tolerance,
                'silence': str(self.silence),
                'hop': str(self.hop),
                'filename': self.filename,
                'samplerate': self.samplerate,
                'max_x': self.max_x,
                'min_y': self.min_y,
                'max_y': self.max_y,
                'max_note': self.max_note,
                'min_note': self.min_note,
                'histogram': self.histogram,
                'unit': self.unit,
                'title': self.title
        }
        return self.analysis

    def analyse(self, json_path=None):
        total_frames = 0
        pitches = []
        confidences = []
        while True:
            samples, read = self.s()
            pitch = self.pitch_o(samples)[0]
            confidence = self.pitch_o.get_confidence()
            if self.clean and confidence < self.clean_tollerance:
                pitch = self.clean_value
            pitches += [pitch]
            confidences += [confidence]
            total_frames += read
            if read < self.hop:
                break
        # clean results
        pitches = np.array(pitches)
        pitches = np.ma.masked_where(
            pitches <= 12.0,
            pitches
        )
        # save and return analysis
        return self.set_analysis(pitches)


import calendar
from datetime import datetime, timedelta

from telebot.types import Message, CallbackQuery

from app.models import Task
from app.services.tasks import get_tasks_by_week
from ...base import base, callback_query_base
from ...helpers import send_message_private, mark_user
from ...keyboards.inline import get_week_inline_markup
from ...loader import bot, bot_username

@bot.message_handler(regexp='^📝 Домашнее задание$')
@bot.message_handler(commands=['homework'])
@base()
def homework_handler(message: Message):
    text, markup = _get_homework_data()

    send_message_private(message, text, reply_markup=markup, disable_web_page_preview=True)

@bot.callback_query_handler(func=lambda call: call.data.startswith('homework'))
@callback_query_base()
def inline_homework_handler(call: CallbackQuery):
    query, option = call.data.split('_')
    if option == 'this':
        date = datetime.today()
    elif option == 'next':
        date = datetime.today() + timedelta(weeks=1)
    elif option == 'previous':
        date = datetime.today() - timedelta(weeks=1)
    else:
        return

    timetable = get_tasks_by_week(date.isocalendar()[1])

    text = _get_text(timetable)
    if not call.inline_message_id and call.message.chat.type != 'private':
        text = mark_user(text, call.from_user.id)

    markup = get_week_inline_markup(query, option, with_previous=True)
    try:
        if call.inline_message_id:
            bot.edit_message_text(text, message_id=call.inline_message_id, inline_message_id=call.inline_message_id,
                                  reply_markup=markup,
                                  disable_web_page_preview=True)
        else:
            bot.edit_message_text(text, call.message.chat.id, call.message.message_id, reply_markup=markup,
                                  disable_web_page_preview=True)
        bot.answer_callback_query(call.id, 'Ок')
    except Exception as e:
        if e.error_code == 400:
            bot.answer_callback_query(call.id, 'Ничего не поменялось', show_alert=True)

def _get_text(timetable: list[list[Task]]):
    deep_link = f'tg://resolve?domain={bot_username}&start=task'

    text = ''
    for i in range(len(timetable)):
        tasks = timetable[i]
        if not tasks:
            continue

        text += calendar.day_name[i].capitalize() + ':\n'

        j = 1
        for task in tasks:
            if task.subject:
                files_view_link = f'\nПосмотреть прикрепленные файлы 👀\n' \
                    if task.files or task.photos else ''

                text += f'{j}) {task.subject.name}⠀\n{task.text.rstrip()}{files_view_link}\n\n'
                j += 1

        text = text.rstrip() + '\n\n'

    text = text.rstrip()

    if not text:
        return 'Нет домашних заданий'

    return text

def _get_homework_data():
    shift = 'this'

    now = datetime.now()
    if now.weekday() > 4 or (now.weekday() == 4 and now.hour > 13):
        shift = 'next'
        now += timedelta(weeks=1)

    timetable = get_tasks_by_week(now.isocalendar()[1])

    text = _get_text(timetable)

    markup = get_week_inline_markup('homework', shift, with_previous=True)

    return text, markup


#!/usr/bin/env python
"""
This module parse arbitrary NIAK opt arguments to run a pipeline with the needed inputs
"""
__author__ = 'poquirion'
__version__ = 1.0

import argparse
import os
import re
import sys

#sys.path.append("{}/..".format(os.path.dirname(os.path.realpath(__file__))))
import pyniak.load_pipeline

OPTION_PREFIX = "--opt"
ESCAPE_STRING = "666_____666_____666"

def build_opt(option):
    """
    Translate all option with prefix --opt to into psom-niak options

    :param option: option of the form  --opt-some-value-meaningful-for-psom VAL
    :return: a string of the form opt.some.value.meaningful.for.psom=VAL
    """
    parser = argparse.ArgumentParser(description="All options")

    opt_dico = {}
    for i, o in enumerate(option):
        if OPTION_PREFIX in o:
            option[i] = re.sub("(\w)-(\w)", "\g<1>{0}\g<2>".format(ESCAPE_STRING), o)
            parser.add_argument(option[i], nargs='?')

    parsed = parser.parse_known_args(option)

    for k, v in parsed[0].__dict__.items():
        if v is None:
            opt_dico["{0}".format(k.replace(ESCAPE_STRING, "."))] = "true"
        else:
            opt_dico["{0}".format(k.replace(ESCAPE_STRING, "."))] = "{0}".format(v)

    return opt_dico

def main(args=None):

    if args is None:
        args = sys.argv[1:]

    print("{0} {1}".format(__file__, " ".join(args)))

    parser = argparse.ArgumentParser(description='Run a niak script',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # parser.add_argument("pipeline", default=None)

    parser.add_argument('bids_dir', help='The directory with the input dataset '
                    'formatted according to the BIDS standard.')

    parser.add_argument('output_dir', help='The directory where the output files '
                    'should be stored. If you are running group level analysis '
                    'this folder should be prepopulated with the results of the'
                    'participant level analysis.')

    parser.add_argument('analysis_level', help='Level of the analysis that will be performed. '
                    'Multiple participant level analyses can be run independently '
                    '(in parallel) using the same output_dir.',
                    choices=['participant', 'group'])

    parser.add_argument('-v', '--version', action='version',
                        version='BIDS-App example version {}'.format(__version__))

    parser.add_argument("--participant_label", nargs="+",
                        help='The label(s) of the participant(s) that should be analyzed. The label '
                        'corresponds to sub-<participant_label> from the BIDS spec '
                        '(so it does not include "sub-"). If this parameter is not '
                        'provided all subjects should be analyzed. Multiple '
                        'participants can be specified with a space separated list.')

    parser.add_argument('--n_thread', default=1, help="Number of threads to compute niak")

    ## Slice timing options
    parser.add_argument('--type_scaner', default="", type=str.capitalize
                        , help="Type of MR scanner. The only value that will change something to the processing here "
                               "is 'Siemens', which has different conventions for interleaved acquisitions."
                        , choices=["", "Siemens"])

    aq_type = ['manual', 'sequential ascending',
               'sequential descending','interleaved ascending',
               'interleaved descending']

    parser.add_argument('--type_acquisition', default="interleaved ascending",
                        help="Type of acquisition used by the scanner. "
                             "Possible choices are 'manual', 'sequential "
                             "ascending', 'sequential descending', 'interleaved ascending', "
                             "'interleaved descending'", choices=aq_type)

    parser.add_argument("--delay_in_tr", default=0, help="The delay (s) between the last slice of the first "
                                                         "volume and the first slice of the following volume.")

    parser.add_argument("--suppress_vol", default=0, help="The number of volumes that are suppressed at the begining of"
                                                          " the time series")

    parser.add_argument("--skip_slice_timing", action="store_true",
                        help="Skip all slice timing procedures")

    # Filtering
    parser.add_argument("--hp", default=0.01, help="Cut-off frequency for high pass")
    parser.add_argument("--lp", default=float('inf'), help="Cut-off frequency for low pass")

    parser.add_argument("--t1_preprocess_nu_correct", default=50, help="Non-uniformity corrections distance. "
                                                                       "Sets the N3 spline distance in mm: suggested "
                                                                       "values: 200 for 1.5T scan, 50 for 3T scan.")

    parser.add_argument("--smooth_vol_fwhm", default=6, help="Full width at half max of the Gaussian kernel (mm) "
                                                               "used for smoothing")

    parsed, unformated_options = parser.parse_known_args(args)

    pipeline_name = None
    options = build_opt(unformated_options)

    if pipeline_name is None:
        pipeline_name = "Niak_fmri_preprocess"
    try:
        os.mkdir('/outputs/tmp')
    except OSError:
        pass

    group = False
    if parsed.analysis_level == "group":
        group = True
    pipeline = pyniak.load_pipeline.FmriPreprocessBids(folder_in=parsed.bids_dir, folder_out=parsed.output_dir,
                                                       subjects=parsed.participant_label,
                                                       smooth_vol_fwhm=parsed.smooth_vol_fwhm,
                                                       t1_preprocess_nu_correct=parsed.t1_preprocess_nu_correct,
                                                       lp=parsed.lp, hp=parsed.hp, suppress_vol=parsed.suppress_vol,
                                                       delay_in_tr=parsed.delay_in_tr,
                                                       type_acquisition=parsed.type_acquisition,
                                                       type_scaner=parsed.type_scaner, n_thread=parsed.n_thread,
                                                       skip_slice_timing=parsed.skip_slice_timing, group=group)
    pipeline.run()

if __name__ == '__main__':
    main()
    #main([ "-p", "Niak_fmri_preprocess",  '--file_in', 'data_test_niak_mnc1', '--folder_out', 'results-directory', '--opt-psom-max_queued', '4', '--opt-slice_timing-type_scanner', 'Bruker', '--opt-slice_timing-type_acquisition', '"interleaved ascending"', '--opt-slice_timing-delay_in_tr', '0', '--opt-resample_vol-voxel_size', '10', '--opt-t1_preprocess-nu_correct-arg', "'-distance 75'", '--opt-time_filter-hp', '0.01', '--opt-time_filter-lp', 'Inf', '--opt-regress_confounds-flag_gsc', 'true', '--opt-regress_confounds-flag_scrubbing','true', '--opt-regress_confounds-thre_fd', '0.5', '--opt-smooth_vol-fwhm', '6'])
    # main(["/home/niak/util/bin/niak_cmd.py","Niak_fmri_preprocess","--file_in","Cambridge_Buckner","--folder_out","results-directory","--opt-psom-max_queued","100","--opt-slice_timing-type_scanner","Siemens","--opt-slice_timing-type_acquisition","interleaved","--opt-slice_timing-delay_in_tr","0","--opt-resample_vol-voxel_size","3","--opt-t1_preprocess-nu_correct-arg","'-distance","50'","--opt-time_filter-hp","0.01","--opt-time_filter-lp","Inf","--opt-regress_confounds-flag_gsc","true","--opt-regress_confounds-flag_scrubbing","true","--opt-regress_confounds-thre_fd","0.5","--opt-smooth_vol-fwhm","6","--opt-corsica-sica-nb_comp","50","--opt-corsica-threshold","0.15","--opt-corsica-flag_skip","false","--opt-size_output","quality_control","--opt-motion_correction-suppress_vol","0","--opt-granularity","max"])


import io
import spacy
import sys
from markdown import markdown
import pytesseract
from PIL import Image
from cnocr import CnOcr
from pypdf import PdfReader
from prompts import Prompt
from core import OCR_LANG, OCR_TYPE, brain
from PyQt6.QtWidgets import QApplication, QMainWindow, QTextBrowser
class OCR():
    def __init__(self) -> None:
        if OCR_LANG == "zh-CN":
            self.spacy_model = "zh_core_web_sm"
            self.rec_model = 'ch_PP-OCRv3'

        if OCR_LANG == "en-US":
            self.spacy_model = "en_core_web_sm"
            self.rec_model = 'en_PP-OCRv3'
        self.nlp = spacy.load(self.spacy_model)
        self.img_file = None
        self.ocr_type = OCR_TYPE

    def ocr_pdf(self, pdf_file, page_number=None, scope=False):
        reader = PdfReader(pdf_file)
        outs = []
        pages = reader.pages

        if type(page_number) is int:
            pages = [reader.pages[page_number]]

        if type(page_number) is list:
            if scope:
                page_number = range(page_number[0],page_number[1])
            for page in page_number:
                for image_file_object in reader.pages[page].images:
                    self.img_file = Image.open(io.BytesIO(image_file_object.data))
                    outs.append(self.paddOCR())
            return "".join(outs)

        for page in pages:
            for image_file_object in page.images:
                self.img_file = Image.open(io.BytesIO(image_file_object.data))
                outs.append(self.paddOCR())
        return "".join(outs)

    def ocr_img(self,img_file):
        self.img_file = img_file
        if self.ocr_type == "tesseract":
            out = self.tessOCR()

        if self.ocr_type == "paddle":
            out = self.paddOCR(self.rec_model)

        self.doc = self.nlp(out)
        sentences = [sent.text for sent in self.doc.sents]
        return sentences

    def tessOCR(self, lang='eng+chi_sim'):
        out = pytesseract.image_to_string(self.img_file, lang)
        return out

    def paddOCR(self, rec_model_name):
        ocr = CnOcr(rec_model_name)
        out = "".join([i['text'] for i in ocr.ocr(self.img_file)])
        return out

if __name__ == "__main__":
    rco = OCR()
    res = "".join(rco.ocr_img('.latest.screenshot.png'))
    conversation = []
    conversation.append(Prompt.screenshot.value)
    conversation.append({
        "role": "user",
        "content": res
    })
    result = brain(conversation)
    app = QApplication(sys.argv)
    window = QMainWindow()
    text_browser = QTextBrowser(window)
    window.setCentralWidget(text_browser)
    window.setGeometry(QApplication.primaryScreen().size().width(), 0, 400, QApplication.primaryScreen().size().height())
    text_browser.setText(markdown(result))
    window.show()
    sys.exit(app.exec())


import requests
import time

# Telegram Bot API credentials
bot_token = 'YOUR_BOT_TOKEN'
chat_id = 'YOUR_CHAT_ID'

# List of servers to check
servers = [
    {'name': 'Server 1', 'url': 'http://server1.com'},
    {'name': 'Server 2', 'url': 'http://server2.com'},
    {'name': 'Server 3', 'url': 'http://server3.com'}
]

# Function to send Telegram message
def send_telegram_message(message):
    url = f'https://api.telegram.org/bot{bot_token}/sendMessage'
    payload = {
        'chat_id': chat_id,
        'text': message
    }
    response = requests.post(url, json=payload)
    if response.status_code == 200:
        print("Telegram message sent successfully!")
    else:
        print("Failed to send Telegram message. Response:", response.text)

# Check server status
def check_server_status(server):
    server_name = server['name']
    server_url = server['url']
    try:
        response = requests.get(server_url)
        if response.status_code == 200:
            print(f"{server_name} is up and running!")
        else:
            send_telegram_message(f"{server_name} is down! Response code: {response.status_code}")
    except requests.ConnectionError:
        send_telegram_message(f"{server_name} is down!")

# Main program
if __name__ == '__main__':
    while True:
        for server in servers:
            check_server_status(server)
            time.sleep(1)  # Delay for 1 second between pings

        time.sleep(60)  # Delay for 60 seconds before rechecking all servers


# Dictionary.py
# tony schwebach
# 11/8/2020
# notes

import json
import difflib

data = json.load(open("dictdata.json"))

def define(word):
    word = word.lower()
    if word in data:
        return(data[word])
    elif word.title() in data:
        return(data[word.title()])
    elif word.upper() in data:
        return(data[word.upper()])
    elif word.capitalize() in data:
        return(data[word.capitalize()])
    else:
        sug = difflib.get_close_matches(word,data.keys())
        return(str(word)+ " not in dictionary. Did you mean " +str(sug) +"?")

while 0==0:
    word = input("Type word: ")
    definition = define(word)
    i=1
    if type(definition) == list:
        for item in definition:
            print(str(i),": ",item)
            i+=1
    else:
        print(definition)