# pylint: disable=too-many-lines

"""
Contains all of the Fides resources modeled as Pydantic models.
"""
from __future__ import annotations

from enum import Enum
from typing import Any, Dict, List, Optional, Union
from warnings import warn

from pydantic import (
    AnyUrl,
    BaseModel,
    ConstrainedStr,
    Field,
    HttpUrl,
    PositiveInt,
    root_validator,
    validator,
)

from fideslang.validation import (
    FidesKey,
    FidesVersion,
    check_valid_country_code,
    deprecated_version_later_than_added,
    has_versioning_if_default,
    is_deprecated_if_replaced,
    matching_parent_key,
    no_self_reference,
    parse_data_type_string,
    sort_list_objects_by_name,
    unique_items_in_list,
    valid_data_type,
)

# Reusable Validators
country_code_validator = validator("third_country_transfers", allow_reuse=True)(
    check_valid_country_code
)
matching_parent_key_validator = validator("parent_key", allow_reuse=True, always=True)(
    matching_parent_key
)
no_self_reference_validator = validator("parent_key", allow_reuse=True)(
    no_self_reference
)
has_versioning_if_default_validator = validator(
    "is_default", allow_reuse=True, always=True
)(has_versioning_if_default)
deprecated_version_later_than_added_validator = validator(
    "version_deprecated", allow_reuse=True
)(deprecated_version_later_than_added)
is_deprecated_if_replaced_validator = validator("replaced_by", allow_reuse=True)(
    is_deprecated_if_replaced
)

# Reusable Fields
name_field = Field(description="Human-Readable name for this resource.")
description_field = Field(
    description="A detailed description of what this resource is."
)
meta_field = Field(
    default=None,
    description="An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects.",
)

class FidesModel(BaseModel):
    """The base model for most top-level Fides objects."""

    fides_key: FidesKey = Field(
        description="A unique key used to identify this resource."
    )
    organization_fides_key: FidesKey = Field(
        default="default_organization",
        description="Defines the Organization that this resource belongs to.",
    )
    tags: Optional[List[str]] = None
    name: Optional[str] = name_field
    description: Optional[str] = description_field

    class Config:
        "Config for the FidesModel"
        extra = "ignore"
        orm_mode = True

class DefaultModel(BaseModel):
    """
    A model meant to be inherited by versioned parts of the Default Taxonomy.
    """

    version_added: Optional[str] = Field(
        default=None,
        description="The version of Fideslang in which this label was added.",
    )
    version_deprecated: Optional[str] = Field(
        default=None,
        description="The version of Fideslang in which this label was deprecated.",
    )
    replaced_by: Optional[FidesKey] = Field(
        default=None,
        description="The new name, if applicable, for this label after deprecation.",
    )
    is_default: bool = Field(
        default=False,
        description="Denotes whether the resource is part of the default taxonomy or not.",
    )

    _has_versioning_if_default: classmethod = has_versioning_if_default_validator
    _deprecated_version_later_than_added: classmethod = (
        deprecated_version_later_than_added_validator
    )
    _is_deprecated_if_replaced: classmethod = is_deprecated_if_replaced_validator

    @validator("version_added")
    @classmethod
    def validate_verion_added(
        cls, version_added: Optional[str], values: Dict
    ) -> Optional[str]:
        """
        Validate that the `version_added` field is a proper FidesVersion
        """
        if not version_added:
            return None

        FidesVersion.validate(version_added)
        return version_added

    @validator("version_deprecated")
    @classmethod
    def validate_version_deprecated(
        cls, version_deprecated: Optional[str], values: Dict
    ) -> Optional[str]:
        """
        Validate that the `version_deprecated` is a proper FidesVersion
        """
        if not version_deprecated:
            return None

        FidesVersion.validate(version_deprecated)
        return version_deprecated

class DataResponsibilityTitle(str, Enum):
    """
    The model defining the responsibility or role over
    the system that processes personal data.

    Used to identify whether the organization is a
    Controller, Processor, or Sub-Processor of the data
    """

    CONTROLLER = "Controller"
    PROCESSOR = "Processor"
    SUB_PROCESSOR = "Sub-Processor"

class IncludeExcludeEnum(str, Enum):
    """
    Determine whether or not defined rights are
    being included or excluded.
    """

    ALL = "ALL"
    EXCLUDE = "EXCLUDE"
    INCLUDE = "INCLUDE"
    NONE = "NONE"

class DataSubjectRightsEnum(str, Enum):
    """
    The model for data subject rights over
    personal data.

    Based upon chapter 3 of the GDPR
    """

    INFORMED = "Informed"
    ACCESS = "Access"
    RECTIFICATION = "Rectification"
    ERASURE = "Erasure"
    PORTABILITY = "Portability"
    RESTRICT_PROCESSING = "Restrict Processing"
    WITHDRAW_CONSENT = "Withdraw Consent"
    OBJECT = "Object"
    OBJECT_TO_AUTOMATED_PROCESSING = "Object to Automated Processing"

class LegalBasisEnum(str, Enum):
    """
    Deprecated. The model for allowable legal basis categories on data uses.
    """

    CONSENT = "Consent"
    CONTRACT = "Contract"
    LEGAL_OBLIGATION = "Legal Obligation"
    VITAL_INTEREST = "Vital Interest"
    PUBLIC_INTEREST = "Public Interest"
    LEGITIMATE_INTEREST = "Legitimate Interests"

class LegalBasisForProcessingEnum(str, Enum):
    """
    The model for allowable legal basis categories on privacy declarations.

    Based upon article 6 of the GDPR
    """

    CONSENT = "Consent"
    CONTRACT = "Contract"
    LEGAL_OBLIGATION = "Legal obligations"
    VITAL_INTEREST = "Vital interests"
    PUBLIC_INTEREST = "Public interest"
    LEGITIMATE_INTEREST = "Legitimate interests"

class LegalBasisForProfilingEnum(str, Enum):
    """The model for describing the legal basis under which profiling is performed"""

    EXPLICIT_CONSENT = "Explicit consent"
    CONTRACT = "Contract"
    AUTHORISED_BY_LAW = "Authorised by law"

class LegalBasisForTransfersEnum(str, Enum):
    """
    The model for describing the legal basis under which data is transferred

    We currently do _not_ enforce this enum on the `legal_basis_for_transfers`
    field, because the set of allowable values seems to be changing frequently
    and without clear notice in upstream, public data sources.
    """

    ADEQUACY_DECISION = "Adequacy Decision"
    SCCS = "SCCs"
    BCRS = "BCRs"
    SUPPLEMENTARY_MEASURES = "Supplementary Measures"
    OTHER = "Other"

class SpecialCategoriesEnum(str, Enum):
    """
    Deprecated. Special Categories Enum that was used on Data Uses.
    """

    CONSENT = "Consent"
    EMPLOYMENT = "Employment"
    VITAL_INTEREST = "Vital Interests"
    NON_PROFIT_BODIES = "Non-profit Bodies"
    PUBLIC_BY_DATA_SUBJECT = "Public by Data Subject"
    LEGAL_CLAIMS = "Legal Claims"
    PUBLIC_INTEREST = "Substantial Public Interest"
    MEDICAL = "Medical"
    PUBLIC_HEALTH_INTEREST = "Public Health Interest"

class SpecialCategoryLegalBasisEnum(str, Enum):
    """
    The model for the legal basis for processing special categories of personal data
    on privacy declarations

    Based upon article 9 of the GDPR
    """

    CONSENT = "Explicit consent"
    EMPLOYMENT = "Employment, social security and social protection"
    VITAL_INTEREST = "Vital interests"
    NON_PROFIT_BODIES = "Not-for-profit bodies"
    PUBLIC_BY_DATA_SUBJECT = "Made public by the data subject"
    LEGAL_CLAIMS = "Legal claims or judicial acts"
    PUBLIC_INTEREST = "Reasons of substantial public interest (with a basis in law)"
    MEDICAL = "Health or social care (with a basis in law)"
    PUBLIC_HEALTH_INTEREST = "Public health (with a basis in law)"
    RESEARCH = "Archiving, research and statistics (with a basis in law)"

# Privacy Data Types
class DataCategory(FidesModel, DefaultModel):
    """The DataCategory resource model."""

    parent_key: Optional[FidesKey]

    _matching_parent_key: classmethod = matching_parent_key_validator
    _no_self_reference: classmethod = no_self_reference_validator

class DataQualifier(FidesModel, DefaultModel):
    """The DataQualifier resource model."""

    parent_key: Optional[FidesKey]

    _matching_parent_key: classmethod = matching_parent_key_validator
    _no_self_reference: classmethod = no_self_reference_validator

class Cookies(BaseModel):
    """The Cookies resource model"""

    name: str
    path: Optional[str]
    domain: Optional[str]

    class Config:
        """Config for the cookies"""

        orm_mode = True

class DataSubjectRights(BaseModel):
    """
    The DataSubjectRights resource model.

    Includes a strategy and optionally a
    list of data subject rights to apply
    via the set strategy.
    """

    strategy: IncludeExcludeEnum = Field(
        description="Defines the strategy used when mapping data rights to a data subject.",
    )
    values: Optional[List[DataSubjectRightsEnum]] = Field(
        description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.",
    )

    @root_validator()
    @classmethod
    def include_exclude_has_values(cls, values: Dict) -> Dict:
        """
        Validate the if include or exclude is chosen, that at least one
        value is present.
        """
        strategy, rights = values.get("strategy"), values.get("values")
        if strategy in ("INCLUDE", "EXCLUDE"):
            assert (
                rights is not None
            ), f"If {strategy} is chosen, rights must also be listed."
        return values

class DataSubject(FidesModel, DefaultModel):
    """The DataSubject resource model."""

    rights: Optional[DataSubjectRights] = Field(description=DataSubjectRights.__doc__)
    automated_decisions_or_profiling: Optional[bool] = Field(
        description="A boolean value to annotate whether or not automated decisions/profiling exists for the data subject.",
    )

class DataUse(FidesModel, DefaultModel):
    """The DataUse resource model."""

    parent_key: Optional[FidesKey] = None
    legal_basis: Optional[LegalBasisEnum] = Field(
        description="Deprecated. The legal basis category of which the data use falls under. This field is used as part of the creation of an exportable data map.",
    )
    special_category: Optional[SpecialCategoriesEnum] = Field(
        description="Deprecated. The special category for processing of which the data use falls under. This field is used as part of the creation of an exportable data map.",
    )
    recipients: Optional[List[str]] = Field(
        description="Deprecated. An array of recipients when sharing personal data outside of your organization.",
    )
    legitimate_interest: Optional[bool] = Field(
        description="Deprecated. A boolean representation of if the legal basis used is `Legitimate Interest`. Validated at run time and looks for a `legitimate_interest_impact_assessment` to exist if true.",
    )
    legitimate_interest_impact_assessment: Optional[AnyUrl] = Field(
        description="Deprecated. A url pointing to the legitimate interest impact assessment. Required if the legal bases used is legitimate interest.",
    )

    _matching_parent_key: classmethod = matching_parent_key_validator
    _no_self_reference: classmethod = no_self_reference_validator

    @root_validator
    @classmethod
    def deprecate_fields(cls, values: Dict) -> Dict:
        """
        Warn of Data Use fields pending deprecation.
        """
        deprecated_fields = [
            "legal_basis",
            "recipients",
            "special_category",
            "legitimate_interest",
            "legitimate_interest_impact_assessment",
        ]
        for field in deprecated_fields:
            if values.get(field) is not None:
                warn(
                    f"The {field} field is deprecated, and will be removed in a future version of fideslang.",
                    DeprecationWarning,
                )
        return values

    @validator("legitimate_interest", always=True)
    @classmethod
    def set_legitimate_interest(cls, value: bool, values: Dict) -> bool:
        """Sets if a legitimate interest is used."""
        if values["legal_basis"] == "Legitimate Interests":
            value = True
        return value

    @validator("legitimate_interest_impact_assessment", always=True)
    @classmethod
    def ensure_impact_assessment(cls, value: AnyUrl, values: Dict) -> AnyUrl:
        """
        Validates an impact assessment is applied if a
        legitimate interest has been defined.
        """
        if values["legitimate_interest"]:
            assert (
                value is not None
            ), "Impact assessment cannot be null for a legitimate interest, please provide a valid url"
        return value

# Dataset
class DatasetFieldBase(BaseModel):
    """Base DatasetField Resource model.

    This model is available for cases where the DatasetField information needs to be
    customized. In general this will not be the case and you will instead want to use
    the DatasetField model.

    When this model is used you will need to implement your own recursive field in
    to adding any new needed fields.

    Example:

    ```py
    from typing import List, Optional
    from fideslang import DatasetFieldBase

    class MyDatasetField(DatasetFieldBase):
        custom: str
        fields: Optional[List[MyDatasetField]] = []
    ```
    """

    name: str = name_field
    description: Optional[str] = description_field
    data_categories: Optional[List[FidesKey]] = Field(
        description="Arrays of Data Categories, identified by `fides_key`, that applies to this field.",
    )
    data_qualifier: FidesKey = Field(
        default="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified",
        description="A Data Qualifier that applies to this field. Note that this field holds a single value, therefore, the property name is singular.",
    )
    retention: Optional[str] = Field(
        description="An optional string to describe the retention policy for a dataset. This field can also be applied more granularly at either the Collection or field level of a Dataset.",
    )

class EdgeDirection(str, Enum):
    """Direction of a FidesDataSetReference"""

    FROM = "from"
    TO = "to"

class FidesDatasetReference(BaseModel):
    """Reference to a field from another Collection"""

    dataset: FidesKey
    field: str
    direction: Optional[EdgeDirection]

class FidesMeta(BaseModel):
    """Supplementary metadata used by the Fides application for additional features."""

    references: Optional[List[FidesDatasetReference]] = Field(
        description="Fields that current field references or is referenced by. Used for drawing the edges of a DSR graph.",
        default=None,
    )
    identity: Optional[str] = Field(
        description="The type of the identity data that should be used to query this collection for a DSR."
    )
    primary_key: Optional[bool] = Field(
        description="Whether the current field can be considered a primary key of the current collection"
    )
    data_type: Optional[str] = Field(
        description="Optionally specify the data type. Fides will attempt to cast values to this type when querying."
    )
    length: Optional[PositiveInt] = Field(
        description="Optionally specify the allowable field length. Fides will not generate values that exceed this size."
    )
    return_all_elements: Optional[bool] = Field(
        description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False."
    )
    read_only: Optional[bool] = Field(
        description="Optionally specify if a field is read-only, meaning it can't be updated or deleted."
    )

    @validator("data_type")
    @classmethod
    def valid_data_type(cls, value: Optional[str]) -> Optional[str]:
        """Validate that all annotated data types exist in the taxonomy"""
        return valid_data_type(value)

class FidesopsMetaBackwardsCompat(BaseModel):
    """Mixin to convert fidesops_meta to fides_meta for backwards compatibility
    as we add DSR concepts to fideslang"""

    def __init__(self, **data: Union[Dataset, DatasetCollection, DatasetField]) -> None:
        """For Datasets, DatasetCollections, and DatasetFields, if old fidesops_meta field is specified,
        convert this to a fides_meta field instead."""
        fidesops_meta = data.pop("fidesops_meta", None)
        fides_meta = data.pop("fides_meta", None)
        super().__init__(
            fides_meta=fides_meta or fidesops_meta,
            **data,
        )

class DatasetField(DatasetFieldBase, FidesopsMetaBackwardsCompat):
    """
    The DatasetField resource model.

    This resource is nested within a DatasetCollection.
    """

    fides_meta: Optional[FidesMeta] = None

    fields: Optional[List[DatasetField]] = Field(
        description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).",
    )

    @validator("fides_meta")
    @classmethod
    def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]:
        """Validate upfront that the return_all_elements flag can only be specified on array fields"""
        if not meta_values:
            return meta_values

        is_array: bool = bool(
            meta_values.data_type and meta_values.data_type.endswith("[]")
        )
        if not is_array and meta_values.return_all_elements is not None:
            raise ValueError(
                "The 'return_all_elements' attribute can only be specified on array fields."
            )
        return meta_values

    @validator("fields")
    @classmethod
    def validate_object_fields(  # type: ignore
        cls,
        fields: Optional[List["DatasetField"]],
        values: Dict[str, Any],
    ) -> Optional[List["DatasetField"]]:
        """Two validation checks for object fields:
        - If there are sub-fields specified, type should be either empty or 'object'
        - Additionally object fields cannot have data_categories.
        """
        declared_data_type = None
        field_name: str = values.get("name")  # type: ignore

        if values.get("fides_meta"):
            declared_data_type = values["fides_meta"].data_type

        if fields and declared_data_type:
            data_type, _ = parse_data_type_string(declared_data_type)
            if data_type != "object":
                raise ValueError(
                    f"The data type '{data_type}' on field '{field_name}' is not compatible with specified sub-fields. Convert to an 'object' field."
                )

        if (fields or declared_data_type == "object") and values.get("data_categories"):
            raise ValueError(
                f"Object field '{field_name}' cannot have specified data_categories. Specify category on sub-field instead"
            )

        return fields

# this is required for the recursive reference in the pydantic model:
DatasetField.update_forward_refs()

class FidesCollectionKey(ConstrainedStr):
    """
    Dataset.Collection name where both dataset and collection names are valid FidesKeys
    """

    @classmethod
    def validate(cls, value: str) -> str:
        """
        Overrides validation to check FidesCollectionKey format, and that both the dataset
        and collection names have the FidesKey format.
        """
        values = value.split(".")
        if len(values) == 2:
            FidesKey.validate(values[0])
            FidesKey.validate(values[1])
            return value
        raise ValueError(
            "FidesCollection must be specified in the form 'FidesKey.FidesKey'"
        )

class CollectionMeta(BaseModel):
    """Collection-level specific annotations used for query traversal"""

    after: Optional[List[FidesCollectionKey]]
    skip_processing: Optional[bool] = False

class DatasetCollection(FidesopsMetaBackwardsCompat):
    """
    The DatasetCollection resource model.

    This resource is nested within a Dataset.
    """

    name: str = name_field
    description: Optional[str] = description_field
    data_categories: Optional[List[FidesKey]] = Field(
        description="Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection.",
    )
    data_qualifier: FidesKey = Field(
        default="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified",
        description="Array of Data Qualifier resources identified by `fides_key`, that apply to all fields in the collection.",
    )
    retention: Optional[str] = Field(
        description="An optional string to describe the retention policy for a Dataset collection. This field can also be applied more granularly at the field level of a Dataset.",
    )
    fields: List[DatasetField] = Field(
        description="An array of objects that describe the collection's fields.",
    )

    fides_meta: Optional[CollectionMeta] = None

    _sort_fields: classmethod = validator("fields", allow_reuse=True)(
        sort_list_objects_by_name
    )
    _unique_items_in_list: classmethod = validator("fields", allow_reuse=True)(
        unique_items_in_list
    )

class ContactDetails(BaseModel):
    """
    The contact details information model.

    Used to capture contact information for controllers, used
    as part of exporting a data map / ROPA.

    This model is nested under an Organization and
    potentially under a system/dataset.
    """

    name: str = Field(
        default="",
        description="An individual name used as part of publishing contact information. Encrypted at rest on the server.",
    )
    address: str = Field(
        default="",
        description="An individual address used as part of publishing contact information. Encrypted at rest on the server.",
    )
    email: str = Field(
        default="",
        description="An individual email used as part of publishing contact information. Encrypted at rest on the server.",
    )
    phone: str = Field(
        default="",
        description="An individual phone number used as part of publishing contact information. Encrypted at rest on the server.",
    )

class DatasetMetadata(BaseModel):
    """
    The DatasetMetadata resource model.

    Object used to hold application specific metadata for a dataset
    """

    resource_id: Optional[str]
    after: Optional[List[FidesKey]]

class Dataset(FidesModel, FidesopsMetaBackwardsCompat):
    """The Dataset resource model."""

    meta: Optional[Dict] = meta_field
    data_categories: Optional[List[FidesKey]] = Field(
        description="Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset.",
    )
    data_qualifier: Optional[FidesKey] = Field(
        description="Deprecated. Array of Data Qualifier resources identified by `fides_key`, that apply to all collections in the Dataset.",
    )
    fides_meta: Optional[DatasetMetadata] = Field(
        description=DatasetMetadata.__doc__, default=None
    )
    joint_controller: Optional[ContactDetails] = Field(
        description="Deprecated. " + ContactDetails.__doc__,
    )
    retention: Optional[str] = Field(
        description="Deprecated. An optional string to describe the retention policy for a dataset. This field can also be applied more granularly at either the Collection or field level of a Dataset.",
    )
    third_country_transfers: Optional[List[str]] = Field(
        description="Deprecated. An optional array to identify any third countries where data is transited to. For consistency purposes, these fields are required to follow the Alpha-3 code set in [ISO 3166-1](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3).",
    )
    collections: List[DatasetCollection] = Field(
        description="An array of objects that describe the Dataset's collections.",
    )

    _sort_collections: classmethod = validator("collections", allow_reuse=True)(
        sort_list_objects_by_name
    )
    _check_valid_country_code: classmethod = country_code_validator
    _unique_items_in_list: classmethod = validator("collections", allow_reuse=True)(
        unique_items_in_list
    )

    @root_validator
    @classmethod
    def deprecate_fields(cls, values: Dict) -> Dict:
        """
        Warn of Dataset fields pending deprecation.
        """
        deprecated_fields = [
            "joint_controller",
            "data_qualifier",
            "retention",
            "third_country_transfers",
        ]
        for field in deprecated_fields:
            if values.get(field) is not None:
                warn(
                    f"The {field} field is deprecated, and will be removed in a future version of fideslang.",
                    DeprecationWarning,
                )
        return values

# Evaluation
class ViolationAttributes(BaseModel):
    "The model for attributes which led to an evaluation violation"

    data_categories: List[str] = Field(
        description="A list of data categories which led to an evaluation violation.",
    )
    data_subjects: List[str] = Field(
        description="A list of data subjects which led to an evaluation violation.",
    )
    data_uses: List[str] = Field(
        description="A list of data uses which led to an evaluation violation.",
    )
    data_qualifier: str = Field(
        description="The data qualifier which led to an evaluation violation.",
    )

class Violation(BaseModel):
    "The model for violations within an evaluation."

    violating_attributes: ViolationAttributes = Field(
        description=ViolationAttributes.__doc__
    )
    detail: str = Field(
        description="A human-readable string detailing the evaluation violation.",
    )

class StatusEnum(str, Enum):
    "The model for possible evaluation results."

    FAIL = "FAIL"
    PASS = "PASS"

class Evaluation(BaseModel):
    """
    The Evaluation resource model.

    This resource is created after an evaluation is executed.
    """

    fides_key: FidesKey = Field(
        description="A uuid generated for each unique evaluation.",
    )
    status: StatusEnum = Field(description=StatusEnum.__doc__)
    violations: List[Violation] = Field(
        default=[],
        description=Violation.__doc__,
    )
    message: str = Field(
        default="",
        description="A human-readable string response for the evaluation.",
    )

    class Config:
        "Config for the Evaluation"
        extra = "ignore"
        orm_mode = True

# Organization
class ResourceFilter(BaseModel):
    """
    The ResourceFilter resource model.
    """

    type: str = Field(
        description="The type of filter to be used (i.e. ignore_resource_arn)",
    )
    value: str = Field(
        description="A string representation of resources to be filtered. Can include wildcards.",
    )

class OrganizationMetadata(BaseModel):
    """
    The OrganizationMetadata resource model.

    Object used to hold application specific metadata for an organization
    """

    resource_filters: Optional[List[ResourceFilter]] = Field(
        description="A list of filters that can be used when generating or scanning systems."
    )

class Organization(FidesModel):
    """
    The Organization resource model.

    This resource is used as a way to organize all other resources.
    """

    # It inherits this from FidesModel but Organizations don't have this field
    organization_parent_key: None = Field(
        default=None,
        description="An inherited field from the FidesModel that is unused with an Organization.",
    )
    controller: Optional[ContactDetails] = Field(
        description=ContactDetails.__doc__,
    )
    data_protection_officer: Optional[ContactDetails] = Field(
        description=ContactDetails.__doc__,
    )
    fidesctl_meta: Optional[OrganizationMetadata] = Field(
        description=OrganizationMetadata.__doc__,
    )
    representative: Optional[ContactDetails] = Field(
        description=ContactDetails.__doc__,
    )
    security_policy: Optional[HttpUrl] = Field(
        description="Am optional URL to the organization security policy."
    )

# Policy
class MatchesEnum(str, Enum):
    """
    The MatchesEnum resource model.

    Determines how the listed resources are matched in the evaluation logic.
    """

    ANY = "ANY"
    ALL = "ALL"
    NONE = "NONE"
    OTHER = "OTHER"

class PrivacyRule(BaseModel):
    """
    The PrivacyRule resource model.

    A list of privacy data types and what match method to use.
    """

    matches: MatchesEnum = Field(
        description=MatchesEnum.__doc__,
    )
    values: List[FidesKey] = Field(
        description="A list of fides keys to be used with the matching type in a privacy rule.",
    )

class PolicyRule(BaseModel):
    """
    The PolicyRule resource model.

    Describes the allowed combination of the various privacy data types.
    """

    name: str
    data_categories: PrivacyRule = Field(
        description=PrivacyRule.__doc__,
    )
    data_uses: PrivacyRule = Field(
        description=PrivacyRule.__doc__,
    )
    data_subjects: PrivacyRule = Field(
        description=PrivacyRule.__doc__,
    )
    data_qualifier: FidesKey = Field(
        default="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified",
        description="The fides key of the data qualifier to be used in a privacy rule.",
    )

class Policy(FidesModel):
    """
    The Policy resource model.

    An object used to organize a list of PolicyRules.
    """

    rules: List[PolicyRule] = Field(
        description=PolicyRule.__doc__,
    )

    _sort_rules: classmethod = validator("rules", allow_reuse=True)(
        sort_list_objects_by_name
    )

# Registry
class Registry(FidesModel):
    """
    The Registry resource model.

    Systems can be assigned to this resource, but it doesn't inherently
    point to any other resources.
    """

# System
class DataProtectionImpactAssessment(BaseModel):
    """
    The DataProtectionImpactAssessment (DPIA) resource model.

    Contains information in regard to the data protection
    impact assessment exported on a data map or Record of
    Processing Activities (RoPA).

    A legal requirement under GDPR for any project that
    introduces a high risk to personal information.
    """

    is_required: bool = Field(
        default=False,
        description="A boolean value determining if a data protection impact assessment is required. Defaults to False.",
    )
    progress: Optional[str] = Field(
        description="The optional status of a Data Protection Impact Assessment. Returned on an exported data map or RoPA.",
    )
    link: Optional[AnyUrl] = Field(
        description="The optional link to the Data Protection Impact Assessment. Returned on an exported data map or RoPA.",
    )

class PrivacyDeclaration(BaseModel):
    """
    The PrivacyDeclaration resource model.

    States a function of a system, and describes how it relates
    to the privacy data types.
    """

    name: Optional[str] = Field(
        description="The name of the privacy declaration on the system.",
    )
    data_categories: List[FidesKey] = Field(
        description="An array of data categories describing a system in a privacy declaration.",
    )
    data_use: FidesKey = Field(
        description="The Data Use describing a system in a privacy declaration.",
    )
    data_qualifier: Optional[FidesKey] = Field(
        description="Deprecated. The fides key of the data qualifier describing a system in a privacy declaration.",
    )
    data_subjects: List[FidesKey] = Field(
        default_factory=list,
        description="An array of data subjects describing a system in a privacy declaration.",
    )
    dataset_references: Optional[List[FidesKey]] = Field(
        description="Referenced Dataset fides keys used by the system.",
    )
    egress: Optional[List[FidesKey]] = Field(
        description="The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied."
    )
    ingress: Optional[List[FidesKey]] = Field(
        description="The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied."
    )
    features: List[str] = Field(
        default_factory=list, description="The features of processing personal data."
    )
    flexible_legal_basis_for_processing: Optional[bool] = Field(
        description="Whether the legal basis for processing is 'flexible' (i.e. can be overridden in a privacy notice) for this declaration.",
    )
    legal_basis_for_processing: Optional[LegalBasisForProcessingEnum] = Field(
        description="The legal basis under which personal data is processed for this purpose."
    )
    impact_assessment_location: Optional[str] = Field(
        description="Where the legitimate interest impact assessment is stored"
    )
    retention_period: Optional[str] = Field(
        description="An optional string to describe the time period for which data is retained for this purpose."
    )
    processes_special_category_data: bool = Field(
        default=False,
        description="This system processes special category data",
    )
    special_category_legal_basis: Optional[SpecialCategoryLegalBasisEnum] = Field(
        description="The legal basis under which the special category data is processed.",
    )
    data_shared_with_third_parties: bool = Field(
        default=False,
        description="This system shares data with third parties for this purpose.",
    )
    third_parties: Optional[str] = Field(
        description="The types of third parties the data is shared with.",
    )
    shared_categories: List[str] = Field(
        default_factory=list,
        description="The categories of personal data that this system shares with third parties.",
    )
    cookies: Optional[List[Cookies]] = Field(
        description="Cookies associated with this data use to deliver services and functionality",
    )

    @validator("data_qualifier")
    @classmethod
    def deprecate_data_qualifier(cls, value: FidesKey) -> FidesKey:
        """
        Warn that the `data_qualifier` field is deprecated, if set.
        """
        if value is not None:
            warn(
                "The data_qualifier field is deprecated, and will be removed in a future version of fideslang.",
                DeprecationWarning,
            )

        return value

    class Config:
        """Config for the Privacy Declaration"""

        orm_mode = True

class SystemMetadata(BaseModel):
    """
    The SystemMetadata resource model.

    Object used to hold application specific metadata for a system
    """

    resource_id: Optional[str] = Field(
        description="The external resource id for the system being modeled."
    )
    endpoint_address: Optional[str] = Field(
        description="The host of the external resource for the system being modeled."
    )
    endpoint_port: Optional[str] = Field(
        description="The port of the external resource for the system being modeled."
    )

class FlowableResources(str, Enum):
    """
    The resource types with which DataFlows can be created.
    """

    DATASET = "dataset"
    SYSTEM = "system"
    USER = "user"

class DataFlow(BaseModel):
    """
    The DataFlow resource model.

    Describes a resource model with which a given System resource communicates.
    """

    fides_key: FidesKey = Field(
        ...,
        description="Identifies the System or Dataset resource with which the communication occurs. May also be 'user', to represent communication with the user(s) of a System.",
    )
    type: str = Field(
        ...,
        description=f"Specifies the resource model class for which the `fides_key` applies. May be any of {', '.join([member.value for member in FlowableResources])}.",
    )
    data_categories: Optional[List[FidesKey]] = Field(
        description="An array of data categories describing the data in transit.",
    )

    @root_validator(skip_on_failure=True)
    @classmethod
    def user_special_case(cls, values: Dict) -> Dict:
        """
        If either the `fides_key` or the `type` are set to "user",
        then the other must also be set to "user".
        """

        if values["fides_key"] == "user" or values["type"] == "user":
            assert (
                values["fides_key"] == "user" and values["type"] == "user"
            ), "The 'user' fides_key is required for, and requires, the type 'user'"

        return values

    @validator("type")
    @classmethod
    def verify_type_is_flowable(cls, value: str) -> str:
        """
        Assert that the value of the `type` field is a member
        of the `FlowableResources` Enum.
        """

        flowables = [member.value for member in FlowableResources]
        assert value in flowables, f"'type' must be one of {', '.join(flowables)}"
        return value

class System(FidesModel):
    """
    The System resource model.

    Describes an application and includes a list of PrivacyDeclaration resources.
    """

    registry_id: Optional[int] = Field(
        description="The id of the system registry, if used.",
    )
    meta: Optional[Dict] = meta_field
    fidesctl_meta: Optional[SystemMetadata] = Field(
        description=SystemMetadata.__doc__,
    )
    system_type: str = Field(
        description="A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc.",
    )
    data_responsibility_title: Optional[DataResponsibilityTitle] = Field(
        description="Deprecated. The responsibility or role over the system that processes personal data",
    )
    egress: Optional[List[DataFlow]] = Field(
        description="The resources to which the system sends data."
    )
    ingress: Optional[List[DataFlow]] = Field(
        description="The resources from which the system receives data."
    )
    privacy_declarations: List[PrivacyDeclaration] = Field(
        description=PrivacyDeclaration.__doc__,
    )
    joint_controller: Optional[ContactDetails] = Field(
        description="Deprecated. " + ContactDetails.__doc__,
    )
    third_country_transfers: Optional[List[str]] = Field(
        description="Deprecated. An optional array to identify any third countries where data is transited to. For consistency purposes, these fields are required to follow the Alpha-3 code set in ISO 3166-1.",
    )
    administrating_department: Optional[str] = Field(
        default="Not defined",
        description="An optional value to identify the owning department or group of the system within your organization",
    )
    data_protection_impact_assessment: Optional[DataProtectionImpactAssessment] = Field(
        description="Deprecated. " + DataProtectionImpactAssessment.__doc__,
    )
    vendor_id: Optional[str] = Field(
        description="The unique identifier for the vendor that's associated with this system."
    )
    dataset_references: List[FidesKey] = Field(
        default_factory=list,
        description="Referenced Dataset fides keys used by the system.",
    )
    processes_personal_data: bool = Field(
        default=True,
        description="This toggle indicates whether the system stores or processes personal data.",
    )
    exempt_from_privacy_regulations: bool = Field(
        default=False,
        description="This toggle indicates whether the system is exempt from privacy regulation if they do process personal data.",
    )
    reason_for_exemption: Optional[str] = Field(
        description="The reason that the system is exempt from privacy regulation."
    )
    uses_profiling: bool = Field(
        default=False,
        description="Whether the vendor uses data to profile a consumer in a way that has a legal effect.",
    )
    legal_basis_for_profiling: List[LegalBasisForProfilingEnum] = Field(
        default_factory=list,
        description="The legal basis (or bases) for performing profiling that has a legal effect.",
    )
    does_international_transfers: bool = Field(
        default=False,
        description="Whether this system transfers data to other countries or international organizations.",
    )
    legal_basis_for_transfers: List[str] = Field(
        default_factory=list,
        description="The legal basis (or bases) under which the data is transferred.",
    )
    requires_data_protection_assessments: bool = Field(
        default=False,
        description="Whether this system requires data protection impact assessments.",
    )
    dpa_location: Optional[str] = Field(
        description="Location where the DPAs or DIPAs can be found."
    )
    dpa_progress: Optional[str] = Field(
        description="The optional status of a Data Protection Impact Assessment"
    )
    privacy_policy: Optional[AnyUrl] = Field(
        description="A URL that points to the system's publicly accessible privacy policy."
    )
    legal_name: Optional[str] = Field(
        description="The legal name for the business represented by the system."
    )
    legal_address: Optional[str] = Field(
        description="The legal address for the business represented by the system."
    )
    responsibility: List[DataResponsibilityTitle] = Field(
        default_factory=list,
        description=DataResponsibilityTitle.__doc__,
    )
    dpo: Optional[str] = Field(
        description="The official privacy contact address or DPO."
    )
    joint_controller_info: Optional[str] = Field(
        description="The party or parties that share the responsibility for processing personal data."
    )  # Use joint_controller_info in favor of joint_controller
    data_security_practices: Optional[str] = Field(
        description="The data security practices employed by this system."
    )
    cookie_max_age_seconds: Optional[int] = Field(
        description="The maximum storage duration, in seconds, for cookies used by this system."
    )
    uses_cookies: bool = Field(
        default=False, description="Whether this system uses cookie storage."
    )
    cookie_refresh: bool = Field(
        default=False,
        description="Whether the system's cookies are refreshed after being initially set.",
    )
    uses_non_cookie_access: bool = Field(
        default=False,
        description="Whether the system uses non-cookie methods of storage or accessing information stored on a user's device.",
    )
    legitimate_interest_disclosure_url: Optional[AnyUrl] = Field(
        description="A URL that points to the system's publicly accessible legitimate interest disclosure."
    )

    _sort_privacy_declarations: classmethod = validator(
        "privacy_declarations", allow_reuse=True
    )(sort_list_objects_by_name)

    _check_valid_country_code: classmethod = country_code_validator

    @root_validator
    @classmethod
    def deprecate_fields(cls, values: Dict) -> Dict:
        """
        Warn of System fields pending deprecation.
        """
        deprecated_fields = [
            "joint_controller",
            "third_country_transfers",
            "data_responsibility_title",
            "data_protection_impact_assessment",
        ]
        for field in deprecated_fields:
            if values.get(field) is not None:
                warn(
                    f"The {field} field is deprecated, and will be removed in a future version of fideslang.",
                    DeprecationWarning,
                )
        return values

    @validator("privacy_declarations", each_item=True)
    @classmethod
    def privacy_declarations_reference_data_flows(
        cls,
        value: PrivacyDeclaration,
        values: Dict,
    ) -> PrivacyDeclaration:
        """
        Any `PrivacyDeclaration`s which include `egress` and/or `ingress` fields must
        only reference the `fides_key`s of defined `DataFlow`s in said field(s).
        """

        for direction in ["egress", "ingress"]:
            fides_keys = getattr(value, direction, None)
            if fides_keys is not None:
                data_flows = values[direction]
                system = values["fides_key"]
                assert (
                    data_flows is not None and len(data_flows) > 0
                ), f"PrivacyDeclaration '{value.name}' defines {direction} with one or more resources and is applied to the System '{system}', which does not itself define any {direction}."

                for fides_key in fides_keys:
                    assert fides_key in [
                        data_flow.fides_key for data_flow in data_flows
                    ], f"PrivacyDeclaration '{value.name}' defines {direction} with '{fides_key}' and is applied to the System '{system}', which does not itself define {direction} with that resource."

        return value

    class Config:
        """Class for the System config"""

        use_enum_values = True

# Taxonomy
class Taxonomy(BaseModel):
    """
    Represents an entire taxonomy of Fides Resources.

    The choice to not use pluralized forms of each resource name
    was deliberate, as this would have caused huge amounts of complexity
    elsewhere across the codebase.
    """

    data_category: List[DataCategory] = Field(default_factory=list)
    data_subject: Optional[List[DataSubject]] = Field(default_factory=list)
    data_use: Optional[List[DataUse]] = Field(default_factory=list)
    data_qualifier: Optional[List[DataQualifier]] = Field(default_factory=list)

    dataset: Optional[List[Dataset]] = Field(default_factory=list)
    system: Optional[List[System]] = Field(default_factory=list)
    policy: Optional[List[Policy]] = Field(default_factory=list)

    registry: Optional[List[Registry]] = Field(default_factory=list)
    organization: List[Organization] = Field(default_factory=list)


from tkinter import *
from PIL import Image, ImageDraw
import numpy as np
np.random.seed(123)  # for reproducibility
from keras.applications.vgg16 import preprocess_input
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.preprocessing.image import img_to_array
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
import cv2
from keras.utils import np_utils
from keras.models import model_from_yaml
import os

import HandwritingNNv2Function as nn
import LetterBreaker
import Prediction

IMAGES = 10

class GUI(Frame):
    def __init__(self, master):
        Frame.__init__(self, master)
        self.master = master
        self.users = ["Hannah", "Hayley", "Ashley"]
        self.mainMenu()

    #resets the GUI so that new things can be displayed on it
    def reset(self):
        for child in self.master.winfo_children():
            child.destroy()

    #the main menu screen
    def mainMenu(self):
        self.reset()
        self.title = Label(self.master, text = "Neural Network Handwriting Analysis").pack(side = TOP)
        self.subtitle = Label(self.master, text = "Can you fool the computer?").pack(side = TOP)
        self.newUserButton = Button(self.master, text = "Create New User", command = self.createNewUser).pack()
        self.determineUserButton = Button(self.master, text = "Determine User", command = self.determineUserScreen).pack()
        #self.debugButton = Button(self.master, text = "DEBUG, straight to the NN baby", command = trainNeuralNetwork([])).pack()
        self.exitbutton = Button(self.master, text = "Exit", fg = "red", command = self.master.destroy).pack(side = BOTTOM)

    #asks the user for a username
    def createNewUser(self):
        self.reset()
        self.title = Label(self.master, text = "Type your name here:").pack(side = TOP)
        self.namebox = Entry(self.master)
        self.namebox.pack()
        self.namebox.focus()
        self.submitbutton = Button(self.master, text = "Submit", command = self.newUserWriting).pack()

    #opens the writing screen 25 times so that 25 images can be created
    def newUserWriting(self):

        username = self.namebox.get()
        self.reset()
        self.users.append(username)
        for i in range(0, IMAGES):
            self.title = Label(self.master, text = "Write something! Anything! In print, and something different each time. (Testing Image {}/{})".format(i+1, IMAGES))
            self.title.grid(row = 0, column = 0, sticky = E+W+N)
            self.writingScreen(username, i)
        self.reset()
        label = Label(self.master, text = "Training Neural Network... \n\n(This may take a bit! Be patient!)").pack()

        data, labels, tempwords = LetterBreaker.imageProcess(self.users, 10)

        nn.trainNeuralNetwork(self.users, data, labels) #########
        self.mainMenu()

    #lets the user draw on the screen while creating an identical image in memory to be saved as a .png
    def writingScreen(self, username, image_number):
        def draw(event):
            x, y = event.x, event.y
            if canvas.old_coords:
                x1, y1 = canvas.old_coords
                #tkinter canvas drawing (visible)
                canvas.create_line(x, y, x1, y1)
                #PIL image draw (in memory)
                invisible_draw.line([x, y, x1, y1], (0,0,0))
            canvas.old_coords = x, y
        def reset_coords(event):
            canvas.old_coords = None

        canvas = Canvas(window, width=400, height=400, bg = "white")
        canvas.grid(row = 1, column = 0, sticky = E+W)
        canvas.old_coords = None

        # PIL create an empty image and draw object to draw on
        # memory only, not visible
        user_image = Image.new("RGB", (400, 400), (255, 255, 255))
        invisible_draw = ImageDraw.Draw(user_image)

        writing = IntVar()
        submit = Button(window, text = "Submit", command = lambda: writing.set(1))
        submit.grid(row = 2, column = 0, sticky = E+W+S)

        self.master.bind('<B1-Motion>', draw)
        self.master.bind('<ButtonRelease-1>', reset_coords)

        submit.wait_variable(writing)

        filename = "{}{}.png".format(username, image_number)
        #user_image.save(filename)
        user_image = np.array(user_image)
        cv2.imwrite(filename, user_image)
        return filename

    #lets the user write something, then calls the NN to figure out which user wrote it
    def determineUserScreen(self):
        self.reset()
        unknown_user_image = self.writingScreen("unknown", 0)
        this_user = self.determineUserNN(unknown_user_image)
        self.reset()
        self.guess = Label(self.master, text = "The AI thinks that {} wrote this!".format(this_user)).pack(side = TOP)
        self.back = Button(self.master, text = "Return to Main Menu", command = self.mainMenu).pack()

    def determineUserNN(self, filename):
        model = self.loadModel()
        temp = ["test"]
        tempdata, templabels, words = LetterBreaker.imageProcess(temp, 1, filename)
        whichUser = Prediction.predictUser(model, words, self.users)
        return whichUser

    def loadModel(self):
        yaml_file = open('model.yaml', 'r')
        loaded_model_yaml = yaml_file.read()
        yaml_file.close()
        loaded_model = model_from_yaml(loaded_model_yaml)
        loaded_model.load_weights("model.h5")
        return loaded_model

#####################################################

window = Tk()
main_menu = GUI(window)
window.mainloop()


from flask import Blueprint, request
from modules.common.gestor_personas import gestor_personas
from modules.common.gestor_comun import exportar

reportes_exel = Blueprint('routes_reportes', __name__)

@reportes_exel.route('/personas/generar_exel', methods=['POST', 'GET'])
def generar_exel_personas():
    personas=gestor_personas().obtener_todo()
    personas_data=[]
    for persona in personas:
        pd={}
        pd["Nombre"] = persona.nombre
        pd["Apellido"] = persona.apellido
        pd["email"] = persona.email
        pd["Edad"] = persona.age
        pd["Fecha nacimiento"]=persona.birthdate.strftime('%d/%m/%Y')
        pd["Genero"]=persona.genero.nombre
        pd["Pais"]=persona.lugar.pais.nombre
        pd["Provincia"]=persona.lugar.provincia.nombre
        pd["Ciudad"]=persona.lugar.ciudad.nombre
        pd["Barrio"]=persona.lugar.barrio.nombre
        personas_data.append(pd)

    return exportar.exportar_excel(personas_data)


import cv2
import numpy as np
import time
import threading

import sys
import queue

import os

camera_matrix = np.array([[528.6062, 0, 332.2342],
                          [0, 529.8388, 234.6237],
                          [0, 0, 1]], dtype=np.float32)

dist_coeffs = np.array([0.0834, 0.1615, 0, 0, -0.5344], dtype=np.float32) # 更新畸变系数

class image_processer():
    def __init__(self,image_display_width,image_display_height ,show_img=0,cap_id=0)  :
        self.cap = cv2.VideoCapture(0)
        self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, image_display_width)
        self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, image_display_height)
        self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))
        self.cap.set(cv2.CAP_PROP_FPS, 60)
        self.img_queue=queue.Queue(maxsize=10)
        self.cap_id=cap_id
        self.show_image=show_img
        fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
        self.videoWriter = cv2.VideoWriter('{}/video_result.mp4'.format('.'), fourcc, 60,(640,480) )

        self.new_camera_matrix, self.roi = cv2.getOptimalNewCameraMatrix(camera_matrix, dist_coeffs, (640, 480), 1, (image_display_width,image_display_height))
        self.start()
        # self.cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.25)
        # self.cap.set(cv2.CAP_PROP_EXPOSURE, 0.01)
        # self.cap.set(cv2.CAP_PROP_BRIGHTNESS, 0.01)
        # self.cap.set(cv2.CAP_PROP_CONTRAST, 0.01)
        # self.cap.set(cv2.CAP_PROP_SATURATION, 0.01)
        # self.cap.set(cv2.CAP_PROP_HUE, 0.01)
        # self.cap.set(cv2.CAP_PROP_GAIN, 0.01)
        # self.cap.set(cv2.CAP_PROP_AUTOFOCUS, 0.01)
        # self.cap.set(cv2.CAP_PROP_AUTO_WB, 0.01)
        # self.cap.set(cv2.CAP_PROP_WB_TEMPERATURE, 0.01)
        # self.cap.set(cv2.CAP_PROP_BACKLIGHT, 0.01)
        # self.cap.set(cv2.CAP_PROP_SHARPNESS, 0.01)
    def start(self):
        self.thread=threading.Thread(target=self.run)
        self.thread.start()

    def run(self):
        while True:
            ret, frame = self.cap.read()
            if ret:
                t1 = time.time()
                if self.img_queue.full():
                    _ = self.img_queue.get()
                self.img_queue.put(frame)
                self.videoWriter.write(frame)
                if self.show_image:
                    cv2.imshow("cap"+str(self.cap_id),frame)
                    cv2.waitKey(1)
            else:
                print("read image error")

            # time.sleep(0.0001)
    def get_image(self):
        return self.img_queue.get()

    def stop(self):
        self.is_running = False
        self.thread.join()  # 等待线程结束
        self.videoWriter.release()
        self.cap.release()  # 关闭摄像头
        while not self.img_queue.empty():  # 清空队列
            self.img_queue.get()

# if __name__ == "__main__":
#     image_processer=image_processer(520,520,0,0)
#     while True:
#         time.sleep(1)


import cv2
import dlib
from imutils import face_utils
from flask import Flask,request,jsonify
from werkzeug.utils import secure_filename

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

(left_Start, left_End) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
# points for left eye and right eye
(right_Start, right_End) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]

def get_image_with_landmarks(file_path: str):
    flag = 0

    try:
        image = cv2.imread(file_path, 1)

        img_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        gray = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2GRAY)

        dlib_faces = detector(gray, 0)

    except Exception:
        return {'error': 'Error while reading the image'}

    for face in dlib_faces:
        eyes = []  #
        (x, y, w, h) = face_utils.rect_to_bb(face)
        cv2.rectangle(img_rgb, (x, y), (x + w, y + h), (255, 0, 0), 1)  # draws blue box over face

        shape = predictor(gray, face)
        shape = face_utils.shape_to_np(shape)

        leftEye = shape[left_Start:left_End]
        # indexes for left eye key points

        rightEye = shape[right_Start:right_End]

        eyes.append(leftEye)  # wrap in a list
        eyes.append(rightEye)

        for index, eye in enumerate(eyes):
            flag += 1
            left_side_eye = eye[0]  # left edge of eye
            right_side_eye = eye[3]  # right edge of eye
            top_side_eye = eye[1]  # top side of eye
            bottom_side_eye = eye[4]  # bottom side of eye

            # calculate height and width of dlib eye keypoints
            eye_width = right_side_eye[0] - left_side_eye[0]
            eye_height = bottom_side_eye[1] - top_side_eye[1]

            # create bounding box with buffer around keypoints
            eye_x1 = int(left_side_eye[0] - 0 * eye_width)
            eye_x2 = int(right_side_eye[0] + 0 * eye_width)

            eye_y1 = int(top_side_eye[1] - 1 * eye_height)
            eye_y2 = int(bottom_side_eye[1] + 0.75 * eye_height)

            # draw bounding box around eye roi

            # cv2.rectangle(img_rgb,(eye_x1, eye_y1), (eye_x2, eye_y2),(0,255,0),2)

            roi_eye = img_rgb[eye_y1:eye_y2, eye_x1:eye_x2]  # desired EYE Region(RGB)
            if flag == 1:
                break

    x = roi_eye.shape
    row = x[0]
    col = x[1]
    # this is the main part,
    # where you pick RGB values from the area just below pupil
    array1 = roi_eye[row // 2:(row // 2) + 1, int((col // 3) + 3):int((col // 3)) + 6]

    array1 = array1[0][2]

    array1 = tuple(array1)  # store it in tuple and pass this tuple to "find_color" Funtion

    return str(array1)

app = Flask(__name__)
@app.route('/',methods=["GET","POST"])
def index():
    if request.method == "POST":
        file = request.files.get("file")
        if file is None or file.filename == "":
            return jsonify({"error": "no file"})
        try:

            f = request.files['file']
            f.save(secure_filename(f.filename))
            return jsonify({'results':get_image_with_landmarks(f.filename)})

        except Exception as e:
            return jsonify({"error": str(e)})

    return "OK"
if __name__ == '__main__':
    app.run(debug=True)


import csv
import json
import os
import io, gzip

from django.http import HttpResponse
from django.views import View

from command.lib.db.admin.admin_options import AdminOptions
from command.lib.db.admin.compendium_database import CompendiumDatabase
from command.lib.db.compendium.bio_feature import BioFeature
from command.lib.db.compendium.bio_feature_reporter import BioFeatureReporter
from command.lib.db.compendium.raw_data import RawData
from command.lib.tasks import export_data
from command.lib.utils.decorators import forward_exception_to_http, check_permission
#from command.lib.utils.group_compendium_permission import GroupCompendiumPermission
from command.lib.utils.message import Message
from command.lib.utils.permission import Permission

class ExportDataView(View):

    def get(self, request, operation, *args, **kwargs):
        method = getattr(self, operation)
        return method(request, *args, **kwargs)

    def post(self, request, operation, *args, **kwargs):
        method = getattr(self, operation)
        return method(request, *args, **kwargs)

    @staticmethod
    @forward_exception_to_http
    def read_file(request, *args, **kwargs):
        path = request.GET['path']
        base_dir = AdminOptions.objects.get(option_name='raw_data_directory').option_value

        if not os.path.isfile(base_dir + path):
            return HttpResponse('File ' + os.path.basename(path) + ' no longer exists!',
                                    content_type='text/plain')
        file = open(base_dir + path, 'rb')
        if path.endswith('tsv.gz'):
            response = HttpResponse(file.read(), content_type='application/x-gzip')
            response['Content-Disposition'] = 'attachment; filename="raw_data.tsv.gz"'
        elif path.endswith('hdf5'):
            response = HttpResponse(file.read(), content_type='application/octet-stream')
            response['Content-Disposition'] = 'attachment; filename="raw_data.hdf5"'
        response['Content-Length'] = str(os.stat(base_dir + path).st_size)

        return response

    @staticmethod
    @forward_exception_to_http
    @check_permission(Permission.EXPORT_COMPENDIUM)
    def export_raw_data(request, *args, **kwargs):
        comp_id = request.POST['compendium_id']
        channel_name = request.session['channel_name']
        view = request.POST['view']
        operation = request.POST['operation']

        compendium = CompendiumDatabase.objects.get(id=comp_id)

        base_dir = AdminOptions.objects.get(option_name='raw_data_directory').option_value
        full_dir = os.path.join(base_dir, compendium.compendium_nick_name, 'export_raw_data')

        # check genes, raw_data and mapping
        bio_features_check = BioFeature.objects.using(compendium.compendium_nick_name).count() > 0
        raw_data_check = RawData.objects.using(compendium.compendium_nick_name).count() > 0
        mapping_check = BioFeatureReporter.objects.using(compendium.compendium_nick_name).\
            exclude(bio_feature__isnull=True).count() > 0

        if bio_features_check and raw_data_check and mapping_check:
            export_data.export_raw_data.apply_async(
                (request.user.id, comp_id, full_dir, channel_name, view, operation)
            )
            return HttpResponse(json.dumps({'success': True}),
                                content_type="application/json")
        else:
            error_msg = Message(type='error', title='Cannot export data',
                                message='To export data you need to:'
                                        '
'
                                        '
import biological features;
'
                                        '
import at least one experiment;
'
                                        '
platform mapped (if requested);
'
                                        '
')
            return HttpResponse(json.dumps(error_msg.to_dict()),
                                content_type="application/json")


import graphene
from django.core.exceptions import ValidationError
from django.db import transaction

from core.graph.mutations import BaseMutation, ModelDeleteMutation, ModelMutation
from . import models
from .enums import NavigationType
from .types import Menu, MenuItem
from ..categories import models as categories_models
from ..categories.types import Category
from ..collections import models as collections_models
from ..collections.types import Collection
from ..core.permissions import MenuPermissions, SitePermissions
from ..pages import models as pages_models
from ..pages.types import Page

class MenuItemMoveInput(graphene.InputObjectType):
    item_id = graphene.ID(description="The menu item ID to move.", required=True)
    parent_id = graphene.ID(
        description="ID of the parent menu. If empty, menu will be top level menu."
    )
    sort_order = graphene.Int(description="Sorting position of the menu item (from 0 to x).")

class MenuItemInput(graphene.InputObjectType):
    name = graphene.String(description="Name of the menu item.")
    url = graphene.String(description="URL of the pointed item.")
    category = graphene.ID(description="Category to which item points.", name="category")
    collection = graphene.ID(description="Collection to which item points.", name="collection")
    page = graphene.ID(description="Page to which item points.", name="page")

class MenuItemCreateInput(MenuItemInput):
    name = graphene.String(description="Name of the menu item.", required=True)
    menu = graphene.ID(description="Menu to which item belongs.", name="menu", required=True)
    parent = graphene.ID(
        description="ID of the parent menu. If empty, menu will be top level menu.", name="parent",
    )

class MenuInput(graphene.InputObjectType):
    name = graphene.String(description="Name of the menu.")

class MenuCreateInput(graphene.InputObjectType):
    name = graphene.String(description="Name of the menu.", required=True)
    items = graphene.List(MenuItemInput, description="List of menu items.")

class MenuCreate(ModelMutation):
    class Arguments:
        input = MenuCreateInput(required=True, description="Fields required to create a menu.")

    class Meta:
        description = "Creates a new Menu."
        model = models.Menu
        permissions = (MenuPermissions.MANAGE_MENUS,)

    @classmethod
    def clean_input(cls, info, instance, data):
        cleaned_input = super().clean_input(info, instance, data)
        items = []
        for item in cleaned_input.get("items", []):
            category = item.get("category")
            collection = item.get("collection")
            page = item.get("page")
            url = item.get("url")
            if len([i for i in [category, collection, page, url] if i]) > 1:
                raise ValidationError({"items": ValidationError("More than one item provided.")})

            if category:
                category = cls.get_node_or_error(info, category, field="items", only_type=Category)
                item["category"] = category
            elif collection:
                collection = cls.get_node_or_error(
                    info, collection, field="items", only_type=Collection
                )
                item["collection"] = collection
            elif page:
                page = cls.get_node_or_error(info, page, field="items", only_type=Page)
                item["page"] = page
            elif not url:
                raise ValidationError({"items": ValidationError("No menu item provided.")})
            items.append(item)

        cleaned_input["items"] = items
        return cleaned_input

    @classmethod
    def _save_m2m(cls, info, instance, cleaned_data):
        super()._save_m2m(info, instance, cleaned_data)
        items = cleaned_data.get("items", [])
        for item in items:
            instance.items.create(**item)

class MenuUpdate(ModelMutation):
    class Arguments:
        id = graphene.ID(required=True, description="ID of a menu to update.")
        input = MenuInput(required=True, description="Fields required to update a menu.")

    class Meta:
        description = "Updates a menu."
        model = models.Menu
        permissions = (MenuPermissions.MANAGE_MENUS,)

class MenuDelete(ModelDeleteMutation):
    class Arguments:
        id = graphene.ID(required=True, description="ID of a menu to delete.")

    class Meta:
        description = "Deletes a menu."
        model = models.Menu
        permissions = (MenuPermissions.MANAGE_MENUS,)

def _validate_menu_item_instance(cleaned_input: dict, field: str, expected_model):
    """Check if the value to assign as a menu item matches the expected model."""
    item = cleaned_input.get(field)
    if item:
        if not isinstance(item, expected_model):
            msg = (
                f"Enter a valid {expected_model._meta.verbose_name} ID "
                f"(got {item._meta.verbose_name} ID)."
            )
            raise ValidationError({field: ValidationError(msg)})

class MenuItemCreate(ModelMutation):
    class Arguments:
        input = MenuItemCreateInput(
            required=True,
            description=(
                "Fields required to update a menu item. Only one of `url`, `category`, "
                "`page`, `collection` is allowed per item."
            ),
        )

    class Meta:
        description = "Creates a new menu item."
        model = models.MenuItem
        permissions = (MenuPermissions.MANAGE_MENUS,)

    @classmethod
    def clean_input(cls, info, instance, data):
        cleaned_input = super().clean_input(info, instance, data)

        _validate_menu_item_instance(cleaned_input, "page", pages_models.Page)
        _validate_menu_item_instance(cleaned_input, "collection", collections_models.Collection)
        _validate_menu_item_instance(cleaned_input, "category", categories_models.Category)

        items = [
            cleaned_input.get("page"),
            cleaned_input.get("collection"),
            cleaned_input.get("url"),
            cleaned_input.get("category"),
        ]
        items = [item for item in items if item is not None]
        if len(items) > 1:
            raise ValidationError("More than one item provided.")
        return cleaned_input

class MenuItemUpdate(MenuItemCreate):
    class Arguments:
        id = graphene.ID(required=True, description="ID of a menu item to update.")
        input = MenuItemInput(
            required=True,
            description=(
                "Fields required to update a menu item. Only one of `url`, `category`, "
                "`page`, `collection` is allowed per item."
            ),
        )

    class Meta:
        description = "Updates a menu item."
        model = models.MenuItem
        permissions = (MenuPermissions.MANAGE_MENUS,)

    @classmethod
    def construct_instance(cls, instance, cleaned_data):
        # Only one item can be assigned per menu item
        instance.page = None
        instance.collection = None
        instance.category = None
        instance.url = None
        return super().construct_instance(instance, cleaned_data)

class MenuItemDelete(ModelDeleteMutation):
    class Arguments:
        id = graphene.ID(required=True, description="ID of a menu item to delete.")

    class Meta:
        description = "Deletes a menu item."
        model = models.MenuItem
        permissions = (MenuPermissions.MANAGE_MENUS,)

class MenuItemMove(BaseMutation):
    menu = graphene.Field(Menu, description="Assigned menu to move within.")

    class Arguments:
        id = graphene.ID(description="The menu item ID to move.", required=True)
        parent_id = graphene.ID(
            description="ID of the parent menu. If empty, menu will be top level menu."
        )
        sort_order = graphene.Int(
            description="Sorting position index based on parent.", required=True
        )

    class Meta:
        description = "Moves items of menus."
        permissions = (MenuPermissions.MANAGE_MENUS,)

    @classmethod
    @transaction.atomic
    def perform_mutation(cls, root, info, **data):
        id = data.get("id")
        parent = data.get("parent_id", None)
        sort_order = data.get("sort_order", 0)
        item = cls.get_node_or_error(info, id, field="id", only_type=MenuItem)
        menu = item.menu

        if parent:
            parent = cls.get_node_or_error(info, parent, field="id", only_type=MenuItem)
            item.parent = parent
            item.save()
            parent.refresh_from_db()
        else:
            item.parent = parent
            item.save()

        # resort order
        qs = (
            models.MenuItem.objects.filter(level=0, menu_id=menu.id)
            if parent is None
            else parent.get_children()
        )
        qs = list(qs)
        old_index = next((index for (index, d) in enumerate(qs) if d.id == item.id), None)
        qs.insert(sort_order, qs.pop(old_index))
        for idx, item in enumerate(qs):
            item.sort_order = idx
        models.MenuItem.objects.bulk_update(qs, ["sort_order"])
        menu.refresh_from_db()

        return MenuItemMove(menu=menu)

class AssignNavigation(BaseMutation):
    menu = graphene.Field(Menu, description="Assigned navigation menu.")

    class Arguments:
        menu = graphene.ID(description="ID of the menu.")
        navigation_type = NavigationType(
            description="Type of the navigation bar to assign the menu to.", required=True,
        )

    class Meta:
        description = "Assigns storefront's navigation menus."
        permissions = (MenuPermissions.MANAGE_MENUS, SitePermissions.MANAGE_SETTINGS)

    @classmethod
    def perform_mutation(cls, _root, info, navigation_type, menu=None):
        site_settings = info.context.site.settings
        menu = cls.get_node_or_error(info, menu, field="menu")

        prev_menu = None
        if navigation_type == NavigationType.MAIN:
            prev_menu = site_settings.top_menu
            site_settings.top_menu = menu
            site_settings.save(update_fields=["top_menu"])
        elif navigation_type == NavigationType.SECONDARY:
            prev_menu = site_settings.bottom_menu
            site_settings.bottom_menu = menu
            site_settings.save(update_fields=["bottom_menu"])

        if menu is not None and prev_menu:
            prev_menu.refresh_from_db()

        return AssignNavigation(menu=menu if menu is not None else prev_menu)


import datetime
import json

import django_rq

from server.apis.guidebox import GuideBox
from server.models import Channel, Content

def inital_database_population_of_content():
    g = GuideBox()
    q = django_rq.get_queue('low')
    result = []

    total_shows = g.get_total_number_of_shows()

    for i in range(0, total_shows, 250):
        show_list = g.get_content_list(i)
        shows_dict = json.loads(show_list)

        for show in shows_dict['results']:
            q.enqueue(g.save_content, show)

            result.append(show)

    return result

def refresh_database(d):
    g = GuideBox()
    q = django_rq.get_queue('high')

    last = datetime.date(2016,1,1)
    milis = time.mktime(last.timetuple())
    milis = int(milis)
    new_shows = g.get_new_shows(milis)
    # for i in range

def inital_database_population_of_channels():
    g = GuideBox()
    q = django_rq.get_queue('low')
    result = []

    total_channels = g.get_total_number_of_channels()

    for i in range(0, total_channels, 24):
        channel_list = g.get_channel_list('all', i)
        channel_dict = json.loads(channel_list)

        for chan in channel_dict['results']:
            q.enqueue(g.save_channel, chan)

            result.append(chan)

    return result

def connect_content_channel_task():
    g = GuideBox()
    q_low = django_rq.get_queue('low')
    q_high = django_rq.get_queue('high')
    print('im splitting stuff')

    all_channels = Channel.objects.all()
    count = 0
    for i in [all_channels[i:i + 5] for i in range(0, len(all_channels), 5)]:
        count += 5
        if count % 2 == 0:
            q_high.enqueue(g.connect_channels_shows, i)
        else:
            q_low.enqueue(g.connect_channels_shows, i)

def add_available_sources_to_shows():
    g = GuideBox()
    q_high = django_rq.get_queue('high')

    # all_shows = Content.objects.all()
    all_shows = Content.objects.all()
    # print(all_shows)
    for show in [all_shows[i: i + 20]  for i in range(0, len(all_shows), 20)]:
        shows = [x.id for x in show]
        q_high.enqueue(g.add_additional_channels_for_show, shows)

def add_detail_to_shows():
    g = GuideBox()
    q_low = django_rq.get_queue('low')

    all_shows = Content.objects.all()

    all_shows = [show for show in all_shows if show.guidebox_data is not None and 'detail' not in show.guidebox_data]
    for shows_chunk in [all_shows[i: i+20] for i in range(0, len(all_shows), 20)]:
        q_low.enqueue(g.process_shows_for_content_detail, shows_chunk)


#!/usr/bin/python

__author__ = 'Jan Nash'
__version__ = "0.2"
__license__ = "MIT"

VERSION = "%s v" + __version__

def parse_args():
    """
    Parses command-line arguments and returns the parsed namespace-object.
    """
    import argparse
    import os

    parser = argparse.ArgumentParser(version=VERSION)

    parser.add_argument(
        '-p', '--path',
        action='store', default=os.getcwd(), dest='project_source_root_path',
        help='The path to the Xcode-project\'s source-root folder that will be walked. '
             'Defaults to the current working directory.')

    parser.add_argument(
        '-u', '--update',
        action='store_true', dest='update',
        help='If set, the keys of missing translations will be written to the strings-files.')

    parser.add_argument(
        '-d', '--debug',
        action='store_true', default=False, dest='debug',
        help='Set logging level to DEBUG. Beware, it\'s gonna be verbose :).')

    parser.add_argument(
        '-c', '--custom-macros',
        action='store', nargs='*', dest='custom_macros', default=(),
        help='Custom macros that are used in the project. For help regarding the format of those custom macros '
             'see documentation of nslocapysation.classes.localized_string.LocalizedString')

    parser.add_argument(
        '-i', '--ignore',
        action='store', nargs='*', dest='ignore_language_codes', default=(),
        help='Language codes for which missing translations should be disregarded.'
             'This is useful if you are lazy and use the UI-string of that language as key.')

    return parser.parse_args()

def main(cmd_args):
    import logging
    from nslocapysation import constants
    from nslocapysation.tools.collect_m_file_and_lproj_dir_paths import collect_implementation_file_and_lproj_dir_paths
    from nslocapysation.tools.collect_localized_strings import collect_localized_strings
    from nslocapysation.tools.collect_localizable_strings_files import collect_localizable_strings_files
    from nslocapysation.tools.check_translations import check_translations
    from nslocapysation.classes.ns_localized_string_macro import NSLocalizedStringMacro

    if cmd_args.debug:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    file_path_dict = collect_implementation_file_and_lproj_dir_paths(cmd_args.project_source_root_path)

    implementation_file_paths = file_path_dict[constants.IMPLEMENTATION_FILE_PATHS_KEY]

    localization_file_paths = file_path_dict[constants.LPROJ_DIR_PATHS_KEY]

    custom_macros = [NSLocalizedStringMacro(format_=fmt) for fmt in cmd_args.custom_macros]

    localized_strings = collect_localized_strings(implementation_file_paths=implementation_file_paths,
                                                  custom_macros=custom_macros)

    localizable_strings_files = collect_localizable_strings_files(localization_dir_paths=localization_file_paths)

    check_translations(translation_files=localizable_strings_files,
                       localized_strings=localized_strings,
                       update=cmd_args.update,
                       ignore_language_codes=cmd_args.ignore_language_codes)

if __name__ == '__main__':
    cl_args = parse_args()
    main(cl_args)


import sys, os, re, pprint
# UDPipe tokenize.py
from .helpers import *
from .config import *

from .utils import Batch

def file2fast_text(in_file_1, in_file_2, out_file, empty_dict=None):
    """
    Converts two tokenized text files into fast_text format
    Empty lines as well original number of lines will be saved to empty_dict dictionary
    args:
        in_file_1 (string): valid path to file 1
        in_file_2 (string): valid path to file 1
        out_file  (string): path where output should be saved to
        empty_dict  (dict): a dictionary where empty line indeces and number of lines in the input files will be saved to
    """
    print('Writing:', out_file)
    with open(in_file_1, 'r') as srcf, \
         open(in_file_2, 'r') as trgf, \
         open(out_file, 'w') as outf:
        empty_lines = []
        for i, (src, trg) in enumerate(zip(srcf, trgf)):
            src = src.strip()
            trg = trg.strip()
            if src and trg:
                outf.write(src + ' ||| ' + trg + '\n')
            else:
                empty_lines.append(i)

    if empty_dict != None:
        filename = os.path.basename(out_file)
        if filename.endswith('.fast_text'):
            filename = filename.rsplit('.', 1)[0]
        empty_dict[filename] = empty_lines
        empty_dict[filename+'.no_lines'] = i+1

def dir2fast_text(input_dir):
    """
    Finds language pairs based on notation in form de-en.de, de-en.en and converts them into fast text format using file2fast_text()
    args:
        input_dir (string): path to input directory
    """
    input_dir, output_dir = create_same_level_output_dir(input_dir, 'merged')
    pairs_dict = get_pairs(input_dir, verbose=False)
    for lang_pair in pairs_dict:
        print(lang_pair)
        for lang in pairs_dict[lang_pair]:
            assert not pairs_dict[lang_pair][lang].endswith('.conll'), 'Fast Text format cannot be applied to .conll-files!'
    print('Pairs found:')
    pprint.pprint(pairs_dict)

    print('Merging..')
    empty_dict = {}
    for pair in pairs_dict:
        lang_1, lang_2 = pair.split('-')
        root = root = pairs_dict[pair][lang_1].split('.')[0]

        filename_d1 = f'{root}.{lang_1}-{lang_2}.fast_text'
        outputpath_d1 = os.path.join(output_dir, filename_d1)

        inputpath_l1 = os.path.join(input_dir, pairs_dict[pair][lang_1])
        inputpath_l2 = os.path.join(input_dir, pairs_dict[pair][lang_2])
        file2fast_text(inputpath_l1, inputpath_l2, outputpath_d1, empty_dict)

        filename_d2 = f'{root}.{lang_2}-{lang_1}.fast_text'

        outputpath_d2 = os.path.join(output_dir, filename_d2)
        file2fast_text(inputpath_l2, inputpath_l1, outputpath_d2, empty_dict)

    empty_dict_path = os.path.join(output_dir, 'empty.dict')
    with open(empty_dict_path, 'w') as f:
        f.write(repr(empty_dict))
    print('.. done.')
    return output_dir

def align(input_dir, use_shell=False, args=None):
    """
    Expects 'input_dir' to be the path to the directory with tokens. Tokens will be converted to fast text and saved
    into a directory 'merged'. Batchfiles for alignment jobs will be created and submitted to slurm job system
    args:
        input_dir (string): path to token directory
        use_shell (bool)  : trys to perform alignment in shell instead of slurm sbatch (experimental)
        args (None or argparser.args): additional arguments to modify parameters such es memory, timelimit or partition to run on

    """

    input_dir = dir2fast_text(input_dir=input_dir)
    # changing input dir to merged_dir for alignment input
    input_dir, output_dir = create_same_level_output_dir(input_dir, 'alignment')

    merged_files = list(filter(lambda x: re.match(r'.*\.[a-z]{2}-[a-z]{2}', x), os.listdir(input_dir)))
    print(f'Valid input files  ("{input_dir}"):')
    pprint.pprint(merged_files)
    print()
    print('Create and submit batchfiles..')
    batch = Batch(name=f'alignment', n_cpus='2', log_dir='alignment', timelimit='04:00:00', args=args)

    for filename in merged_files:
        outputname = filename
        if filename.endswith('.fast_text'):
            outputname = filename.rsplit('.', 1)[0]
        pair = 'al_'+outputname.split('.')[-1]
        batch.name = pair

        batch.align(input_dir=input_dir, output_dir=output_dir, filename=filename, outputname=outputname)
        batch.shell() if use_shell else batch.submit()


import sqlite3
from logger import *

def connect():
    conn=sqlite3.connect('routine.db')
    cur=conn.cursor()
    cur.execute("CREATE TABLE IF NOT EXISTS routine (Id INTEGER PRIMARY KEY , date text , earnings integer ,excercize text , study text , diet text , expense integer)")
    conn.commit()
    conn.close()
    logging.info("Connection established")

def insert(date,earnings,excercize,study,diet,expense):
    conn=sqlite3.connect('routine.db')
    cur=conn.cursor()
    cur.execute("INSERT INTO routine VALUES (NULL , ?,?,?,?,?,?)",(date,earnings,excercize,study,diet,expense))
    conn.commit()
    conn.close()
    logging.info("Data Insersection intitalized")

def view():
    conn=sqlite3.connect('routine.db')
    cur=conn.cursor()
    cur.execute("SELECT * FROM routine")
    rows=cur.fetchall()
    conn.commit()
    conn.close()
    logging.info("View intitalized")
    return(rows)

def delete(id):
    conn=sqlite3.connect('routine.db')
    cur=conn.cursor()
    cur.execute("DELETE FROM routine WHERE id=?",(id,))
    conn.commit()
    conn.close()
    logging.info("Delete")

def search(date='',earnings='',excercize='',study='',diet='',expense=''):
    conn=sqlite3.connect('routine.db')
    cur=conn.cursor()
    cur.execute("SELECT * FROM routine WHERE date=?  OR earnings=? OR excercize=? OR study=? OR diet=? OR expense=?" , (date , earnings , excercize , study , diet , expense))
    rows=cur.fetchall()
    conn.commit()
    conn.close()
    logging.info("Searched routine")
    return(rows)

#insert('2-3-20',200,'done','done','yes',300)
#view()
# insert('2-3-20',400,'done','done','yes',500)
# insert('2-3-20',600,'done','done','yes',700)
# insert('2-3-20',800,'done','done','yes',900)
# print(search(earnings=400))


"""
https://leetcode.com/problems/find-minimum-in-rotated-sorted-array/description/
"""
from typing import List
class Solution:
    def findMin(self, nums: List[int]) -> int:
        bsearch=self.helper(nums,0,len(nums)-1)
        return nums[bsearch]

    def helper(self,nums,left,right):

        mid=left+((right-left)//2)

        if left==right or (nums[mid-1]>nums[mid]):
            return mid

        else:
            if nums[mid]>nums[left] and nums[mid]>nums[right]:
                return self.helper(nums,mid+1,right)
            else:
                return self.helper(nums, left, mid-1)

if __name__ == '__main__':
    obj=Solution()
    print(obj.findMin([3,4,5,1,2]))
    print(obj.findMin([4,5,6,7,0,1,2]))
    print(obj.findMin([11,13,15,17]))
    print(obj.findMin([2,1]))
    print(obj.findMin([0,1]))


# -*- coding: utf-8 -*-

import six
from django.core.exceptions import NON_FIELD_ERRORS, ValidationError
from django.forms.fields import Field
from django.forms.forms import DeclarativeFieldsMetaclass
from django.forms.utils import ErrorDict, ErrorList

class BaseDeserializer(Field):
    """
    The main implementation of Deserializers logic. Note that this class is
    slightly different from the classic Form.
    """

    __slots__ = ()

    empty_values = (None, "")

    def __init__(self, data=None, **kwargs):
        super(BaseDeserializer, self).__init__(**kwargs)
        self._init_data(data)

    def _init_data(self, data):
        # type:(Optional[Dict]) -> None
        self._data = data
        self._errors = None  # Stores the errors after `full_clean()` has been called.
        self._cleaned_data = None

    @classmethod
    def fields(cls):
        # type:() -> Iterator[str, Field]
        for name in cls.declared_fields:
            yield name, cls.declared_fields[name]

    @property
    def errors(self):
        # type:() -> ErrorDict
        """Return an ErrorDict for the data provided for the deserializer."""
        if self._errors is None:
            self.full_clean()
        return self._errors

    def is_valid(self):
        """Return True if the deserializer has no errors, or False otherwise."""
        return not self.errors

    def _get_error_dict(self, field_name, error):
        # type:(str, ValidationError) -> Dict
        if not hasattr(error, "error_dict"):
            # In case the error comes from simple ("flat") Field
            return {field_name or NON_FIELD_ERRORS: error.messages}

        # Nested field case (the Field itself is a Deserializer instance)
        return {field_name: dict(error)}

    def add_error(self, field_name, error):
        # type:(str, ValidationError) -> None
        """
        Updates the content of `self._errors`. Each `self._errors` is a `dict` instance.
        """
        error = self._get_error_dict(field_name, error)
        for field_name, errors in error.items():
            self._errors[field_name] = errors

    def full_clean(self):
        """
        Clean all of self._data and populate self._errors and self._cleaned_data.
        """
        self._errors = ErrorDict()
        self._cleaned_data = {}
        self._clean_fields()

    def _clean_fields(self):
        if self._data in self.empty_values:
            self._cleaned_data = self._data
            return
        if not isinstance(self._data, dict):
            error_list = ErrorList(["This field should be an object."])
            raise ValidationError(error_list)
        for name, field in self.declared_fields.items():
            value = self._data.get(name)
            try:
                value = field.clean(value)
                self._cleaned_data[name] = value
                if hasattr(self, "post_clean_{}".format(name)):
                    value = getattr(self, "post_clean_{}".format(name))(value)
                    self._cleaned_data[name] = value
            except ValidationError as e:
                if field.required:
                    self.add_error(name, e)

    def clean(self, data):
        """
        Overriden version of superclass Field.clean(). Called in case of nested
        Deserializers. Performs the validation and cleaning process on the data
        parameter. Returns the cleaned data to the parent deserializer.
        In case of error, Raises `ValidationError` with the whole ErrorDict()
        of its errors in order to be catched in the parent.
        """

        # Raises an error if `data` is empty and the current (nested)
        # deserializer is required.
        self.validate(data)

        self._init_data(data)
        if self.errors:
            raise ValidationError(self.errors)
        return self._cleaned_data

    @property
    def data(self):
        # type:() -> dict
        if not self._cleaned_data:
            self.full_clean()
        return self._cleaned_data

class Deserializer(six.with_metaclass(DeclarativeFieldsMetaclass, BaseDeserializer)):
    pass

class AllPassDeserializer(Deserializer):
    """Deserializer that accepts any given data. the clean always returns
    the data received as input.
    """

    def _clean_fields(self):
        for name, value in self._data.items():
            self._cleaned_data[name] = value


import numpy as np
import pickle
import threading
import time

from stream import Stream, StreamArray, run

def door_example():
    #from stream import Stream, StreamArray, run
    from example_operators import subtract_mean_from_StreamArray
    from example_operators import join_synch, detect_anomaly
    from example_operators import append_item_to_StreamArray

    accelerometers = ['i2c1_0x53', 'i2c1_0x1d']
    NUM_ACCELEROMETERS = len(accelerometers)
    NUM_AXES=3

    DEMEAN_WINDOW_SIZE = 4

    def cloud_func(window, ):
        print ('')
        print ('anomaly!')
        print ('window ', window)

    #-------------------------------------------
    # SPECIFY STREAMS
    #-------------------------------------------

    # Specify acceleration streams, one stream for
    # each accelerometer.
    acceleration_streams = [StreamArray(
        name='acceleration_streams['+ str(i) + ']',
        dtype=float,
        dimension=NUM_AXES)
            for i in range(NUM_ACCELEROMETERS)
        ]

    # Specify zero_mean_streams streams, one stream for
    # each accelerometer. These are the acceleration
    # streams after subtracting the mean.
    zero_mean_streams = [StreamArray(
        name='zero_mean_streams '+str(i),
        dtype=float,
        dimension=NUM_AXES)
            for i in range(NUM_ACCELEROMETERS)
        ]

    # Specify joined_stream which is the stream after
    # joining the inputs from all accelerometers.
    joined_stream = StreamArray(
        name="joined_stream", dtype="float",
        dimension=(NUM_ACCELEROMETERS, NUM_AXES))

    #-------------------------------------------
    # SPECIFY AGENTS
    #-------------------------------------------

    # Create an agent to subtract mean from each acceleration_stream
    # and generate zero_mean_streams
    for i in range(NUM_ACCELEROMETERS):
        subtract_mean_from_StreamArray(
            in_stream=acceleration_streams[i],
            window_size=DEMEAN_WINDOW_SIZE,
            func=append_item_to_StreamArray,
            out_stream=zero_mean_streams[i]
            )

    # Create an agent to join zero_mean_streams from all
    # accelerometers and generate joined_stream
    join_synch(in_streams=zero_mean_streams,
                   out_stream=joined_stream,
                   func=append_item_to_StreamArray)

    # Create an agent to input joined_stream and to detect anomalies
    # in the stream. Detected anomalies are passed to cloud_func which
    # prints the anomalies or puts them in the cloud.
    detect_anomaly(in_stream=joined_stream, window_size=2,
                       anomaly_size=1, anomaly_factor=1.01,
                       cloud_data_size=2, cloud_func=cloud_func)

    Stream.scheduler.start()
    return

#------------------------------------------------
# TEST BY PUTTING DATA INTO ACCELEROMETER STREAMS
#-------------------------------------------------
def put_data_into_accelerometer_0():
    pickled_data = pickle.dumps((
        'acceleration_streams[0]',
        np.array(
        [
            [1.0, 1.0, 1.0],
            [1.0, 1.0, 1.0],
            [2.0, 2.0, 2.0],
            [2.0, 2.0, 2.0],
            [2.0, 2.0, 2.0],
            [3.0, 3.0, 3.0],
            [101.0, 121.0, 201.0]
        ])))
    Stream.scheduler.input_queue.put(
        pickled_data)

    # Sleep for some time
    time.sleep(2)

    pickled_data = pickle.dumps((
        'acceleration_streams[0]',
        np.array(
        [
            [0.0, 0.0, 0.0],
            [1.0, 1.0, 1.0],
            [0.0, 0.0, 0.0],
            [2.0, 2.0, 2.0],
            [0.0, 0.0, 0.0],
            [3.0, 3.0, 3.0],
            [80.0, 80.0, 80.0]
        ])))
    Stream.scheduler.input_queue.put(
        pickled_data)

def put_data_into_accelerometer_1():
    pickled_data = pickle.dumps((
        'acceleration_streams[1]',
        np.array(
        [
            [1.0, 1.0, 1.0],
            [1.0, 1.0, 1.0],
            [2.0, 2.0, 2.0],
            [2.0, 2.0, 2.0],
            [2.0, 2.0, 2.0],
            [3.0, 3.0, 3.0],
            [201.0, 221.0, 301.0]
        ])))
    Stream.scheduler.input_queue.put(
        pickled_data)

    # Sleep for some time
    time.sleep(1)

    pickled_data = pickle.dumps((
        'acceleration_streams[1]',
        np.array(
        [
            [0.0, 0.0, 0.0],
            [1.0, 1.0, 1.0],
            [0.0, 0.0, 0.0],
            [2.0, 2.0, 2.0],
            [0.0, 0.0, 0.0],
            [3.0, 3.0, 3.0],
            [90.0, 90.0, 90.0]
        ])))
    Stream.scheduler.input_queue.put(
        pickled_data)

    # Sleep and then shut down
    time.sleep(3)
    pickled_data = pickle.dumps((
        'scheduler', 'halt'))
    Stream.scheduler.input_queue.put(
        pickled_data)

    return

if __name__ == '__main__':
    # This is the compute thread that identifies
    # anomalies
    main_thread = threading.Thread(
        target=door_example, args=())

    # This thread puts data into accelerometer_0
    accelerometer_0_thread = threading.Thread(
        target=put_data_into_accelerometer_0, args=())

    # This thread puts data into accelerometer_1
    accelerometer_1_thread = threading.Thread(
        target=put_data_into_accelerometer_1, args=())

    # Start threads in any order
    accelerometer_0_thread.start()
    accelerometer_1_thread.start()
    main_thread.start()

    # Join threads.
    accelerometer_0_thread.join()
    accelerometer_1_thread.join()
    main_thread.join()


import psycopg2
import pandas as pd
from .conf import config

# Obtain the configuration parameters
params = config()
# Connect to the PostgreSQL database
conn = psycopg2.connect(**params)
# Create a new cursor
cur = conn.cursor()

# A function that takes in a PostgreSQL query and outputs a pandas database
def create_pandas_table(sql_query, database=conn):
    table = pd.read_sql_query(sql_query, database)
    return table

# Utilize th24re create_pandas_table function to create a Pandas data frame
# Store the data as a variable
data_emas = create_pandas_table("SELECT * FROM emas")
data_perak = create_pandas_table("SELECT * FROM perak")
data_dolar = create_pandas_table("SELECT * FROM dolar")
# Close the cursor and connection to so the server can allocate

# bandwidth to other requests
cur.close()
conn.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
name: svn源码泄露扫描
referer: unknown
author: Lucifer
description: 忘记了删除.svn目录而导致的漏洞。
'''
import re
import sys
import requests
import warnings
from termcolor import cprint

class svn_check_BaseVerify:
    def __init__(self, url):
        self.url = url

    def run(self):
        headers = {
            "User-Agent":"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50"
        }
        payload = "/.svn/entries"
        vulnurl = self.url + payload
        try:
            req = requests.get(vulnurl, headers=headers, timeout=10, verify=False, allow_redirects=False)
            try:
                contents = str(req.text).split('\x0c')
                pattern = re.compile(r'has-props|file|dir')
                for content in contents:
                    match = len(pattern.search(content).group(0))
                    if req.status_code == 200 and match > 0:
                        cprint("[+]存在svn源码泄露漏洞...(高危)\tpayload: "+vulnurl, "red")
                        break
                    else:
                        cprint("[-]不存在svn_check漏洞", "white", "on_grey")
            except:
                cprint("[-] "+__file__+"====>可能不存在漏洞", "cyan")

        except:
            cprint("[-] "+__file__+"====>可能不存在漏洞", "cyan")

if __name__ == "__main__":
    warnings.filterwarnings("ignore")
    testVuln = svn_check_BaseVerify(sys.argv[1])
    testVuln.run()


import sys
from datetime import timedelta

def info(message, *args):
    message = str(message)
    message = message.format(*args)
    message = '\r' + message

    sys.stdout.write(message)
    sys.stdout.flush()

    lineBreak()

def lineBreak():
    sys.stdout.write('\n')
    sys.stdout.flush()

def progress(messageFormat, index=None, count=None, *args):
    if index == None or count == None:
        index = 1
        count = 1

    maxFrequency = 10000
    body = count - count % maxFrequency
    maxFrequency = count / maxFrequency
    if maxFrequency != 0 and index % maxFrequency > 0 and index < body:
        return

    index = float(index)
    count = float(count)
    percentage = 100 * index / count
    percentage = min(100, percentage)
    args = [percentage] + list(args)

    messageFormat = messageFormat.format(*args)
    messageFormat = '\r' + messageFormat

    sys.stdout.write(messageFormat)
    sys.stdout.flush()

def delta(seconds):
    deltaString = str(timedelta(seconds=seconds))
    deltaString = deltaString.split('.')[0]

    return deltaString