# SPDX-License-Identifier: MIT
import re
import warnings
from dataclasses import dataclass
from functools import cached_property
from itertools import chain
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar, Union, cast
from xml.etree import ElementTree

from deprecation import deprecated

from .additionalaudience import AdditionalAudience
from .admindata import AdminData
from .companydata import CompanyData
from .comparaminstance import ComparamInstance
from .comparamspec import ComparamSpec
from .diagcomm import DiagComm
from .diagdatadictionaryspec import DiagDataDictionarySpec
from .diaglayerraw import DiagLayerRaw
from .diaglayertype import DiagLayerType
from .diagservice import DiagService
from .ecuvariantpattern import EcuVariantPattern
from .exceptions import DecodeError, OdxWarning, odxassert
from .functionalclass import FunctionalClass
from .message import Message
from .nameditemlist import NamedItemList, OdxNamed
from .odxlink import OdxDocFragment, OdxLinkDatabase, OdxLinkId, OdxLinkRef
from .parentref import ParentRef
from .protstack import ProtStack
from .request import Request
from .response import Response
from .servicebinner import ServiceBinner
from .singleecujob import SingleEcuJob
from .specialdatagroup import SpecialDataGroup
from .statechart import StateChart
from .table import Table
from .unitgroup import UnitGroup
from .unitspec import UnitSpec

T = TypeVar("T")
TNamed = TypeVar("TNamed", bound=OdxNamed)

PrefixTree = Dict[int, Union[List[DiagService], "PrefixTree"]]

@dataclass
class DiagLayer:
    """This class represents a "logical view" upon a diagnostic layer
    according to the ODX standard.

    i.e. it handles the value inheritance, communication parameters,
    encoding/decoding of data, etc.
    """

    diag_layer_raw: DiagLayerRaw

    @staticmethod
    def from_et(et_element: ElementTree.Element, doc_frags: List[OdxDocFragment]) -> "DiagLayer":
        diag_layer_raw = DiagLayerRaw.from_et(et_element, doc_frags)

        # Create DiagLayer
        return DiagLayer(diag_layer_raw=diag_layer_raw)

    def _build_odxlinks(self) -> Dict[OdxLinkId, Any]:
        """Construct a mapping from IDs to all objects that are contained in this diagnostic layer."""
        result = self.diag_layer_raw._build_odxlinks()

        # we want to get the full diag layer, not just the raw layer
        # when referencing...
        result[self.odx_id] = self

        return result

    def _resolve_odxlinks(self, odxlinks: OdxLinkDatabase) -> None:
        """Recursively resolve all references."""

        self.diag_layer_raw._resolve_odxlinks(odxlinks)

    def _finalize_init(self, odxlinks: OdxLinkDatabase) -> None:
        """This method deals with everything inheritance related and
        -- after the final set of objects covered by the diagnostic
        layer is determined -- resolves any short name references in
        the diagnostic layer.

        TODO: In some corner cases, the short name resolution is not
        correct: E.g. Given three layers A, B, and C, where B and C
        derive from A and A defines the diagnostic communication
        SA. If now B overrides SA and there are short name references
        to SA in A, the object to which this reference is resolved is
        undefined. An easy fix for this problem is to copy all
        inherited objects in derived layers, but that would lead to
        excessive memory consumption for large databases...
        """

        #####
        # fill in all applicable objects that use value inheritance
        #####

        # diagnostic communication objects with the ODXLINKs resolved
        diag_comms = self._compute_available_diag_comms(odxlinks)
        self._diag_comms = NamedItemList(diag_comms)

        # filter the diag comms for services and single-ECU jobs
        services = [dc for dc in diag_comms if isinstance(dc, DiagService)]
        single_ecu_jobs = [dc for dc in diag_comms if isinstance(dc, SingleEcuJob)]
        self._services = NamedItemList(services)
        self._single_ecu_jobs = NamedItemList(single_ecu_jobs)

        global_negative_responses = self._compute_available_global_neg_responses(odxlinks)
        self._global_negative_responses = NamedItemList(global_negative_responses)

        functional_classes = self._compute_available_functional_classes()
        self._functional_classes = NamedItemList(functional_classes)

        additional_audiences = self._compute_available_additional_audiences()
        self._additional_audiences = NamedItemList(additional_audiences)

        state_charts = self._compute_available_state_charts()
        self._state_charts = NamedItemList(state_charts)

        ############
        # create a new unit_spec object. This is necessary because
        # unit_groups are subject to value inheritance.

        # unit groups applicable to this diaglayer (i.e., including
        # value inheritance)
        unit_groups = self._compute_available_unit_groups()

        # convenience variable for the locally-defined unit spec
        local_unit_spec: Optional[UnitSpec]
        if self.diag_layer_raw.diag_data_dictionary_spec is not None:
            local_unit_spec = self.diag_layer_raw.diag_data_dictionary_spec.unit_spec
        else:
            local_unit_spec = None

        unit_spec: Optional[UnitSpec]
        if local_unit_spec is None and not unit_groups:
            # no locally defined unit spec and no inherited unit groups
            unit_spec = None
        elif local_unit_spec is None:
            # no locally defined unit spec but inherited unit groups
            unit_spec = UnitSpec(
                unit_groups=NamedItemList(unit_groups),
                units=NamedItemList([]),
                physical_dimensions=NamedItemList([]),
                sdgs=[])
        else:
            # locally defined unit spec and inherited unit groups
            unit_spec = UnitSpec(
                unit_groups=NamedItemList(unit_groups),
                units=local_unit_spec.units,
                physical_dimensions=local_unit_spec.physical_dimensions,
                sdgs=[])
        ############

        dops = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.data_object_props,
            lambda parent_ref: parent_ref.not_inherited_dops,
        )
        structures = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.structures,
            lambda parent_ref: parent_ref.not_inherited_dops,
        )
        dtc_dops = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.dtc_dops,
            lambda parent_ref: parent_ref.not_inherited_dops,
        )
        end_of_pdu_fields = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.end_of_pdu_fields,
            lambda parent_ref: parent_ref.not_inherited_dops,
        )
        dynamic_length_fields = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.dynamic_length_fields,
            lambda parent_ref: parent_ref.not_inherited_dops,
        )
        env_data_descs = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.env_data_descs,
            lambda parent_ref: parent_ref.not_inherited_dops,
        )
        env_datas = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.env_datas, lambda parent_ref: parent_ref.not_inherited_dops)
        muxs = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.muxs, lambda parent_ref: parent_ref.not_inherited_dops)
        tables = self._compute_available_ddd_spec_items(
            lambda ddd_spec: ddd_spec.tables, lambda parent_ref: parent_ref.not_inherited_tables)
        ddds_sdgs: List[SpecialDataGroup]
        if self.diag_layer_raw.diag_data_dictionary_spec:
            ddds_sdgs = self.diag_layer_raw.diag_data_dictionary_spec.sdgs
        else:
            ddds_sdgs = []

        # create a DiagDataDictionarySpec which includes all the
        # inherited objects. To me, this seems rather inelegant, but
        # hey, it's described like this in the standard.
        self._diag_data_dictionary_spec = DiagDataDictionarySpec(
            data_object_props=dops,
            dtc_dops=dtc_dops,
            structures=structures,
            end_of_pdu_fields=end_of_pdu_fields,
            dynamic_length_fields=dynamic_length_fields,
            tables=tables,
            env_data_descs=env_data_descs,
            env_datas=env_datas,
            muxs=muxs,
            unit_spec=unit_spec,
            sdgs=ddds_sdgs,
        )

        #####
        # compute the communication parameters applicable to the
        # diagnostic layer. Note that communication parameters do
        # *not* use value inheritance, but a slightly different
        # scheme, cf the docstring of
        # _compute_available_commmunication_parameters().
        #####
        self._comparams = NamedItemList(self._compute_available_commmunication_parameters())

        #####
        # resolve all SNREFs. TODO: We allow SNREFS to objects that
        # were inherited by the diaglayer. This might not be allowed
        # by the spec (So far, I haven't found any definitive
        # statement...)
        #####
        self.diag_layer_raw._resolve_snrefs(self)

    #####
    # <convenience functionality>
    #####
    @cached_property
    def service_groups(self) -> ServiceBinner:
        return ServiceBinner(self.services)

    #####
    # </convenience functionality>
    #####

    #####
    # <properties forwarded to the "raw" diag layer>
    #####
    @property
    def variant_type(self) -> DiagLayerType:
        return self.diag_layer_raw.variant_type

    @property
    def odx_id(self) -> OdxLinkId:
        return self.diag_layer_raw.odx_id

    @property
    def short_name(self) -> str:
        return self.diag_layer_raw.short_name

    @property
    def long_name(self) -> Optional[str]:
        return self.diag_layer_raw.long_name

    @property
    def description(self) -> Optional[str]:
        return self.diag_layer_raw.description

    @property
    def admin_data(self) -> Optional[AdminData]:
        return self.diag_layer_raw.admin_data

    @property
    def company_datas(self) -> NamedItemList[CompanyData]:
        return self.diag_layer_raw.company_datas

    @property
    def requests(self) -> NamedItemList[Request]:
        return self.diag_layer_raw.requests

    @property
    def positive_responses(self) -> NamedItemList[Response]:
        return self.diag_layer_raw.positive_responses

    @property
    def negative_responses(self) -> NamedItemList[Response]:
        return self.diag_layer_raw.negative_responses

    @property
    def import_refs(self) -> List[OdxLinkRef]:
        return self.diag_layer_raw.import_refs

    @property
    def sdgs(self) -> List[SpecialDataGroup]:
        return self.diag_layer_raw.sdgs

    @property
    def parent_refs(self) -> List[ParentRef]:
        return self.diag_layer_raw.parent_refs

    @property
    def ecu_variant_patterns(self) -> List[EcuVariantPattern]:
        return self.diag_layer_raw.ecu_variant_patterns

    @property
    def comparam_spec_ref(self) -> Optional[OdxLinkRef]:
        return self.diag_layer_raw.comparam_spec_ref

    @property
    def prot_stack_snref(self) -> Optional[str]:
        return self.diag_layer_raw.prot_stack_snref

    @property
    def comparam_spec(self) -> Optional[ComparamSpec]:
        return self.diag_layer_raw.comparam_spec

    @property
    def prot_stack(self) -> Optional[ProtStack]:
        return self.diag_layer_raw.prot_stack

    #####
    # </properties forwarded to the "raw" diag layer>
    #####

    #######
    # <stuff subject to value inheritance>
    #######
    @property
    def diag_comms(self) -> NamedItemList[DiagComm]:
        """All diagnostic communication primitives applicable to this DiagLayer

        Diagnostic communication primitives are diagnostic services as
        well as single-ECU jobs. This list has all references
        resolved.
        """
        return self._diag_comms

    @property
    def services(self) -> NamedItemList[DiagService]:
        """All diagnostic services applicable to this DiagLayer

        This is a subset of all diagnostic communication
        primitives. All references are resolved in the list returned.
        """
        return self._services

    @property
    def single_ecu_jobs(self) -> NamedItemList[SingleEcuJob]:
        """All single-ECU jobs applicable to this DiagLayer

        This is a subset of all diagnostic communication
        primitives. All references are resolved in the list returned.
        """
        return self._single_ecu_jobs

    @property
    def global_negative_responses(self) -> NamedItemList[Response]:
        """All global negative responses applicable to this DiagLayer"""
        return self._global_negative_responses

    @property
    @deprecated(details="use diag_data_dictionary_spec.tables")
    def tables(self) -> NamedItemList[Table]:
        return self.diag_data_dictionary_spec.tables

    @property
    def functional_classes(self) -> NamedItemList[FunctionalClass]:
        """All functional classes applicable to this DiagLayer"""
        return self._functional_classes

    @property
    def state_charts(self) -> NamedItemList[StateChart]:
        """All state charts applicable to this DiagLayer"""
        return self._state_charts

    @property
    def additional_audiences(self) -> NamedItemList[AdditionalAudience]:
        """All audiences applicable to this DiagLayer"""
        return self._additional_audiences

    @property
    def diag_data_dictionary_spec(self) -> DiagDataDictionarySpec:
        """The DiagDataDictionarySpec applicable to this DiagLayer"""
        return self._diag_data_dictionary_spec

    #######
    # </stuff subject to value inheritance>
    #######

    #####
    # <value inheritance mechanism helpers>
    #####
    def _get_parent_refs_sorted_by_priority(self, reverse: bool = False) -> Iterable[ParentRef]:
        return sorted(
            self.diag_layer_raw.parent_refs,
            key=lambda pr: pr.layer.variant_type.inheritance_priority,
            reverse=reverse)

    def _compute_available_objects(
        self,
        get_local_objects: Callable[["DiagLayer"], Iterable[T]],
        get_not_inherited: Callable[[ParentRef], Iterable[str]],
    ) -> Iterable[T]:
        """Helper method to compute the set of all objects applicable
        to the DiagLayer if these objects are subject to the value
        inheritance mechanism

        Note that all objects subject to the value inheritance
        mechanism exhibit a short_name attribute.

        :param get_local_objects: Function mapping a DiagLayer to the
        set of objects that are locally defined by that DiagLayer. If
        any of these objects have already been defined in any of the
        parents, the locally defined instance overrides them.

        :param get_not_inherited: Function mapping a ParentRef to the
        set of short names of the objects which shall not be inherited
        from the parents.

        """

        result_dict: Dict[str, T] = {}

        # populate the result dictionary with the inherited objects
        #
        # TODO (?): make sure that there are no "illegal" collisions
        # i.e., different objects with the same short name stemming
        # from parent layers exhibiting the same priority that are not
        # overwritten by a locally defined object. (IMO, this is quite
        # a corner case.)
        for parent_ref in self._get_parent_refs_sorted_by_priority():
            parent_dl = parent_ref.layer
            for dc in parent_dl._compute_available_objects(get_local_objects, get_not_inherited):
                result_dict[dc.short_name] = dc  # type: ignore[attr-defined]

            # remove the explictly not inherited objects
            for sn in get_not_inherited(parent_ref):
                if sn in result_dict:
                    del result_dict[sn]

        # consider the locally defined objects (override the
        # inherited entries or add new ones)
        for obj in get_local_objects(self):
            result_dict[obj.short_name] = obj  # type: ignore[attr-defined]

        return result_dict.values()

    def _get_local_diag_comms(self, odxlinks: OdxLinkDatabase) -> Iterable[DiagComm]:
        """Return the list of locally defined diagnostic communications.

        This is not completely trivial as it requires to resolving the
        references specified in the <DIAG-COMMS> XML tag.
        """
        result_dict: Dict[str, DiagComm] = {}

        # TODO (?): add objects from the import-refs

        for dc_proxy in self.diag_layer_raw.diag_comms:
            if isinstance(dc_proxy, OdxLinkRef):
                dc = odxlinks.resolve(dc_proxy)
            else:
                dc = dc_proxy

            odxassert(isinstance(dc, DiagComm))
            odxassert(
                dc.short_name not in result_dict,
                f"Multiple definitions of DIAG-COMM '{dc.short_name}' in "
                f"layer '{self.short_name}'")
            result_dict[dc.short_name] = dc

        return result_dict.values()

    def _get_local_unit_groups(self) -> Iterable[UnitGroup]:
        if self.diag_layer_raw.diag_data_dictionary_spec is None:
            return []

        unit_spec = self.diag_layer_raw.diag_data_dictionary_spec.unit_spec
        if unit_spec is None:
            return []

        return unit_spec.unit_groups

    def _compute_available_diag_comms(self, odxlinks: OdxLinkDatabase) -> Iterable[DiagComm]:

        def get_local_objects_fn(dl: DiagLayer) -> Iterable[DiagComm]:
            return dl._get_local_diag_comms(odxlinks)

        def not_inherited_fn(parent_ref: ParentRef) -> List[str]:
            return parent_ref.not_inherited_diag_comms

        return self._compute_available_objects(get_local_objects_fn, not_inherited_fn)

    def _compute_available_global_neg_responses(self, odxlinks: OdxLinkDatabase) \
            -> Iterable[Response]:

        def get_local_objects_fn(dl: DiagLayer) -> Iterable[Response]:
            return dl.diag_layer_raw.global_negative_responses

        def not_inherited_fn(parent_ref: ParentRef) -> List[str]:
            return parent_ref.not_inherited_global_neg_responses

        return self._compute_available_objects(get_local_objects_fn, not_inherited_fn)

    def _compute_available_ddd_spec_items(
        self,
        include: Callable[[DiagDataDictionarySpec], Iterable[TNamed]],
        exclude: Callable[["ParentRef"], List[str]],
    ) -> NamedItemList[TNamed]:

        def get_local_objects_fn(dl: DiagLayer) -> Iterable[TNamed]:
            if dl.diag_layer_raw.diag_data_dictionary_spec is None:
                return []
            return include(dl.diag_layer_raw.diag_data_dictionary_spec)

        found = self._compute_available_objects(get_local_objects_fn, exclude)
        return NamedItemList(found)

    def _compute_available_functional_classes(self) -> Iterable[FunctionalClass]:

        def get_local_objects_fn(dl: DiagLayer) -> Iterable[FunctionalClass]:
            return dl.diag_layer_raw.functional_classes

        def not_inherited_fn(parent_ref: ParentRef) -> List[str]:
            return []

        return self._compute_available_objects(get_local_objects_fn, not_inherited_fn)

    def _compute_available_additional_audiences(self) -> Iterable[AdditionalAudience]:

        def get_local_objects_fn(dl: DiagLayer) -> Iterable[AdditionalAudience]:
            return dl.diag_layer_raw.additional_audiences

        def not_inherited_fn(parent_ref: ParentRef) -> List[str]:
            return []

        return self._compute_available_objects(get_local_objects_fn, not_inherited_fn)

    def _compute_available_state_charts(self) -> Iterable[StateChart]:

        def get_local_objects_fn(dl: DiagLayer) -> Iterable[StateChart]:
            return dl.diag_layer_raw.state_charts

        def not_inherited_fn(parent_ref: ParentRef) -> List[str]:
            return []

        return self._compute_available_objects(get_local_objects_fn, not_inherited_fn)

    def _compute_available_unit_groups(self) -> Iterable[UnitGroup]:

        def get_local_objects_fn(dl: DiagLayer) -> Iterable[UnitGroup]:
            return dl._get_local_unit_groups()

        def not_inherited_fn(parent_ref: ParentRef) -> List[str]:
            return []

        return self._compute_available_objects(get_local_objects_fn, not_inherited_fn)

    #####
    # </value inheritance mechanism helpers>
    #####

    #####
    # <communication parameter handling>
    #####
    def _compute_available_commmunication_parameters(self) -> List[ComparamInstance]:
        """Compute the list of communication parameters that apply to
        the diagnostic layer

        Be aware that the inheritance scheme for communication
        parameters is slightly different than for objects that are
        subject to value inheritance:

        - The ODXLINK ID id of communication parameters is used to
          override inherited parameters instead of the short name.
        - A parameter is only overridden if the specified protocol
          matches.

        Note that the specification leaves some room for
        interpretation here: It says that if no protocol is specified,
        the parameter shall apply to any protocol. But what happens if
        the the same comparam is specified with and without a
        protocol? Is this allowed at all? If yes, which of these
        definitions gets priority? How does this interact with
        inheritance? The approach taken here is to allow such cases
        and to use the specific comparams if possible whilst the ones
        without a specified protocol are taken as fallbacks...

        """
        com_params_dict: Dict[Tuple[str, Optional[str]], ComparamInstance] = {}

        # Look in parent refs for inherited communication
        # parameters. First fetch the communication parameters from
        # low priority parents, then update with increasing priority.
        for parent_ref in self._get_parent_refs_sorted_by_priority():
            for cp in parent_ref.layer._compute_available_commmunication_parameters():
                com_params_dict[(cp.spec_ref.ref_id, cp.protocol_snref)] = cp

        # finally, handle the locally defined communication parameters
        for cp in self.diag_layer_raw.comparams:
            com_params_dict[(cp.spec_ref.ref_id, cp.protocol_snref)] = cp

        return list(com_params_dict.values())

    @property
    def comparams(self) -> NamedItemList[ComparamInstance]:
        """All communication parameters applicable to this DiagLayer

        Note that, although communication parameters use inheritance,
        it is *not* the "value inheritance" scheme used by e.g. DOPs,
        tables, state charts, ...
        """
        return self._comparams

    @cached_property
    def protocols(self) -> NamedItemList["DiagLayer"]:
        """Return the set of all protocols which are applicable to the diagnostic layer

        Note that protocols are *not* explicitly inherited objects,
        but the parent diagnostic layers of variant type "PROTOCOL".
        """
        result_dict: Dict[str, DiagLayer] = {}

        for parent_ref in self._get_parent_refs_sorted_by_priority():
            for prot in parent_ref.layer.protocols:
                result_dict[prot.short_name] = prot

        if self.diag_layer_raw.variant_type == DiagLayerType.PROTOCOL:
            result_dict[self.diag_layer_raw.short_name] = self

        return NamedItemList(result_dict.values())

    def get_comparam(
        self,
        cp_short_name: str,
        *,
        protocol: Optional[Union[str, "DiagLayer"]] = None,
    ) -> Optional[ComparamInstance]:
        """Find a specific communication parameter according to some criteria.

        Setting a given parameter to `None` means "don't care"."""

        protocol_name: Optional[str]
        if isinstance(protocol, DiagLayer):
            protocol_name = protocol.short_name
        else:
            protocol_name = protocol

        # determine the set of applicable communication parameters
        cps = [cp for cp in self.comparams if cp.short_name == cp_short_name]
        if protocol_name is not None:
            cps = [cp for cp in cps if cp.protocol_snref in (None, protocol_name)]

        if len(cps) > 1:
            warnings.warn(
                f"Communication parameter `{cp_short_name}` specified more "
                f"than once. Using first occurence.",
                OdxWarning,
                stacklevel=1,
            )
        elif len(cps) == 0:
            return None

        return cps[0]

    def get_max_can_payload_size(self,
                                 protocol: Optional[Union[str,
                                                          "DiagLayer"]] = None) -> Optional[int]:
        """Return the maximum size of a CAN frame payload that can be
        transmitted in bytes.

        For classic CAN busses, this is basically always 8. CAN-FD can
        send up to 64 bytes per frame.

        """
        com_param = self.get_comparam("CP_CANFDTxMaxDataLength", protocol=protocol)
        if com_param is None:
            rx_id = self.get_can_receive_id(protocol=protocol)
            if rx_id is not None:
                # bus is CAN. We also use 8 bytes for CAN-FD if this
                # parameter was not specified.
                return 8

            # bus is not CAN
            return None

        val = com_param.value
        if not isinstance(val, str):
            return None

        m = re.search("TX_DL *= *([0-9]*)", val)
        if m:
            return int(m.group(1))

        # unexpected format of parameter value
        return 8

    def uses_can(self, protocol: Optional[Union[str, "DiagLayer"]] = None) -> bool:
        """
        Check if CAN ought to be used as the link layer protocol.
        """
        return self.get_can_receive_id(protocol=protocol) is not None

    def uses_can_fd(self, protocol: Optional[Union[str, "DiagLayer"]] = None) -> bool:
        """Check if CAN-FD ought to be used.

        If the ECU is not using CAN-FD for the specified protocol, `False`
        is returned.

        If this method returns `True`, `.uses_can() == True` is implied
        for the protocol.
        """

        if not self.uses_can(protocol):
            return False

        com_param = self.get_comparam("CP_CANFDTxMaxDataLength", protocol=protocol)
        if com_param is None:
            return False

        return "CANFD" in com_param.value

    def get_can_baudrate(self, protocol: Optional[Union[str, "DiagLayer"]] = None) -> Optional[int]:
        """Baudrate of the CAN bus which is used by the ECU [bits/s]

        If the ECU is not using CAN for the specified protocol, None
        is returned.

        """
        com_param = self.get_comparam("CP_Baudrate", protocol=protocol)
        if com_param is None:
            return None

        val = com_param.value
        if not isinstance(val, str):
            return None

        return int(val)

    def get_can_fd_baudrate(self,
                            protocol: Optional[Union[str, "DiagLayer"]] = None) -> Optional[int]:
        """Data baudrate of the CAN bus which is used by the ECU [bits/s]

        If the ECU is not using CAN-FD for the specified protocol,
        None is returned.
        """
        if not self.uses_can_fd(protocol=protocol):
            return None

        com_param = self.get_comparam("CP_CANFDBaudrate", protocol=protocol)
        if com_param is None:
            return None

        val = com_param.value
        if not isinstance(val, str):
            return None

        return int(val)

    def get_can_receive_id(self,
                           protocol: Optional[Union[str, "DiagLayer"]] = None) -> Optional[int]:
        """CAN ID to which the ECU listens for diagnostic messages"""
        com_param = self.get_comparam("CP_UniqueRespIdTable", protocol=protocol)
        if com_param is None:
            return None

        with warnings.catch_warnings():
            # depending on the protocol, we may get
            # "Communication parameter 'CP_UniqueRespIdTable' does not
            # specify 'CP_CanPhysReqId'" warning here. we don't want this
            # warning and simply return None...
            warnings.simplefilter("ignore", category=OdxWarning)
            result = com_param.get_subvalue("CP_CanPhysReqId")
        if result is None:
            return None
        odxassert(isinstance(result, str))

        return int(result)

    @deprecated(details="use get_can_receive_id()")
    def get_receive_id(self) -> Optional[int]:
        return self.get_can_receive_id()

    def get_can_send_id(self, protocol: Optional[Union[str, "DiagLayer"]] = None) -> Optional[int]:
        """CAN ID to which the ECU sends replies to diagnostic messages"""

        # this hopefully resolves to the 'CP_UniqueRespIdTable'
        # parameter from the ISO_15765_2 comparam subset. (There is a
        # parameter with the same name in the ISO_13400_2_DIS_2015
        # subset for DoIP. If the wrong one is retrieved, try
        # specifying the protocol.)
        com_param = self.get_comparam("CP_UniqueRespIdTable", protocol=protocol)
        if com_param is None:
            return None

        with warnings.catch_warnings():
            # depending on the protocol, we may get
            # "Communication parameter 'CP_UniqueRespIdTable' does not
            # specify 'CP_CanRespUSDTId'" warning here. we don't want this
            # warning and simply return None...
            warnings.simplefilter("ignore", category=OdxWarning)
            result = com_param.get_subvalue("CP_CanRespUSDTId")
        if result is None:
            return None
        odxassert(isinstance(result, str))

        return int(result)

    @deprecated(details="use get_can_send_id()")
    def get_send_id(self) -> Optional[int]:
        return self.get_can_send_id()

    def get_can_func_req_id(self,
                            protocol: Optional[Union[str, "DiagLayer"]] = None) -> Optional[int]:
        """CAN Functional Request Id."""
        com_param = self.get_comparam("CP_CanFuncReqId", protocol=protocol)
        if com_param is None:
            return None

        result = com_param.get_value()
        if result is None:
            return None
        odxassert(isinstance(result, str))

        return int(result)

    def get_doip_logical_ecu_address(self,
                                     protocol: Optional[Union[str, "DiagLayer"]] = None
                                    ) -> Optional[int]:
        """Return the address of the ECU when using functional addressing.

        The parameter protocol is used to distinguish between
        different interfaces, e.g., offboard and onboard DoIP
        Ethernet.
        """

        # this hopefully resolves to the 'CP_UniqueRespIdTable'
        # parameter from the ISO_13400_2_DIS_2015 comparam
        # subset. (There is a parameter with the same name in the
        # ISO_15765_2 subset for CAN. If the wrong one is retrieved,
        # try specifying the protocol.)
        com_param = self.get_comparam("CP_UniqueRespIdTable", protocol=protocol)

        if com_param is None:
            return None

        # The CP_DoIPLogicalEcuAddress is specified by the
        # "CP_DoIPLogicalEcuAddress" subvalue of the complex comparam
        # CP_UniqueRespIdTable of the ISO_13400_2_DIS_2015 comparam
        # subset. Depending of the underlying transport protocol,
        # (i.e., CAN using ISO-TP) this subvalue might not exist.
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", category=OdxWarning)
            ecu_addr = com_param.get_subvalue("CP_DoIPLogicalEcuAddress")
        if ecu_addr is None:
            return None
        return int(ecu_addr)

    def get_doip_logical_gateway_address(self,
                                         protocol: Optional[Union[str, "DiagLayer"]] = None
                                        ) -> Optional[int]:
        """The logical gateway address for the diagnosis over IP transport protocol"""

        # retrieve CP_DoIPLogicalGatewayAddress from the
        # ISO_13400_2_DIS_2015 subset. hopefully.
        com_param = self.get_comparam("CP_DoIPLogicalGatewayAddress", protocol=protocol)
        if com_param is None:
            return None

        result = com_param.get_value()
        if result is None:
            return None
        odxassert(isinstance(result, str))

        return int(result)

    def get_doip_logical_tester_address(self,
                                        protocol: Optional[Union[str, "DiagLayer"]] = None
                                       ) -> Optional[int]:
        """DoIp logical gateway address"""

        # retrieve CP_DoIPLogicalTesterAddress from the
        # ISO_13400_2_DIS_2015 subset. hopefully.
        com_param = self.get_comparam("CP_DoIPLogicalTesterAddress", protocol=protocol)
        if com_param is None:
            return None

        result = com_param.get_value()
        if result is None:
            return None
        odxassert(isinstance(result, str))

        return int(result)

    def get_doip_logical_functional_address(self,
                                            protocol: Optional[Union[str, "DiagLayer"]] = None
                                           ) -> Optional[int]:
        """The logical functional DoIP address of the ECU."""

        # retrieve CP_DoIPLogicalFunctionalAddress from the
        # ISO_13400_2_DIS_2015 subset. hopefully.
        com_param = self.get_comparam(
            "CP_DoIPLogicalFunctionalAddress",
            protocol=protocol,
        )
        if com_param is None:
            return None

        result = com_param.get_value()
        if result is None:
            return None
        odxassert(isinstance(result, str))

        return int(result)

    def get_doip_routing_activation_timeout(self,
                                            protocol: Optional[Union[str, "DiagLayer"]] = None
                                           ) -> Optional[float]:
        """The timout for the DoIP routing activation request in seconds"""

        # retrieve CP_DoIPRoutingActivationTimeout from the
        # ISO_13400_2_DIS_2015 subset. hopefully.
        com_param = self.get_comparam("CP_DoIPRoutingActivationTimeout", protocol=protocol)
        if com_param is None:
            return None

        result = com_param.get_value()
        if result is None:
            return None
        odxassert(isinstance(result, str))

        return float(result) / 1e6

    def get_doip_routing_activation_type(self,
                                         protocol: Optional[Union[str, "DiagLayer"]] = None
                                        ) -> Optional[int]:
        """The DoIP routing activation type

        The number returned has the following meaning:

        0          Default
        1          WWH-OBD (worldwide harmonized onboard diagnostic).
        2-0xDF     reserved
        0xE0-0xFF  OEM-specific
        """

        # retrieve CP_DoIPRoutingActivationType from the
        # ISO_13400_2_DIS_2015 subset. hopefully.
        com_param = self.get_comparam("CP_DoIPRoutingActivationType", protocol=protocol)
        if com_param is None:
            return None

        result = com_param.get_value()
        if result is None:
            return None
        odxassert(isinstance(result, str))

        return int(result)

    def get_tester_present_time(self,
                                protocol: Optional[Union[str,
                                                         "DiagLayer"]] = None) -> Optional[float]:
        """Timeout on inactivity in seconds.

        This is defined by the communication parameter "CP_TesterPresentTime".

        Description of the comparam: "Time between a response and the
        next subsequent tester present message (if no other request is
        sent to this ECU) in case of physically addressed requests."
        """

        # retrieve CP_TesterPresentTime from either the
        # ISO_13400_2_DIS_2015 or the ISO_15765_2 subset.
        com_param = self.get_comparam("CP_TesterPresentTime", protocol=protocol)
        if com_param is None:
            return None

        result = com_param.get_value()
        if result is None:
            return None
        odxassert(isinstance(result, str))

        # the comparam specifies microseconds. convert this to seconds
        return float(result) / 1e6

    #####
    # </communication parameter handling>
    #####

    #####
    # <PDU decoding>
    #####

    @cached_property
    def _prefix_tree(self) -> PrefixTree:
        """Constructs the coded prefix tree of the services.

        Each leaf node is a list of `DiagService`s.  (This is because
        navigating from a service to the request/ responses is easier
        than finding the service for a given request/response object.)

        Example:
        Let there be four services with corresponding requests:
        * Request 1 has the coded constant prefix `12 34`.
        * Request 2 has the coded constant prefix `12 34`.
        * Request 3 has the coded constant prefix `12 56`.
        * Request 4 has the coded constant prefix `12 56 00`.

        Then, the constructed prefix tree is the dict
        ```
        {0x12: {0x34: {-1: [<Service 1>, <Service 2>]},
                0x56: {-1: [<Service 3>],
                       0x0: {-1: [<Service 4>]}
                       }}}
        ```
        Note, that the inner `-1` are constant to distinguish them
        from possible service IDs.

        Also note, that it is actually allowed that
        (a) SIDs for different services are the same like for service 1 and 2 (thus each leaf node is a list) and
        (b) one SID is the prefix of another SID like for service 3 and 4 (thus the constant `-1` key).

        """
        prefix_tree: PrefixTree = {}
        for s in self.services:
            # Compute prefixes for the service's request and all
            # possible responses. We need to consider the global
            # negative responses here, because they might contain
            # MATCHING-REQUEST parameters. If these global responses
            # do not contain such parameters, this will potentially
            # result in an enormous amount of decoded messages for
            # global negative responses. (I.e., one for each
            # service. This can be avoided by specifying the
            # corresponding request for `decode_response()`.)
            request_prefix = b''
            if s.request is not None:
                request_prefix = s.request.coded_const_prefix()
            prefixes = [request_prefix]
            prefixes += [
                x.coded_const_prefix(request_prefix=request_prefix) for x in chain(
                    s.positive_responses, s.negative_responses, self.global_negative_responses)
            ]
            for coded_prefix in prefixes:
                self._extend_prefix_tree(prefix_tree, coded_prefix, s)

        return prefix_tree

    @staticmethod
    def _extend_prefix_tree(prefix_tree: PrefixTree, coded_prefix: bytes,
                            service: DiagService) -> None:

        # make sure that tree has an entry for the given prefix
        sub_tree = prefix_tree
        for b in coded_prefix:
            if b not in sub_tree:
                sub_tree[b] = {}
            sub_tree = cast(PrefixTree, sub_tree[b])

        # Store the object as in the prefix tree. This is done by
        # assigning the list of possible objects to the key -1 of the
        # dictionary (this is quite hacky...)
        if sub_tree.get(-1) is None:
            sub_tree[-1] = [service]
        else:
            cast(List[DiagService], sub_tree[-1]).append(service)

    def _find_services_for_uds(self, message: bytes) -> List[DiagService]:
        prefix_tree = self._prefix_tree

        # Find matching service(s) in prefix tree
        possible_services: List[DiagService] = []
        for b in message:
            if b in prefix_tree:
                odxassert(isinstance(prefix_tree[b], dict))
                prefix_tree = cast(PrefixTree, prefix_tree[b])
            else:
                break
            if -1 in prefix_tree:
                possible_services += cast(List[DiagService], prefix_tree[-1])
        return possible_services

    def _decode(self, message: bytes, candidate_services: Iterable[DiagService]) -> List[Message]:
        decoded_messages: List[Message] = []

        for service in candidate_services:
            try:
                decoded_messages.append(service.decode_message(message))
            except DecodeError:
                # check if the message can be decoded as a global
                # negative response for the service
                for gnr in self.global_negative_responses:
                    try:
                        decoded_gnr = gnr.decode(message)
                        if not isinstance(decoded_gnr, dict):
                            raise DecodeError(f"Expected the decoded value of a global "
                                              f"negative response to be a dictionary, "
                                              f"got {type(decoded_gnr)} for {self.short_name}")

                        decoded_messages.append(
                            Message(
                                coded_message=message,
                                service=service,
                                coding_object=gnr,
                                param_dict=decoded_gnr))
                    except DecodeError:
                        pass

        if len(decoded_messages) == 0:
            raise DecodeError(
                f"None of the services {candidate_services} could parse {message.hex()}.")

        return decoded_messages

    def decode(self, message: bytes) -> List[Message]:
        candidate_services = self._find_services_for_uds(message)

        return self._decode(message, candidate_services)

    def decode_response(self, response: bytes, request: Union[bytes, Message]) -> List[Message]:
        if isinstance(request, Message):
            candidate_services = [request.service]
        else:
            if not isinstance(request, (bytes, bytearray)):
                raise TypeError(f"Request parameter must have type "
                                f"Message, bytes or bytearray but was {type(request)}")
            candidate_services = self._find_services_for_uds(request)
        if candidate_services is None:
            raise DecodeError(f"Couldn't find corresponding service for request {request.hex()}.")

        return self._decode(response, candidate_services)

    #####
    # </PDU decoding>
    #####


import requests
from bs4 import BeautifulSoup
import json
data = {}
postIndex = 1

firstpost = "https://www.commitstrip.com/en/2012/09/03/best-of-les-codeurs-au-cinema-avatar/"
r = requests.get(firstpost)
soup = BeautifulSoup(r.content, features="html.parser")
title = soup.find("h1", class_="entry-title").text
datetime = soup.find("time", class_="entry-date")
imageDiv = soup.find("div", class_="entry-content")
if(imageDiv.find("img")):
    imgSrc =  imageDiv.find("img")['src']
elif(soup.find("img", class_="size-full")):
    imgSrc = soup.find("img", class_="size-full")['src']
else:
    imgSrc=""
postData = {"title":title, "datetime":datetime['datetime'], "date":datetime.text, "imageLink":imgSrc, "postLink":firstpost}
data[str(postIndex)] = postData
spanNext = soup.find("span", class_="nav-next")
if(spanNext.find("a")):
    print(spanNext.find("a")['href'])
else:
    print("done")
print(postData)


#from keras.models import  Sequential
import pandas as pd
from keras.layers import Activation,Dense
from keras import optimizers
from keras.models import Sequential
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

housing_data=pd.read_csv('Housing.csv')
#print(housing_data.isnull().sum())
# We do not have any null data

for column in ['mainroad','guestroom','basement','hotwaterheating','airconditioning','prefarea']:
    housing_data[column] = housing_data[column].map(
        {'yes': 1, 'no': 0})

y=housing_data['price'].values
X=housing_data.drop(['price','furnishingstatus'],axis=1).values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=101)

s_scaler = StandardScaler()
X_train = s_scaler.fit_transform(X_train.astype(np.float32))
X_test = s_scaler.transform(X_test.astype(np.float32))

model = Sequential()
model.add(Dense(10, input_shape = (11,)))
model.add(Activation('sigmoid'))
model.add(Dense(10))                         # Hidden layer => only output dimension should be designated
model.add(Activation('sigmoid'))
model.add(Dense(1))

sgd = optimizers.SGD(lr = 0.01)    # stochastic gradient descent optimizer

model.compile(optimizer = sgd, loss = 'mean_squared_error', metrics = ['mse'])
model.fit(X_train, y_train, batch_size = 50, epochs = 50, verbose = 1)
model.summary()

results = model.evaluate(X_test, y_test)

print('loss: ', results[0])
print('mse: ', results[1])


from typing import List

import numpy as np

def topic_json_maker(mallet_path: str) -> List[dict]:
    """Make a list of d3 json dict for the mallet file in mallet path.

    This function read and parse the mallet file,
    then for each topic in file, we make a d3 json dict to draw word cloud
    :param mallet_path: the path of the mallet file
    :return: a list of d3 json dict
    """
    # Configuration
    # Top N number of words in each topic that will appear in output
    num_top_words = 100
    mallet_vocab = []
    word_topic_counts = []
    topics = []

    # Calculate the number of topics in the model
    n_topics = []
    with open(mallet_path, encoding='utf-8') as data:
        for line in data:
            try:
                # split the line into data
                line_data = line.rstrip().split(' ')
                line_data[0:2] = []  # Delete the ID and word from the list
                # New list topic-count pairs
                topic_count_pairs = [pair.split(':') for pair in line_data]
                for topic, count in topic_count_pairs:
                    n_topics.append(int(topic))  # New list with topics
            except ValueError:
                raise ValueError(
                    'Your source data cannot be parsed into a regular number '
                    'of columns. Please ensure that there are no spaces in '
                    'your file names or file paths. It may be easiest to open'
                    ' the output_state file in a spreadsheet using a space as '
                    'the delimiter to ensure that there are a regular number '
                    'of columns. Please fix any misaligned data and upload '
                    'the data again.')

    n_topics.sort()  # Sort the topics
    # The number of topics in the model is the highest in the list
    num_topics = max(n_topics) + 1

    # Re-shape the file data
    with open(mallet_path, encoding='utf-8') as f:
        for line in f:
            line_data = line.rstrip().split(' ')
            word = line_data[1]
            line_data[0:2] = []
            topic_count_pairs = [pair.split(':') for pair in line_data]
            mallet_vocab.append(word)
            counts = np.zeros(num_topics)
            for topic, count in topic_count_pairs:
                counts[int(topic)] = int(count)
            word_topic_counts.append(counts)

    word_topic = np.array(word_topic_counts)
    word_topic = word_topic / np.sum(word_topic, axis=0)
    mallet_vocab = np.array(mallet_vocab)

    # Generate a topics dictionary
    for t in range(num_topics):
        top_words_idx = np.argsort(word_topic[:, t])[::-1]
        top_words_idx = top_words_idx[:num_top_words]
        top_words = mallet_vocab[top_words_idx]
        top_words_shares = word_topic[top_words_idx, t]
        topics.append({})
        for word, share in zip(top_words, top_words_shares):
            # Create the topics dictionary
            topics[t].update({word: np.round(share, 3)})

    # Begin Topics to Document Files Conversion

    # If the convert topics check box is checked create topic files
    from flask import request
    checked = request.form.getlist('convertTopics')
    if len(checked) != 0:

        # Generate a full topics dictionary for topic files
        topics_full = []
        for t in range(num_topics):
            top_words_idx = np.argsort(word_topic[:, t])[::-1]
            top_words = mallet_vocab[top_words_idx]
            top_words_shares = word_topic[top_words_idx, t]
            topics_full.append({})
            for word, share in zip(top_words, top_words_shares):
                # Create the full topics dictionary
                topics_full[t].update({word: np.round(share, 3)})

        import lexos.managers.utility
        file_manager = lexos.managers.utility.load_file_manager()
        for i in range(len(topics_full)):
            fn = "Topic" + str(i) + ".txt"
            text = ""
            for name, size in topics_full[i].items():
                count = int(size * 1000)
                term = ""
                for c in range(count):
                    term += name + " "
                text += term + " "
            # Save the topic file to the file manager
            file_manager.add_upload_file(text, fn)
            lexos.managers.utility.save_file_manager(file_manager)

    # End Topics to Document Files Conversion

    # For Lexos, build the json string
    json_str = ""

    for i in range(len(topics)):
        json_str += '{"name": "Topic' + str(i) + '.txt", "children": ['
        children = ""
        for name, size in topics[i].items():
            children += ', {"text": "%s", "size": %s}' % (name, size * 1000)
            children = children.lstrip(', ')
        json_str += children
        json_str += ']}, '

    # Send the json_str variable to the template
    json_obj = []

    for i in range(len(topics)):
        new_children_list = []

        for name, size in topics[i].items():
            new_children_list.append({"text": name, "size": size * 1000})

        json_obj.append({"name": "Topic" + str(i) + ".txt",
                        "children": new_children_list})

    return json_obj


import requests
from dotenv import load_dotenv
import json
import os
from datetime import datetime
from collections import defaultdict
load_dotenv()

API_ME = "https://scrapbox.io/api/users/me"
API_IMPORT = "https://scrapbox.io/api/page-data/import/{project}.json"
API_PAGE_LIST = "https://scrapbox.io/api/pages/{project}"
API_PAGE = "https://scrapbox.io/api/pages/{project}/{title}"
API_QUERY = "https://scrapbox.io/api/pages/{project}/search/query"

# init
sid = os.getenv("SID")
cookie = "connect.sid=" + sid
r = requests.get(API_ME, headers={"Cookie": cookie})
r.raise_for_status()
csrfToken = r.json()["csrfToken"]

def import_to(project, data):
    """
    data: string
    """

    url = API_IMPORT.format(project=project)
    r = requests.post(
        url,
        files={"import-file": data},
        headers={
            "Cookie": cookie,
            "Accept": "application/json, text/plain, */*",
            "X-CSRF-TOKEN": csrfToken,
        }
    )
    r.raise_for_status()

def get_page_list(project):
    url = API_PAGE_LIST.format(project=project)
    r = requests.get(
        url,
        headers={
            "Cookie": cookie,
            "Accept": "application/json, text/plain, */*",
            "X-CSRF-TOKEN": csrfToken,
        }
    )
    r.raise_for_status()
    return r

def get_page(project, title):
    url = API_PAGE.format(project=project, title=title)
    r = requests.get(
        url,
        headers={
            "Cookie": cookie,
            "Accept": "application/json, text/plain, */*",
            "X-CSRF-TOKEN": csrfToken,
        }
    )
    r.raise_for_status()
    return r

def query(project, q, skip=0, sort="pageRank", limit=100):
    url = API_QUERY.format(project=project)
    r = requests.get(
        url,
        params=dict(q=q, skip=skip, sort=sort, limit=limit),
        headers={
            "Cookie": cookie,
            "Accept": "application/json, text/plain, */*",
            "X-CSRF-TOKEN": csrfToken,
        }
    )
    r.raise_for_status()
    return r

def write_pages(project, pages):
    data = json.dumps({"pages": pages})
    import_to(project, data)

def get_recent_pages(project):
    r = get_page_list("nishio")
    data = r.json()
    pages = data["pages"]
    now = datetime.now()
    for p in pages:
        updated = datetime.fromtimestamp(p["updated"])
        if (now - updated).days < 1:
            yield p

def get_recent_keyword(project):
    link_to = defaultdict(list)
    for page in get_recent_pages(project):
        title = page["title"]
        print(title)
        page = get_page(project, title).json()
        links = page["links"]
        for k in links:
            link_to[k].append(title)

    return link_to

def ex1():
    active_project = os.getenv("ACTIVE_PROJECT")
    target_projects = os.getenv("TARGET_PROJECTS").split(",")
    target_project = target_projects[0]
    output_project = os.getenv("OUTPUT_PROJECT")

    link_to = get_recent_keyword(active_project)
    lines = []
    print("-----\n\n")
    for keyword in link_to:
        data = query(target_project, keyword).json()
        query_link = f"[query https://scrapbox.io/{target_project}/search/page?q={keyword}]"
        if data["count"] > 0:
            lines.append(
                f"[{keyword}]:"
            )
            lines.append(
                f" [/{target_project}/{keyword}]({query_link}: {data['count']} hits)"
            )
            if data["count"] > 7:
                # too many
                pass
            else:
                for page in data["pages"]:
                    title = page['title']
                    lines.append(f" [/{target_project}/{title}]:")
                    for line in page["lines"]:
                        lines.append(f"  `{line}`")
    print("\n".join(lines))

    # write_pages(output_project, dict(title="tmp", lines=lines))

def _test():
    pages = [
        {
            "title": "Scbot Home",
            "lines": ["Scbot Home", "Hello world!"]
        }
    ]
    write_pages("nishio", pages)


import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as ss

folds = np.loadtxt("files.dat", dtype=bytes, delimiter='\t').astype(str)

order_str = "_008.norm.dat"

specs=[]
corrs=[]
phases=[]
for i in range(0, len(folds[:,0])):
    specs.append(np.loadtxt("../EXTRACTED/"+folds[i,0]+"/"+folds[i,1]+"/"+folds[i,1]+order_str))
    corrs.append(float(folds[i,3]))
    phases.append(float(folds[i,2]))

xx=[]
yy=[]
v0=-11.8
lam0=6562.8
wing=4 # wing in angstrems
for k in range(0, len(folds)):
    xx1=[]
    yy1=[]
    x0=lam0*(1.0+(v0-corrs[k])/300000.0)
    xlim1=x0-wing; xlim2=x0+wing
    for i in range(0, len(specs[k][:,0])):
        if((specs[k][i,0]>=xlim1) & (specs[k][i,0]<=xlim2)):
            xx1.append(specs[k][i,0])
            yy1.append(specs[k][i,1])
    yyf = ss.savgol_filter(yy1, 5, 2)
    xx.append(xx1)
    yy.append(yyf)

for i in range(0, len(xx)):
    x0=lam0*(1.0+(v0-corrs[i])/300000.0)
    for j in range(0, len(xx[i])):
        xx[i][j] = (xx[i][j] - x0)*300000/x0

period = 1.76159 #days
expos = (30/60)/24/period
pp =  np.linspace(0,1,100)
vv = np.linspace(-100, 100, 200)
xxm, yym = np.meshgrid(vv, pp)
zz=np.zeros((len(pp),len(vv)))

for i in range(0,len(pp)):
    filled = False
    for p in range(0, len(phases)):
        if (phases[p]<=(pp[i]+expos/2)) & (phases[p]>(pp[i]-expos/2)):
            for k in range(0, len(vv)):
                zz[i,k]=np.interp(vv[k], xx[p], yy[p])
            filled = True
    if filled==False:
        for k in range(0, len(vv)):
                zz[i,k]=1

plt.imshow(zz, interpolation="nearest", cmap=plt.get_cmap("seismic"), extent=[vv[0],vv[len(vv)-1],1,0], aspect=300)
plt.xlabel("Velosity, km/s")
plt.ylabel("Phase")
plt.colorbar()
plt.show()


import os
import sys
import cv2
import random
import numpy as np
import pandas as pd
from tqdm import tqdm

output = "dataset"
num_classes = 5
line_width = 2
num_imgs_per_class = int(sys.argv[1]) if len(sys.argv)>1 else 1000
img_size = int(sys.argv[2]) if len(sys.argv)>2 else 64
shape_names = ["circle", "square", "rectangle", "triangle", "polygon"]
df = pd.DataFrame(columns=["path", "label"])
if not os.path.exists(output):
    os.makedirs(output)

def generate_(shape_name):
    img = np.zeros((img_size, img_size), dtype=np.uint8)
    center = (random.randint(img_size // 4, 3 * img_size // 4), random.randint(img_size // 4, 3 * img_size // 4))
    if shape_name == 'circle':
        radius = img_size // 4 - line_width // 2
        cv2.circle(img, center, radius, 255, line_width)
    elif shape_name == 'square':
        side = img_size // 2 - line_width // 2
        side = min(center[0], center[1], img_size - center[0], img_size - center[1])  - line_width // 2
        pt1 = (center[0] - side // 2, center[1] - side // 2)
        pt2 = (center[0] + side // 2, center[1] + side // 2)
        cv2.rectangle(img, pt1, pt2, 255, line_width)
    elif shape_name == 'rectangle':
        width = img_size // 4 - line_width // 2
        height = img_size // 4 - line_width // 2
        width = min(center[0], img_size - center[0]) - line_width // 2
        height = min(center[1], img_size - center[1]) - line_width // 2
        pt1 = (center[0] - width // 2, center[1] - height // 2)
        pt2 = (center[0] + width // 2, center[1] + height // 2)
        cv2.rectangle(img, pt1, pt2, 255, line_width)
    elif shape_name == 'triangle':
        side = img_size // 4 - line_width // 2
        side = min(center[0], center[1], img_size - center[0], img_size - center[1]) * 2 - line_width // 2
        pt1 = (center[0], center[1] - side // 2)
        pt2 = (center[0] - side // 2, center[1] + side // 2)
        pt3 = (center[0] + side // 2, center[1] + side // 2)
        pts = np.array([pt1, pt2, pt3], dtype=np.int32)
        cv2.polylines(img, [pts], True, 255, line_width)
    elif shape_name == 'polygon':
        pts = []
        poly_side = 5
        r = min(center[0], center[1], img_size - center[0], img_size - center[1]) - line_width // 2
        for i in range(poly_side):
            theta = i * (360 // poly_side)
            #r = random.randint(img_size // 4 - line_width, img_size // 2)
            x = center[0] + int(r * np.cos(np.deg2rad(theta)))
            y = center[1] + int(r * np.sin(np.deg2rad(theta)))
            pts.append([x, y])
        pts = np.array(pts, dtype=np.int32)
        cv2.polylines(img, [pts], True, 255, line_width)
    return img

count = 0
print("----- Generating Dataset -----")
for i in tqdm(range(num_classes), total=num_classes):
    shape_name = shape_names[i]
    for j in range(num_imgs_per_class):
        img = generate_(shape_name)
        label = i
        filename = f'{shape_name}_{j}.png'
        df.loc[count, 'path'] = filename.split(".")[0]
        df.loc[count, 'label'] = i
        cv2.imwrite(f"{output}/{filename}", img)
        count += 1
print(f'{num_classes*num_imgs_per_class} images generated')
print("Distribution of the data:")
print(df.label.value_counts())
df.to_csv("./dataset.csv", index=False)


from flask_sqlalchemy import SQLAlchemy
import json
from sqlalchemy import Table, Column, MetaData
from sqlalchemy.sql import select

class MetabaseExport:
    def __init__(self, metabase_db_credentials):
        """
        Setup the required connection for the metabase backend database.

        Input: A dictionary with credentials for the metabase databases.
        Example Input:
        {
            "database_type": "postgresql",  # One of the following https://docs.sqlalchemy.org/en/14/core/engines.html
            "database_driver": "psycopg2",  # Specific to the database type https://docs.sqlalchemy.org/en/14/core/engines.html (This is optional)
            "user": "postgres",
            "pass": "password",
            "host": "localhost",
            "port": "5432",
            "database": "metabase_output"
        }
        """

        # Generate the start of the DB connection URI. Example: "postgresql+psycopg2".
        connection_details_db = ""
        if "database_driver" in metabase_db_credentials.keys():
            connection_details_db = "{}+{}".format(
                metabase_db_credentials["database_type"],
                metabase_db_credentials["database_driver"],
            )
        else:
            connection_details_db = metabase_db_credentials["database_type"]

        # Create the DB connection URI.
        db_uri = "{}://{}:{}@{}:{}/{}".format(
            connection_details_db,
            metabase_db_credentials["user"],
            metabase_db_credentials["pass"],
            metabase_db_credentials["host"],
            metabase_db_credentials["port"],
            metabase_db_credentials["database"],
        )

        # Create connection to the Metabase DB.
        sqlalchemy = SQLAlchemy()
        self.engine_input = sqlalchemy.create_engine(sa_url=db_uri, engine_opts={})
        self.connection_input = self.engine_input.connect()

    def get_databases(self, skip_default=False):
        """
        Get the credentials to all databases that Metabase uses.
        - If skip_default is set to True, this class (MetabaseExport) will not export
          questions that come from default databases.

        Output: Two dictionaries -- self.database_connections and
                self.databases_to_skip_questions (would include sample databases if skip_default = True)
        """

        self.database_connections = {}
        self.databases_to_skip_questions = (
            []
        )  # These would be sample databases if skip_default=True.
        metadata = MetaData()
        query_table = Table(
            "metabase_database",
            metadata,
            Column("name"),
            Column("details"),
            Column("engine"),
            Column("is_sample"),
            Column("id"),
        )
        query = select(query_table)
        result = self.connection_input.execute(query).fetchall()

        for res in result:
            if res[3] is False or not skip_default:
                # Runs if DB is not a sample DB provided by Metabase or if Skip Default is set to False.
                self.database_connections[res[4]] = json.loads(res[1])
            else:
                # Runs if Skip Default is set to True and the DB is a sample one.
                self.databases_to_skip_questions.append(res[4])

    def get_questions(self):
        """
        Get all the questions used and save them in two arrays:
            - Native Questions
            - MBQL Questions

        Output: Two arrays -- self.native_questions and self.mbql_questions
        """

        self.native_questions = []
        self.mbql_questions = []
        metadata = MetaData()
        query_table = Table(
            "report_card",
            metadata,
            Column("name"),
            Column("dataset_query"),
            Column("database_id"),
            Column("query_type"),
        )
        query = select(query_table)
        result = self.connection_input.execute(query).fetchall()

        for res in result:
            if res[2] in self.databases_to_skip_questions:
                # Skip all questions that come from a sample database.
                continue
            if res[3] == "native":
                # Get all SQL Queries.
                query_only = json.loads(res[1])["native"]["query"]
                self.native_questions.append(
                    {"name": res[0], "query": query_only, "database_id": res[2]}
                )
            else:
                # Get all MBQL Queries which needs to be converted to SQL.
                query_only = json.loads(res[1])["query"]
                self.mbql_questions.append(
                    {"name": res[0], "query": query_only, "database_id": res[2]}
                )

    def process_native(self, write_json=True):
        """
        Process all data in self.native_questions and write to JSON native_questions.json

        Output: native_questions.json
        """

        all_data = []
        for question in self.native_questions:
            data = {}
            data["name"] = question["name"]
            data["query"] = question["query"]
            # Get information to connect to DB that contains the table(s) used.
            data["data_source"] = self.database_connections[question["database_id"]]
            all_data.append(data)

        # Write processed data to JSON file.
        if write_json:
            with open("native_questions.json", "w") as file:
                file.writelines(json.dumps(all_data, indent=4))
        else:
            # Write processed data to CSV file.
            self.write_to_csv(all_data, "native_questions.csv")

    def process_mbql(self, write_json=True):
        """
        Process all data in self.mbql_questions and write to JSON mbql_questions.json

        Output: mbql_questions.json
        """

        def get_metabase_field(id):
            metadata = MetaData()
            query_table = Table(
                "metabase_field", metadata, Column("name"), Column("id")
            )
            query = select(query_table.c.name).where(query_table.c.id == id)
            return self.connection_input.execute(query).fetchall()[0][0]

        all_data = []
        id = 1
        for question in self.mbql_questions:
            final_sql_query = []
            question_query = question["query"]
            data = {}
            data["id"] = id
            id += 1
            data["name"] = question["name"]

            # Have empty dictionary that will be used in case there is an error processing a question.
            empty_data = {
                "id": id,
                "name": question["name"],
                "kpi_query": None,
                "data_source": None,
                "datetime_column": None,
                "metric": None,
                "metric_precision": None,
                "aggregation": None,
                "filters": {},
                "dimensions": [],
            }

            # Get the name of the table where the question is coming from.
            source_table = question_query["source-table"]

            # If source_table is not a number, return empty data as the code below will fail.
            if type(source_table) != int and not source_table.isdigit():
                all_data.append(empty_data)
                continue

            metadata = MetaData()
            query_table = Table(
                "metabase_table",
                metadata,
                Column("name"),
                Column("schema"),
                Column("id"),
            )
            query = select(query_table.c.name, query_table.c.schema).where(
                query_table.c.id == source_table
            )
            source_table_name, source_table_schema = self.connection_input.execute(
                query
            ).fetchall()[0]
            # Save the name of the table used.
            data["kpi_query"] = f"{source_table_schema}.{source_table_name}"

            # Get information to connect to DB that contains the table(s) used.
            data["data_source"] = self.database_connections[question["database_id"]]

            # Get all columns in source table.
            metadata = MetaData()
            source_table_columns_query_table = Table(
                "metabase_field",
                metadata,
                Column("name"),
                Column("base_type"),
                Column("table_id"),
            )
            source_table_columns_query = select(
                source_table_columns_query_table.c.name,
                source_table_columns_query_table.c.base_type,
            ).where(source_table_columns_query_table.c.table_id == source_table)
            source_table_columns_query_output = self.connection_input.execute(
                source_table_columns_query
            ).fetchall()

            # Get possible datetime columns.
            possible_datetime = []
            for table_column in source_table_columns_query_output:
                if (
                    "date" in table_column[1].lower()
                    or "time" in table_column[1].lower()
                ):
                    possible_datetime.append(table_column[0])
            # Handle different cases for the array possible_datetime.
            if len(possible_datetime) == 0:
                data["datetime_column"] = None
            elif len(possible_datetime) == 1:
                data["datetime_column"] = possible_datetime[0]
            else:
                data["datetime_column"] = possible_datetime

            # Get all information possible for metrics and aggregations.
            try:
                metrics = []
                aggregations = []
                for aggregation in question_query["aggregation"]:
                    aggregations.append(aggregation[0])
                    try:
                        metrics.append(get_metabase_field(aggregation[1][1]))
                    except:
                        pass

                # Convert to string if not multiple options.
                if len(metrics) == 1:
                    metrics = metrics[0]
                if len(aggregations) == 1:
                    aggregations = aggregations[0]

                data["metric"] = metrics
                data["metric_precision"] = 2
                data["aggregation"] = aggregations
            except KeyError:
                # This error will occur if no aggregation was used.
                data["metric"] = None
                data["metric_precision"] = 2
                data["aggregation"] = None

            # Convert filters into usable dictionary.
            try:
                filters = {}
                if question_query["filter"][0] == "and":
                    # This runs if there are multiple filters.
                    for filter in question_query["filter"][1:]:
                        filter_data = [filter[0]]
                        for i in filter[2:]:
                            filter_data.append(i)
                        filter_table_name = get_metabase_field(filter[1][1])
                        # Check if there are already filters for the filter_table_name.
                        # If there are none, initialize it with an array.
                        if filter_table_name not in filters.keys():
                            filters[filter_table_name] = []
                        filters[filter_table_name].append(filter_data)
                else:
                    # This runs if there is only one filter.
                    filter = question_query["filter"]
                    filter_data = [filter[0]]
                    for i in filter[2:]:
                        filter_data.append(i)
                    filter_table_name = get_metabase_field(filter[1][1])
                    filters[filter_table_name] = filter_data

                data["filters"] = filters
            except KeyError:
                # This error occurs if there were no filters used.
                data["filters"] = {}

            # Put all table names in dictionary.
            data["dimensions"] = [
                value[0] for value in source_table_columns_query_output
            ]

            all_data.append(data)

        # Write processed data to JSON file.
        if write_json:
            with open("mbql_questions.json", "w") as file:
                file.writelines(json.dumps(all_data, indent=4))
        else:
            # Write processed data to CSV file.
            self.write_to_csv(all_data, "mbql_questions.csv")

    def write_to_csv(self, all_data, file_name):
        # Write processed data to CSV file.
        delimiter = "   "
        with open(file_name, "w", encoding="utf-8") as file:
            if len(all_data) != 0:
                file.write(f"{delimiter}".join(all_data[0].keys()))
                file.write("\n")
                for current_data in all_data:
                    row = []
                    for current_data_column in current_data.values():
                        row.append(str(current_data_column))
                    file.write(f"{delimiter}".join(row))
                    file.write("\n")
            else:
                # If there are no MBQL Questions that have been imported, return an empty CSV.
                file.write("")

    def close_connections(self):
        """
        Runs at the end and closes the SQLAlchemy connection.
        """
        self.connection_input.close()

    def export(self, write_json=True):
        # Runs all the functions above to save all of the Metabase Questions into a JSON or CSV.
        self.get_databases()
        self.get_questions()
        self.process_mbql(write_json=write_json)
        self.process_native(write_json=write_json)
        self.close_connections()


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 27 12:08:23 2019

@author: lille_000
"""
import numpy as np

a_n = lambda n: (1+(1)**n)

N = 100
p_n = np.zeros(N)
term = 1

for ii in range(1,N+1):
    term = term * a_n(ii)
    p_n[ii-1] = term

#print("The first 15 terms are:", terms[:15])
print("The last 15 terms are:", p_n[-15:])


"""Couting without counting with recursion call."""

from neighbors import neighbors

def count_sequences(start_pos, num_hops):
    """Recursion solution."""
    if num_hops is 0:
        return 1

    num_sequences = 0
    for pos in neighbors(start_pos):
        num_sequences += count_sequences(pos, num_hops - 1)
    return num_sequences

if __name__ == '__main__':
    import args
    a = args.parse_arguments()
    print(count_sequences(a.start_position, a.num_hops))


from fastapi import Depends, Response, status
from fastapi.routing import APIRouter
from sqlalchemy.orm import Session

from src.apps.products.schemas import (CategoryInputSchema,
                                       CategoryOutputSchema,
                                       ProductInputSchema, ProductOutputSchema)
from src.apps.products.services.category_services import (
    create_category, delete_single_category, get_all_categories,
    get_single_category, update_single_category)
from src.apps.products.services.product_services import (create_product,
                                                         delete_single_product,
                                                         get_all_products,
                                                         get_single_product,
                                                         update_single_product)
from src.core.pagination.models import PageParams
from src.core.pagination.schemas import PagedResponseSchema, T
from src.dependencies.get_db import get_db
from src.dependencies.user import authenticate_user

category_router = APIRouter(prefix="/categories", tags=["category"])
product_router = APIRouter(prefix="/products", tags=["product"])

@category_router.post(
    "/",
    dependencies=[Depends(authenticate_user)],
    response_model=CategoryInputSchema,
    status_code=status.HTTP_201_CREATED,
)
def post_category(
    category: CategoryInputSchema, db: Session = Depends(get_db)
) -> CategoryOutputSchema:
    db_category = create_category(db, category)
    return db_category

@category_router.get(
    "/",
    response_model=PagedResponseSchema[CategoryOutputSchema],
    dependencies=[Depends(authenticate_user)],
    status_code=status.HTTP_200_OK,
)
def get_categories(
    db: Session = Depends(get_db), page_params: PageParams = Depends()
) -> PagedResponseSchema[CategoryOutputSchema]:
    db_categories = get_all_categories(db, page_params)
    return db_categories

@category_router.get(
    "/{category_id}",
    dependencies=[Depends(authenticate_user)],
    response_model=CategoryOutputSchema,
    status_code=status.HTTP_200_OK,
)
def get_category(
    category_id: int, db: Session = Depends(get_db)
) -> CategoryOutputSchema:
    db_category = get_single_category(db, category_id)
    return db_category

@category_router.put(
    "/{category_id}",
    dependencies=[Depends(authenticate_user)],
    response_model=CategoryOutputSchema,
    status_code=status.HTTP_200_OK,
)
def update_category(
    category_id: int, category: CategoryInputSchema, db: Session = Depends(get_db)
) -> CategoryOutputSchema:
    db_category = update_single_category(db, category, category_id)
    return db_category

@category_router.delete(
    "/{category_id}",
    dependencies=[Depends(authenticate_user)],
    status_code=status.HTTP_204_NO_CONTENT,
)
def delete_category(category_id: int, db: Session = Depends(get_db)) -> Response:
    delete_single_category(db, category_id)
    return Response(status_code=status.HTTP_204_NO_CONTENT)

@product_router.post(
    "/",
    dependencies=[Depends(authenticate_user)],
    response_model=ProductInputSchema,
    status_code=status.HTTP_201_CREATED,
)
def post_product(
    product: ProductInputSchema, db: Session = Depends(get_db)
) -> ProductOutputSchema:
    db_product = create_product(db, product)
    return db_product

@product_router.get(
    "/",
    response_model=PagedResponseSchema[ProductOutputSchema],
    dependencies=[Depends(authenticate_user)],
    status_code=status.HTTP_200_OK,
)
def get_products(
    db: Session = Depends(get_db), page_params: PageParams = Depends()
) -> PagedResponseSchema[ProductOutputSchema]:
    db_products = get_all_products(db, page_params)
    return db_products

@product_router.get(
    "/{product_id}",
    dependencies=[Depends(authenticate_user)],
    response_model=ProductOutputSchema,
    status_code=status.HTTP_200_OK,
)
def get_product(product_id: int, db: Session = Depends(get_db)) -> ProductOutputSchema:
    db_product = get_single_product(db, product_id)
    return db_product

@product_router.put(
    "/{product_id}",
    dependencies=[Depends(authenticate_user)],
    response_model=ProductOutputSchema,
    status_code=status.HTTP_200_OK,
)
def update_product(
    product_id: int, product: ProductInputSchema, db: Session = Depends(get_db)
) -> ProductOutputSchema:
    db_product = update_single_product(db, product, product_id)
    return db_product

@product_router.delete(
    "/{product_id}",
    dependencies=[Depends(authenticate_user)],
    status_code=status.HTTP_204_NO_CONTENT,
)
def delete_product(product_id: int, db: Session = Depends(get_db)) -> Response:
    delete_single_product(db, product_id)
    return Response(status_code=status.HTTP_204_NO_CONTENT)


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from auto_nag import people
from auto_nag.bugbug_utils import get_bug_ids_classification
from auto_nag.bzcleaner import BzCleaner
from auto_nag.utils import nice_round

class SpamBug(BzCleaner):
    def __init__(self):
        super().__init__()
        self.people = people.People.get_instance()

    def description(self):
        return "[Using ML] Detect spam bugs"

    def columns(self):
        return ["id", "summary", "confidence"]

    def sort_columns(self):
        return lambda p: (-p[2], -int(p[0]))

    def handle_bug(self, bug, data):
        reporter = bug["creator"]
        if self.people.is_mozilla(reporter):
            return None

        return bug

    def get_bz_params(self, date):
        start_date, _ = self.get_dates(date)

        return {
            "include_fields": ["id", "groups", "summary", "creator"],
            # Ignore closed bugs.
            "bug_status": "__open__",
            "f1": "reporter",
            "v1": "%group.editbugs%",
            "o1": "notsubstring",
            "f2": "creation_ts",
            "o2": "greaterthan",
            "v2": start_date,
        }

    def get_bugs(self, date="today", bug_ids=[]):
        # Retrieve the bugs with the fields defined in get_bz_params
        raw_bugs = super().get_bugs(date=date, bug_ids=bug_ids, chunk_size=7000)

        if len(raw_bugs) == 0:
            return {}

        # Extract the bug ids
        bug_ids = list(raw_bugs.keys())

        # Classify those bugs
        bugs = get_bug_ids_classification("spambug", bug_ids)

        results = {}

        for bug_id in sorted(bugs.keys()):
            bug_data = bugs[bug_id]

            if not bug_data.get("available", True):
                # The bug was not available, it was either removed or is a
                # security bug
                continue

            if not {"prob", "index"}.issubset(bug_data.keys()):
                raise Exception(f"Invalid bug response {bug_id}: {bug_data!r}")

            bug = raw_bugs[bug_id]
            prob = bug_data["prob"]

            if prob[1] < self.get_config("confidence_threshold"):
                continue

            results[bug_id] = {
                "id": bug_id,
                "summary": bug["summary"],
                "confidence": nice_round(prob[1]),
            }

        return results

    def get_autofix_change(self):
        return {}

if __name__ == "__main__":
    SpamBug().run()


import morfessor
import nltk
import sys

class RunMorfessor():
    u"""Training and decoding using Morfessor"""

    unsupervised = None
    semisupervised = None
    tokens = []
    io = None
    acw = 0
    segs = None

    def __init__(self, unsuper_file, semisuper_file,acw):
        self.io = morfessor.MorfessorIO()
        self.acw = acw

        # build models
        self.unsupervised = self.io.read_binary_model_file(unsuper_file)

        self.semisupervised = self.io.read_binary_model_file(unsuper_file)
        a = self.io.read_annotations_file(semisuper_file)
        annotations = {}
        for word in a:
            hypotheses = []
            for hypothesis in a[word]:
                h = ""
                for morpheme in hypothesis:
                    h += morpheme + " "
                hypotheses.append(h[:-1])
            annotations[word] = hypotheses
        self.semisupervised.set_annotations(annotations,acw)

        # write segmentation
        with open('models/bible-segmentation','w') as f:
            for word in annotations:
                construction = ""
                for hypothesis in annotations[word]:
                    construction += hypothesis + ", "
                construction = construction[:-2]
                f.write(construction+"\n")
        self.segs = self.io.read_segmentation_file('models/bible-segmentation',has_counts=False)

    def get_segs(self):
        return self.segs

    def train_batch_recursive(self):
        """batch training on given model files with recursive algorithm"""

        self.unsupervised.train_batch(algorithm='recursive')
        self.semisupervised.train_batch(algorithm='recursive')

    def read_testset(self,test_file):
        all_tokens = []
        with open(test_file,'r') as f:
            for sentence in f:
                tokens = nltk.word_tokenize(sentence)
                all_tokens.extend(tokens)

        self.tokens = all_tokens

    def decode_unsupervised(self):
        with open('results/result_segs_unsupervised','w') as f1:
            for token in self.tokens:
                morphemes = self.unsupervised.viterbi_segment(token)[0]
                decomposed = ""
                for morpheme in morphemes:
                    decomposed += morpheme + "+"
                decomposed = decomposed[:-1]
                f1.write(token+": "+decomposed+"\n")

        with open('results/result_segs_semi_'+str(self.acw),'w') as f2:
            f2.write(str(self.acw)+"\n-----------------\n")
            for token in self.tokens:
                morphemes = self.unsupervised.viterbi_segment(token)[0]
                decomposed = ""
                for morpheme in morphemes:
                    decomposed += morpheme + "+"
                decomposed = decomposed[:-1]
                f2.write(token+": "+decomposed+"\n")
            f2.write("\n")

def evaluate(gold_data,morf,acw,segs):
    print(str(acw) + " weighted annotated corpus")
    io = morfessor.MorfessorIO()
    gold = io.read_annotations_file(gold_data)
    ev = morfessor.MorfessorEvaluation(gold)

    models = [morf.unsupervised, morf.semisupervised]

    # evaluate models
    config = morfessor.evaluation.EvaluationConfig(10,25)
    results = [ev.evaluate_model(m,config) for m in models]
    print("model evaluation")
    print(results[0])
    print(results[1])

    wsr = morfessor.evaluation.WilcoxonSignedRank()
    r = wsr.significance_test(results)
    wsr.print_table(r)

    # evaluate segmentations
    # config2 = morfessor.evaluation.EvaluationConfig(5,15)
    # eval_seg = [ev.evaluate_segmentation(segs,config2) for m in models]
    # print("segmentation evaluation")
    # print(eval_seg[0])
    # print(eval_seg[1])

    # wsr = morfessor.evaluation.WilcoxonSignedRank()
    # r_seg = wsr.significance_test(eval_seg)
    # wsr.print_table(r_seg)

if __name__ == '__main__':
    weights = [0.0, 0.1, 0.25, 0.5, 0.75, 0.95, 1.0, 2.0, 2.5, 5.0, 7.5, 10.0]

    for w in weights:
        m = RunMorfessor(sys.argv[1],sys.argv[2],w)

        m.train_batch_recursive()
        m.read_testset(sys.argv[3])
        m.decode_unsupervised()

        evaluate(sys.argv[4],m,w,m.get_segs())


import yaml
import os
from common import do_contants

class DoYaml:
    def __init__(self, yamlfilepath):
        if os.path.exists(yamlfilepath):
            self.yamlfilepath = yamlfilepath
        else:
            raise FileNotFoundError('文件不存在！')

    def read_yaml(self):
        with open(self.yamlfilepath, 'rb') as f:
            y = yaml.load(f, Loader=yaml.FullLoader)
        return y

    def write_yaml(self, num, key, value):
        with open(self.yamlfilepath, 'rb') as f:
            y = yaml.load(f, Loader=yaml.FullLoader)
            y[int(num - 1)][key] = value

        with open(self.yamlfilepath, 'w', encoding='utf-8') as w_f:
            # 覆盖原先的配置文件
            yaml.dump(y, w_f, allow_unicode=True)
            # y = yaml.load(f, Loader=yaml.FullLoader)
        return y

if __name__ == '__main__':
    a = DoYaml(do_contants.case_file + '\\wap_day.yaml')
    b = a.read_yaml()
    print(b)
    # print(
    # a.write_yaml(b,'result','false'))


import time

# elapsed time
start = time.time()

#evaluation
assignment5_output=list()
assignment6_output=list()
TeamProject_output=list()
ground_truth=list()

def calculate_F1_score(cluster):
    f1_score_list=list()
    sum_f1_score=0
    for cluster_index in range(len(cluster)):
        cluster_set=set(cluster[cluster_index])
        f1_score_list.clear()
        for ground_truth_index in range(len(ground_truth)):
            ground_truth_set=set(ground_truth[ground_truth_index])
            TP=len(ground_truth_set&cluster_set)
            if TP==0:
                continue
            FP=len(cluster_set-ground_truth_set)
            FN=len(ground_truth_set-cluster_set)
            precision=TP/(TP+FP)
            recall=TP/(TP+FN)
            precision = TP/len(cluster_set)
            recall = TP/len(ground_truth_set)
            f1_score=2*precision*recall/(precision+recall)
            f1_score_list.append(f1_score)
        if len(f1_score_list)==0:
            continue
        else:
            sum_f1_score+=max(f1_score_list)
    return sum_f1_score/len(cluster)

def bfs(graph, start):  # 탐색 알고리즘
    visited = list()
    queue = [start]
    while queue:
        n = queue.pop(0)
        if n not in visited:
            visited.append(n)
            queue += graph[n] - set(visited)
    return visited

def calculate_clustering_coefficient(graph, subgraph):
    clustering_coefficient_list = list()
    for key in subgraph:
        actual_edge = 0
        temp = list()
        Vi = graph[key]
        NVi = len(Vi)
        neighbors_list = list(Vi)
        for neighbors_index1 in range(len(Vi) - 1):
            for neighbors_index2 in range(neighbors_index1, len(Vi)):
                if neighbors_list[neighbors_index1] in graph[neighbors_list[neighbors_index2]]:
                    actual_edge += 1
        if actual_edge == 0:
            continue
        temp.append(key)
        temp.append(actual_edge / (NVi * (NVi - 1) / 2))
        clustering_coefficient_list.append(temp)
    return clustering_coefficient_list

def make_seed_cluster(graph, seed):
    temp_c = list()
    temp_c.append(seed)
    temp_c.extend(list(graph[seed]))
    return temp_c

def check_neighbors(graph, seed_cluster):
    temp_c = list()
    for i in range(1, len(seed_cluster)):
        temp_c.extend(list(graph[seed_cluster[i]]))
    temp_set = set(temp_c)
    seed_cluster_set = set(seed_cluster)
    temp_set = temp_set - seed_cluster_set
    temp_c = list(set(temp_set))
    return temp_c

def find_max_edge_vertex(graph, seed_cluster, neighbors_list):
    max_edge_vertex_list = list()
    temp = list()
    for key in neighbors_list:
        max_edge_num = 0
        for seed_vertex in seed_cluster:
            if key in graph[seed_vertex]:
                max_edge_num += 1
        temp.append(key)
        temp.append(max_edge_num)
        max_edge_vertex_list.append(temp)
        temp = list()
    return max_edge_vertex_list

def check_vertex_degree(graph, max_edge_vertex):
    max_degree_list = list()
    for i in range(len(max_edge_vertex)):
        max_edge_vertex[i][1] = len(graph[max_edge_vertex[i][0]])

    max_degree_list = sorted(max_edge_vertex, key=lambda x: x[1], reverse=True)
    max_degree_num = max_degree_list[0][1]
    for i in range(len(max_degree_list)):
        if max_degree_list[i][1] < max_degree_num:
            del max_degree_list[i:len(max_degree_list)]
            break
    return max_degree_list

def calculate_density(graph, subgraph):
    vertex_num = len(subgraph)
    if vertex_num == 1:
        return 0
    edge_num = 0
    for node1 in range(vertex_num - 1):
        for node2 in range(node1 + 1, vertex_num):
            if subgraph[node2] in graph[subgraph[node1]]:
                edge_num += 1
    return 2 * edge_num / (vertex_num * (vertex_num - 1))

def make_output_txt_file(final_cluster):
    cluster_f = open('TeamProject_output.txt', 'w')
    for cluster in final_cluster:
        cluster_f.write(str(len(cluster)))
        cluster_f.write(": ")
        for point in cluster:
            cluster_f.write(point)
            cluster_f.write(" ")
        cluster_f.write("\n")
    cluster_f.close()

def main():

    final_cluster = list()

    # graph 생성
    G = dict()
    with open("gene.txt", 'r') as file:
        for line in file:
            n1, n2 = line.strip().split('\t')
            try:
                G[n1].add(n2)
            except KeyError:
                G[n1] = {n2}
            try:
                G[n2].add(n1)
            except KeyError:
                G[n2] = {n1}

    # input data 로부터 subgraph 분류
    subgraphs = list()
    nodes = list(G.keys())
    count = 0
    while len(nodes) > 0:
        subgraphs.append(bfs(G, nodes[0]))
        for value in subgraphs[count]:
            nodes.remove(value)
        count += 1
    for i in range(len(subgraphs)):
        subgraphs[i] = sorted(subgraphs[i])

    # subgraph 선택 후 clustering 작업 수행
    for i in range(len(subgraphs)):

        # clustering coefficient 측정
        clustering_coefficient_list = list()
        clustering_coefficient_list = calculate_clustering_coefficient(G, subgraphs[i])

        # clustering coefficient가 측정되지 않을 경우 final cluster로 분리 후 다음 subgraph로 이동
        if len(clustering_coefficient_list) == 0:
            final_cluster.append(subgraphs[i])
            continue

        # max clustering coefficient 측정
        value_list = list()
        for j in range(len(clustering_coefficient_list)):
            value_list.append(clustering_coefficient_list[j][1])
        max_clustering_coefficient = max(value_list)

        # clustering coefficient 값이 최대치인 node들 골라내기
        seed_list = list()
        for j in range(len(clustering_coefficient_list)):
            if clustering_coefficient_list[j][1] == max_clustering_coefficient:
                seed_list.append(clustering_coefficient_list[j][0])

        # 골라진 seed node들로 cluster 만들기
        for j in range(len(seed_list)):

            # 만약 해당 seed node가 만들어진 cluster 안에 포함되어 있다면 다음 seed node로 넘어가기
            temp_list = list()
            for k in range(len(final_cluster)):
                temp_list.extend(final_cluster[k])
            if seed_list[j] in temp_list:
                continue

            # seed cluster 생성
            seed_cluster = list()
            seed_cluster = make_seed_cluster(G, seed_list[j])

            # density 저장
            pre_density = calculate_density(G, seed_cluster)

            # iterative part
            while True:

                # seed cluster의 neighbor들 골라내기
                neighbors = list()
                neighbors = check_neighbors(G, seed_cluster)

                # neighbor가 하나도 없을 경우 stop
                if len(neighbors) == 0:
                    break

                # neighbor들 중 seed cluster와 연결된 edge가 가장 많은 neighbor 골라내기
                max_edge_list = list()
                max_edge_list = find_max_edge_vertex(G, seed_cluster, neighbors)

                # 병합 과정
                # max edge를 가진 node가 하나만 있을 경우
                if len(max_edge_list) == 1:
                    seed_cluster.append(max_edge_list[0][0])

                    # density 측정
                    density = calculate_density(G, seed_cluster)

                    # density가 증가하지 않을 시 stop
                    if density < pre_density:
                        seed_cluster.remove(max_edge_list[0][0])
                        break
                    # 증가할 시 density 저장 후 continue
                    else:
                        pre_density = density
                # max edge를 가진 node가 여러 개일 경우
                else:
                    # max edge를 가진 node들의 degree 측정
                    max_degree_list = list()
                    max_degree_list = check_vertex_degree(G, max_edge_list)

                    # degree가 가장 큰 node가 하나만 있을 경우
                    if len(max_degree_list) == 1:
                        seed_cluster.append(max_degree_list[0][0])

                        # density 측정
                        density = calculate_density(G, seed_cluster)

                        # density가 증가하지 않을 시 stop
                        if density < pre_density:
                            seed_cluster.remove(max_degree_list[0][0])
                            break
                        # 증가할 시 density 저장 후 continue
                        else:
                            pre_density = density
                    # degree가 가장 큰 node가 여러 개일 경우
                    else:
                        for k in range(len(max_degree_list)):
                            seed_cluster.append(max_degree_list[k][0])

                        # density 측정
                        density = calculate_density(G, seed_cluster)

                        # density가 증가하지 않을 시 stop
                        if density < pre_density:
                            for k in range(len(max_degree_list)):
                                seed_cluster.remove(max_degree_list[k][0])
                            break
                        # 증가할 시 density 저장 후 continue
                        else:
                            pre_density = density

            # 병합이 끝났을 시 final cluster로 분리
            final_cluster.append(seed_cluster)

    # output 파일 추출
    final_cluster.sort(key=len,reverse=True)
    make_output_txt_file(final_cluster)

    # make evaluation set
    with open("assignment5_output.txt", 'r') as file:
        for line in file:
            assignment5_output.append(line.split())
        for i in range(len(assignment5_output)):
            del assignment5_output[i][0]

    with open("assignment6_output.txt", 'r') as file:
        for line in file:
            assignment6_output.append(line.split())
        for i in range(len(assignment6_output)):
            del assignment6_output[i][0]

    with open("ground_truth.txt", 'r') as file:
        for line in file:
            ground_truth.append(line.split())

    with open("TeamProject_output.txt", 'r') as file:
        for line in file:
            TeamProject_output.append(line.split())
        for i in range(len(TeamProject_output)):
            del TeamProject_output[i][0]

    # f score 계산
    print(calculate_F1_score(assignment5_output))
    print(calculate_F1_score(assignment6_output))
    print(calculate_F1_score(TeamProject_output))

    # elapsed 시간 측정
    print("elapsed time : ", end='')
    print(f"{time.time() - start:.6f} sec")

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 25 14:33:12 2021

@author: thomas.aronson
"""

# Complete the lambda function below.  It begins in the non-alterable code above
    # lambda x : (x[i]*x[i] for i in range(len(x)))
def lambdaMap(arr):
        ans = map(lambda x : (x[i]*x[i] for i in range(len(x)) if x[i] > 0), arr)
        return ans

if __name__ == '__main__':
    n = int(input())
    arr = []
    for _ in range(n):
        arr.append(list(map(int, input().split())))

    ans = lambdaMap(arr)
    for row in ans:
        print(' '.join(map(str, row)))


# Python标准库
from collections import OrderedDict
# 根据插入的顺序来维护项目的顺序
favorite_languages = OrderedDict()

favorite_languages = {
    'jen': 'python',
    'sarah': 'c',
    'edward': 'ruby',
    'phil': 'python',
}

for name, language in favorite_languages.items():
    print(f'{name.title()}\'s favorite language is {language.title()}.')

print('--------------------------')

from collections import OrderedDict

# Creating an ordered dictionary
ordered_dict = OrderedDict()

# Adding items to the ordered dictionary
ordered_dict['one'] = 1
ordered_dict['three'] = 3
ordered_dict['five'] = 5
ordered_dict['two'] = 2

# Displaying the ordered dictionary
print("OrderedDict 1:", ordered_dict)

# Output:
# OrderedDict 1: OrderedDict([('one', 1), ('three', 3), ('five', 5), ('two', 2)])

print('--------------------------')

from collections import OrderedDict

# Creating an ordered dictionary with items in a specific order
ordered_dict = OrderedDict([('one', 1), ('three', 3), ('five', 5), ('two', 2)])

# Displaying the ordered dictionary
print("OrderedDict 2:", ordered_dict)

# Output:
# OrderedDict 2: OrderedDict([('one', 1), ('three', 3), ('five', 5), ('two', 2)])

print('--------------------------')

from collections import OrderedDict

# Creating an ordered dictionary
ordered_dict = OrderedDict([('one', 1), ('three', 3), ('five', 5), ('two', 2)])

# Iterating over key-value pairs in order
for key, value in ordered_dict.items():
    print(key, value)

# Output:
# one 1
# three 3
# five 5
# two 2

print('--------------------------')


from PyObjCTools.TestSupport import *
from Quartz.PDFKit import *

class TestPDFAnnotationPopup (TestCase):
    @min_os_level('10.5')
    def testMethods(self):
        self.assertResultIsBOOL(PDFAnnotationPopup.isOpen)
        self.assertArgIsBOOL(PDFAnnotationPopup.setIsOpen_, 0)

if __name__ == "__main__":
    main()