"""Support functions to write and read data."""
import os
from enum import Enum
from pathlib import Path
import yaml

from scm.plams import Molecule as PlamsMolecule
from scm.plams.interfaces.adfsuite.ams import AMSResults

from osp.core.cuds import Cuds
from osp.core.namespaces import crystallography, emmo
# from osp.core.utils import pretty_print

class GeometryType(str, Enum):

    XYZ = "xyz"
    MOL = "mol"
    PDB = "pdb"

def read_molecule(filename: str, TS: bool = False, GO_optimized: bool = False,
                  read_from_object: bool = False, geometry_type: GeometryType = None):
    """
    Read geometry from an external file or from PLAMS object.

    Supported formats: xyz, mol and pdb.

    :param filename: Name of the file to be read.

    :param TS: wether the output CUDS object should be a emmo.TransitionStateGeometry or not.

    :param GO_optimized: in case the output be a emmo.OptimizedMolecularGeometry.

    :param read_from_object: Optionally, emmo.MolecularGeometry is retrieved from a
                             PLAMSMolecule object.

    :param return: emmo.MolecularGeometry or emmo.TransitionStateGeometry or
                   emmo.OptimizedMolecularGeometry CUDS object.
    """

    if not read_from_object:
        if not geometry_type:
            geometry_type = assign_geom_type(filename)

        if geometry_type == GeometryType.XYZ:
            return molecule_from_xyz(filename, TS)

        elif geometry_type == GeometryType.MOL:
            return molecule_from_mol(filename, TS)

        elif geometry_type == GeometryType.PDB:
            raise_error(
                file=os.path.basename(__file__),
                function=read_molecule.__name__,
                type="NotImplementedError",
                message="pdb format not implemented yet.",
            )
    else:
        # Read from PLAMS.molecule format.
        return molecule_from_PlamsMolecule(filename, GO_optimized)

def read_lattice(filename: str) -> crystallography.UnitCell:
    """
    Function to read lattice vectors from different file formats.

    Supported formats: xyz, mol and pdb.

    :param filename: Name of the file to be read.

    :param return: crystallography.UnitCell object with added LatticeVectors.
    """

    geometry_type = assign_geom_type(filename)

    if geometry_type == "xyz":
        return lattice_from_xyz(filename)

    elif geometry_type == "mol":
        raise_error(
            file=os.path.basename(__file__),
            function=read_molecule.__name__,
            type="NotImplementedError",
            message="mol format not implemented yet.",
        )

    elif geometry_type == "pdb":
        raise_error(
            file=os.path.basename(__file__),
            function=read_molecule.__name__,
            type="NotImplementedError",
            message="pdb format not implemented yet.",
        )
    return

def assign_geom_type(filename: str) -> str:
    """
    Assign a geometry type format to a entry filename file.

    :param filename: name containing the path and name to file.

    :param return: String with the format of the geometry file.
    """

    path = Path(filename)

    suffix = path.suffix.lower()

    if suffix not in (".xyz", ".mol", ".pdb"):
        raise NameError(f"Unknown geometry format: {suffix}.")

    return suffix.strip(".")

def assign_chemical_name(root_cuds_object: Cuds, labels: list) -> Cuds:
    """
    Assign a ChemicalName to a Molecular Geometry.

    :param root_cuds_object: emmo.Molecule

    :param labels: list with atom labels.

    :param return: same CUDS object with the ChemicalName added.
    """
    counts = []
    labels_no_redundant = []
    label = str()

    for lb in labels:
        counts.append(labels.count(lb))

    for index, lb in enumerate(labels):
        if counts[index] == 1:
            labels[index] = lb
        else:
            labels[index] = lb + str(counts[index])

    # Eliminate redundancy on labels:
    labels_no_redundant = sorted(list(dict.fromkeys(labels)))

    # Get the final molecular label:
    for lb in labels_no_redundant:
        if lb[:3] != "VEC":
            label = label + lb

    chemical_name = emmo.ChemicalName(hasSymbolData=label)
    root_cuds_object.add(chemical_name, rel=emmo.hasSign)

    return

def molecule_from_xyz(filename: str, TS: bool):
    """
    Create emmo.MolecularGeometry or emmo.TransitionStateGeometry from .xyz file.

    It adds single CUDS atom containers with CUDS atom_labels and CUDS positions.

    :param filename: name containing the path and name to file.

    :param TS: option to return emmo.TransitionStateGeometry instead of emmo.MolecularGeometry.

    :param return: emmo.MolecularGeometry containing atoms, labels and position objects.
    """

    if TS:
        transition_state = emmo.TransitionStateGeometry()
    else:
        molecule = emmo.MolecularGeometry()

    labels = []
    coordinates = []
    regions = []
    xyz = open(filename)
    n_atoms = int(xyz.readline())
    _ = xyz.readline()

    # Reading the .xyz file:
    for line in xyz:

        params = line.split()
        label, x, y, z = params[0:4]
        try:
            if params[4]:
                region = params[4].strip()
        except IndexError:
            region = "x=NIL"

        labels.append(label)
        coordinates.append([float(x), float(y), float(z)])
        regions.append(region.split("=")[1])
    xyz.close()
    print("molecule_from_xyz: External", filename, "file has been read.")

    # Creating atom containers and adding them to the root_cuds_object
    # (i.e. MolecularGeometry)
    atoms = []
    for iatom in range(n_atoms):
        atom = emmo.AtomEntity()
        atom_symbol = emmo.ChemicalElement(hasSymbolData=labels[iatom])

        position_vec = emmo.PositionVector()
        coordinate_x = emmo.Real(hasNumericalData=coordinates[iatom][0])
        coordinate_y = emmo.Real(hasNumericalData=coordinates[iatom][1])
        coordinate_z = emmo.Real(hasNumericalData=coordinates[iatom][2])
        position_vec.add(coordinate_x, rel=emmo.hasSpatialFirst)
        position_vec.add(coordinate_y, rel=emmo.hasSpatialNext)
        position_vec.add(coordinate_z, rel=emmo.hasSpatialLast)

        if regions[iatom] == "NIL":
            atom.add(position_vec, atom_symbol, rel=emmo.hasPart)
        else:
            region = emmo.Region(hasSymbolData=regions[iatom].lower())
            atom.add(position_vec, atom_symbol, region, rel=emmo.hasPart)
        if iatom == 0:
            rel = emmo.hasSpatialFirst
        elif iatom == n_atoms:
            rel = emmo.hasSpatialLast
        else:
            rel = emmo.hasSpatialDirectPart
            atoms[iatom-1].add(atom, rel=emmo.hasSpatialNext)
        atoms.append(atom)
        if TS:
            transition_state.add(atom, rel=rel)
        else:
            molecule.add(atom, rel=rel)
    if TS:
        assign_chemical_name(transition_state, labels)
        return transition_state
    else:
        assign_chemical_name(molecule, labels)
        return molecule

def molecule_from_mol(filename: str, TS: bool = False):
    """
    Create emmo.MolecularGeometry from .mol file.

    It adds single CUDS atom containers with CUDS atom_labels and CUDS positions.

    :param filename: name containing the path and name to file.

    :param TS: option to return emmo.TransitionStateGeometry instead of emmo.MolecularGeometry.

    :param return: emmo.MolecularGeometry or emmo.TransitionStateGeometry containing atoms,
                   labels and position objects.

    #TODO, info on the Bonding structure is not read.
    """

    if TS:
        transition_state = emmo.TransitionStateGeometry()
    else:
        molecule = emmo.MolecularGeometry()

    labels = []
    coordinates = []
    with open(filename, "r") as file:
        mol = file.readlines()
        n_atoms = int(mol[3].split()[0])

        # Reading atoms from .mol file:
        for line in mol[4: 4 + n_atoms]:

            params = line.split()
            x, y, z, label = params[0:4]

            labels.append(label)
            coordinates.append([float(x), float(y), float(z)])

    print("molecule_from_mol External", filename, "file has been read.")

    # Creating atom containers and adding them to the root_cuds_object
    # (i.e. MolecularGeometry)
    for iatom in range(n_atoms):
        atom = emmo.AtomEntity()
        atom_symbol = emmo.ChemicalElement(hasSymbolData=labels[iatom])

        position_vec = emmo.PositionVector()
        coordinate_x = emmo.Real(hasNumericalData=coordinates[iatom][0])
        coordinate_y = emmo.Real(hasNumericalData=coordinates[iatom][1])
        coordinate_z = emmo.Real(hasNumericalData=coordinates[iatom][2])
        position_vec.add(coordinate_x, rel=emmo.hasSpatialFirst)
        position_vec.add(coordinate_y, rel=emmo.hasSpatialNext)
        position_vec.add(coordinate_z, rel=emmo.hasSpatialLast)

        atom.add(position_vec, atom_symbol, rel=emmo.hasPart)
        if TS:
            transition_state.add(atom, rel=emmo.hasSpatialPart)
        else:
            molecule.add(atom, rel=emmo.hasSpatialPart)
    if TS:
        assign_chemical_name(transition_state, labels)
        return transition_state
    else:
        assign_chemical_name(molecule, labels)
        return molecule

def molecule_from_PlamsMolecule(syntactic_molecule: PlamsMolecule, GO_optimized: bool):
    """
    Create emmo.MolecularGeometry from a string from a PlamsMolecule.

    It adds single CUDS atom containers with CUDS atom_labels and CUDS positions.

    :param filename: bulk string.

    :param GO_optimized: in case the output be a emmo.OptimizedMolecularGeometry.

    :param return: emmo.MolecularGeometry or emmo.OptimizedMolecularGeometry containing atoms,
                   labels and position objects.
    """
    labels = []

    if GO_optimized:
        optimized_geometry = emmo.OptimizedMolecularGeometry()
    else:
        molecule = emmo.MolecularGeometry()

    # Reading the .xyz file:
    for atom in syntactic_molecule:

        labels.append(atom.symbol)

        semantic_atom = emmo.AtomEntity()

        atom_symbol = emmo.ChemicalElement(hasSymbolData=atom.symbol)
        position_vec = emmo.PositionVector()
        coordinate_x = emmo.Real(hasNumericalData=atom.coords[0])
        coordinate_y = emmo.Real(hasNumericalData=atom.coords[1])
        coordinate_z = emmo.Real(hasNumericalData=atom.coords[2])
        position_vec.add(coordinate_x, rel=emmo.hasSpatialFirst)
        position_vec.add(coordinate_y, rel=emmo.hasSpatialNext)
        position_vec.add(coordinate_z, rel=emmo.hasSpatialLast)
        semantic_atom.add(position_vec, atom_symbol, rel=emmo.hasPart)

        if GO_optimized:
            optimized_geometry.add(semantic_atom, rel=emmo.hasSpatialPart)
        else:
            molecule.add(semantic_atom, rel=emmo.hasSpatialPart)

    if GO_optimized:
        assign_chemical_name(optimized_geometry, labels)
        return optimized_geometry
    else:
        assign_chemical_name(molecule, labels)
        return molecule

def lattice_from_xyz(filename: str) -> crystallography.UnitCell:
    """
    Create a list of emmo.LatticeVectors from .xyz file.

    :param filename: name containing the path and name to file.

    :param return: List of emmo.LatticeVectors.
    """

    lattice = crystallography.UnitCell()
    labels = []
    coordinates = []
    regions = []
    xyz = open(filename, "r")
    n_atoms = int(xyz.readline())
    _ = xyz.readline()
    n_lines = 2  # to account for the second .xyz blank line.

    # Reading the .xyz file:
    for line in xyz:

        params = line.split()
        label, x, y, z = params[0:4]
        try:
            if params[4]:
                region = params[4].strip()
        except IndexError:
            region = "x=NIL"

        labels.append(label)
        coordinates.append([float(x), float(y), float(z)])
        regions.append(region.split("=")[1])
        n_lines += 1
    xyz.close()
    print("lattice_from_xyz: External", filename, "file has been read.")

    n_vectors = n_lines - n_atoms - 2  # n_atoms line + blank line

    # Raise error if no vectors
    if n_vectors == 0:
        raise_error(
            file=os.path.basename(__file__),
            function=lattice_from_xyz.__name__,
            type="NameError",
            message="LatticeVectors not found in .xyz file.",
        )

    for ivector in range(n_vectors):
        lattice_vec = crystallography.LatticeVector()
        coordinate_x = emmo.Real(
            hasNumericalData=coordinates[len(coordinates) - 1 - ivector][0]
        )
        coordinate_y = emmo.Real(
            hasNumericalData=coordinates[len(coordinates) - 1 - ivector][1]
        )
        coordinate_z = emmo.Real(
            hasNumericalData=coordinates[len(coordinates) - 1 - ivector][2]
        )
        lattice_vec.add(coordinate_x, rel=emmo.hasSpatialFirst)
        lattice_vec.add(coordinate_y, rel=emmo.hasSpatialNext)
        lattice_vec.add(coordinate_z, rel=emmo.hasSpatialLast)
        if ivector == 0:
            lattice.add(lattice_vec, rel=emmo.hasSpatialNext)
        if ivector == 1:
            lattice.add(lattice_vec, rel=emmo.hasSpatialFirst)
        if ivector == 2:
            lattice.add(lattice_vec, rel=emmo.hasSpatialLast)
    return lattice

def read_mechanism(
    filename: str, read_from_file: bool = True, cuds=None,
) -> emmo.ChemicalReactionMechanism:
    """
    Create emmo.ChemicalReactionMechanism from a Zacros-formatted .dat file.

    :param filename: name containing the path and name to file.

    :param return: emmo.ChemicalReactionMechanism with all the semantic information.
    """

    mechanism = cuds or emmo.ChemicalReactionMechanism()

    if read_from_file is True:
        dat = open(filename, "r")
    elif read_from_file is False:
        dat = (
            filename.splitlines()
        )  # For cases where the file names contains the full string.

    initial = False
    final = False

    for count, line in enumerate(dat):
        params = line.split()
        if params:
            if params[0] == "reversible_step":
                reaction = emmo.ReversibleReactionEquation()

            if params[0] == "gas_reacs_prods":
                filedir = os.path.dirname(os.path.abspath(__file__))
                dictionary = os.path.join(
                    filedir, "../dictionaries/energies/gas_energies.yaml"
                )
                with open(dictionary, "r") as f:
                    energy_list = yaml.load(f, Loader=yaml.FullLoader)
                    formation_energy_value_from_dic = energy_list["Gas_energies"][
                        params[1]
                    ]
                if float(params[2]) < 0:
                    gas_reactant = emmo.GasReactantSpecies()
                    atom_symbol = emmo.ChemicalElement(hasSymbolData=params[1])
                    formation_energy = emmo.FormationEnergy()
                    formation_energy_value = emmo.Real(
                        hasNumericalData=formation_energy_value_from_dic
                    )
                    formation_energy.add(
                        formation_energy_value, rel=emmo.hasQuantityValue
                    )
                    coefficient = emmo.StoichiometricCoefficient()
                    coefficient_value = emmo.Real(hasNumericalData=params[2])
                    coefficient.add(coefficient_value, rel=emmo.hasQuantityValue)
                    gas_reactant.add(
                        atom_symbol,
                        coefficient,
                        formation_energy,
                        rel=emmo.hasSpatialDirectPart
                    )
                    reaction.add(gas_reactant, rel=emmo.hasSpatialPart)
                if float(params[2]) > 0:
                    gas_reactant = emmo.GasProductSpecies()
                    atom_symbol = emmo.ChemicalElement(hasSymbolData=params[1])
                    formation_energy = emmo.FormationEnergy()
                    formation_energy_value = emmo.Real(
                        hasNumericalData=formation_energy_value_from_dic
                    )
                    formation_energy.add(
                        formation_energy_value, rel=emmo.hasQuantityValue
                    )
                    coefficient = emmo.StoichiometricCoefficient()
                    coefficient_value = emmo.Real(hasNumericalData=params[2])
                    coefficient.add(coefficient_value, rel=emmo.hasQuantityValue)
                    gas_reactant.add(
                        atom_symbol,
                        coefficient,
                        formation_energy,
                        rel=emmo.hasSpatialDirectPart,
                    )
                    reaction.add(gas_reactant, rel=emmo.hasSpatialPart)

            if params[0] == "sites":
                num_of_sites = int(params[1])

            if params[0] == "neighboring":
                if num_of_sites == 2:
                    neighbors = emmo.Neighboring()
                    for i in range(num_of_sites):
                        neighbor_site = emmo.Real(
                            hasNumericalData=params[1].split("-")[i]
                        )
                        if i == 0:
                            neighbors.add(neighbor_site, rel=emmo.hasSpatialFirst)
                        else:
                            neighbors.add(neighbor_site, rel=emmo.hasSpatialNext)
                    reaction.add(neighbors, rel=emmo.hasSpatialPart)

            if params[0] == "initial":
                record = count
                initial = True

            if initial:
                if count == record:
                    continue
                elif count <= record + num_of_sites:
                    reactant = emmo.ChemicalReactionEquationReactant()
                    atom_symbol = emmo.ChemicalElement(hasSymbolData=params[1])
                    coefficient = emmo.StoichiometricCoefficient()
                    coefficient_value = emmo.Real(hasNumericalData=params[2])
                    coefficient.add(coefficient_value, rel=emmo.hasQuantityValue)
                    reactant.add(
                        atom_symbol, coefficient, rel=emmo.hasSpatialDirectPart
                    )
                    reaction.add(reactant, rel=emmo.hasSpatialPart)
                else:
                    initial = False
                    #

            if params[0] == "final":
                record = count
                final = True

            if final:
                if count == record:
                    continue
                elif count <= record + num_of_sites:
                    reactant = emmo.ChemicalReactionEquationProduct()
                    atom_symbol = emmo.ChemicalElement(hasSymbolData=params[1])
                    coefficient = emmo.StoichiometricCoefficient()
                    coefficient_value = emmo.Real(hasNumericalData=params[2])
                    coefficient.add(coefficient_value, rel=emmo.hasQuantityValue)
                    reactant.add(
                        atom_symbol, coefficient, rel=emmo.hasSpatialDirectPart
                    )
                    reaction.add(reactant, rel=emmo.hasSpatialPart)
                else:
                    final = False
                    #

            if params[0] == "site_types":
                site_type = crystallography.Site()
                atom_symbol = emmo.ChemicalElement(hasSymbolData=params[1])
                site_type.add(atom_symbol, rel=emmo.hasSpatialDirectPart)
                reaction.add(site_type, rel=emmo.hasSpatialPart)
                if num_of_sites == 2:
                    site_type = crystallography.Site()
                    atom_symbol = emmo.ChemicalElement(hasSymbolData=params[2])
                    site_type.add(atom_symbol, rel=emmo.hasSpatialDirectPart)
                    reaction.add(site_type, rel=emmo.hasSpatialPart)

            if params[0] == "pre_expon":
                coefficient = emmo.ArrheniusCoefficient()
                coefficient_value = emmo.Real(hasNumericalData=params[1])
                coefficient.add(coefficient_value, rel=emmo.hasQuantityValue)
                reaction.add(coefficient, rel=emmo.hasSpatialPart)

            if params[0] == "pe_ratio":
                coefficient = emmo.Constant()
                coefficient_value = emmo.Real(hasNumericalData=params[1])
                coefficient.add(coefficient_value, rel=emmo.hasQuantityValue)
                reaction.add(coefficient, rel=emmo.hasSpatialPart)

            if params[0] == "activ_eng":
                coefficient = emmo.ActivationEnergy()
                coefficient_value = emmo.Real(hasNumericalData=params[1])
                coefficient.add(coefficient_value, rel=emmo.hasQuantityValue)
                reaction.add(coefficient, rel=emmo.hasSpatialPart)

            if params[0] == "end_reversible_step":
                mechanism.add(reaction, rel=emmo.hasPart)

    #    pretty_print(mechanism)

    if read_from_file is True:
        dat.close()

    print("read_mechanism: External", filename, "file has been read.")

    return mechanism

def read_cluster_expansion(filename: str, read_from_file: bool = True, cuds=None) -> list:
    """
    Create a [emmo.ClusterExpansion] list from a Zacros-formatted .dat file.

    :param filename: name containing the path and name to file.

    :param return: list of emmo.ClusterExpansion
    """

    if read_from_file is True:
        dat = open(filename, "r")
    elif read_from_file is False:
        dat = (
            filename.splitlines()
        )  # For cases where the file names contains the full string.

    cluster_list = []
    read_lattice_state = False

    for count, line in enumerate(dat):
        params = line.split()
        if params:

            if params[0] == "cluster":
                cluster = cuds or emmo.ClusterExpansion()

            if params[0] == "sites":
                num_of_sites = int(params[1])

            if params[0] == "neighboring":
                if num_of_sites == 2:
                    neighbors = emmo.Neighboring()
                    for i in range(num_of_sites):
                        neighbor_site = emmo.Real(
                            hasNumericalData=params[1].split("-")[i]
                        )
                        if i == 0:
                            neighbors.add(neighbor_site, rel=emmo.hasSpatialFirst)
                        else:
                            neighbors.add(neighbor_site, rel=emmo.hasSpatialNext)
                    cluster.add(neighbors, rel=emmo.hasSpatialPart)

            if params[0] == "lattice_state":
                record = count
                read_lattice_state = True

            if read_lattice_state:
                if count == record:
                    lattice_state = emmo.LatticeState()
                    continue
                elif count <= record + num_of_sites:
                    adsorbed_species = emmo.AdsorbedSpecies()
                    species_number = emmo.Real(hasNumericalData=params[0])
                    atom_symbol = emmo.ChemicalElement(hasSymbolData=params[1])

                    denticity_number = emmo.Real(hasNumericalData=params[2])
                    denticity = emmo.Denticity()
                    denticity.add(denticity_number, rel=emmo.hasQuantityValue)

                    adsorbed_species.add(
                        species_number,
                        atom_symbol,
                        denticity,
                        rel=emmo.hasSpatialDirectPart,
                    )

                    lattice_state.add(adsorbed_species, rel=emmo.hasSpatialDirectPart)
                else:
                    cluster.add(lattice_state, rel=emmo.hasPart)
                    read_lattice_state = False

            if params[0] == "site_types":
                site_type = crystallography.Site()
                atom_symbol = emmo.ChemicalElement(hasSymbolData=params[1])
                site_type.add(atom_symbol, rel=emmo.hasSpatialDirectPart)
                cluster.add(site_type, rel=emmo.hasSpatialPart)
                if num_of_sites == 2:
                    site_type = crystallography.Site()
                    atom_symbol = emmo.ChemicalElement(hasSymbolData=params[2])
                    site_type.add(atom_symbol, rel=emmo.hasSpatialDirectPart)
                    cluster.add(site_type, rel=emmo.hasSpatialPart)

            if params[0] == "cluster_eng":
                cluster_energy = emmo.ClusterEnergy()
                cluster_energy_value = emmo.Real(hasNumericalData=params[1])
                cluster_energy.add(cluster_energy_value, rel=emmo.hasQuantityValue)
                cluster.add(cluster_energy, rel=emmo.hasSpatialPart)

            if params[0] == "end_cluster":
                cluster_list.append(cluster)

    #    for i in cluster_list:
    #        pretty_print(i)

    if read_from_file is True:
        dat.close()

    print("read_cluster_expansion: External", filename, "file has been read.")

    return cluster_list

def print_results(plams_results: AMSResults, result: str):
    """
    Print results of a PLAMS/AMS/ADF calculation.

    :param plams_results: PLAMS result object to be analyzed.

    :param result: String to select what information to print.
    """

    if result == "total_energy":
        total_energy = plams_results.get_energy(unit="eV")
        print("Total electronic energy:", total_energy, "eV")

    if result == "energy_per_go_step":
        print("Total energy per geometry optimization step:")
        nEntries = plams_results.readrkf("History", "nEntries")
        for k in range(nEntries):
            s = "Energy({})".format(k + 1)
            x = plams_results.readrkf("History", s)
            print("Optimization cycle", k + 1, ":", x, "Hartree")
    return

def raise_error(file: str, function: str, type: str, message: str):
    """
    Raise errors according to arguments:

    :param file: Python file where the error occurs.

    :param function: Name of the function calling the error.

    :param type: Error type.

    :param message: String describing the error.
    """

    nl = "\n"

    msg = (
        f"{nl}### ERROR ### {function} function/method"
        f" in {file}:"
        f'{nl}              "{message}"'
    )
    if type == "NameError":
        raise NameError(msg)
    elif type == "ValueError":
        raise ValueError(msg)
    elif type == "NotImplementedError":
        raise NameError(message)

def raise_warning(file: str, function: str, message: str, message2=None):
    """
    Raise warning according to arguments:

    :param file: Python file where the error occurs.

    :param function: Name of the function calling the error.

    :param message: String describing the error.
    """

    nl = "\n"
    if not message2:
        msg = (
            f"{nl}### WARNING! ### {function} function/method"
            f" in {file}:"
            f'{nl}              "{message}"'
        )
    else:
        msg = (
            f"{nl}### WARNING! ### {function} function/method"
            f" in {file}:"
            f'{nl}              "{message}:", {message2}'
        )
    print(msg)
    return


from django.test import TestCase

from datasets.models import RADataEntry
from datasets.serializers import RADataEntrySerializer

class RADataRecordSerializerTestCase(TestCase):
    fixtures = ['set.json', 'record.json', 'entry.json', 'member.json']
    serializer_class = RADataEntrySerializer

    @classmethod
    def setUpTestData(cls):
        cls.instance = RADataEntry.objects.get(number=1, record__number=1, record__dataset__number=1)

    def test_serialize(self):
        serializer = RADataEntrySerializer(instance=self.instance)
        print(serializer.data)

    def test_create(self):
        task_data = {
            'dataset_number': 1,
            'record_number': 1,
            # 'member': '39d6a1e6-e500-46c1-a6ed-acfcc6c485f6',
            'key': 'marked',
            'value': {'has_activities': True}
        }
        serializer = self.serializer_class(data=task_data)
        self.assertTrue(serializer.is_valid(raise_exception=True))
        serializer.save()


from flask import Flask, render_template, request, redirect
from mysqlconnection import connectToMySQL
app = Flask(__name__)

mysql = connectToMySQL('friendsdb')
@app.route('/')
def index():
    result = mysql.query_db("select * from players")
    return render_template("index.html", entry = result)

if __name__=="__main__":
    app.run(debug=True)


# -*- coding : utf-8 -*-

from dateutil.relativedelta import relativedelta
from datetime import datetime, timedelta
from odoo import fields, models, api, tools, _
from odoo.exceptions import Warning, UserError, ValidationError

class custom_partner(models.Model):
    _inherit = "res.partner"

    @api.multi
    def _draft_invoice_total(self):
        account_invoice_report = self.env['account.invoice.report']
        if not self.ids:
            self.total_draft_invoices = 0.0
            return True

    @api.depends('his_birthdate')
    def _calc_age(self):
        for line in self:
            line.age = self.compute_age_from_dates(line.his_birthdate)

    def compute_age_from_dates(self, customer_dob):
        now=datetime.strptime(fields.Datetime.now()[:10], '%Y-%m-%d')
        if (customer_dob):
            dob=datetime.strptime(customer_dob, '%Y-%m-%d')
            delta =relativedelta (now, dob)
            years_months_days = str(delta.year) + "y " + str(delta.month) + "m " + str(delta.day) + "d"
        else:
            years_months_days = "No Birthdate"
        return years_months_days

    #Custom Fields Definitions
    last_name = fields.Char(string='Last Name')
    first_name = fields.Char(string='First Name')
    middle_name = fields.Char(string='Middle Name')
    is_customer = fields.Boolean(string='Customer')
    gender = fields.Selection([('MALE','male'),('FEMALE','female')], string="Gender")
    civil_status = fields.Selection([('SINGLE','single'),('MARRIED','married'),('SEPARATED','separated'),('DIVORCED','divorced'),('WIDOW','widowed')],string="Civil Status")
    his_birthdate = fields.Date('Birthdate')
    age = fields.Char(compute='_calc_age', string='Age', size=50, readonly=True)
    age_yrs = fields.Integer('Age (Years)')
    nationality = fields.Many2one('ejob.nationality',string='Nationality')
    religion = fields.Many2one('ejob.religion', string='Religion')

    barangay = fields.Many2one('ejob.barangays',string='Barangay')
    city_id = fields.Many2one('ejob.cities',string='City')
    zip_id = fields.Many2one('ejob.zip.ref',string='Zip')

    #credit_plus_draft_inv = fields.Float(compute='_compute_credit_plus_draft_inv', string="Total Credit with Draft Invoices", digits=(12,2))
    #total_draft_invoices = fields.Float(compute='_draft_invoice_total', string="Total Draft Invoices", digits=(14,2))

    total_draft_charges = fields.Float(compute='_draft_charges_total', string="Total Draft Charges", digits=(14,2))
    total_charges_bal = fields.Float(compute='_draft_charges_bal_total', string="Current Balance", digits=(14,2))
    date_last_visit = fields.Datetime('Last Visit',default=lambda self: fields.datetime.now())

    @api.onchange('ref','last_name','first_name','middle_name')
    def customer_name_change(self):
        vals = {}
        if self.customer:
            name = ''
            if self.ref:
                name += '[' + self.ref + '] '
            if self.last_name:
                name += self.last_name + ', '
            if self.first_name:
                name += self.first_name + ' '
            if self.middle_name:
                name += self.middle_name

            vals.update({'name': name.upper()})
        self.update(vals)

    @api.onchange('barangay')
    def customer_barangay_change(self):
        if self.barangay:
            if self.barangay.city_id:
                vals = {'city_id':self.barangay.city_id.id}
                self.update(vals)

    @api.onchange('city_id')
    def customer_city_id_change(self):
        if self.city_id:
            vals = {'city':self.city_id.name}
            self.update(vals)

    @api.onchange('zip_id')
    def customer_zip_id_change(self):
        if self.zip_id:
            vals = {'zip':self.zip_id.name}
            if self.zip_id.city_id:
                vals.update({'city_id':self.zip_id.city_id.id})
            if self.zip_id.state_id:
                vals.update({'state_id':self.zip_id.state_id.id})
            if self.zip_id.country_id:
                vals.update({'country_id':self.zip_id.country_id.id})
            self.update(vals)

    @api.multi
    def gen_id_number(self):
        for rec in self:
            if not rec.ref or rec.ref == '':
                ref = self.env['ir.sequence'].next_by_code('ejob.id.number')
                name = '[' + ref + '] '
                if rec.last_name:
                    name += rec.last_name + ', '
                if rec.first_name:
                    name += rec.first_name + ' '
                if rec.middle_name:
                    name += rec.middle_name
                rec.update({'ref':ref, 'name': name.upper()})

    @api.one
    def _compute_credit_plus_draft_inv(self):
        self.credit_plus_draft_inv = self.credit + self.total_draft_invoices

    @api.multi
    def _draft_charges_total(self):
        if not self.ids:
            self.total_draft_pos_orders = 0.0
            return True

        charges_obj = self.env['e.job.orders']

        #user_currency_id = self.env.user.company_id.currency_id.id
        for partner in self:
            total = 0
            charges = charges_obj.search([('partner_id', '=', partner.id), ('state', '=', 'bill'), ('company_id', '=', self.env.user.company_id.id)])
            for charge in charges:
                total += charge.total_services_fee

            partner.total_draft_charges = total

    @api.multi
    def _draft_charges_bal_total(self):
        if not self.ids:
            self.total_charges_bal = 0.0
            return True

        payments_obj = self.env['ejob.cashier']

        #user_currency_id = self.env.user.company_id.currency_id.id
        for partner in self:
            total = 0
            charges = payments_obj.search([('partner_id', '=', partner.id), ('state', 'in', ('inv','partial')), ('company_id', '=', self.env.user.company_id.id)])
            for charge in charges:
                total += charge.balance

            partner.total_charges_bal = total

    @api.multi
    def manage_charges(self):
        for rec in self:
            partner_id = rec.id
            pricelist = rec.property_product_pricelist
            currency = self.env.user.company_id and self.env.user.company_id.currency_id or False
            #Check if the customer have a current (NEW) Charge
            charge_obj=self.env['e.job.orders']
            new_charges = charge_obj.search([('partner_id','=',partner_id),('state','in',('new','submit'))])
            if new_charges:
                #A NEW Charge Record is already created, get charge_id
                current_charge_id = new_charges[0].id

            else:

                #No NEW charge, create a new Charge Record
                srv_ref = self.env['ir.sequence'].next_by_code('e.job.orders')
                #srv_line_ref = self.env['ir.sequence'].next_by_code('eHC.charge.line.number')
                charge = charge_obj.create({'name': srv_ref, 'partner_id':partner_id, 'jo_date':fields.Datetime.now(), 'pricelist_id':pricelist.id})
                current_charge_id = charge.id

            #Display Charge Form
            imd = self.env['ir.model.data']
            action = imd.xmlid_to_object('ejob_order.action_e_job_orders')
            tree_view_id = imd.xmlid_to_res_id('ejob_order.e_job_order_tree_view')
            form_view_id = imd.xmlid_to_res_id('ejob_order.e_job_order_form_view')

            result = {
                'name': action.name,
                'help': action.help,
                'type': action.type,
                'views': [[form_view_id, 'form'],[tree_view_id, 'tree']],
                'target': action.target,
                #'context': action.context,
                'res_model': action.res_model,
                'res_id': current_charge_id,
            }
            result['domain'] = "[('id','=',%s)]" % current_charge_id
            return result

    @api.multi
    def manage_payments(self):
        for rec in self:
            if rec.total_draft_charges > 0:
                partner_id = rec.id
                orders_obj = self.env['e.job.orders']
                #Check if the customer have a current (NEW) payment record
                payment_obj = self.env['ejob.cashier']
                new_payments = payment_obj.search([('partner_id','=',partner_id),('state','=',('new','inv'))])
                if new_payments:
                    current_payment_id = new_payments[0].id
                    #A NEW payment is already created, let's check for additional charges
                    orders = orders_obj.search([('partner_id', '=', partner_id),('state', '=', 'bill'), ('company_id', '=', self.env.user.company_id.id)])
                    for order in orders:
                        order.payment_id = current_payment_id
                else:
                    orders = orders_obj.search([('partner_id', '=', partner_id), ('state', '=', 'bill'), ('company_id', '=', self.env.user.company_id.id)])
                    if orders:
                        #payment_ref = self.env['ir.sequence'].next_by_code('eAC.payment.id')
                        payment_ref = 'NEW'
                        pricelist_id = rec.property_product_pricelist and rec.property_product_pricelist.id or orders[0].pricelist_id.id
                        payment_vals = {
                            'name': payment_ref,
                            'partner_id': partner_id,
                            'payment_date': fields.Datetime.now(),
                            'pricelist_id': pricelist_id,
                            'payment_type': 'Cash',
                        }
                        payment_id = payment_obj.create(payment_vals)
                        current_payment_id = payment_id.id
                        for order in orders:
                            order.payment_id = current_payment_id

                    else:
                        raise UserError('There are no Charges entered for this customer.')
                    #return {'type': 'ir.actions.act_window_close'}
                #update date last visit
                rec.date_last_visit = fields.datetime.now()
                #Open Payment Entry Form
                imd = self.env['ir.model.data']
                action = imd.xmlid_to_object('ejob_order.action_ejob_cashier')
                form_view_id = imd.xmlid_to_res_id('ejob_order.ejob_soa_form_view')

                result = {
                    'name': action.name,
                    'help': action.help,
                    'type': action.type,
                    'views': [[form_view_id, 'form']],
                    'target': action.target,
                    #'context': action.context,
                    'res_model': action.res_model,
                    'res_id': current_payment_id,
                }
                result['domain'] = "[('id','=',%s)]" % current_payment_id
                return result
            else:
                raise UserError('Payment Record Creation Failed! No job orders entered for this customer.')


from ConfigSpace import Constant, UniformFloatHyperparameter, UniformIntegerHyperparameter, CategoricalHyperparameter, \
    EqualsCondition
from skorch.callbacks import LRScheduler
from torch import nn
from torch.optim import AdamW, Adam

from .skorch_base import SkorchBaseModel

class MLP(nn.Module):

    def __init__(self, metadata, num_classes, activation_ff, dropout, n_layers, d_model):
        super(MLP, self).__init__()
        input_dim = metadata.input_shape.max_sequence_len * metadata.input_shape.channels * metadata.input_shape.height * metadata.input_shape.width
        layers = []
        if activation_ff == 'gelu':
            activation_class = nn.GELU
        elif activation_ff == 'relu':
            activation_class = nn.ReLU
        else:
            raise ValueError("Unknown activation function.")

        layer_dim = [input_dim] + [d_model] * (n_layers - 1) + [num_classes]
        print("initializing mlp with architecture", layer_dim)
        for i in range(n_layers):
            if i > 0:
                layers.append(nn.Dropout(dropout))
            layers.append(nn.Linear(layer_dim[i], layer_dim[i + 1]))
            if i < n_layers - 1:
                layers.append(activation_class())

        self.net = nn.Sequential(*layers)

    def forward(self, x):
        x = x.reshape((x.shape[0], -1))
        return self.net(x)

class MLP_Model(SkorchBaseModel):
    def __init__(self, metadata, input_cfg=None, seed=None):
        super().__init__(metadata, input_cfg, seed)

        if self.cfg['optimizer'] == 'Adam':
            self.skorch_kwargs['optimizer'] = Adam
        elif self.cfg['optimizer'] == 'AdamW':
            self.skorch_kwargs['optimizer'] = AdamW
            self.skorch_kwargs['optimizer__weight_decay'] = self.cfg['weight_decay']

        if self.cfg['lr_scheduler']:
            self.callbacks.append(
                self.callbacks.append(LRScheduler(policy='CosineAnnealingLR', T_max=self.cfg['max_epochs'])))

        self.model = self.model_class(
            MLP,
            max_epochs=self.cfg['max_epochs'],
            lr=self.cfg['lr'],
            # problem dimensions
            module__metadata=self.metadata,
            module__num_classes=self.output_shape[0],
            # model parameters
            module__d_model=self.cfg['d_model'],
            module__n_layers=self.cfg['n_layers'],
            module__dropout=self.cfg['dropout'],
            module__activation_ff=self.cfg['activation_ff'],
            callbacks=self.callbacks,
            **self.skorch_kwargs,
        )

    @classmethod
    def get_config_space(cls):
        cs = super().get_config_space()
        name = Constant(name="model_name", value="mlp")
        lr = UniformFloatHyperparameter(
            name="lr", lower=1e-5, upper=1e-2, default_value=3e-3, log=True
        )
        optimizer = CategoricalHyperparameter(
            name="optimizer", choices=['Adam', 'AdamW'], default_value='AdamW'
        )
        lr_scheduler = CategoricalHyperparameter(
            name="lr_scheduler", choices=[True, False], default_value=True
        )
        weight_decay = UniformFloatHyperparameter(
            name="weight_decay", lower=1e-7, upper=1e-1, default_value=0.03, log=True
        )
        dropout = UniformFloatHyperparameter(
            name="dropout", lower=0., upper=0.5, default_value=0.1, log=False
        )
        max_epochs = UniformIntegerHyperparameter(
            name="max_epochs", lower=1, upper=300, default_value=200, log=False
        )
        n_layers = UniformIntegerHyperparameter(  # 6 for submission 10 for paper
            name="n_layers", lower=1, upper=10, default_value=4, log=False
        )
        d_model = UniformIntegerHyperparameter(  # 104 for submission 128 for paper
            name="d_model", lower=32, upper=1024, default_value=64, log=False
        )
        activation_ff = CategoricalHyperparameter(
            name="activation_ff", choices=['gelu', 'relu'], default_value='gelu'
        )
        cs.add_hyperparameters(
            [name, lr, max_epochs, d_model, dropout, n_layers, weight_decay, lr_scheduler, optimizer, activation_ff])
        weight_decay_condition = EqualsCondition(weight_decay, optimizer, 'AdamW')
        cs.add_conditions([weight_decay_condition])
        return cs

    @classmethod
    def is_applicable(cls, metadata):
        return metadata.input_shape.width * metadata.input_shape.height * metadata.input_shape.max_sequence_len * metadata.input_shape.channels < 1024


"""This submodule handles the cli-like utilities such as the parser

"""

import argparse

class RadolanParser(argparse.ArgumentParser):
    def __init__(self):
        super().__init__(
            description="Training U-Net model for segmentation of RADOLAN precipitation images"
        )
        self.add_arguments(self)

    @staticmethod
    def add_arguments(parent_parser, argument_group=False):
        if argument_group:
            parser = parent_parser.add_argument_group("Radolan Parser")
        else:
            parser = parent_parser
        parser.add_argument(
            "--device",
            type=str,
            default="cuda:0",
            help="device for training (default: cuda:0)",
        )
        parser.add_argument(
            "--workers",
            type=int,
            default=4,
            help="number of workers for data loading (default: 4)",
        )
        parser.add_argument(
            "--weights", type=str, default="./weights", help="folder to save weights"
        )
        parser.add_argument(
            "--logs", type=str, default="./logs", help="folder to save logs"
        )
        parser.add_argument(
            "--images", type=str, default="../Radolan", help="root folder with images"
        )
        parser.add_argument(
            "--save",
            type=bool,
            default=False,
            help="Save the Unet at the end of training. Path is RADOLAN_PATH/Models/RUN-TIMESTAMP"
        )
        parser.add_argument(
            "--verbose",
            type=bool,
            default=False,
            help="Verbose setting. Either true or false."
        )
        if argument_group:
            return parent_parser
        else:
            return parser

class DeprecatedRadolanParser(argparse.ArgumentParser):
    def __init__(self):
        super().__init__(
            description="OldArgparseArguments"
        )

        self.add_argument(
            "--batch-size",
            type=int,
            default=6,
            help="input batch size for training (default: 6)",
        )
        self.add_argument(
            "--epochs",
            type=int,
            default=100,
            help="number of epochs to train (default: 100)",
        )
        self.add_argument(
            "--lr",
            type=float,
            default=0.001,
            help="initial learning rate (default: 0.001)",
        )
        self.add_argument(
            "--device",
            type=str,
            default="cuda:0",
            help="device for training (default: cuda:0)",
        )
        self.add_argument(
            "--workers",
            type=int,
            default=4,
            help="number of workers for data loading (default: 4)",
        )
        self.add_argument(
            "--vis-images",
            type=int,
            default=200,
            help="number of visualization images to save in log file (default: 200)",
        )
        self.add_argument(
            "--vis-freq",
            type=int,
            default=10,
            help="frequency of saving images to log file (default: 10)",
        )
        self.add_argument(
            "--weights", type=str, default="./weights", help="folder to save weights"
        )
        self.add_argument(
            "--logs", type=str, default="./logs", help="folder to save logs"
        )
        self.add_argument(
            "--images", type=str, default="../Radolan", help="root folder with images"
        )
        self.add_argument(
            "--image-size",
            type=int,
            default=256,
            help="target input image size (default: 256)",
        )
        self.add_argument(
            "--aug-scale",
            type=int,
            default=0.05,
            help="scale factor range for augmentation (default: 0.05)",
        )
        self.add_argument(
            "--save",
            type=bool,
            default=False,
            help="Save the Unet at the end of training. Path is RADOLAN_PATH/Models/RUN-TIMESTAMP"
        )
        self.add_argument(
            "--cat",
            type=bool,
            default=False,
            help="Whether the skips should be implemented with torch.cat or with a simple sum. "
                 "False (default) means sum."
        )
        self.add_argument(
            "--verbose",
            type=bool,
            default=False,
            help="Verbose setting. Either true or false."
        )
        self.add_argument(
            "--filename",
            type=str,
            default='no_name.h5',
            help='Name for the h5 file.'
        )


from django.shortcuts import render, redirect
from . import models
from . import forms
from django.contrib.auth.decorators import permission_required

# Create your views here.

def Product(request, _product_id):
    category_parents = models.Category_parent.objects.order_by("order")
    product = models.Product.objects.get(product_id = _product_id)
    context = {
        "category_parents": category_parents,
        "product": product
    }
    return render(request, "products/Product.html", context)

def Popular(request):
    category_parents = models.Category_parent.objects.order_by("order")
    products = models.Product.objects.order_by("views")[::-1]
    context = {
        "products": products,
        "category_parents": category_parents
    }
    return render(request, "products/PopularProducts.html", context)

@permission_required("is_superuser")
def List(request):
    _get = request.GET
    if ("search" in _get):
        products = models.Product.objects.filter(nombre__contains=_get["search"])
    else:
        products = models.Product.objects
    products = products.order_by("views")[::-1]
    context = {
        "products": products,
    }
    return render(request, "products/List.html", context)

@permission_required("is_superuser")
def Edit(request, _product_id):
    if (request.method == "GET"):
        product = models.Product.objects.get(product_id = _product_id)
        data = {
            "product_id": product.product_id,
            "category_id": product.category_id,
            "nombre": product.nombre,
            "imagen_principal": product.imagen_principal,
            "precio": product.precio,
            "stock": product.stock,
            "description": product.description,
            "specs": product.specs,
            "views": product.views,
            "garantia": product.garantia,
            }
        form = forms.Product(initial=data)
        context = {
            "product": product,
            "form": form
        }
        return render(request, "products/EditProduct.html", context)
    elif (request.method == "POST"):
        _instance = models.Product.objects.get(product_id = _product_id)
        _form = forms.Product(request.POST, request.FILES)

        if (request.POST['action'] == "Update"):
            if (_form.is_valid()):
                if (_form.cleaned_data['imagen_principal']):
                    _instance.imagen_principal = _form.cleaned_data["imagen_principal"]
                _instance.category_id = _form.cleaned_data["category_id"]
                _instance.nombre = _form.cleaned_data["nombre"]
                _instance.precio = _form.cleaned_data["precio"]
                _instance.stock = _form.cleaned_data["stock"]
                _instance.description = _form.cleaned_data["description"]
                _instance.specs = _form.cleaned_data["specs"]
                _instance.garantia = _form.cleaned_data["garantia"]
                _instance.save()
        elif (request.POST['action'] == "Delete"):
            _instance.delete()

        return redirect("products:list")

@permission_required("is_superuser")
def Add(request):
    if (request.method == "GET"):
        data = {
            "product_id": 0,
            "views": 0,
            }
        _form = forms.Product(initial=data)

    elif (request.method == "POST"):
        _instance = models.Product()
        _form = forms.Product(request.POST, request.FILES)

        if (_form.is_valid()):
            if (_form.cleaned_data['imagen_principal']):
                _instance.imagen_principal = _form.cleaned_data["imagen_principal"]
            _instance.category_id = _form.cleaned_data["category_id"]
            _instance.nombre = _form.cleaned_data["nombre"]
            _instance.precio = _form.cleaned_data["precio"]
            _instance.stock = _form.cleaned_data["stock"]
            _instance.description = _form.cleaned_data["description"]
            _instance.specs = _form.cleaned_data["specs"]
            _instance.garantia = _form.cleaned_data["garantia"]
            _instance.save()

            return redirect("products:list")
    context = {
            "form": _form
        }
    return render(request, "products/AddProduct.html", context)


from loaders.dataloader_mnist import load_mnist
from models.model_vae import VariationalInference, VariationalAutoencoder
from models.distributions import ReparameterizedDiagonalGaussian

import torch
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
sns.set_style("whitegrid")

### TEST DISTRIBUTION

def test_normal_distribution():
    """a few safety checks for your implementation"""
    N = 1000000
    ones = torch.ones(torch.Size((N,)))
    mu = 1.224 * ones
    sigma = 0.689 * ones
    dist = ReparameterizedDiagonalGaussian(mu, sigma.log())
    z = dist.sample()

    # Expected value E[N(0, 1)] = 0
    expected_z = z.mean()
    diff = (expected_z - mu.mean())**2
    assert diff < 1e-3, f"diff = {diff}, expected_z = {expected_z}"

    # Variance E[z**2 - E[z]**2]
    var_z = (z**2 - expected_z**2).mean()
    diff = (var_z - sigma.pow(2).mean())**2
    assert diff < 1e-3, f"diff = {diff}, var_z = {var_z}"

    # log p(z)
    from torch.distributions import Normal
    base = Normal(loc=mu, scale=sigma)
    diff = ((base.log_prob(z) - dist.log_prob(z))**2).mean()
    assert diff < 1e-3, f"diff = {diff}"

test_normal_distribution()

n_samples = 10000
mu = torch.tensor([[0, 1]])
sigma = torch.tensor([[0.5 , 3]])
ones = torch.ones((1000,2))
p = ReparameterizedDiagonalGaussian(mu=mu*ones, log_sigma=(sigma*ones).log())
samples = p.sample()
data = pd.DataFrame({"x": samples[:, 0], "y": samples[:, 1]})
g = sns.jointplot(
    data=data,
    x="x",y="y",
    kind="hex",
    ratio=10
)
plt.subplots_adjust(top=0.9)
g.fig.suptitle(r"$\mathcal{N}(\mathbf{y} \mid \mu, \sigma)$")
plt.show()

### TEST LOADER
classes = [3, 7]

path_train = "~/Documents/PhD/courses/deep_learning/assignments/7_Unsupervised"
path_test  = "~/Documents/PhD/courses/deep_learning/assignments/7_Unsupervised"

train_loader, test_loader = load_mnist(path_train, path_test, classes)

#plot a few MNIST examples
f, axarr = plt.subplots(4, 16, figsize=(16, 4))

# Load a batch of images into memory
images, labels = next(iter(train_loader))

for i, ax in enumerate(axarr.flat):
    ax.imshow(images[i].view(28, 28), cmap="binary_r")
    ax.axis('off')

plt.suptitle('MNIST handwritten digits')
plt.show()

vi = VariationalInference(beta=1)
latent_features = 2
vae = VariationalAutoencoder(images[0].shape, latent_features)
loss, diagnostics, outputs = vi(vae, images)
print(f"{'loss':6} | mean = {loss:10.3f}, shape: {list(loss.shape)}")
for key, tensor in diagnostics.items():
    print(f"{key:6} | mean = {tensor.mean():10.3f}, shape: {list(tensor.shape)}")


import xml.etree.ElementTree
import xml_share as xshare
import os

class xml_share_repository:
    """
    Stores the information read from xml for easy access.
    """
    xml_shares = []

    def built_up_repository(self, share_file=os.path.dirname(__file__) + '/shares.xml'):
        """
        Fill the repository at start.
        """
        xml_shares_root = xml.etree.ElementTree.parse(share_file).getroot()

        for xml_share_in_file in xml_shares_root.findall('share'):
            xml_share = xshare.xml_share()

            for xml_name_temp in xml_share_in_file.findall('name'):
                if "&" in xml_name_temp:
                    xml_name_temp.replace('&', '&')
                else:
                    xml_share.xml_name = str(xml_name_temp.text)

            for xml_units_temp in xml_share_in_file.findall('units'):
                xml_share.xml_units = str(xml_units_temp.text)

            for xml_buy_price_temp in xml_share_in_file.findall('buyPrice'):
                xml_share.xml_buy_price = str(xml_buy_price_temp.text)

            for xml_buy_date_temp in xml_share_in_file.findall('buyDate'):
                xml_share.xml_buy_date = str(xml_buy_date_temp.text)

            for xml_trailing_stop_date_temp in xml_share_in_file.findall('trailingStopDate'):
                xml_share.xml_trailing_stop_date = str(xml_trailing_stop_date_temp.text)

            for xml_trailing_stop_percentage_temp in xml_share_in_file.findall('trailingStopPercentage'):
                xml_share.xml_trailing_stop_percentage = str(xml_trailing_stop_percentage_temp.text)

            for xml_trailing_stop_absolute_temp in xml_share_in_file.findall('trailingStopAbsolute'):
                xml_share.xml_trailing_stop_absolute = str(xml_trailing_stop_absolute_temp.text)

            for xml_trailing_stop_init_temp in xml_share_in_file.findall('trailingStopInit'):
                xml_share.xml_trailing_stop_init = str(xml_trailing_stop_init_temp.text)

            if len(xml_share_in_file.findall('stopExpiration')) < 1:
                xml_share.xml_stop_expiration = ''

            for xml_stop_expiration_temp in xml_share_in_file.findall('stopExpiration'):
                xml_share.xml_stop_expiration = str(xml_stop_expiration_temp.text)

            positionToInsert = 0

            for entry in self.xml_shares:
                if entry.xml_name < xml_share.xml_name:
                    positionToInsert = positionToInsert + 1
                else:
                    break

            self.xml_shares.insert(positionToInsert, xml_share)


######################################################
# 8,330: Machine Learning (MiQEF)
# Assignment 3: kNN Algorithm
# Niklas Leander Kampe | 16-611-618
######################################################

# Utility Libraries
import math
import warnings
import numpy as np
import pandas as pd
from sklearn.model_selection import KFold
pd.set_option('display.max_columns', None)
warnings.filterwarnings('ignore')

def accuracy(data):
    n = len(data)
    correct = 0
    for i in range(n):
        if data.iloc[i,0] == data.iloc[i,1]:
            correct += 1
        else:
            continue
    acc = round((correct/n) * 100, 2)
    return acc

def data():
    df = pd.read_csv('Data/diabetes.csv')
    df = df[['Outcome', 'Glucose']]
    df.rename(columns = {'Outcome': 'y', 'Glucose': 'x'}, inplace = True)
    y = df['y']
    X = df['x']
    print("\n############ Raw Data ############\n")
    print(df)
    return y, X, df

class kNN_Algorithm():
    def __init__(self, y, X, df, n_splits):
        self.y = y
        self.X = X
        self.df = df
        self.n_splits = n_splits
    # Function to Split Train/Test Data for Cross-Validation
    def split(self):
        data = self.df
        kf_n = KFold(n_splits = self.n_splits, shuffle = False)
        split = list(kf_n.split(data))
        data_train = []
        data_test = []
        test_indices = {n: "" for n in range(1,len(split)+1)}
        for i in range(0, len(split)):
            train_index = data.iloc[split[i][0]]
            test_index = data.iloc[split[i][1]]
            data_train.append(train_index)
            data_test.append(test_index)
            test_indices[i+1] = str(test_index.index[0]) + "-" + str(test_index.index[-1])
        print("\n############ Test Indices ############\n")
        for i in range(0, len(split)):
            print(f'Split {i+1}: {test_indices[i+1]}')
        return data_train, data_test
    # Utility Function to Sort Values in a Data Frame Ascending
    def sort(self):
        data_train, data_test = self.split()
        for i in range(0, len(data_train)):
            data_train[i].sort_values(by = 'x', axis = 0, inplace = True)
            data_test[i].sort_values(by = 'x', axis = 0, inplace = True)
        return data_train, data_test
    # kNN Classification Algorithm based on Cross-Validation
    def classifier(self):
        data_train, data_test = self.sort()
        k_benchmark = math.floor(np.sqrt(len(data_train[0])))
        if k_benchmark % 2 == 0:
            k_benchmark = k_benchmark - 1
        k_arr = np.arange(3, k_benchmark * 3, 4)
        if k_benchmark not in k_arr:
            k_arr = np.append(k_arr, k_benchmark)
        accuracies_dict = {k: 0 for k in k_arr}
        y_preds = []
        for k in k_arr:
            accuracies = []
            for i in range(0, len(data_train)):
                y_train = data_train[i]['y'].to_numpy()
                X_train = data_train[i]['x'].to_numpy()
                X_test = data_test[i]['x'].to_numpy()
                y_pred = []
                for obs in X_test:
                    closest = min(X_train, key = lambda x: abs(x - obs))
                    closest_index = list(X_train).index(closest)
                    lower_bound = closest_index - math.floor(k/2)
                    upper_bound = closest_index + math.floor(k/2)
                    k_closest = y_train[lower_bound:upper_bound]
                    if np.mean(k_closest) >= 0.5:
                        y_pred.append(1.0)
                    else:
                        y_pred.append(0.0)
                y_preds.append(y_pred)
                data_pred = pd.DataFrame(columns = ['y_pred', 'y', 'x'])
                data_pred['y_pred'] = y_pred
                data_pred['y'] = data_test[i]['y'].to_numpy()
                data_pred['x'] = X_test
                acc = accuracy(data_pred)
                accuracies.append(acc)
            accuracies_dict[k] = round(np.mean(accuracies), 2)
        print("\n############ Classification Accuracies ############\n")
        for k in k_arr:
            print(f"k={k}: {accuracies_dict[k]}%")
        print("\n############ Best Classifier ############\n")
        k_optimal = max(accuracies_dict, key = accuracies_dict.get)
        print(f"Optimal k: {k_optimal}")
        print(f"Optimal Accuracy: {accuracies_dict[k_optimal]}%")
        print("-----")
        print(f"Benchmark k: {k_benchmark}")
        print(f"Behcmark Accuracy: {accuracies_dict[k_benchmark]}%")

if __name__ == '__main__':
    n_splits = 5
    y, X, df = data()
    kNN_Algorithm(y, X, df, n_splits).classifier()

Assignment 3/kNN Algorithm - Niklas Kampe.py


def app():
    import pandas as pd
    from operator import itemgetter
    import glob
    import streamlit as st

    pd.options.mode.chained_assignment = None
    st.header('Master List of LPI')
    files = glob.glob('*.xlsx')

    appended_data = []
    leagueList = []
    for file in files:
        leagueName = file.split(".xlsx")[0]
        leagueList.append(leagueName)
        df = pd.read_excel(file, sheet_name="Louie Power Index")
        df["League"] = leagueName
        appended_data.append(df)

    dfFINAL = pd.concat(appended_data)
    dfFINAL = dfFINAL.iloc[: , 1:]
    # dfFINAL.index += 1
    df1 = dfFINAL.sort_values(by=['Louie Power Index (LPI)'], ascending=False)

    df3 = df1.reset_index(drop=True).style.background_gradient(subset=['Louie Power Index (LPI)']).apply(lambda x: ["background-color: purple; color: white"
                            if i == leagueList[0]
                            else "" for i in x], axis = 1).apply(lambda x: ["background-color: skyblue" if i == leagueList[1]
                            else "" for i in x], axis = 1).apply(lambda x: ["background-color: goldenrod" if i == leagueList[2]
                            else "" for i in x], axis = 1).apply(lambda x: ["background-color: gray; color: white" if i == leagueList[3]
                            else "" for i in x], axis = 1).apply(lambda x: ["background-color: green; color: white" if i == leagueList[4]
                            else "" for i in x], axis = 1).apply(lambda x: ["background-color: maroon; color: white" if i == leagueList[5]
                            else "" for i in x], axis = 1)
    st.dataframe(df3, height=2150)

app()


'''
Created on Feb 13, 2014

@author: theo
'''
import os
from django.core.management.base import BaseCommand, CommandError
from acacia.data.models import Project, ProjectLocatie, MeetLocatie, Datasource, Parameter, Series
from django.conf import settings

class Command(BaseCommand):
    args = ''
    help = 'Deletes unused files from upload area'

    def handle(self, *args, **options):
        # get all files in use
        inuse = ([p.image.path for p in Project.objects.exclude(image='')])
        inuse.extend([l.image.path for l in ProjectLocatie.objects.exclude(image='')])
        inuse.extend([m.image.path for m in MeetLocatie.objects.exclude(image='')])
        inuse.extend([f.filepath() for ds in Datasource.objects.all() for f in ds.sourcefiles.all()])
        inuse.extend([p.thumbpath() for p in Parameter.objects.exclude(thumbnail='')])
        inuse.extend([s.thumbpath() for s in Series.objects.exclude(thumbnail='')])

        count = 0
        for path, folders, files in os.walk(settings.MEDIA_ROOT):
            for f in files:
                name = os.path.join(path,f)
                if not name in inuse:
                    self.stdout.write('Deleting %s\n' % name)
                    try:
                        os.remove(name)
                        count = count+1
                    except Exception as e:
                        self.stdout.write('Error deleting %s: %s\n' % (name,e))
                else:
                    #self.stdout.write('Keeping %s\n' % name)
                    pass

        self.stdout.write('%d files deleted\n' % count)


from django.conf.urls import url

from . import views

from django.contrib.auth import views as auth_views

urlpatterns = [
    url(r'^$', views.index, name="index"),
    url(r'^login/$', views.authentication, name="authentication"),
    url(r'^logout$', auth_views.logout, {'next_page': '/'}, name="logout", ),
    url(r'^report/$', views.report, name='report'),
    url(r'^reportnivelsalud/$', views.reportnivelsalud, name='reportnivelsalud'),
    url(r'^reportfactorescondicionantes/$', views.reportfactorescondicionantes, name='reportfactorescondicionantes'),
    url(r'^NSr1/$', views.NSr1, name='NSr1'),
    url(r'^NSr2/$', views.NSr2, name='NSr2'),
    url(r'^NSr3/$', views.NSr3, name='NSr3'),
    url(r'^NSr4/$', views.NSr4, name='NSr4'),
    url(r'^NSr5/$', views.NSr5, name='NSr5'),
    url(r'^NSr6/$', views.NSr6, name='NSr6'),
    url(r'^NSr7/$', views.NSr7, name='NSr7'),
    url(r'^NSr8/$', views.NSr8, name='NSr8'),
    url(r'^FCr1/$', views.FCr1, name='FCr1'),
    url(r'^FCr2/$', views.FCr2, name='FCr2'),
    url(r'^FCr3/$', views.FCr3, name='FCr3'),
    url(r'^NSf1/$', views.NSf1, name='NSf1'),
    url(r'^FCf1/$', views.FCf1, name='FCf1'),
    url(r'^ingresar/$', views.ingresardatos, name='ingresar'),

    # url(r'^$', views.login),
    # url(r'^$', views.index, name='index'),
    # url(r'^index/$', views.index, name='index'),
    # url(r'^factorescondicionantes/$', views.factorescondicionantes, name='factorescondicionantes'),

]


import os
from datetime import date

import dateutil

from src.examples.program.traccar.config.config import Config
from src.examples.program.traccar.config.keys import Keys

class MediaManager:

    _LOGGER = "LoggerFactory.getLogger(MediaManager.class)"

    def __init__(self, config):

        self._path = None

        self._path = config.getString(Keys.MEDIA_PATH)

    def _createFile(self, uniqueId, name):
        filePath = os.O_PATH.get(self._path, uniqueId, name)
        directoryPath = filePath.getParent()
        if directoryPath is not None:
            filePath.createDirectories(directoryPath)
        return filePath.toFile()

    def createFileStream(self, uniqueId, name, extension):
        return "FileOutputStream(self._createFile(uniqueId, name + \".\" + extension))"

    def writeFile(self, uniqueId, buf, extension):
        if self._path is not None:
            size = buf.readableBytes()
            name = (dateutil.formatDate("yyyyMMddHHmmss")).format(date()) + "." + extension
            try:
                with "FileOutputStream(createFile(uniqueId, name))" as output, output.getChannel() as fileChannel:
                    byteBuffer = buf.nioBuffer()
                    written = 0
                    while written < size:
                        written += fileChannel.write(byteBuffer)
                    fileChannel.force(False)
                    return name
            except Exception as e:
                MediaManager._LOGGER.warn("Save media file error", e)
        return None


import numpy as np
import matplotlib.pyplot as plt

def test_polyreg_univariate():
    '''
        Test polynomial regression
    '''

    # load the data
    filepath = "./test.txt"
    df = pd.read_csv(filepath, header=None)

    X = df[df.columns[:-1]]
    y = df[df.columns[-1]]

    # regression with degree = d
    d = 8
    model = PolynomialRegression(degree=d, tuneLambda=True, regLambdaValues= [ 0.001, 0.003, 0.001])
    model.fit(X, y)

    # output predictions
    xpoints = pd.DataFrame(np.linspace(np.max(X), np.min(X), 100))
    ypoints = model.predict(xpoints)
    # plot curve
    plt.figure()
    plt.plot(X, y, 'rx')
    plt.title('PolyRegression with d = '+str(d))
    plt.plot(xpoints, ypoints, 'b-')
    plt.xlabel('X')
    plt.ylabel('Y')
    plt.show()
    print(model.theta)

    #  # # Compute the closed form solution
    # n,d = X.shape
    # X = np.asmatrix(X.to_numpy())
    # X = np.c_[np.ones((n,1)), X]     # Add a row of ones for the bias term
    # y = np.asmatrix(y.to_numpy())

    # y = y.reshape(n,1)
    # thetaClosedForm = inv(X.T*X)*X.T*y
    # print("thetaClosedForm: ", thetaClosedForm.T)

test_polyreg_univariate()


from datetime import date
from typing import List, Optional

from pydantic import BaseModel

class UserBase(BaseModel):
    email: str
    username: str
    tanggal_lahir: date
    alamat: str
    status: Optional[str]
    no_telp: str

class UserUpdate(BaseModel):
    email: Optional[str]
    username: Optional[str]
    tanggal_lahir: Optional[date]
    alamat: Optional[str]
    status: Optional[str]
    no_telp: Optional[str]

class UserCreate(UserBase):
    password: str

class User(UserBase):
    id: int

    class Config:
        orm_mode = True

class ProductBase(BaseModel):
    nama_product: Optional[str]
    harga: Optional[int]
    deskripsi: Optional[str]
    stok: Optional[int]
    gambar: Optional[str]

class Product(ProductBase):
    product_id: int

    class Config:
        orm_mode = True

class Token(BaseModel):
    access_token: str
    token_type: str

class TokenData(BaseModel):
    username: Optional[str] = None
    email: Optional[str] = None

class MemberBase(BaseModel):
    member_id: int
    poin: int


from django.conf.urls import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
from django.http import HttpResponseRedirect

admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'tsune.views.home', name='home'),
    # url(r'^tsune/', include('tsune.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
    # Uncomment the next line to enable the admin:go.contrib.admindocs.urls')),

    url(r'^$', lambda x: HttpResponseRedirect('/cardbox/')),  # redirect to /cardbox/
    url(r'^admin/', include(admin.site.urls)),
    url(r'^cardbox/', include('deckglue.deckurls', namespace="deck")),
    url(r'^user/', include('authentication.urls')),
    url(r'^learning/', include('deckglue.learningurls', namespace="learning")),
    url(r'^cardbox/cards/', include('deckglue.cardurls', namespace="card")),
    url(r'^markitup/', include('markitup.urls')),
    url(r'^profile/', include('profiles.urls', namespace="profile")),
    url(r'^inplaceeditform/', include('inplaceeditform.urls')),
    url(r'^upload/', include('upload.urls', namespace="upload")),
    url(r'^comic/', include('comic.urls'))
)


from django.db import models
from django.utils import timezone
from embed_video.fields import EmbedVideoField
from multiselectfield import MultiSelectField

CATEGORY_CHOICES =(
    ('action','ACTION'),
    ('biography','BIOGAPHY'),
    ('drama','DRAMA',),
    ('comedy','COMEDY'),
    ('romance','ROMANCE',),
)

LANGUAGE_CHOICES=(
    ('english','ENGLISH'),
    ('german','GERMAN'),
)
STATUS_CHOICES=(
    ('RA','RECENTLY ADDED'),
    ('MW','MOST WATCHED'),
    ('TR','TOP RATED'),

)
# Create your models here.
class movie(models.Model):
    title=models.CharField(max_length=100)
    description=models.TextField(max_length=1000)
    image=models.ImageField(upload_to='movies')
    banner=models.ImageField(upload_to='movies_banner')
    category=MultiSelectField(choices=CATEGORY_CHOICES)
    language=models.CharField(choices=LANGUAGE_CHOICES,max_length=10)
    status=models.CharField(choices=STATUS_CHOICES,max_length=2)
    year_of_production=models.DateField()
    views_count=models.IntegerField(default=0)
    slug=models.SlugField(blank=True,null=True)
    video = EmbedVideoField()
    created=models.DateTimeField(default=timezone.now)

    def __str__(self):
        return self.title
LINK_CHOICES=(
('D','DOWNLOAD LINK'),
('W','WATCH LINK'),
)

class movielinks(models.Model):
    movie=models.ForeignKey(movie,related_name='movie_watch_link',on_delete=models.CASCADE)
    link=models.URLField()
    type=models.CharField(choices=LINK_CHOICES,max_length=1)

    def __str__(self):
        return  str(self.movie)

class cast(models.Model):
    movie=models.ForeignKey(movie,related_name='cast',on_delete=models.CASCADE)
    name =models.CharField(max_length=100)

    def __str__(self):
        return self.name


from pyspark import SparkContext
from pyspark.sql import SparkSession

class SparkSpawner:
    @classmethod
    def get_spark(self) -> SparkSession:
        spark_builder = SparkSession.builder.appName('BigDW')\
            .enableHiveSupport() \
            .config("spark.jars.packages","io.delta:delta-core_2.11:0.5.0")  \
            .config("spark.jars.packages", "com.microsoft.sqlserver:mssql-jdbc:7.0.0.jre8,io.delta:delta-core_2.11:0.5.0")
            #.config("hive.metastore.warehouse.dir","d:/AppData/Metastore/")
            #.config("spark.sql.extensions","io.delta.sql.DeltaSparkSessionExtension")
        spark = spark_builder.getOrCreate()

        return spark


import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
import os
import gc

array_length = 2**20
num_arrays = 10
filename = "test.parquet"

# Only run this if the data file hasn't been already generated
if(not os.path.isfile(filename)):

    # Generate arrays and write them to a dataframe
    print("Generating the dataframe...")
    columns = np.arange(1, array_length + 1)
    data = pd.DataFrame(columns = columns)
    for i in range(1,num_arrays+1):
        array = np.random.rand(array_length)
        data.loc[i] = array
    print("Done")

    # Convert the dataframe to arrow and then save to disk as parquet
    print("Saving to disk...")
    arrow_data = pa.Table.from_pandas(data)
    pq.write_table(arrow_data, filename, use_dictionary=False)
    print("Done")

# Reading the data
try:
    print("Option 1:")
    pq.read_table(filename)
except OSError as e:
    print("Option 1 failed with the following error:\n", e)

'''
try:
    print("Option 2: read column by column")
    div = 2**18
    limit = 2**31-1 # Maximum value I can choose
    col = pq.read_table(filename, columns = ['1'], thrift_string_size_limit = limit, thrift_container_size_limit = limit)
    #col = pq.read_table(filename, columns = ['1'])
    print(col)
    for i in range(1,array_length+1):
        print(i)
        if (i % div == 0):
            print(i)
        pq.read_table(filename, columns = [str(i)], thrift_string_size_limit = limit, thrift_container_size_limit = limit)
except OSError as e:
    print("Option 2 failed with the following error:\n", e)

exit()
'''

print("Option 3:")
print("Memory usage is blowing up here...")
limit = 2**31-1 # Maximum value I can choose
pq.read_table(filename, thrift_string_size_limit = limit, thrift_container_size_limit = limit)
gc.collect()

print("Option 4:")
print("This option either takes too long or it just gets stuck...")
print("Notice how the memory usage is still high from option 2, even after calling the garbage collector.")
pd.read_parquet(filename, engine = 'fastparquet')


"""
https://leetcode.com/problems/sort-array-by-parity/

"""
from typing import List

class Solution:
    def sortArrayByParity(self, A: List[int]) -> List[int]:
        write_p = 0

        for i in range(len(A)):
            if A[i] % 2 == 0:
                A[i], A[write_p] = A[write_p], A[i]
                write_p += 1

        return A


import requests
from bs4 import BeautifulSoup
import pandas as pd
import pathlib
import datetime

def fetch(which):
    url = 'https://isin.twse.com.tw/isin/C_public.jsp'
    param = dict(
        exchange={'strMode':'2'},
        otc={'strMode':'4'})
    response = requests.get(url, param[which])
    assert response.status_code == 200
    return response.text

def get_html(which):
    print(f'overview GET {which:8} html:', end='\t')
    path = agent.joinpath(which).with_suffix('.html')
    if path.exists():
        print('EXIST', end='\t')
    else:
        print('REQUEST', end='\t')
        text = fetch(which)
        path.write_text(text,encoding='utf8')
    print('SUCCESS!')
    return None

def read_html(which):
    path = agent.joinpath(which).with_suffix('.html')
    text = path.read_text(encoding='utf8')
    soup = BeautifulSoup(text, 'lxml')

    labels = [''.join(t.text.split())
                for t in soup.find_all('b', text=lambda x: x != ' ')]

    table = pd.read_html(text, header=0,converters={})[0]
    table.loc[table['備註'].isin(labels), 'labels'] = labels
    table['labels'] = table['labels'].ffill()
    table = table.loc[~table['備註'].isin(labels)]

    split = table['有價證券代號及名稱'].str.split(expand=True)

    table['ID'] = split[0]

    table['name'] = split.loc[:,1:].fillna('').apply(lambda row: ''.join(row),axis=1)

    table['is_stock'] = table['labels'] == '股票'

    table['industry'] = table['產業別']
    table['market'] = which

    table = table[['ID', 'name', 'is_stock','industry','market']]

    table = table.reset_index(drop=True)
    return table

def html_to_csv(which):
    print(f'PARSE {which:8} html',end='\t')
    path = agent.joinpath(which).with_suffix('.csv')
    table = read_html(which)
    table.to_csv(path.as_posix(),encoding='utf8')
    print('SUCCESS!')
    return table

def read_csv(which):
    path = agent.joinpath(which).with_suffix('.csv')
    if path.exists():
        print(f'{which:8} CSV EXIST!')
        table = pd.read_csv(path.as_posix(),encoding='utf8',dtype={'ID':str},index_col=0)
    else:
        table = html_to_csv(which)
    return table

def merge_csv():
    path = agent.joinpath('merge').with_suffix('.csv')
    exchange = read_csv('exchange')
    otc = read_csv('otc')
    print('MERGE')
    table = exchange.append(otc, ignore_index=False)
    duplicate = table.loc[table.ID.duplicated(False)]
    if not duplicate.empty:
        print('\n','='*25,'合併上市上櫃一覽表時發現重複代碼','='*25)
        display(duplicate)
        assert False, f'duplicate: {duplicate.ID.unique()}'
    table = table.reset_index(drop=True)
    table.to_csv(path.as_posix(),encoding='utf8')
    return table

def read_merge():
    path = agent.joinpath('merge').with_suffix('.csv')
    if path.exists():
        print('MERGE CSV EXIST!')
        table = pd.read_csv(path.as_posix(),encoding='utf8',dtype={'ID':str},index_col=0)
    else:
        table = merge_csv()
    return table

def release():
    release_file = ['*.csv','*.html']
    for file in release_file:
        for item in agent.glob(file):
            item.unlink()

def update():
    print('='*10,str(datetime.date.today()),'='*10)
    get_html('exchange')
    get_html('otc')
    path = depository.joinpath('data.csv')
    new_data = read_merge()
    new_data['status'] = 'new'
    print('UPDATE')
    if path.exists():
        previous_data = pd.read_csv(path.as_posix(),encoding='utf8',dtype={'ID':str}, index_col=0)
        previous_data['status'] = 'data'

        merge_data = previous_data.append(new_data, ignore_index=False)
        merge_data = merge_data.drop_duplicates(subset=['ID','name','is_stock','market'])

        duplicate = merge_data.loc[merge_data.ID.duplicated(False)]
        if not duplicate.empty:
            print('\n','='*25,'一覽表添加新資料時發現內容不重複，但代碼重複的資料','='*25)
            display(duplicate)
            assert False, f'duplicate: {duplicate.ID.unique()}'
    else:
        merge_data = new_data
    merge_data = merge_data.reset_index(drop=True)
    merge_data = merge_data.drop('status',axis=1)
    merge_data.to_csv(path,encoding='utf8')
    print('RELEASE')
    release()
    print('FINISH!')


from django.test import TestCase

from core.models import User, ActivityPeriod
from core.views import get_user_data_by_id, get_all_user_data, get_user_activity_details

class UserTestCase(TestCase):

    def test_user(self):
        with self.assertRaises(User.DoesNotExist):
            user_1 = User.objects.get(id="W07QCRPA4")
            user_1_expected_output = '{"id": "W07QCRPA4", "real_name": "Glinda Southgood", "tz": "Asia/Kolkata"}'
            self.assertEquals(user_1.__str__(), user_1_expected_output)
        with self.assertRaises(User.DoesNotExist):
            User.objects.get(id="W07QCSQA6")

class ViewTestCase(TestCase):

    def test_get_user_data_by_id(self):
        user_1_id = "W07QCSQA6"
        self.assertEquals(get_user_data_by_id(user_1_id), None)

    def test_get_all(self):
        user_1_id = "W07QCSQA6"
        self.assertEquals(get_all_user_data(), None)

    def test_get_user_activity_details(self):
        user_1 = None
        self.assertEquals(get_user_activity_details(user_1), [])


import matplotlib.pyplot as plt
import numpy as np

from fluids.fittings import entrance_rounded, entrance_rounded_methods

rcs = np.linspace(0,0.4, 1000)
for method in entrance_rounded_methods:
    Ks = [entrance_rounded(Di=1.0, rc=rc, method=method) for rc in rcs]
    plt.plot(rcs, Ks, label=method)
plt.legend()
plt.title('Comparison of available methods for rounded flush entrances to pipes')
plt.xlabel('rc/Di')
plt.ylabel('K')
#plt.show()


import cv2
import os

for root, dirs, files in os.walk('pai/1/'):
    # print(root)
    # print(dirs)
    # print(files)
    if (files):
        for item in files:
            img = cv2.imread(os.path.join('pai/1', item))
            rows, cols = img.shape[:2]
            M = cv2.getRotationMatrix2D((cols / 2, rows / 2), 15, 1.2)
            dst = cv2.warpAffine(img, M, (cols, rows))
            cv2.imwrite('pai/11/' + item + '_warp.jpg', dst)


# Released under the MIT License. See LICENSE for details.
#
"""Functionality related to managing cloud based assets."""

from __future__ import annotations

from typing import TYPE_CHECKING, Annotated
from dataclasses import dataclass, field
from pathlib import Path
import threading
import urllib.request
import logging
import weakref
import time
import os
import sys

from efro.dataclassio import (
    ioprepped,
    IOAttrs,
    dataclass_from_json,
    dataclass_to_json,
)
import _babase

if TYPE_CHECKING:
    from bacommon.assets import AssetPackageFlavor

@ioprepped
@dataclass
class FileValue:
    """State for an individual file."""

@ioprepped
@dataclass
class State:
    """Holds all persistent state for the asset-manager."""

    files: Annotated[dict[str, FileValue], IOAttrs('files')] = field(
        default_factory=dict
    )

class AssetManager:
    """Wrangles all assets."""

    _state: State

    def __init__(self, rootdir: Path) -> None:
        print('AssetManager()')
        assert isinstance(rootdir, Path)
        self.thread_ident = threading.get_ident()
        self._rootdir = rootdir
        self._started = False
        if not self._rootdir.is_dir():
            raise RuntimeError(f'Provided rootdir does not exist: "{rootdir}"')

        self.load_state()

    def __del__(self) -> None:
        print('~AssetManager()')
        if self._started:
            logging.warning('AssetManager dying in a started state.')

    def launch_gather(
        self,
        packages: list[str],
        flavor: AssetPackageFlavor,
        account_token: str,
    ) -> AssetGather:
        """Spawn an asset-gather operation from this manager."""
        print(
            'would gather',
            packages,
            'and flavor',
            flavor,
            'with token',
            account_token,
        )
        return AssetGather(self)

    def update(self) -> None:
        """Can be called periodically to perform upkeep."""

    def start(self) -> None:
        """Tell the manager to start working.

        This will initiate network activity and other processing.
        """
        if self._started:
            logging.warning('AssetManager.start() called on running manager.')
        self._started = True

    def stop(self) -> None:
        """Tell the manager to stop working.

        All network activity should be ceased before this function returns.
        """
        if not self._started:
            logging.warning('AssetManager.stop() called on stopped manager.')
        self._started = False
        self.save_state()

    @property
    def rootdir(self) -> Path:
        """The root directory for this manager."""
        return self._rootdir

    @property
    def state_path(self) -> Path:
        """The path of the state file."""
        return Path(self._rootdir, 'state')

    def load_state(self) -> None:
        """Loads state from disk. Resets to default state if unable to."""
        print('ASSET-MANAGER LOADING STATE')
        try:
            state_path = self.state_path
            if state_path.exists():
                with open(self.state_path, encoding='utf-8') as infile:
                    self._state = dataclass_from_json(State, infile.read())
                    return
        except Exception:
            logging.exception('Error loading existing AssetManager state')
        self._state = State()

    def save_state(self) -> None:
        """Save state to disk (if possible)."""

        print('ASSET-MANAGER SAVING STATE')
        try:
            with open(self.state_path, 'w', encoding='utf-8') as outfile:
                outfile.write(dataclass_to_json(self._state))
        except Exception:
            logging.exception('Error writing AssetManager state')

class AssetGather:
    """Wrangles a gathering of assets."""

    def __init__(self, manager: AssetManager) -> None:
        assert threading.get_ident() == manager.thread_ident
        self._manager = weakref.ref(manager)
        # self._valid = True
        print('AssetGather()')
        # url = 'https://files.ballistica.net/bombsquad/promo/BSGamePlay.mov'
        # url = 'http://www.python.org/ftp/python/2.7.3/Python-2.7.3.tgz'
        # fetch_url(url,
        #           filename=Path(manager.rootdir, 'testdl'),
        #           asset_gather=self)
        # print('fetch success')
        thread = threading.Thread(target=self._run)
        thread.run()

    def _run(self) -> None:
        """Run the gather in a background thread."""
        print('hello from gather bg')

        # First, do some sort of.

    # @property
    # def valid(self) -> bool:
    #     """Whether this gather is still valid.

    #     A gather becomes in valid if its originating AssetManager dies.
    #     """
    #     return True

    def __del__(self) -> None:
        print('~AssetGather()')

def fetch_url(url: str, filename: Path, asset_gather: AssetGather) -> None:
    """Fetch a given url to a given filename for a given AssetGather."""
    # pylint: disable=consider-using-with
    import socket

    # We don't want to keep the provided AssetGather alive, but we want
    # to abort if it dies.
    assert isinstance(asset_gather, AssetGather)
    # weak_gather = weakref.ref(asset_gather)

    # Pass a very short timeout to urllib so we have opportunities
    # to cancel even with network blockage.
    req = urllib.request.urlopen(
        urllib.request.Request(
            url, None, {'User-Agent': _babase.user_agent_string()}
        ),
        context=_babase.app.net.sslcontext,
        timeout=1,
    )
    file_size = int(req.headers['Content-Length'])
    print(f'\nDownloading: {filename} Bytes: {file_size:,}')

    def doit() -> None:
        time.sleep(1)
        print('dir', type(req.fp), dir(req.fp))
        print('WOULD DO IT', flush=True)
        # req.close()
        # req.fp.close()

    threading.Thread(target=doit).run()

    with open(filename, 'wb') as outfile:
        file_size_dl = 0

        block_sz = 1024 * 1024 * 1000
        time_outs = 0
        while True:
            try:
                data = req.read(block_sz)
            except ValueError:
                import traceback

                traceback.print_exc()
                print('VALUEERROR', flush=True)
                break
            except socket.timeout:
                print('TIMEOUT', flush=True)
                # File has not had activity in max seconds.
                if time_outs > 3:
                    print('\n\n\nsorry -- try back later')
                    os.unlink(filename)
                    raise
                print(
                    '\nHmmm... little issue... '
                    'I\'ll wait a couple of seconds'
                )
                time.sleep(3)
                time_outs += 1
                continue

            # We reached the end of the download!
            if not data:
                sys.stdout.write('\rDone!\n\n')
                sys.stdout.flush()
                break

            file_size_dl += len(data)
            outfile.write(data)
            percent = file_size_dl * 1.0 / file_size
            status = f'{file_size_dl:20,} Bytes [{percent:.2%}] received'
            sys.stdout.write('\r' + status)
            sys.stdout.flush()
    print('done with', req.fp)


"""
2005B DVD租赁
考虑如下的在线DVD租赁问题。顾客缴纳一定数量的月费成为会员，订购DVD租赁服务。会员对哪些DVD有兴趣，只要在线提交订单，网站就会通过快递的方式尽可能满足要求。会员提交的订单包括多张DVD，这些DVD是基于其偏爱程度排序的。网站会根据手头现有的DVD数量和会员的订单进行分发。每个会员每个月租赁次数不得超过2次，每次获得3张DVD。请考虑以下问题：
数据中会在测试点中给出。数据包括：网站n种DVD的现有张数和当前需要处理的m位会员的在线订单。 如何对这些DVD进行分配，才能使会员获得最大的满意度。
具体数据大家可以参考2005年全国大学生数学建模竞赛B题。
"""
import numpy as np
import cvxpy as cp
m,n=[eval(x) for x in input().split()]
p=np.zeros((m,n))
for i in range(m):
    p[i,:]=[eval(x) for x in input().split()]
d=[eval(x) for x in input().split()]
s=np.zeros((m,n))
for i in range(m):
    for j in range(n):
        if p[i,j]>0:
            s[i,j]=2**(11-p[i,j])
x=cp.Variable((m,n),boolean=True)
obj=cp.Maximize(cp.sum(cp.multiply(s,x)))
cons=[cp.sum(x,axis=0)<=d,cp.sum(x,axis=1)<=3]
prob=cp.Problem(obj,cons)
prob.solve(solver='GLPK_MI')
print(int(prob.value))

山东大学 数学模型 程序汇总/2005年全国大学生数学建模竞赛B题 DVD租赁.py

2005年全国大学生数学建模竞赛B题 DVD租赁.py


from __future__ import absolute_import

from flask import current_app

from .builder import JenkinsBuilder

class JenkinsGenericBuilder(JenkinsBuilder):
    def __init__(self, master_urls=None, *args, **kwargs):
        self.script = kwargs.pop('script')
        self.cluster = kwargs.pop('cluster')
        self.path = kwargs.pop('path', '')
        self.workspace = kwargs.pop('workspace', '')

        if not master_urls:
            # if we haven't specified master urls, lets try to take the default
            # for this given cluster
            master_urls = current_app.config['JENKINS_CLUSTERS'].get(self.cluster)

        super(JenkinsGenericBuilder, self).__init__(master_urls, *args, **kwargs)

    def get_job_parameters(self, job, script=None, target_id=None, path=None):
        params = super(JenkinsGenericBuilder, self).get_job_parameters(
            job, target_id=target_id)

        if path is None:
            path = self.path

        if script is None:
            script = self.script

        project = job.project
        repository = project.repository

        vcs = repository.get_vcs()
        if vcs:
            repo_url = vcs.remote_url
        else:
            repo_url = repository.url

        params.extend([
            {'name': 'CHANGES_PID', 'value': project.slug},
            {'name': 'REPO_URL', 'value': repo_url},
            {'name': 'SCRIPT', 'value': script},
            {'name': 'REPO_VCS', 'value': repository.backend.name},
            {'name': 'CLUSTER', 'value': self.cluster},
            {'name': 'WORK_PATH', 'value': path},
            {'name': 'C_WORKSPACE', 'value': self.workspace},
        ])

        return params


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error,mean_squared_error
from keras.models import Model,Sequential
from keras.layers import Dense,Activation

df = pd.read_excel('AL_WIND_07_12.xlsx')

df['Time'] = df['DateTime'].dt.hour

df = df.drop(['DateTime'],axis=1)

Y = df['Power generated by system | (kW)']
X = df.drop(['Power generated by system | (kW)'],axis=1)

X_train,X_test,Y_train,Y_test = train_test_split(X,Y, test_size=0.33, random_state=42)

def basemodel():
	model = Sequential()
	model.add(Dense(12,input_dim=5,activation='relu'))
	model.add(Dense(12,activation='relu'))
	model.add(Dense(1,activation='linear'))
	model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mae'])
	return model

regr = basemodel()

regr.fit(X_train,Y_train,batch_size=32,epochs=400,shuffle=True)
predictions = regr.predict(X_test)
df0=pd.DataFrame(predictions)
df0[df0 < 0] = 0
df['NN_predictions'] = df0
df.to_csv('NN_wind_predictions.csv')

print(mean_absolute_error(predictions,Y_test))
print(np.sqrt(mean_squared_error(predictions,Y_test)))

Zindi Africa/Wind Power predictions/lineareg.py


import argparse

from ccc_client.dcs.DcsRunner import DcsRunner
from ccc_client.utils import print_API_response

def run(args):
    runner = DcsRunner(args.host, args.port, args.authToken)
    for i in args.cccId:
        r = runner.create_link(args.setId, i)
        print_API_response(r)

parser = argparse.ArgumentParser(
    description='Assign one or more resources to a set'
)
parser.set_defaults(runner=run)
parser.add_argument(
    '--setId', '-p',
    required=True,
    type=str,
    help='CCC_ID of new or existing set'
)
parser.add_argument(
    '--cccId', '-c',
    type=str,
    nargs='+',
    help='CCC_ID(s) of data to be assigned to set'
)