# -*- coding: utf-8 -*-
"""
Tools for retrieving catchment data.

"""
import paths

import os
import pandas as pd

import geopandas as gpd
import numpy as np
import numpy.ma as ma
from math import sqrt

from shapely.geometry import Point
from shapely.geometry import box
from shapely.ops import nearest_points

import rasterio
from rasterio.windows import Window
from rasterio.plot import show
from rasterio.plot import show_hist
from rasterio.mask import mask

from fiona.crs import from_epsg

import json

import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm

# LCM2015 dataset is split into layers, named at these coordinates.
LCM2015_EASTING_LAYERS = (35000, 135000, 235000, 335000, 435000, 535000,
                          635000, 735000)
LCM2015_NORTHING_LAYERS = (50000, 150000, 250000, 350000, 450000, 550000,
                           650000, 750000, 850000, 950000, 1050000, 1150000,
                           1250000)

# Map for how many extra depths to check when finding closest grids. (See
# closest_CCAR_above_val for more details).
DEPTH_CHECK_MAP = {
    0: 0,
    1: 0,
    2: 0,
    3: 1,
    4: 1,
    5: 2,
    6: 2,
    7: 2,
    8: 3,
    9: 3,
    10: 4,
    11: 4,
    12: 4,
    13: 5,
    14: 5,
    15: 5,
    16: 4,
    17: 3,
    18: 2,
    19: 1,
    20: 0,
}

class CatchmentData(object):
    _descriptor_type_dirs = {
        "FEH_gb": paths.GB_FEH_DIR,
        "FEH_ni": paths.NI_FEH_DIR,
        "LCM_2000_gb": paths.GB_LCM_2000_DIR,
        "LCM_2000_ni": paths.NI_LCM_2000_DIR,
        "LCM_2007_gb": paths.GB_LCM_2007_DIR,
        "LCM_2007_ni": paths.NI_LCM_2007_DIR,
        "LCM_2015_gb": paths.GB_LCM_2015_DIR,
        "LCM_2015_ni": paths.NI_LCM_2015_DIR,
    }

    # Aggregates of LCM classes used for NRFA. I.e. LCM and many classes but
    # NRFA groups them into a smaller subset.
    _LCM_class_aggregates = {
        "Woodland": [
            "2000_11",
            "2000_21",
            "2007_1",
            "2007_2",
            "2015_1",
            "2015_2",
        ],
        "Arable and Horticulture": [
            "2000_41",
            "2000_42",
            "2000_43",
            "2007_3",
            "2015_3",
        ],
        "Grassland": [
            "2000_51",
            "2000_52",
            "2000_61",
            "2000_71",
            "2000_81",
            "2000_91",
            "2000_111",
            "2007_4",
            "2007_5",
            "2007_6",
            "2007_7",
            "2007_8",
            "2007_9",
            "2015_4",
            "2015_5",
            "2015_6",
            "2015_7",
            "2015_8",
        ],
        "Heath/Bog": [
            "2000_101",
            "2000_102",
            "2000_121",
            "2000_151",
            "2007_10",
            "2007_11",
            "2007_12",
            "2007_13",
            "2015_9",
            "2015_10",
            "2015_11",
        ],
        "Bareground": [
            "2000_161",
        ],
        "Inland Rock": [
            "2007_14",
            "2015_12",
        ],
        "Water": [
            "2000_121",
            "2000_131",
            "2007_15",
            "2007_16",
            "2015_13",
            "2015_14",
        ],
        "Coastal": [
            "2000_181",
            "2000_191",
            "2000_201",
            "2000_211",
            "2000_212",
            "2007_17",
            "2007_18",
            "2007_19",
            "2007_20",
            "2007_21",
            "2015_15",
            "2015_16",
            "2015_17",
            "2015_18",
            "2015_19",
        ],
        "Urban": [
            "2000_171",
            "2000_172",
            "2007_22",
            "2007_23",
            "2015_20",
            "2015_21",
        ],
        "Unknown": [
            "2000_9999",
            "2007_9999",
            "2015_9999",
        ],
    }

    # The full names of the FEH codes
    FEH_code_map = {
        "CCAR": "ihdtm-catchment-area",
        "HGHT": "ihdtm-height",
        "Q__C": "ddf-c-catchment",
        "Q__E": "ddf-e-catchment",
        "Q__F": "ddf-f-catchment",
        "Q_D1": "ddf-d1-catchment",
        "Q_D2": "ddf-d2-catchment",
        "Q_D3": "ddf-d3-catchment",
        "QALT": "altbar",
        "QASB": "aspbar",
        "QASV": "aspvar",
        "QB19": "BFIHOST19",
        "QBFI": "bfihost",
        "QDPB": "dplbar",
        "QDPS": "dpsbar",
        "QFAR": "farl",
        "QFPD": "mean-flood-plain-depth",
        "QFPL": "mean-flood-plain-location",
        "QFPX": "mean-flood-plain-extent",
        "QLDP": "ldp",
        "QPRW": "propwet",
        "QR1D": "rmed-1d",
        "QR1H": "rmed-1h",
        "QR2D": "rmed-2d",
        "QS47": "saar-1941-1970",
        "QS69": "saar-1961-1990",
        "QSPR": "sprhost",
        "QUC2": "urbconc-2000",
        "QUCO": "urbconc-1990",
        "QUE2": "urbext-2000",
        "QUEX": "urbext-1990",
        "QUL2": "urbloc-2000",
        "QULO": "urbloc-1990",
        "RM_C": "ddf-c",
        "RM_E": "ddf-e",
        "RM_F": "ddf-f",
        "RMD1": "ddf-d1",
        "RMD2": "ddf-d2",
        "RMD3": "ddf-d3",
    }

    def __init__(self, easting, northing, station="", snap_to_river=False):
        self.easting_raw = easting
        self.northing_raw = northing

        if snap_to_river:
            self.easting, self.northing = self._river_snapping()
        else:
            self.easting = self.easting_raw
            self.northing = self.northing_raw

        self.region = self._get_region()

        self.station = station

        # Set up catchment descriptor attributes
        self.FEH_data = None
        self.LCM_2000_data = None
        self.LCM_2007_data = None
        self.LCM_2015_data = None
        self.QCN_data = None

        self._valid_desc_types = ["FEH", "LCM2000", "LCM2007", "LCM2015"]
        self._valid_LCM_years = [2000, 2007, 2015]

    def _river_snapping(self):
        cell_dict = closest_ccar_above_val(self.easting_raw,
                                           self.northing_raw,
                                           min_ccar=200,
                                           max_depths=20)
        if cell_dict is None:
            raise UserWarning("No cell found close enough for river snapping")

        print("Coords snapped to river. From (%s, %s) to (%s, %s)" % (
            self.easting_raw, self.northing_raw, cell_dict["easting"],
            cell_dict["northing"]
        ))

        return cell_dict["easting"], cell_dict["northing"]

    def _get_region(self):
        """
        Work out if coordinates are for Great Britian (GB) or Northern Ireland
        (NI). Do this by checking if the fall within one of the two bounding
        boxes that cover NI. If not, assume GB.

        Bounding boxes are defined as [x_min,y_min,x_max,y_max]

        """
        # West and east bounding boxes for NI
        west_ni = [000000, 469190, 143723, 614827]
        east_ni = [143723, 469190, 185797, 597050]

        region = "gb"

        # Test west bounding box
        if self.easting >= west_ni[0] and self.easting <= west_ni[2]:
            if self.northing >= west_ni[1] and self.northing <= west_ni[3]:
                region = "ni"

        # Test east bounding box
        elif self.easting >= east_ni[0] and self.easting <= east_ni[2]:
            if self.northing >= east_ni[1] and self.northing <= east_ni[3]:
                region = "ni"

        return region

    def _read_directory_tifs(self, directory, desc_type):
        """
        Read the tif files in the given directory. Each tif contains data for
        a particluar catchment descriptor. For each, establish the descriptor
        type (PROPERTY_ITEM) and extract the value for it at the class
        coordinates.
        Append all the values (and their type) to a dataframe.

        """
        # Create a pandas dataframe to hold the data.
        df = pd.DataFrame(columns=['PROPERTY_ITEM', 'PROPERTY_VALUE'])
        for dir_path, dir_name_list, file_name_list in os.walk(directory):
            for filename in file_name_list:
                # If this is not a tif file.
                if not filename.endswith('.tif'):
                    # Skip it
                    continue

                # Open the tif file.
                file_path = os.path.join(dir_path, filename)
                raster_file = rasterio.open(file_path)

                if desc_type == "FEH":
                    # Get the descriptor code (from filename).
                    descriptor_code = str(filename[-11:-7])
                elif desc_type == "LCM":
                    lcm_year = str(filename.split("_")[1])
                    lcm_code = str(filename.split("_")[2])[:-4]
                    descriptor_code = lcm_year + "_" + lcm_code

                # Use the class coordinates and get the value of raster.
                val = list(raster_file.sample([
                    (self.easting, self.northing)]))[0][0]

                # Create a list to use to append the data to the dataframe.
                values_list = [descriptor_code, val]
                # Make a series to make a row.
                value_series = pd.Series(values_list, index=df.columns)
                df = df.append(value_series, ignore_index=True)
                raster_file.close()

        # Make sure values are floats
        df["PROPERTY_VALUE"] = df["PROPERTY_VALUE"].astype(float)

        return df

    def _add_db_columns(self, desc_df, group="", method="", comment="",
                        title="", units="", source="", remove_errors=True):
        """
        Restructure dataframe to have columns required for DB table.

        """
        if remove_errors:
            desc_df = desc_df[desc_df["PROPERTY_VALUE"] > -5000]

        desc_df["STATION"] = self.station
        desc_df["PROPERTY_GROUP"] = group
        desc_df["PROPERTY_METHOD"] = method
        desc_df["PROPERTY_COMMENT"] = comment
        desc_df["TITLE"] = title
        desc_df["UNITS"] = units
        desc_df["SOURCE_VALUE"] = source

        # Arrange column order.
        return desc_df[["STATION", "PROPERTY_GROUP", "PROPERTY_ITEM",
                        "PROPERTY_VALUE", "PROPERTY_METHOD",
                        "PROPERTY_COMMENT", "TITLE", "UNITS", "SOURCE_VALUE"]]

    def _aggregate_LCM_classes(self, LCM_data, year):
        """
        LCM data has a set of classes, e.g. Improved grassland and Neutral
        grassland, with assciated codes (which can vary between years). For
        this work we define a simplfied set of classes which aggregates LCM
        classes, e.g. Grassland, which includes both 'Improved' and 'Neutral'.

        The mapping between the simplied classes and the LCM class codes is
        given in the dictionary self._LCM_class_aggregates.

        """
        group = "lcm%snrfav2021" % year

        for agg_class, lcm_classes in self._LCM_class_aggregates.items():
            # Extract details from the LCM classes within the aggregate class
            valid_classes = LCM_data[
                LCM_data["PROPERTY_ITEM"].isin(lcm_classes)]

            if len(valid_classes) == 0:
                continue

            # LCM classes have format 'year_class', extract just the class
            # numbers so we can create a sting for teh SOURCE_VALUE column
            valid_cls_nums = [cls.split("_")[-1]
                              for cls in valid_classes["PROPERTY_ITEM"]]
            source = "+".join(valid_cls_nums)

            # Sum the total percentage across aggregate classes
            agg_total = valid_classes["PROPERTY_VALUE"].sum()

            new_row = {
                "STATION": self.station,
                "PROPERTY_GROUP": group,
                "PROPERTY_ITEM": agg_class,
                "PROPERTY_VALUE": agg_total,
                "PROPERTY_METHOD": "automatic",
                "PROPERTY_COMMENT": "",
                "TITLE": agg_class,
                "UNITS": "proportion",
                "SOURCE_VALUE": source
            }

            LCM_data = LCM_data.append(new_row, ignore_index=True)

        return LCM_data

    def get_FEH_data(self, convert_codes=False):
        """
        Extract FEH data for catchment. Data for each grid are saved in a tif
        files.

        Arg:
            convert_codes: Bool
                Optionally convert FEH codes to their full names.

        """
        # Get appropriate tif directory.
        desc_dir_key = "FEH_%s" % self.region
        desc_dir = self._descriptor_type_dirs.get(desc_dir_key)
        if desc_dir is None:
            raise UserWarning("region attribute: '%s', is not valid"
                              % self.region)

        if self.FEH_data is None:
            FEH_data = self._read_directory_tifs(desc_dir, "FEH")

            if convert_codes is True:
                FEH_data = FEH_data.replace(self.FEH_code_map)

            self.FEH_data = self._add_db_columns(FEH_data,
                                                 group="FEH",
                                                 method="automatic")

        return self.FEH_data

    def get_LCM_data(self, year=2015):
        """
        Extract land cover map (LCM) data for catchment, specifying the LCM
        version by its year. Data for each grid are saved in a tif files.

        """
        # Get appropriate tif directory.
        desc_dir_key = "LCM_%s_%s" % (year, self.region)
        desc_dir = self._descriptor_type_dirs.get(desc_dir_key)
        if desc_dir is None:
            if year not in self._valid_LCM_years:
                raise UserWarning("Invalid year: %s. Valid years are: %s" % (
                                      self.region,
                                      ", ".join(self._valid_LCM_years)
                                  ))
            else:
                raise UserWarning("region attribute: '%s', is not valid"
                                  % self.region)

        # Establish the LCM attribute by year
        if year == 2000:
            LCM_data = self.LCM_2000_data
            attr = "LCM_2000_data"
        elif year == 2007:
            LCM_data = self.LCM_2007_data
            attr = "LCM_2007_data"
        elif year == 2015:
            LCM_data = self.LCM_2015_data
            attr = "LCM_2015_data"
        else:
            raise UserWarning("Invalid year: %s. Valid years are: %s" % (
                                  self.region,
                                  ", ".join(self._valid_LCM_years)
                              ))

        if LCM_data is None:
            LCM_data = self._read_directory_tifs(desc_dir, "LCM")

            group = "lcm%sv2021" % year
            source = LCM_data["PROPERTY_ITEM"]
            LCM_data = self._add_db_columns(LCM_data,
                                            group=group,
                                            method="automatic",
                                            units="proportion",
                                            source=source)

            # Divide by 400 to get the area in square km.
            LCM_data["PROPERTY_VALUE"] = LCM_data["PROPERTY_VALUE"] / 400.
            # Get the sum of area.
            catchment_area = LCM_data["PROPERTY_VALUE"].sum()
            # Calculate the percentage of each category.
            LCM_data["PROPERTY_VALUE"] = (
                LCM_data["PROPERTY_VALUE"] / catchment_area) * 100.

            # Join LCM classes into simplified classes
            LCM_data = self._aggregate_LCM_classes(LCM_data, year)

            setattr(self, attr, LCM_data)

        return LCM_data

    def get_data(self, desc_types="all", savepath=None):
        """
        Extract catchment data and convert to format for database.

        """
        if isinstance(desc_types, str):
            desc_types = [desc_types]
        if desc_types[0] == "all":
            desc_types = self._valid_desc_types
        else:
            for desc_type in desc_types:
                if desc_type not in self._valid_desc_types:
                    raise UserWarning("Invalid descriptor type: %s. Valid "
                                      "descriptor types are: %s" % (
                                          desc_type,
                                          ", ".join(self._valid_desc_types)
                                      ))

        # Fetch given descriptor types, convert to DB format and combine.
        descs_df = None
        for desc_type in desc_types:
            if desc_type == "FEH":
                self.get_FEH_data()
                if descs_df is None:
                    descs_df = self.FEH_data
                else:
                    descs_df = descs_df.append(self.FEH_data,
                                               ignore_index=True)

            elif desc_type == "LCM2000":
                self.get_LCM_data(2000)
                if descs_df is None:
                    descs_df = self.LCM_2000_data
                else:
                    descs_df = descs_df.append(self.LCM_2000_data,
                                               ignore_index=True)

            elif desc_type == "LCM2007":
                self.get_LCM_data(2007)
                if descs_df is None:
                    descs_df = self.LCM_2007_data
                else:
                    descs_df = descs_df.append(self.LCM_2007_data,
                                               ignore_index=True)

            elif desc_type == "LCM2015":
                self.get_LCM_data(2015)
                if descs_df is None:
                    descs_df = self.LCM_2015_data
                else:
                    descs_df = descs_df.append(self.LCM_2015_data,
                                               ignore_index=True)

        if savepath is not None:
            descs_df.to_csv(savepath)

        return descs_df

def get_QCN_data(stations):
    """
    Create QCN (Polygon centroids) Dataset.
    Centroids csv file is a direct export from the centroid calculation in
    ArcGIS.
    For every new catchment you have to re-export this file after adding
    the new catchment.
    (If it's only one station it may be quicker to copy and paste the
    coordinated directly from ArcGIS).

    """
    qcn_data = pd.read_csv(paths.QCN_DIR + "catchments_all.csv")
    qcn_data = qcn_stations_df[qcn_data["STATION"].isin(stations)]
    if len(qcn_data) == 0:
        print("No valid QCN station IDs given")
        return

    print("Creating QCN (Polygon centroids) dataset *************************")
    qcndf = pd.DataFrame(
        columns=["STATION", "PROPERTY_ITEM", "PROPERTY_VALUE"])

    for index, row in qcn_data.iterrows():
        station_name = row["STATION"]
        QCNE = int(row["QCNE"])
        QCNN = int(row["QCNN"])
        list_qcne = [station_name, "QCNE", QCNE]
        list_qcnn = [station_name, "QCNN", QCNN]

        # Make a series per row
        value_series_qcne = pd.Series(list_qcne, index=qcndf.columns)
        value_series_qcnn = pd.Series(list_qcnn, index=qcndf.columns)

        qcndf = qcndf.append(value_series_qcne, ignore_index=True)
        qcndf = qcndf.append(value_series_qcnn, ignore_index=True)

    # Add Columns to match NRFA Oracle table
    qcndf["PROPERTY_GROUP"] = "FEH"
    qcndf["PROPERTY_METHOD"] = "automatic"
    qcndf["PROPERTY_COMMENT"] = ""
    qcndf["TITLE"] = ""
    qcndf["UNITS"] = ""
    qcndf["SOURCE_VALUE"] = ""

    # Rearrange columns to match Oracle table
    return qcndf[["STATION", "PROPERTY_GROUP", "PROPERTY_ITEM",
                  "PROPERTY_VALUE", "PROPERTY_METHOD", "PROPERTY_COMMENT",
                  "TITLE", "UNITS", "SOURCE_VALUE"]]

def base_round(x, base):
    """
    Returns a value rounded to base number.

    """
    return base * round(x / base)

def get_layer(easting, northing):
    """
    This function returns a string based on two lists for easting and northing.
    It is used to derive the name of the LCM2015 dataset, based on the
    coordinates given by the user.

    Layer name has format: "T(easting)_(northing)"

    """
    label_easting = min(LCM2015_EASTING_LAYERS, key=lambda x:abs(x-easting))
    label_northing = min(LCM2015_NORTHING_LAYERS, key=lambda x:abs(x-northing))
    return "T" + str(label_easting) + "_" + str(label_northing)

def read_ccar(easting, northing, raster_file=None):
    """
    Get CCAR value from the dataset.
    CCAR is the number of grid cells (at 50m^2) that flow into the cell at the
    given coordinates.

    """
    if raster_file is None:
        raster_file = rasterio.open(paths.CCAR_FILE)
        close_raster = True
    else:
        # If raster file is given, do not close it in this function
        close_raster = False

    # Round easting and northing to the closest 50m
    easting = base_round(easting, 50)
    northing = base_round(northing, 50)

    # Using the rasterio.sample method to get CCAR at specific point
    val = list(raster_file.sample([(easting, northing)]))[0][0]

    if close_raster:
        raster_file.close()

    if val == 2147483647:
        # Special bad value, convert to -999
        return -999
    else:
        return val

def read_ccar_square(easting, northing, depth, border_only=False,
                     raster_file=None):
    """
    Read in the cells around the central easting and northing given, at the
    given depth.
    If border_only is True, the cells withing the border are not read.
    For example, a depth of 1 means the 8 cells the surround the central cell
    are read, as oppose to all 9. At depth 2, the 16 cells that surround the
    central 9 are read.. and so on.

    """
    cell_dicts = []

    if depth == 0:
        # Handle 0 case separately (is just the given centre cell).
        ccar = read_ccar(easting, northing, raster_file)
        cell_dict = {}
        cell_dict["easting"] = easting
        cell_dict["northing"] = northing
        cell_dict["ccar"] = ccar
        cell_dict["distance"] = 0
        cell_dicts.append(cell_dict)

    else:
        if border_only is True:
            # To pick out only the border cells, we constrain easting or
            # northing to always be at max or min depth.
            e_depth_range = [-depth, depth]
        else:
            e_depth_range = range(-depth, depth + 1)

        for e_depth in e_depth_range:
            for n_depth in range(-depth, depth + 1):
                this_easting = easting + (50 * e_depth)
                this_northing = northing + (50 * n_depth)

                ccar = read_ccar(this_easting, this_northing, raster_file)
                dist = sqrt((this_easting - easting)**2 +
                            (this_northing - northing)**2)

                cell_dict = {}
                cell_dict["easting"] = this_easting
                cell_dict["northing"] = this_northing
                cell_dict["ccar"] = ccar
                cell_dict["distance"] = dist
                cell_dicts.append(cell_dict)

        if border_only is True:
            # Do remaining on cells on top and bottom borders (leaving out the
            # corners as they are already done).
            for n_depth in [-depth, depth]:
                for e_depth in range(-depth + 1, depth):
                    this_easting = easting + (50 * e_depth)
                    this_northing = northing + (50 * n_depth)

                    ccar = read_ccar(this_easting, this_northing, raster_file)
                    dist = sqrt((this_easting - easting)**2 +
                                (this_northing - northing)**2)

                    cell_dict = {}
                    cell_dict["easting"] = this_easting
                    cell_dict["northing"] = this_northing
                    cell_dict["ccar"] = ccar
                    cell_dict["distance"] = dist
                    cell_dicts.append(cell_dict)

    return cell_dicts

def largest_ccar_close_by(easting, northing, depth=1):
    """
    Search the cells around the given cell to a given depth (e.g. 1 gives
    surrounding 9 cells, 2 is 25 cells).

    The absolute highest value is always returned, however, if > 1 max values
    are found, the closest is returned.
    If > 1 max values at the same distance are returned, the first found is
    returned, which is southern, then western, most.

    """
    # Round easting and northing to the closest 50m
    easting = base_round(easting, 50)
    northing = base_round(northing, 50)

    # Open the raster file
    raster_file = rasterio.open(paths.CCAR_FILE)

    best_cell = None

    cell_dicts = read_ccar_square(easting, northing, depth,
                                  raster_file=raster_file)

    for cell_dict in cell_dicts:
        if best_cell is None:
            best_cell = cell_dict

        elif cell_dict["ccar"] >= best_cell["ccar"]:
            # If equal maxes, check distance.
            if cell_dict["ccar"] == best_cell["ccar"] and \
                    cell_dict["distance"] >= best_cell["distance"]:
                # Is no closer so do not replace.
                continue

            best_cell = cell_dict

    raster_file.close()

    return best_cell

def closest_ccar_above_val(easting, northing, min_ccar=10, max_depths=20):
    """
    Find the cell nearest to a given cell that has at least the given CCAR
    value.

    """
    # Round easting and northing to the closest 50m
    easting = base_round(easting, 50)
    northing = base_round(northing, 50)

    # Open the raster file
    raster_file = rasterio.open(paths.CCAR_FILE)

    best_cell = None
    extra_depths = None
    for depth in range(0, max_depths + 1):
        if extra_depths is not None:
            if extra_depths == 0:
                break
            else:
                extra_depths -= 1

        # Gather cell data for growing squares around the centre.
        cell_dicts = read_ccar_square(easting, northing, depth,
                                      border_only=True,
                                      raster_file=raster_file)

        for cell_dict in cell_dicts:
            if cell_dict["ccar"] >= min_ccar:
                # We found a cell that passes the min ccar test, but there
                # could be others in the square and we want the closest.
                if best_cell is None:
                    best_cell = cell_dict

                elif cell_dict["distance"] <= best_cell["distance"]:
                    if cell_dict["distance"] == best_cell["distance"]:
                        # If equal distance, check CCAR
                        if cell_dict["ccar"] <= best_cell["ccar"]:
                            # CCAR no larger so do not replace.
                            continue

                    best_cell = cell_dict

                # Our search expands by square size. This means as the square
                # gets bigger, there is a chance that the corners of an inner
                # square and further away than the sides of the next outer
                # square.
                # This means, once the depth is a certain size, we need to keep
                # checking further depths to make sure we actually have the
                # closest. How many extra depths to check depends on the
                # current depth and has been worked out and stored in
                # DEPTH_CHECK_MAP.
                extra_depths = DEPTH_CHECK_MAP[depth]

    raster_file.close()

    if best_cell is None:
        print("No cell found with CCAR > %s in surrounding area. %s depths "
              "search" % (min_ccar, max_depths))

    return best_cell

def getFeatures(gdf):
    """
    Function to parse features from GeoDataFrame in such a manner that rasterio
    wants them.

    """
    return [json.loads(gdf.to_json())['features'][0]['geometry']]

def crop_grid(easting, northing, depth):
    """
    This function creates a clipped version of CCAR file, since CCAR file is
    large and kernel crashes (low memory).
    It creates a file called "out_tif.tif". Always overwrite previous.
    Use this to get the cropped version of CCAR below, for plotting
    input is easting, northing of centre cell, and depth: number of cells
    either side.

    """
    raster_file = rasterio.open(paths.CCAR_FILE)
    radius = depth * 50
    bbox = box(easting - radius, northing - radius,
               easting + radius, northing + radius)
    geo  = gpd.GeoDataFrame({'geometry': bbox}, index=[0],
                            crs=from_epsg(27700))
    geo = geo.to_crs(crs=raster_file.crs.data)
    coords = getFeatures(geo)
    out_img, out_transform = mask(dataset=raster_file, shapes=coords,
                                  crop=True)
    out_meta = raster_file.meta.copy()
    epsg_code = int(raster_file.crs.data['init'][5:])
    out_meta.update({"driver": "GTiff",
                     "height": out_img.shape[1],
                     "width": out_img.shape[2],
                     "transform": out_transform,
                     "crs": 27700})
    with rasterio.open(paths.TEMP_CCAR_FILE, "w", **out_meta) as dest:
        dest.write(out_img)

def plot_grid_and_values(pts):
    # Usually 3 points, colour them red, orange and blue
    cols = ['red', 'orange', 'blue']
    raster_file = rasterio.open(paths.TEMP_CCAR_FILE)
    left = raster_file.bounds[0] + 25
    top = raster_file.bounds[3] - 25
    fig, ax = plt.subplots(figsize=(20, 20))
    show(raster_file, ax=ax, cmap="Greens", norm=LogNorm())
    arr = raster_file.read(1)
    arr_masked = ma.array(arr)
    null_val = 2147483647
    arr_masked[arr == null_val] = ma.masked

    for (j, i), label in np.ndenumerate(arr_masked):
        if label != null_val:
            ax.text((left + (i * 50)), (top - (j * 50)), label, ha='center',
                    va='center')

    for i in range(len(pts)):
        plt.scatter(pts[i][0], pts[i][1], s=300, c=cols[i], marker='o')


from unittest.mock import AsyncMock

import pytest

from shared.models.webhook_topics.base import CloudApiWebhookEventGeneric
from webhooks.dependencies.redis_service import RedisService

wallet_id = "test_wallet"
topic = "test_topic"
payload = '"payload": {"test":"1"}'

json_entries = [
    f'{{"wallet_id": "{wallet_id}", "topic": "{topic}", "origin": "a", {payload}}}',
    f'{{"wallet_id": "{wallet_id}", "topic": "{topic}", "origin": "a", {payload}}}',
]
cloudapi_entries = [
    CloudApiWebhookEventGeneric(
        wallet_id=wallet_id, topic=topic, origin="a", payload={"test": "1"}
    ),
    CloudApiWebhookEventGeneric(
        wallet_id=wallet_id, topic=topic, origin="a", payload={"test": "1"}
    ),
]

@pytest.mark.anyio
async def test_add_webhook_event():
    event_json = json_entries[0]

    # Create a mock Redis client with the methods you want to test
    redis_client = AsyncMock()
    redis_client.zadd = AsyncMock()
    redis_client.publish = AsyncMock()

    # Initialize RedisService with the mocked Redis client
    redis_service = RedisService(redis_client)

    # Call the method you want to test
    await redis_service.add_webhook_event(event_json, wallet_id)

    # Assert that the mocked methods were called as expected
    redis_client.zadd.assert_called_once()
    redis_client.publish.assert_called_once()

@pytest.mark.anyio
async def test_get_json_webhook_events_by_wallet():
    redis_client = AsyncMock()
    redis_client.zrange = AsyncMock(return_value=[e.encode() for e in json_entries])
    redis_service = RedisService(redis_client)

    events = await redis_service.get_json_webhook_events_by_wallet(wallet_id)

    assert events == json_entries
    redis_client.zrange.assert_called_once()

@pytest.mark.anyio
async def test_get_webhook_events_by_wallet(mocker):
    expected_events = cloudapi_entries

    redis_service = RedisService(AsyncMock())
    mocker.patch.object(
        redis_service, "get_json_webhook_events_by_wallet", return_value=json_entries
    )
    mocker.patch(
        "shared.util.rich_parsing.parse_with_error_handling",
        side_effect=expected_events,
    )

    events = await redis_service.get_webhook_events_by_wallet(wallet_id)

    assert events == expected_events

@pytest.mark.anyio
async def test_get_json_webhook_events_by_wallet_and_topic(mocker):
    different_json = [
        '{"payload":{"test":"1"},"topic":"test_topic"}',
        '{"payload":{"test":"2"},"topic":"other_topic"}',
    ]
    expected_events = ['{"payload":{"test":"1"},"topic":"test_topic"}']

    redis_service = RedisService(AsyncMock())
    mocker.patch.object(
        redis_service, "get_json_webhook_events_by_wallet", return_value=different_json
    )

    events = await redis_service.get_json_webhook_events_by_wallet_and_topic(
        wallet_id="test_wallet_id", topic=topic
    )

    assert events == expected_events

@pytest.mark.anyio
async def test_get_webhook_events_by_wallet_and_topic(mocker):
    event_instances = [
        CloudApiWebhookEventGeneric(
            wallet_id=wallet_id, topic=topic, origin="a", payload={"test": "1"}
        ),
        CloudApiWebhookEventGeneric(
            wallet_id=wallet_id, topic="other", origin="a", payload={"test": "2"}
        ),
    ]
    expected_events = [event_instances[0]]

    redis_service = RedisService(AsyncMock())
    mocker.patch.object(
        redis_service, "get_webhook_events_by_wallet", return_value=event_instances
    )

    events = await redis_service.get_webhook_events_by_wallet_and_topic(
        wallet_id, topic
    )

    assert events == expected_events

@pytest.mark.anyio
async def test_get_json_events_by_timestamp():
    start_timestamp = 1609459200
    end_timestamp = 1609545600

    redis_client = AsyncMock()
    redis_client.zrangebyscore = AsyncMock(
        return_value=[e.encode() for e in json_entries]
    )

    redis_service = RedisService(redis_client)

    events = await redis_service.get_json_events_by_timestamp(
        wallet_id, start_timestamp, end_timestamp
    )

    assert events == json_entries
    redis_client.zrangebyscore.assert_called_once_with(
        f"wallet_id:{wallet_id}", min=start_timestamp, max=end_timestamp
    )

@pytest.mark.anyio
async def test_get_events_by_timestamp(mocker):
    start_timestamp = 1609459200
    end_timestamp = 1609545600

    redis_service = RedisService(AsyncMock())
    mocker.patch.object(
        redis_service, "get_json_events_by_timestamp", return_value=json_entries
    )
    mocker.patch(
        "shared.util.rich_parsing.parse_with_error_handling",
        side_effect=cloudapi_entries,
    )

    events = await redis_service.get_events_by_timestamp(
        wallet_id, start_timestamp, end_timestamp
    )

    assert events == cloudapi_entries

@pytest.mark.anyio
async def test_get_all_wallet_ids():
    expected_wallet_ids = ["wallet1", "wallet2", "wallet3"]
    scan_results = [
        (
            1,
            [
                f"wallet_id:{wallet_id}".encode()
                for wallet_id in expected_wallet_ids[:2]
            ],
        ),
        (0, [f"wallet_id:{expected_wallet_ids[2]}".encode()]),
    ]

    redis_client = AsyncMock()
    redis_client.scan = AsyncMock(side_effect=scan_results)

    redis_service = RedisService(redis_client)

    wallet_ids = await redis_service.get_all_wallet_ids()

    assert set(wallet_ids) == set(expected_wallet_ids)
    assert redis_client.scan.call_count == 2


# -*- coding: utf-8 -*-
# __author__ = 'zoujiaxun'
# last_modify:20200812
from biocluster.api.database.base import Base, report_check
from bson.objectid import ObjectId
from types import StringTypes
from bson.son import SON
from biocluster.config import Config
import datetime
import json
import os
import pandas as pd
import glob
import re
from mbio.api.database.ref_rna_v2.api_base import ApiBase

class Gsva(ApiBase):
    def __init__(self, bind_object):
        super(Gsva, self).__init__(bind_object)
        self._project_type = 'tool_lab'

    def add_gsva(self, main_id, es_matrix, gmx, params=None, project_sn='medical_transcriptome', task_id='medical_transcriptome'):
        """
        dump exp data into database
        :param exp_matrix: expression matrix path or an express matrix in pandas DataFrame format.
        :param exp_level: str, transcript or gene
        :param exp_type: str, usually is tpm or fpkm or count. default tpm
        :param group_dict: ordered dict of group info
        :param quant_method: the method to be used to quant expression.
        :param project_sn: project id
        :param task_id: task id
        :param main_id: 主表id，如果提供，则本函数不创建主表
        :param group_id: 包含分组方案信息的主表id
        :param add_distribution: 是否添加表达分布信息到数据库
        :param params: parameters dict for expression quant.
        :return: main table id
        :version:version info
        """
        # add main table info
        if main_id is None:
            name = "Gsva" + '_'
            time_now = datetime.datetime.now()
            name += time_now.strftime("%Y%m%d_%H%M%S")
            if type(params) == dict:
                params = json.dumps(params, sort_keys=True, separators=(',', ':'))
            main_info = dict(
                project_sn=project_sn,
                task_id=task_id,
                name=name,
                created_ts=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                desc='Gsva',
                params=params,
                version="v1",
                status="start",
            )
            main_id = self.create_db_table('sg_geneset_gsva', [main_info])
        if not isinstance(main_id, ObjectId):
            main_id = ObjectId(main_id)
        df = pd.read_table(es_matrix, header=0, sep='\t')
        gsva_columns = df.columns
        columns_list = list()
        columns_list.append({'field': 'geneset', 'filter': False, 'sort':False, 'title': 'geneset', 'type': 'string'})
        for i in gsva_columns[1:]:
            columns_list.append({'field': i, 'filter': False, 'sort': False, 'title': i, 'type': 'float'})
        data_columns = {'column': columns_list, 'condition': {}}
        columns_data = json.dumps(data_columns)
        df['gsva_id'] =main_id
        df['type'] = 'heatmap'
        detail = df.to_dict('r')
        self.create_db_table('sg_geneset_gsva_detail', detail)

        main_collection = self.db['sg_geneset_gsva']
        main_collection.update({"_id": main_id},
                               {"$set": {"status": "end", 'column_data_detail': columns_data}})
        return main_id

    def add_gsva_cluster(self, cluster_output_dir, main_id=None, project_sn='medical_transcriptome',
                            task_id='medical_transcriptome',
                            params=None, title=None):
        # prepare main_table data
        results = os.listdir(cluster_output_dir)
        gene_cluster, sample_cluster = False, False
        genes, samples = list(), list()
        gene_tree, sample_tree = "", ""

        if "seq.cluster_tree.txt" in results:
            target_file = os.path.join(cluster_output_dir, "seq.cluster_tree.txt")
            with open(target_file) as f:
                gene_cluster = True
                gene_tree = f.readline().strip()
                genes = f.readline().strip().split(";")
        #
        if "sample.cluster_tree.txt" in results:
            target_file = os.path.join(cluster_output_dir, "sample.cluster_tree.txt")
            with open(target_file) as f:
                sample_cluster = True
                sample_tree = f.readline().strip()
                sample_tree_list = re.findall('[(,]([^(]*?):', sample_tree)
                samples = f.readline().strip().split(";")
        #
        if "seq.kmeans_cluster.txt" in results:
            gene_cluster = True
            target_file = os.path.join(cluster_output_dir, "seq.kmeans_cluster.txt")
            with open(target_file) as f:
                genes = list()
                for line in f:
                    if not line.strip():
                        continue
                    genes += line.strip().split('\t')[1].split(";")

        if 'gene_name' in samples:
            samples.remove('gene_name')

        if isinstance(main_id, ObjectId):
            main_id = ObjectId(main_id)
        # update main table

        tree_info_sample = dict(
            name=title,
            direction="h",
            data=sample_tree,
            type="tree",
            gsva_id=main_id,
        )
        tree_info_feature = dict(
            name=title,
            direction="v",
            data=gene_tree,
            type="tree",
            gsva_id=main_id,
        )
        tree_data = dict(name="name", condition={'type':'tree'})
        tree_data = json.dumps(tree_data)
        heatmap_data = dict(name='geneset', data=sample_tree_list, condition={'type': 'heatmap'})
        heatmap_data = json.dumps(heatmap_data)
        self.create_db_table('sg_geneset_gsva_cluster_heatmap_tree', [tree_info_sample])
        self.create_db_table('sg_geneset_gsva_cluster_heatmap_tree', [tree_info_feature])
        self.update_db_record('sg_geneset_gsva', main_id, status="end", main_id=main_id, tree_data=tree_data, heatmap_data=heatmap_data,)

    def add_gsva_diff(self, diff_output, main_id, group):
        group_dict = dict()
        with open(group, 'r') as g:
            for line in g.readlines():
                if line.startswith('#'):
                    continue
                else:
                    sample, group = line.strip().split('\t')
                    if group not in group_dict:
                        group_dict[group]=[sample]
                    else:
                        group_dict[group].append(sample)
        diff_files = glob.glob(os.path.join(diff_output, '*_vs_*.*.xls'))
        df_list = list()
        compare_list = list()
        for each in diff_files:
            diff_pd = pd.read_table(each, header=0, sep='\t')
            fname = os.path.basename(each)
            ctrl, test = re.match('(.*)_vs_(.*).limma.xls', fname).groups()
            diff_pd['compare'] = '{}|{}'.format(ctrl, test)
            compare_list.append({'field': '{}|{}'.format(ctrl, test), 'title': '{}|{}'.format(ctrl, test)})
            need_cols = ['geneset', 'log2fc', 'compare', 'fc', 'pvalue', 'padjust']
            diff_pd_new = diff_pd.loc[:,need_cols]
            df_list.append(diff_pd_new)
        df_total = pd.concat(df_list)
        df_total['gsva_id'] = main_id
        detail = df_total.to_dict('r')
        self.create_db_table('sg_geneset_gsva_diff', detail)
        gsva_columns = {'column': [{'field': 'geneset', 'filter': False, 'sort': False, 'title': 'geneset',
                                    'type': 'string'},
                                   {'field': 'log2fc', 'filter': False, 'sort': False,
                                    'title': 'log2fc',
                                    'type': 'float'},
                                   {'field': 'fc', 'filter': False, 'sort': False, 'title': 'fc',
                                    'type': 'float'},
                                   {'field': 'pvalue', 'filter': False, 'sort': False, 'title': 'pvalue',
                                    'type': 'float'},
                                   {'field': 'padjust', 'filter': False, 'sort': False, 'title': 'padjust',
                                    'type': 'float'},
                                   ], 'condition': {}}
        column_data = json.dumps(gsva_columns)
        main_collection = self.db['sg_geneset_gsva']
        main_collection.update({"_id": main_id},
                               {"$set": {"status": "end", 'column_data_diff': column_data}})
        compare_list_data = json.dumps(compare_list)
        main_collection.update({"_id": main_id},
                               {"$set": {"status": "end", 'compare': compare_list_data}})


#
# hw0pr2.py ~ phonebook analysis
#
# Name(s):Tiancheng(Tony) Jiang
#

#
# be sure your file runs from this location,
# relative to the "phonebook" directories
#

#
# hw0pr2.py ~ phonebook analysis
#
# Name(s): Tiancheng(Tony) Jiang

# The Main may take a while to run all the main tests

#
# be sure your file runs from this location,
# relative to the "phonebook" directories
#

import os
import os.path
import shutil

def how_many_txt_files(path):
    """ walks a whole directory structure
        and returns how many txt files are in it!

        call it with: how_many_txt_files(".")

        the (v1) (v2) etc. are different versions, to illustrate
        the process of _trying things out_ and _taking small steps_
    """
    # return 42  # just to check that it's working (v1)

    AllFiles = list(os.walk(path))
    # print(AllFiles)    # just to check out what's up (v2)

    #print("AllFiles has length: ", len(AllFiles), "\n")
    totalCount = 0

    for item in AllFiles:
        # print("item is", item, "\n")    (v3)
        foldername, LoDirs, LoFiles = item   # cool!
        #print("In", foldername, "there are", end=" ")

        count = 0
        for filename in LoFiles:
            if filename[-3:] == "txt":
                count += 1
        #print(count, ".txt files")
        totalCount += count

    return totalCount   # this is not _quite_ correct!

def countdepth(s):
    """return the number of '/' contained in the path to find how deep the path is
    """
    count = 0
    for i in range(len(s)):
        if (s[i] == '/'):
            count += 1
    return count

def max_depth_dir(path):
    """returns the maximum number of times that it's possible
    to move deeper into subdirectory
    """
    AllFiles = list(os.walk(path))
    longestPath = AllFiles[0][0]
    print(longestPath)
    for i in range(len(AllFiles)-1):
        if (countdepth(longestPath) < countdepth(AllFiles[i+1][0])):
            longestPath = AllFiles[i+1][0]
    return longestPath

def count_numInstring(s):
    """Returns the number is digits in a string
    """
    count = 0
    for i in range(len(s)):
        if (s[i].isdigit()):
            count += 1
    return count

def find_files_with10dig_phonenum(path):
    """return the number files that have 10 digit phonenumber in its content
    """
    fileWith10dig = 0
    ListOfContents = os.listdir(path)

    for item in ListOfContents:
        newPath = path + "/" + item
        if os.path.isdir(newPath):
            fileWith10dig = fileWith10dig + find_files_with10dig_phonenum(newPath)
        else:
            filepath = os.path.join(path, item)
            try:
                f = open(filepath,"r", encoding="latin1") # latin1 is a very safe encoding
                data = f.readline()   # read all of the file's data
                if (count_numInstring(data) == 10):
                    fileWith10dig = fileWith10dig + 1
                f.close()         # close the file

            except PermissionError:  # example of "exceptions": atypical errors
                print("file", item, "couldn't be opened: permission error")
                data = ""

            except UnicodeDecodeError:
                print("file", item, "couldn't be opened: encoding error")
                data = "" # no data

            except FileNotFoundError:  # try it with and without this block...
                print("file", item, "couldn't be opened: not found!")
                print("Check that you have the correct path...")
                data = ""
    return fileWith10dig

def find_files_909areacode_phonenum(path):
    """return the number files that have 909 as the phone number's area code
    """
    count = 0
    ListOfContents = os.listdir(path)

    for item in ListOfContents:
        newPath = path + "/" + item
        if os.path.isdir(newPath):
            count = count + find_files_909areacode_phonenum(newPath)
        else:
            filepath = os.path.join(path, item)
            try:
                f = open(filepath,"r", encoding="latin1") # latin1 is a very safe encoding
                data = f.readline()   # read all of the file's data
                if (data[0:3] == "909" and count_numInstring(data) == 10):
                    count = count + 1
                f.close()         # close the file

            except PermissionError:  # example of "exceptions": atypical errors
                print("file", item, "couldn't be opened: permission error")
                data = ""

            except UnicodeDecodeError:
                print("file", item, "couldn't be opened: encoding error")
                data = "" # no data

            except FileNotFoundError:  # try it with and without this block...
                print("file", item, "couldn't be opened: not found!")
                print("Check that you have the correct path...")
                data = ""

    return count

def how_many_lastname(s, path):
    """ returns the number of files that contain the last name s(a string) in this path
    """
    AllFiles = list(os.walk(path))

    count = 0

    for item in AllFiles:

        for filename in item[2]:
            filepath = item[0] + '/' + filename

            try:
                f = open(filepath,"r", encoding="latin1")
                data = f.readlines()[1]   # read all of the file's data
                if(',' in data):
                    if (data.startswith(s)):
                        count += 1
                else:
                    if (data.endswith(s)):
                        count += 1
                f.close()

            except PermissionError:
                print("file", item, "couldn't be opened: permission error")
                data = ""

            except UnicodeDecodeError:
                print("file", item, "couldn't be opened: encoding error")
                data = "" # no data

            except FileNotFoundError:
                print("file", item, "couldn't be opened: not found!")
                print("Check that you have the correct path...")
                data = ""

    return count

def how_many_firstname(s, path):
    """ returns the number of files that contain the first name s(a string) in this path
    """

    AllFiles = list(os.walk(path))

    count = 0

    for item in AllFiles:

        for filename in item[2]:
            filepath = item[0] + '/' + filename
            try:
                f = open(filepath,"r", encoding="latin1")
                data = f.readlines()[1]
                if(',' in data):
                    if (data.endswith(s)):
                        count += 1
                else:
                    if (data.startswith(s)):
                        count += 1
                f.close()         # close the file

            except PermissionError:  # example of "exceptions": atypical errors
                print("file", item, "couldn't be opened: permission error")
                data = ""

            except UnicodeDecodeError:
                print("file", item, "couldn't be opened: encoding error")
                data = "" # no data

            except FileNotFoundError:  # try it with and without this block...
                print("file", item, "couldn't be opened: not found!")
                print("Check that you have the correct path...")
                data = ""

    return count

def how_many_more_than_10dig(path):
    """ returns the number of files that contain phone number with greater than 10 digits
    """

    AllFiles = list(os.walk(path))

    count = 0

    for item in AllFiles:

        for filename in item[2]:
            filepath = item[0] + '/' + filename
            try:
                f = open(filepath,"r", encoding="latin1")
                data = f.readlines()[0]   # read all of the file's data
                if(count_numInstring(data) > 10):
                    count += 1
                f.close()         # close the file

            except PermissionError:  # example of "exceptions": atypical errors
                print("file", item, "couldn't be opened: permission error")
                data = ""

            except UnicodeDecodeError:
                print("file", item, "couldn't be opened: encoding error")
                data = "" # no data

            except FileNotFoundError:  # try it with and without this block...
                print("file", item, "couldn't be opened: not found!")
                print("Check that you have the correct path...")
                data = ""

    return count

#my own three questions
#1 what is the most common area code here
from collections import defaultdict      # be sure to import it!

def most_common_areacode(path):
    """
    Question: what is the most common area code here?
    Answer: to this question is area code (442)

    I did this question by creating defaultdict with all the areacodes as key
    by looping through all files. Then I pick the area code have the most values to find
    the one that appears the most often

    """

    AllFiles = list(os.walk(path))

    areacodes = defaultdict(int)
    for item in AllFiles:

        for filename in item[2]:
            filepath = item[0] + '/' + filename #os.path.join(path, item)
            try:
                f = open(filepath,"r", encoding="latin1") # latin1 is a very safe encoding
                data = f.readlines()[0]   # read all of the file's data
                ac = data[0:3]
                areacodes[ac] += 1
                f.close()         # close the file

            except PermissionError:  # example of "exceptions": atypical errors
                print("file", item, "couldn't be opened: permission error")
                data = ""

            except UnicodeDecodeError:
                print("file", item, "couldn't be opened: encoding error")
                data = "" # no data

            except FileNotFoundError:  # try it with and without this block...
                print("file", item, "couldn't be opened: not found!")
                print("Check that you have the correct path...")
                data = ""
    counter = 0
    mc = ""
    for code in areacodes.keys():
        if (areacodes[code] > counter):
            mc = code
            counter = areacodes[code]
    return  mc

#2
def how_many_substring42(path):
    """
    Question: How many phone numbers have substring 42 in it
    Answer: 2500

    I looped through all the phonebook files and check if each phone number contains 42 or not

    """

    AllFiles = list(os.walk(path))

    count = 0
    for item in AllFiles:

        for filename in item[2]:
            filepath = item[0] + '/' + filename #os.path.join(path, item)
            try:
                f = open(filepath,"r", encoding="latin1") # latin1 is a very safe encoding
                data = f.readlines()[0]   # read all of the file's data
                if ("42" in data):
                    count += 1
                f.close()         # close the file

            except PermissionError:  # example of "exceptions": atypical errors
                print("file", item, "couldn't be opened: permission error")
                data = ""

            except UnicodeDecodeError:
                print("file", item, "couldn't be opened: encoding error")
                data = "" # no data

            except FileNotFoundError:  # try it with and without this block...
                print("file", item, "couldn't be opened: not found!")
                print("Check that you have the correct path...")
                data = ""

    return count

#3
def firstnamesize(s):
    """return the number of letters for the first name
    """
    if(s == ""):
        return 0
    else:
        if(',' in s):
            if (s[-1] == ' ' ):
                return -1
            else:
                return 1 + firstnamesize(s[:-1])
        else:
            if (s[0] == ' ' ):
                return 0
            else:
                return 1 + firstnamesize(s[1:])

def size_longest_firstname(path):
    """
    Question: what is the name with longest firstname in phonebook?
    Answer: Providencia Hamlett and has length 11 for firstname

    I did this question by creating defaultdict with all the names as key
    by looping through all files. Then I pick the name that has the longest
    first name by using the helper function above

    """

    AllFiles = list(os.walk(path))

    longest = 0
    names = defaultdict(int)
    for item in AllFiles:

        for filename in item[2]:
            filepath = item[0] + '/' + filename #os.path.join(path, item)
            try:
                f = open(filepath,"r", encoding="latin1") # latin1 is a very safe encoding
                data = f.readlines()[1]   # read all of the file's data
                names[data] = firstnamesize(data)

                f.close()         # close the file

            except PermissionError:  # example of "exceptions": atypical errors
                print("file", item, "couldn't be opened: permission error")
                data = ""

            except UnicodeDecodeError:
                print("file", item, "couldn't be opened: encoding error")
                data = "" # no data

            except FileNotFoundError:  # try it with and without this block...
                print("file", item, "couldn't be opened: not found!")
                print("Check that you have the correct path...")
                data = ""

    counter = 0
    n = ""
    for i in names.keys():
        if (names[i] > counter):
            n = i
            counter = names[i]
    return n, counter

def main():
    """ overall function to run all examples """

    print("Start of main()\n")

    num_txt_files = how_many_txt_files(".")
    print("num_txt_files in . is:", num_txt_files)
    print('\n')
    max_depth_directories = countdepth(max_depth_dir("."))
    print("max_depth_directories in . is:", max_depth_directories)
    print('\n')
    path_deepest_directory = max_depth_dir(".")
    print("path_deepest_directory in . is:", path_deepest_directory)
    print('\n')
    num_files_with10dig_phone_number = find_files_with10dig_phonenum('.')
    print("num_files_with10dig_phone_numer in . is:", num_files_with10dig_phone_number)
    print('\n')
    num_files_with10dig_with909areacode = find_files_909areacode_phonenum('.')
    print("num_files_with10dig_with909areacode in . is:", num_files_with10dig_with909areacode)
    print('\n')
    result = how_many_lastname('Davis', '.')
    print("number of people with last name 'Davis':", result)
    print('\n')
    result0 = how_many_lastname('Canel', '.')
    print("number of people with last name 'Canel':", result0)
    print('\n')
    result1 = how_many_firstname('Davis','.')
    print("number of people with first name 'Davis':", result1)
    print('\n')
    result3 = how_many_firstname('Oliva','.')
    print("number of people with first name 'Oliva':", result3)
    print('\n')
    result4 = how_many_more_than_10dig('.')
    print("number of people with phonenumber of greater than 10 digits :", result4)
    print('\n')
    print("THE FOLLOWING ARE MY OWN QUESTIONS:")
    print("1. How many people have substring 42 in his or her phone number?")
    result5 = how_many_substring42('.')
    print("number of people with phonenumber that has '42' in it :", result5)
    print('\n')
    result6 = most_common_areacode('.')
    print("2. what is the most common area code out of all phone numbers?")
    print("The most common area code of all phonenumbers is :", result6)
    print('\n')
    result7, size = size_longest_firstname('.')
    print("3. what is the longest firstname in the phonebook and its length?")
    print("The size of longest firstname is:", result7, "with length", size)
    print('\n')

if __name__ == "__main__":
    main()


import time
import unittest
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class ProfessorTests(unittest.TestCase):

    def setUp(self):
        # Initialize the WebDriver (e.g., for Chrome)
        self.driver = webdriver.Chrome(executable_path='C://programming//chromedriver_win32//chromedriver.exe')

    def tearDown(self):
        # Close the WebDriver after each test
        self.driver.quit()

    def test_add_professor(self):
        driver = self.driver
        driver.get("http://localhost/WebProgramming")  # Replace with your website URL

        # Locate and click the "Add Professor" button
        add_professor_button = driver.find_element(By.ID, "addProfessorButton")
        add_professor_button.click()

        # Wait for the modal to appear (you may need to adjust the wait time)
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.ID, "addProfessorModal"))
        )

        # Locate the input fields in the modal
        fullname_input = driver.find_element(By.NAME, "fullname")
        email_input = driver.find_element(By.NAME, "email")
        phone_input = driver.find_element(By.NAME, "phone")
        gender_select = driver.find_element(By.NAME, "gender")
        dob_input = driver.find_element(By.NAME, "dateofbirth")

        # Enter professor information
        fullname_input.send_keys("John Doe")
        email_input.send_keys("johndoe@example.com")
        phone_input.send_keys("1234567890")
        gender_select.select_by_value("male")  # Replace with the appropriate value
        dob_input.send_keys("2000-01-01")  # Replace with the appropriate date

        # Locate and click the "Submit" button in the modal
        submit_button = driver.find_element(By.XPATH, "//button[text()='Submit']")
        submit_button.click()

        # Wait for the modal to close (you may need to adjust the wait time)
        WebDriverWait(driver, 10).until(
            EC.invisibility_of_element_located((By.ID, "addProfessorModal"))
        )

        # Verify that the new professor is displayed on the page
        new_professor = driver.find_element(By.XPATH, "//h5[text()='John Doe']")
        self.assertEqual(new_professor.text, "John Doe")

    # Add more test cases for other professor-related functionalities (e.g., editing, deleting, viewing profile, etc.)

if __name__ == "__main__":
    unittest.main()


from django.urls import path

from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('results/', views.results, name='results'),
    path('sendmail/', views.sendmail, name='sendmail'),
    path('addcontact/', views.addcontact, name='addcontact'),
    path('ajaxForm/', views.ajaxForm, name='ajaxForm')
]


from codepack import CodePack, CodePackSnapshot, ArgPack
from fastapi import APIRouter, HTTPException
from ..models.scheduler import CodePackIDJob, CodePackJSONJob, SnapshotJSONJob, IDPairJob
from apscheduler.jobstores.base import JobLookupError, ConflictingIdError
from ..dependencies import common

router = APIRouter(
    prefix='/scheduler',
    tags=['scheduler'],
    responses={404: {'description': 'Not Found'},
               409: {'description': 'Conflict'}},
)

@router.post('/register')
async def register(params: CodePackJSONJob):
    codepack = CodePack.from_dict(params.codepack)
    argpack = ArgPack.from_dict(params.argpack)
    try:
        common.scheduler.add_codepack(codepack=codepack, argpack=argpack, job_id=params.job_id,
                                      trigger=params.trigger, **params.trigger_config)
    except ConflictingIdError:
        _job_id = params.job_id if params.job_id else codepack.id
        raise HTTPException(status_code=409, detail='%s already exists' % _job_id)
    return {'serial_number': codepack.serial_number}

@router.post('/register/id')
async def register_by_id(params: CodePackIDJob):
    codepack = CodePack.load(params.id)
    if codepack is None:
        raise HTTPException(status_code=404, detail="'%s' not found" % params.id)
    argpack = ArgPack.from_dict(params.argpack)
    try:
        common.scheduler.add_codepack(codepack=codepack, argpack=argpack, job_id=params.job_id,
                                      trigger=params.trigger, **params.trigger_config)
    except ConflictingIdError:
        _job_id = params.job_id if params.job_id else codepack.id
        raise HTTPException(status_code=409, detail='%s already exists' % _job_id)
    return {'serial_number': codepack.serial_number}

@router.post('/register/id-pair')
async def register_by_id_pair(params: IDPairJob):
    codepack = CodePack.load(params.codepack_id)
    if codepack is None:
        raise HTTPException(status_code=404, detail="'%s' not found" % params.codepack_id)
    argpack = ArgPack.load(params.argpack_id)
    if argpack is None:
        raise HTTPException(status_code=404, detail="'%s' not found" % params.argpack_id)
    try:
        common.scheduler.add_codepack(codepack=codepack, argpack=argpack, job_id=params.job_id,
                                      trigger=params.trigger, **params.trigger_config)
    except ConflictingIdError:
        _job_id = params.job_id if params.job_id else codepack.id
        raise HTTPException(status_code=409, detail='%s already exists' % _job_id)
    return {'serial_number': codepack.serial_number}

@router.post('/register/snapshot')
async def register_by_snapshot(params: SnapshotJSONJob):
    snapshot = CodePackSnapshot.from_dict(params.snapshot)
    try:
        common.scheduler.add_codepack(snapshot=snapshot, job_id=params.job_id,
                                      trigger=params.trigger, **params.trigger_config)
    except ConflictingIdError:
        _job_id = params.job_id if params.job_id else snapshot.id
        raise HTTPException(status_code=409, detail='%s already exists' % _job_id)
    return {'serial_number': snapshot.serial_number}

@router.delete('/unregister/{id}')
async def unregister(id: str):
    try:
        common.scheduler.remove_job(job_id=id)
    except JobLookupError:
        raise HTTPException(status_code=404, detail='%s not found' % id)
    return {'id': id}

@router.get('/alive')
async def alive():
    is_alive = common.scheduler.is_running()
    return {'alive': is_alive}


"""
Integration test for replace_standard_id_in_domain_tables module

Original Issues: DC-808, DC-1170

The intent is to map the unmapped survey answers (value_as_concept_ids=0) using
value_source_concept_id through 'Maps to' relationship
"""

# Python Imports
import os

# Project Imports
from common import CONDITION_OCCURRENCE, CONCEPT, CONCEPT_RELATIONSHIP
from common import JINJA_ENV
from app_identity import PROJECT_ID
from cdr_cleaner.cleaning_rules.replace_standard_id_in_domain_tables import \
    ReplaceWithStandardConceptId, SRC_CONCEPT_ID_TABLE_NAME
from tests.integration_tests.data_steward.cdr_cleaner.cleaning_rules.bigquery_tests_base import \
    BaseTest

class ReplaceWithStandardConceptIdTest(BaseTest.CleaningRulesTestBase):

    @classmethod
    def setUpClass(cls):
        print('**************************************************************')
        print(cls.__name__)
        print('**************************************************************')

        super().initialize_class_vars()

        # Set the test project identifier
        cls.project_id = os.environ.get(PROJECT_ID)

        # Set the expected test datasets
        cls.dataset_id = os.environ.get('COMBINED_DATASET_ID')
        cls.sandbox_id = cls.dataset_id + '_sandbox'

        cls.rule_instance = ReplaceWithStandardConceptId(
            cls.project_id, cls.dataset_id, cls.sandbox_id)

        # Generates list of fully qualified table names and their corresponding sandbox table names
        for table_name in cls.rule_instance.affected_tables:
            cls.fq_table_names.append(
                f'{cls.project_id}.{cls.dataset_id}.{table_name}')
            cls.fq_table_names.append(
                f'{cls.project_id}.{cls.dataset_id}._mapping_{table_name}')
            sandbox_table_name = cls.rule_instance.sandbox_table_for(table_name)
            cls.fq_sandbox_table_names.append(
                f'{cls.project_id}.{cls.sandbox_id}.{sandbox_table_name}')

        cls.fq_table_names.extend([
            f'{cls.project_id}.{cls.dataset_id}.{table_name}'
            for table_name in [CONCEPT, CONCEPT_RELATIONSHIP]
        ])

        # Add _logging_standard_concept_id_replacement to fq_sandbox_table_names for cleanup in
        # teardown
        cls.fq_sandbox_table_names.append(
            f'{cls.project_id}.{cls.sandbox_id}.{SRC_CONCEPT_ID_TABLE_NAME}')

        # call super to set up the client, create datasets
        cls.up_class = super().setUpClass()

    def setUp(self):
        """
        Create empty tables for the rule to run on
        """
        # Create tables required for the test
        super().setUp()

        condition_occurrence_data_template = JINJA_ENV.from_string("""
        INSERT INTO `{{project_id}}.{{dataset_id}}.condition_occurrence` (
                condition_occurrence_id,
                person_id,
                condition_concept_id,
                condition_start_date,
                condition_start_datetime,
                condition_end_date,
                condition_end_datetime,
                condition_type_concept_id,
                stop_reason,
                provider_id,
                visit_occurrence_id,
                condition_source_value,
                condition_source_concept_id,
                condition_status_source_value,
                condition_status_concept_id)
        VALUES
              (1, 1, 319835, '2020-05-10', timestamp('2020-05-10 00:00:00 UTC'), null, null, 0, null, null, null, null, 45567179, null, null),
               (2, 1, 0, '2020-05-10', timestamp('2020-05-10 00:00:00 UTC'), null, null, 0, null, null, null, null, 45567179, null, null),
               (3, 1, 45567179, '2020-05-10', timestamp('2020-05-10 00:00:00 UTC'), null, null, 0, null, null, null, null, 0, null, null),
               (4, 1, 45567179, '2020-05-10', timestamp('2020-05-10 00:00:00 UTC'), null, null, 0, null, null, null, null, 45567179, null, null),
               (5, 1, 40398862, '2020-05-10', timestamp('2020-05-10 00:00:00 UTC'), null, null, 0, null, null, null, null, 40398862, null, null),
               (6, 1, 45587397, '2020-05-10', timestamp('2020-05-10 00:00:00 UTC'), null, null, 0, null, null, null, null, 45587397, null, null)
        """)

        mapping_condition_occurrence_data_template = JINJA_ENV.from_string("""
        INSERT INTO `{{project_id}}.{{dataset_id}}._mapping_condition_occurrence`
        (
          condition_occurrence_id, src_table_id, src_dataset_id, src_condition_occurrence_id, src_hpo_id)
        VALUES
               (1, 'hpo_condition_occurrence', 'hpo_dataset', 10, 'test_hpo'),
               (2, 'hpo_condition_occurrence', 'hpo_dataset', 20, 'test_hpo'),
               (3, 'hpo_condition_occurrence', 'hpo_dataset', 30, 'test_hpo'),
               (4, 'hpo_condition_occurrence', 'hpo_dataset', 40, 'test_hpo'),
               (5, 'hpo_condition_occurrence', 'hpo_dataset', 50, 'test_hpo'),
               (6, 'hpo_condition_occurrence', 'hpo_dataset', 60, 'test_hpo')
        """)

        concept_data_template = JINJA_ENV.from_string("""
        INSERT INTO `{{project_id}}.{{dataset_id}}.concept`
        (
          concept_id, concept_name, standard_concept, domain_id, vocabulary_id, concept_class_id, concept_code, valid_start_date, valid_end_date)
        VALUES
               (319835, 'Congested Heart Failure', 'S', 'foo', 'bar', 'baz', 'alpha', '2020-01-01', '2099-12-31'),
               (45567179, 'Congested Heart Failure', 'C', 'foo', 'bar', 'baz', 'beta', '2020-01-01', '2099-12-31'),
               (45587397, 'Anaemia complicating pregnancy, childbirth and the puerperium', 'C', 'foo', 'bar', 'baz', 'gamma', '2020-01-01', '2099-12-31'),
               (434701, 'Anemia in mother complicating pregnancy, childbirth AND/OR puerperium', 'S', 'foo', 'bar', 'baz', 'rho', '2020-01-01', '2099-12-31'),
               (444094, 'Finding related to pregnancy', 'S', 'foo', 'bar', 'baz', 'phi', '2020-01-01', '2099-12-31'),
               (40398862, 'Ischemic chest pain', 'C', 'foo', 'bar', 'baz', 'zed', '2020-01-01', '2099-12-31')
        """)

        concept_relationship_data_template = JINJA_ENV.from_string("""
        INSERT INTO `{{project_id}}.{{dataset_id}}.concept_relationship`
        (
          concept_id_1, concept_id_2, relationship_id, valid_start_date, valid_end_date)
        VALUES
               (319835, 319835, 'Maps to', '2020-01-01', '2099-12-31'),
               (45567179, 319835, 'Maps to', '2020-01-01', '2099-12-31'),
               (45587397, 434701, 'Maps to', '2020-01-01', '2099-12-31'),
               (45587397, 444094, 'Maps to', '2020-01-01', '2099-12-31')
        """)

        concept_data_query = concept_data_template.render(
            project_id=self.project_id, dataset_id=self.dataset_id)

        concept_relationship_data_query = concept_relationship_data_template.render(
            project_id=self.project_id, dataset_id=self.dataset_id)

        condition_occurrence_data_query = condition_occurrence_data_template.render(
            project_id=self.project_id, dataset_id=self.dataset_id)

        mapping_condition_occurrence_data_query = mapping_condition_occurrence_data_template.render(
            project_id=self.project_id, dataset_id=self.dataset_id)

        # Load test data
        self.load_test_data([
            f'''{concept_data_query};
                {concept_relationship_data_query};
                {condition_occurrence_data_query};
                {mapping_condition_occurrence_data_query}'''
        ])

    def test_replace_standard_id_in_domain_tables(self):
        """
        condition_occurrence

        record 1: condition_concept_id is already standard, so do nothing

        record 2: condition_concept_id is 0, but condition_source_concept_id 45567179 can be
        mapped to the standard concept 319835. So condition_concept_id should be replaced with
        319835 and this record is sandboxed.

        record 3: condition_concept_id is the non-standard 45567179 and
        condition_source_concept_id 0. 45567179 can be mapped to the standard concept 319835. So
        condition_concept_id should be replaced with 319835 and condition_source_concept_id
        should be replaced by the original condition_concept_id 45567179 and this record is
        sandboxed.

        record 4: condition_concept_id is the non-standard 45567179 and
        condition_source_concept_id the same non-standard 45567179. 45567179 can be mapped to the
        standard concept 319835. So condition_concept_id should be replaced with 319835 and
        condition_source_concept_id remains the same and this record is sandboxed.

        record 5: condition_concept_id is the non-standard 40398862 and
        condition_source_concept_id the same non-standard 40398862, but there is no standard
        concept mapping for 40398862. Although condition_concept_id and
        condition_source_concept_id remain the same, this record is still sandboxe because this
        record is flagged as being non-standard.

        record 6: condition_concept_id is the non-standard 45587397 and
        condition_source_concept_id the same non-standard 45587397. 45567179 can be mapped to two
        standard concepts 434701 and 444094. So this record is split into two new records,
        the new ids are generated using this logic MAX(condition_occurrence_id) + ROW_NUMBER().
        Two new records with condition_occurrence_id 7 and 8 are generated and the corresponding
        mapping table _mapping_condition_occurrence is updated as well. This record is sandboxed.

        :return:
        """
        # Expected results list
        tables_and_counts = [{
            'fq_table_name':
                f'{self.project_id}.{self.dataset_id}.{CONDITION_OCCURRENCE}',
            'fq_sandbox_table_name':
                f'{self.project_id}.{self.sandbox_id}.{self.rule_instance.sandbox_table_for(CONDITION_OCCURRENCE)}',
            'loaded_ids': [1, 2, 3, 4, 5, 6],
            'sandboxed_ids': [2, 3, 4, 5, 6],
            'fields': [
                'condition_occurrence_id', 'person_id', 'condition_concept_id',
                'condition_source_concept_id'
            ],
            'cleaned_values': [
                (1, 1, 319835, 45567179), (2, 1, 319835, 45567179),
                (3, 1, 319835, 45567179), (4, 1, 319835, 45567179),
                (5, 1, 40398862, 40398862), (7, 1, 434701, 45587397),
                (8, 1, 444094, 45587397)
            ]
        }, {
            'fq_table_name':
                f'{self.project_id}.{self.dataset_id}._mapping_{CONDITION_OCCURRENCE}',
            'loaded_ids': [1, 2, 3, 4, 5, 6],
            'fields': [
                'condition_occurrence_id', 'src_condition_occurrence_id'
            ],
            'cleaned_values': [(1, 10), (2, 20), (3, 30), (4, 40), (5, 50),
                               (7, 60), (8, 60)]
        }]

        self.default_test(tables_and_counts)


# -*- coding: utf-8 -*-
from osv import osv
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta

class GiscedataFacturacioValidationValidator(osv.osv):
    _inherit = 'giscedata.facturacio.validation.validator'
    _name = 'giscedata.facturacio.validation.validator'

    def check_origin_readings_by_contract_category(self, cursor, uid, fact, parameters):

        if 'categoria' not in parameters:
            return None

        pcat_obj = self.pool.get('giscedata.polissa.category')
        pcat_ids = pcat_obj.search(cursor, uid,[('name','like','%'+parameters['categoria']+'%')])

        if pcat_ids and pcat_ids[0] in [x.id for x in fact.polissa_id.category_id]:

            lect_obj = self.pool.get('giscedata.lectures.lectura')

            lo_obj = self.pool.get('giscedata.lectures.origen')
            loc_obj = self.pool.get('giscedata.lectures.origen_comer')

            not_alowed_origins_ids = lo_obj.search(cursor, uid,[('codi','in', eval(parameters['lectures_origen_codes']))])
            not_alowed_origins_comer_ids = loc_obj.search(cursor, uid,[('codi','in', eval(parameters['lectures_origen_comer_codes']))])

            data_inici = datetime.strptime(fact.data_inici, '%Y-%m-%d') - timedelta(days=1)

            clause = [
                ('comptador.polissa', '=', fact.polissa_id.id),
                ('comptador.active', '=', True),
                ('tipus', '=', 'A'),
                ('name', '>=', data_inici.strftime('%Y-%m-%d')),
                ('name', '<=', fact.data_final),
            ]
            if not_alowed_origins_ids:
                clause.append(('origen_id', 'in', not_alowed_origins_ids))
            if not_alowed_origins_comer_ids:
                clause.append(('origen_comer_id', 'in', not_alowed_origins_comer_ids))

            lectures_ids = lect_obj.q(cursor, uid).read(
                ['id', 'name', 'origen_id', 'origen_comer_id'], order_by=['name.asc']
            ).where(clause)

            if len(lectures_ids) == 0:
                return None

            origen_id = list(set([x['origen_id']for x in lectures_ids]))
            origen_comer_id = list(set([x['origen_comer_id']for x in lectures_ids]))

            origins = [x['name'] for x in lo_obj.read(cursor, uid, origen_id, ['name'])]
            origins_comer = [x['name'] for x in loc_obj.read(cursor, uid, origen_comer_id, ['name'])]

            return {
                'categoria': parameters['categoria'],
                'origen': ','.join(origins),
                'origen_distri': ','.join(origins_comer)
            }

        return None

    def check_min_periods_and_teoric_maximum_consum(self, cursor, uid, fact, parameters):

        if 'category' not in parameters:
            return None

        pcat_obj = self.pool.get('giscedata.polissa.category')
        pcat_ids = pcat_obj.search(cursor, uid,[('name','like','%'+parameters['category']+'%')])

        if pcat_ids and pcat_ids[0] in [x.id for x in fact.polissa_id.category_id]:

            fact_obj = self.pool.get('giscedata.facturacio.factura')

            pol_obj = self.pool.get('giscedata.polissa')
            teoric_maximum_consume_gc = pol_obj.read(cursor, uid, fact.polissa_id.id, ['teoric_maximum_consume_gc'])['teoric_maximum_consume_gc']

            n_months = parameters['n_months']
            min_periods = parameters.get('min_periods', False)
            to_date = datetime.strptime(fact.data_inici, '%Y-%m-%d')
            from_date = to_date - relativedelta(months=n_months)

            context = {}
            if parameters.get("min_invoice_len"):
                context['min_invoice_len'] = parameters.get("min_invoice_len")

            parameter_by_date = fact_obj.get_parameter_by_contract(
                cursor, uid, fact.polissa_id.id, 'energia_kwh', from_date,
                to_date, min_periods, context=context
            )

            max_consume = False
            number_of_invoices = len(parameter_by_date)
            if number_of_invoices > 0:
                max_consume = max(parameter_by_date.values())

            if (not max_consume or number_of_invoices < n_months) and (not teoric_maximum_consume_gc or teoric_maximum_consume_gc == 0):
                return {
                    'invoice_consume': fact.energia_kwh,
                }

        return None

    def check_consume_by_percentage_and_category(self, cursor, uid, fact, parameters):

        if 'category' not in parameters:
            return None

        pcat_obj = self.pool.get('giscedata.polissa.category')
        pcat_ids = pcat_obj.search(cursor, uid,[('name','like','%'+parameters['category']+'%')])

        if pcat_ids and pcat_ids[0] in [x.id for x in fact.polissa_id.category_id]:

            fact_obj = self.pool.get('giscedata.facturacio.factura')

            if parameters.get('min_amount', False) and abs(fact.amount_total) <= parameters.get('min_amount', 0.0):
                return None

            n_months = parameters['n_months']
            min_periods = parameters.get('min_periods', False)
            to_date = datetime.strptime(fact.data_inici, '%Y-%m-%d')
            from_date = to_date - relativedelta(months=n_months)

            context = {}
            if parameters.get("min_invoice_len"):
                context['min_invoice_len'] = parameters.get("min_invoice_len")

            parameter_by_date = fact_obj.get_parameter_by_contract(
                cursor, uid, fact.polissa_id.id, 'energia_kwh', from_date,
                to_date, min_periods, context=context
            )

            max_consume = False
            number_of_invoices = len(parameter_by_date)
            if number_of_invoices > 0:
                max_consume = max(parameter_by_date.values())

            pol_obj = self.pool.get('giscedata.polissa')
            teoric_maximum_consume_gc = pol_obj.read(cursor, uid, fact.polissa_id.id, ['teoric_maximum_consume_gc'])['teoric_maximum_consume_gc']

            if not max_consume or number_of_invoices < n_months:
                max_consume = False

            if not max_consume and teoric_maximum_consume_gc and teoric_maximum_consume_gc > 0:
                max_consume = teoric_maximum_consume_gc

            if max_consume:
                percentage_margin = parameters['overuse_percentage']

                inv_consume = fact.energia_kwh
                if inv_consume > (max_consume * (100.0 + percentage_margin))/100.0:
                    return {
                        'invoice_consume': inv_consume,
                        'percentage': percentage_margin,
                        'maximum_consume': max_consume,
                        'n_months': n_months,
                        'maximum_teoric_consume_GC': teoric_maximum_consume_gc if teoric_maximum_consume_gc else 0,
                    }

        return None

    def check_consume_by_percentage(self, cursor, uid, fact, parameters):

        pcat_obj = self.pool.get('giscedata.polissa.category')
        pcat_ids = pcat_obj.search(cursor, uid,[('name','like','%Gran Contracte%')])

        if pcat_ids and pcat_ids[0] in [x.id for x in fact.polissa_id.category_id]:
            return None

        return super(GiscedataFacturacioValidationValidator,
            self).check_consume_by_percentage(cursor, uid, fact, parameters)

    def check_consume_by_amount(self, cursor, uid, fact, parameters):

        pcat_obj = self.pool.get('giscedata.polissa.category')
        pcat_ids = pcat_obj.search(cursor, uid,[('name','like','%%Gran Contracte%')])

        if pcat_ids and pcat_ids[0] in [x.id for x in fact.polissa_id.category_id]:
            return None

        return super(GiscedataFacturacioValidationValidator,
            self).check_consume_by_amount(cursor, uid, fact, parameters)

    def validate_one_invoice(self, cursor, uid, fact_id, validation_id, context=None):
        if context is None:
            context = {}

        fact_obj = self.pool.get('giscedata.facturacio.factura')
        tmpl_obj = self.pool.get('giscedata.facturacio.validation.warning.template')

        tmpl_fields = ['code', 'method', 'parameters', 'description', 'active']
        template_vals = tmpl_obj.read(cursor, uid, validation_id, tmpl_fields, context)

        fact = fact_obj.browse(cursor, uid, fact_id)
        vals = getattr(self, template_vals['method'])(
            cursor, uid, fact, template_vals['parameters']
        )

        ret = {
            'active': template_vals['active'],
            'code': template_vals['code'],
        }
        if vals is not None:
            ret.update({
                'message': template_vals['description'].format(**vals),
                'validation_warning': True,
            })
        else:
            ret.update({
                'message': '',
                'validation_warning': False,
            })
        return ret

GiscedataFacturacioValidationValidator()


from flask import Flask, render_template, request, redirect, url_for, flash
from flask_mysqldb import MySQL

app = Flask(__name__)
app.config['MYSQL_HOST'] = 'localhost'
app.config['MYSQL_USER'] = 'root'
app.config['MYSQL_PASSWORD'] = ''
app.config['MYSQL_DB'] = 'db_invoice'
mysql = MySQL(app)

app.secret_key = 'qwerty'

@app.route('/')
def Index():
    return render_template('index.html')

@app.route('/clientes')
def Clientes():
    cur = mysql.connection.cursor()
    cur.execute('SELECT * FROM  customer')
    data = cur.fetchall()
    return render_template('clientes.html', customers = data)

@app.route('/facturar')
def facturar():
    cur = mysql.connection.cursor()
    cur.execute('SELECT * FROM facturas')
    data = cur.fetchall()
    return render_template('facturar.html', invoices = data)

@app.route('/add', methods=['POST'])
def add_contact():
    if request.method == 'POST':
        id = request.form['id']
        name = request.form['name']
        status = request.form['status']
        mobile = request.form['mobile']
        cur = mysql.connection.cursor()
        cur.execute('INSERT INTO customer (id, name, status, mobile) VALUES (%s, %s, %s, %s)', (id,name, status, mobile))
        mysql.connection.commit()
        flash('Contacto Agregado de Forma Correcta')
        return redirect(url_for('Clientes'))

@app.route('/addfact', methods=['POST'])
def add_factura():
    if request.method == 'POST':
      date = request.form['date']
      id = request.form['idcli']
      price = request.form['price']
      balance = request.form['balance']
      cur = mysql.connection.cursor()
      cur.execute('INSERT INTO facturas (fecha, id_cli, price, balance) VALUES (%s, %s, %s, %s)', (date, id, price, balance))
      mysql.connection.commit()
      flash('Factura Agregada de Forma Correcta')
      return redirect(url_for('facturar'))

@app.route('/edit/<id>', methods = ['POST', 'GET'])
def editContact(id):
    cur = mysql.connection.cursor()
    cur.execute('SELECT * FROM customer WHERE id = %s', [id])
    data = cur.fetchall()
    cur.close()
    return render_template('edit.html', customers = data[0])

@app.route('/editfact/<number>', methods = ['POST', 'GET'])
def editFactura(number):
    cur = mysql.connection.cursor()
    cur.execute('SELECT * FROM facturas WHERE number = %s', [number])
    data2 = cur.fetchall()
    cur.close()
    return render_template('editfact.html', invoices = data2[0])

@app.route('/update/<id>', methods = ['POST', 'GET'])
def update(id):
    if request.method == 'POST':
        name = request.form['name']
        status = request.form['status']
        mobile = request.form['mobile']
        cur = mysql.connection.cursor()
        cur.execute("""
            UPDATE customer
            SET name = %s,
                status = %s,
                mobile = %s
            WHERE id = %s
        """, (name, status, mobile, id))
        mysql.connection.commit()
        flash('Cliente Actualizado')
        return redirect(url_for('Clientes'))

@app.route('/updatefact/<number>', methods = ['POST' , 'GET'])
def updatefact(number):
    if request.method == 'POST':
        #number = request.form['number']
        date = request.form['date']
        idcli = request.form['idcli']
        price = request.form['price']
        balance = request.form['balance']
        cur = mysql.connection.cursor()
        cur.execute("""
             UPDATE facturas
             SET fecha = %s,
                 id_cli = %s,
                 price = %s,
                 balance = %s
           WHERE number = %s
         """, (date, idcli, price, balance, number))
        mysql.connection.commit()
        flash('Factura Actualizada')
        return redirect(url_for('facturar'))

@app.route('/delete/<string:id>')
def delete_contact(id):
    cur = mysql.connection.cursor()
    cur.execute('DELETE FROM customer WHERE id = {0}'.format(id))
    mysql.connection.commit()
    flash('Contacto Removido Satisfactoriamente')
    return redirect(url_for('Clientes'))

@app.route('/deletefact/<string:number>')
def delete_factura(number):
    cur = mysql.connection.cursor()
    cur.execute('DELETE FROM facturas WHERE number = {0}'.format(number))
    mysql.connection.commit()
    flash('Factura Removida Satisfactoriamente')
    return redirect(url_for('facturar'))

if __name__ == '__main__':
    app.run(port = 3000, debug = True)