import re
import os
import json
import docx
import matplotlib

matplotlib.use("Agg")

from collections import defaultdict
from docx import Document
from docx.shared import Cm
from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_BREAK
from docx.oxml.shared import OxmlElement, qn

from api.map_client import *
from api.charts import get_pie_chart, get_line_chart

DATA_DIR = os.getenv("DATA_DIR", "./public/data")

TEMPLATE = "api/template.docx"

# TODO: Improve custom table style in template.docx
DOC_STYLE = "SALCC_style"

# Example placeholder: {{label:title}}
PLACEHOLDER_REGEX = re.compile(r"{{(?P<scope>\w+):(?P<key>\w+)}}")

TOTAL_ROW_COLOR = "cacdd1"

def create_report(unit_id, path, config):
    """
        Creates report for summary unit in docx format.

    Parameters
    ----------
    unit_id: string id of a summary unit
    path: string location where docx to be saved
    config: dict of json data from src/config

    Returns
    -------
    report: docx file

    """
    doc = Document(TEMPLATE)

    # per convention, all marine units begin with M, e.g., M100
    is_marine = unit_id[0].lower() == 'm'
    context = generate_report_context(unit_id, config, is_marine)

    table_counter = 0

    for p in doc.paragraphs:
        # Assumption: placeholders are always completely contained within a run
        for r in p.runs:
            # a run may have multiple placeholders
            for match in re.finditer(PLACEHOLDER_REGEX, r.text):
                scope, key = match.groups()

                item = _resolve(scope, key, context)

                if isinstance(item, str):
                    # match.group() has the original text to replace
                    if scope == "table":
                        r.add_break(WD_BREAK.LINE)
                    r.text = r.text.replace(match.group(), item)

                    if scope in {"table", "chart"}:
                        r.font.italic = True

                elif scope in {"map", "chart"}:
                    # Pictures are added at the run level
                    r.text = ""
                    size = 11.0
                    if scope == "map":
                        size = 16.5
                    if scope == "chart" and key == "priorities":  # If chart is present, put on its own page
                        r.add_break(WD_BREAK.PAGE)
                    if not isinstance(context[scope][key], str):
                        run = p.add_run()
                        run.add_picture(context[scope][key], width=Cm(size))

                        # add a caption if we have one in context
                        caption_name = "{0}_{1}".format(scope, key)
                        # if key in context["caption"]:
                        if caption_name in context["caption"]:
                            caption = doc.add_paragraph(
                                context["caption"][caption_name], style="TableCaption"
                            )
                            _move_p_after(caption, p)

                elif scope == "table":
                    r.text = ""
                    table_counter += 1
                    create_table(doc, item, p)

        if "{{INDICATORS}}" in p.text:
            # Remove '{{INDICATORS}}'placeholder and add the report content above empty p.text
            p.text = ""

            for ecosystem in context["ecosystem_indicators"]:
                name = ecosystem["ecosystem_name"]
                percent = ecosystem["ecosystem_percentage"]

                if percent is not "":
                    heading = "{0}: {1}% of area".format(name, percent)
                    p.insert_paragraph_before(heading, style="Heading13")
                else:
                    heading = name
                    p.insert_paragraph_before(heading, style="Heading13")

                if name == 'Inland waterbodies':
                    p.insert_paragraph_before("These are lakes, reservoirs, and ponds not included in the Blueprint 2.2 priorities. Ecosystem-specific indicators have not yet been developed for this area", style="NoInfo")

                elif ecosystem["indicators"]:

                    for indicator in ecosystem["indicators"]:
                        table_counter += 1

                        p.insert_paragraph_before(
                            indicator["value"]["indicator_name"], style="Heading11"
                        )
                        p.insert_paragraph_before(
                            indicator["value"]["indicator_description"]
                        )

                        caption_text = "Table {0}: {1}".format(
                            table_counter, indicator["value"]["indicator_caption"]
                        )
                        caption = p.insert_paragraph_before(
                            caption_text, style="TableCaption"
                        )

                        create_table(
                            doc,
                            indicator["table"]["indicator_table"],
                            caption,
                        )

                else:
                    p.insert_paragraph_before("No indicators are present in this area", style="NoInfo")

            # Delete the placeholder para
            delete_paragraph(p)

        if "{{THREATS}}" in p.text:
            # Remove '{{THREATS}}'placeholder and add the report content
            p.text = ""
            if is_marine:
                # Marine units do not have any threats data
                p.insert_paragraph_before(context["no_info"]["threats_marine_blocks"], style="NoInfo")

            else:
                for key in ("slr", "urban"):
                    p.insert_paragraph_before(context["heading"][key], style="Heading14")
                    if isinstance(context["chart"][key], str):
                        # unit does not have information for this threat
                        p.insert_paragraph_before(context["chart"][key], style="NoInfo")

                    else:
                        i = p.insert_paragraph_before()
                        run = i.add_run()
                        run.add_picture(context["chart"][key], width=Cm(10))

                        chart_key = "chart_{}".format(key)
                        caption = doc.add_paragraph(
                            context["caption"][chart_key], style="TableCaption"
                        )
                        if key == "slr":
                            add_hyperlink(caption, "https://coast.noaa.gov/digitalcoast/data/slr.html", "NOAA sea-level rise inundation data.")
                        else:
                            add_hyperlink(caption, "http://www.basic.ncsu.edu/dsl/urb.html", "SLEUTH urban growth model.")

                        _move_p_after(caption, i)

                    # insert blank line
                    p.insert_paragraph_before()

            # Delete the placeholder para
            delete_paragraph(p)

        if "{{PARTNERS}}" in p.text:
            # Remove '{{PARTNERS}}'placeholder and add the report content
            p.text = ""

            if is_marine:
                # Marine units do not have partner info available
                p.insert_paragraph_before(context["no_info"]["partners_marine_blocks"],
                                          style="NoInfo")

            else:
                if context["partners"]:
                    for category in context["partners"]:
                        p.insert_paragraph_before(
                            context["partner_headers"][category], style="Heading14"
                        )

                        for partner in context["partners"][category]:
                            text, url = partner

                            if url:
                                part = p.insert_paragraph_before(style="List Bullet 2")
                                add_hyperlink(part, url, text)
                            else:
                                p.insert_paragraph_before(style="List Bullet 2").text = text

                else:
                    # no partner info available
                    p.insert_paragraph_before(context["no_info"]["partners"], style="NoInfo")

                # Counties
                if "counties" in context:
                    p.insert_paragraph_before("Land Trusts (by county)", style="Heading14")
                    for county in context["counties"]:
                        county_name = p.insert_paragraph_before(style="List Bullet 2")
                        add_hyperlink(county_name, county["url"], county["name"])

            # Delete the placeholder para
            delete_paragraph(p)

        if "{{OWNERSHIP}}" in p.text:
            # Remove '{{OWNERSHIP}}'placeholder and add the report content
            p.text = ""

            if is_marine:
                # Marine units do not have ownership info
                p.insert_paragraph_before(context["no_info"]["ownership_marine_blocks"], style="NoInfo")

            else:
                # Add source info
                para = p.insert_paragraph_before("Values derived from: ")
                add_hyperlink(para,
                              "https://www.conservationgateway.org/ConservationByGeography/NorthAmerica/UnitedStates/edc/reportsdata/terrestrial/secured/Pages/default.aspx",
                              "Secured Lands From TNC Eastern Division - 2015 Edition")
                p.insert_paragraph_before()

                p.insert_paragraph_before("Conserved lands ownership", style="Heading14")
                if not isinstance(context["table"]["ownership"], str):
                    para = p.insert_paragraph_before(context["caption"]["table_ownership"], style="TableCaption")
                    create_table(doc, context["table"]["ownership"], para)
                else:
                    p.insert_paragraph_before(context["no_info"]["no_info"], style="NoInfo")

                p.insert_paragraph_before("Land protection status", style="Heading14")
                if not isinstance(context["table"]["protection"], str):
                    para = p.insert_paragraph_before(context["caption"]["table_protection"], style="TableCaption")
                    create_table(doc, context["table"]["protection"], para)
                else:
                    p.insert_paragraph_before(context["no_info"]["no_info"], style="NoInfo")

            # Delete the placeholder para
            delete_paragraph(p)

    report = doc.save(path)

    return report

def generate_report_context(unit_id, config, is_marine):
    """
        Create json of data points required for report

    Parameters
    ----------
    unit_id: string id of a summary unit
    config: dict of json data from src/config

    Returns
    -------
    context: dict of data in report structure

    """

    with open(os.path.join(DATA_DIR, "{0}.json".format(unit_id))) as json_file:
        data = json.loads(json_file.read())

    total_acres = data["acres"]
    summary_unit_type = "marine lease block" if "M" in unit_id else "subwatershed"

    context = dict()

    context["value"] = {
        "summary_unit_name": data["name"],
        "summary_unit_type": summary_unit_type,
        "acres": "{:,}".format(data["acres"]),
    }
    context["table"] = {}
    context["chart"] = {}
    context["map"] = {}
    context["caption"] = {
        "map_priorities": "Figure 1: Map of Blueprint priority categories within the {0} {1}.".format(
            data["name"], summary_unit_type
        )
    }
    figure_num = 2
    context["heading"] = {
        "slr": "Sea level rise",
        "urban": "Urban growth"
    }
    context["no_info"] = {
        "threats_marine_blocks": "No information on threats is available for marine lease blocks.",
        "ownership_marine_blocks": "No information on ownership is available for marine lease blocks.",
        "partners_marine_blocks": "No information on partners is available for marine lease blocks.",
        "partners": "No information on partners is available for this {0}.".format(summary_unit_type),
        "slr": "No sea level rise data are available for this {0}.".format(summary_unit_type),
        "urban": "No urban growth data are available for this {0}".format(summary_unit_type),
        "priority": "No priority information is available for this {0}".format(summary_unit_type),
        "no_info": "No information available"
    }

    # Priorities map

    priorities_config = config["priorities"]
    priorities_map = get_map(unit_id, data, priorities_config)

    context["map"]["priorities"] = priorities_map

    # Priorities table
    table_num = 1

    if data["blueprint"]:
        priorities = dict(col_names=["Priority Category", "Acres", "Percent of Area"])

        priorities["rows"] = [
            (
                priorities_config[str(i)]["label"],
                "{:,}".format(percent_to_acres(percentage, total_acres)),
                "{0}%".format(percentage),
            )
            for i, percentage in enumerate(data.get("blueprint"))
        ]
        priorities["rows"].reverse()

        context["table"]["priorities"] = priorities

        indices = [i for i, v in enumerate(data["blueprint"]) if v > 0]
        indices.reverse()
        values = [data["blueprint"][i] for i in indices]
        colors = [priorities_config[str(i)]["color"] for i in indices]
        labels = [
            "{0} ({1}%)".format(
                priorities_config[str(i)]["label"], data["blueprint"][i]
            )
            for i in indices
        ]
        chart = get_pie_chart(values, colors=colors, labels=labels)
        context["chart"]["priorities"] = chart
        context["caption"]["chart_priorities"] = "Figure {0}: Proportion of each Blueprint category within the {1} {2}.".format(
            figure_num, data["name"], summary_unit_type)
        figure_num += 1
        context["caption"]["table_priorities"] = "Table {0}: Extent of each Blueprint priority category within the {1} {2}".format(
            table_num, data["name"], summary_unit_type)
        table_num += 1

    else:
        context["table"]["priorities"] = context["no_info"]["priority"]
        context["chart"]["priorities"] = ""
        context["caption"]["table_priorities"] = ""

    # Ecosystem table

    ecosystems_table = dict(col_names=["Ecosystems", "Acres", "Percent of Area"])
    ecosystems = data.get("ecosystems", [])  # make sure we always have a list
    context["caption"]["table_ecosystems"] = "Table {0}: Extent of each ecosystem within the {1} {2}".format(
        table_num, data["name"], summary_unit_type
        )
    table_num += 1

    ecosystems_with_area = [
        (e, ecosystems[e]["percent"]) for e in ecosystems if "percent" in ecosystems[e]
    ]
    # filter the list to those with percents, and make a new list of tuples of ecosystem ID and percent
    ecosystems_with_area = sorted(
        ecosystems_with_area, key=lambda x: x[1], reverse=True
    )  # sort by percent, from largest percent to smallest

    ecosystems_table["rows"] = [
        (
            config["ecosystems"][e]["label"],
            "{:,}".format(percent_to_acres(percent, total_acres)),
            "{0}%".format(percent),
        )
        for e, percent in ecosystems_with_area
    ]

    context["table"]["ecosystems"] = ecosystems_table

    # Indicators

    context["ecosystem_indicators"] = []

    ind_ecosystems = [
        (key, data["ecosystems"][key].get("percent", None))
        for key in data["ecosystems"]
    ]

    ecosystems_with_percent = [e for e in ind_ecosystems if e[1] is not None]
    ecosystems_with_percent = sorted(
        ecosystems_with_percent, key=lambda x: x[1], reverse=True
    )  # sort descending percent

    ecosystems_without_percent = [e for e in ind_ecosystems if e[1] is None]
    ecosystems_without_percent = sorted(
        ecosystems_without_percent, key=lambda x: x[0]
    )  # sort ascending label

    ecosystems = ecosystems_with_percent + ecosystems_without_percent

    for ecosystem in ecosystems:
        key = ecosystem[0]
        ecosystem_data = data["ecosystems"][key]
        ecosystem_config = config["ecosystems"][key]

        ecosystem_indicators = {
            "ecosystem_name": ecosystem_config["label"],
            "indicators": [],
            "ecosystem_percentage": ecosystem_data.get("percent", ""),
        }

        # Indicator tables

        for indicator in ecosystem_data.get("indicators", []):
            indicator_config = ecosystem_config["indicators"][indicator]
            indicator_data = ecosystem_data["indicators"][indicator]
            indicator_context = {
                "value": {
                    "indicator_name": indicator_config["label"],
                    "indicator_description": indicator_config["description"],
                    "indicator_caption": indicator_config["reportCaption"],
                },
                "table": {
                    "indicator_table": {
                        "col_names": [
                            "",
                            "Indicator Values",
                            "Acres",
                            "Percent of Area",
                        ],
                        "rows": [],
                    }
                },
            }

            good_threshold = indicator_config.get("goodThreshold", None)

            indicator_keys = [int(v) for v in indicator_config["valueLabels"].keys()]
            indicator_keys.sort()  # make sure keys are sorted in ascending order
            indicator_values = list(zip(indicator_keys, indicator_data["percent"]))
            indicator_values.reverse()  # Reverse order so highest priorities at top of table

            if good_threshold is not None:
                # Find threshold position in indicator values so total will be added after
                threshold_position = indicator_keys[::-1].index(good_threshold)

                total_good_percent = 0
                total_not_good_percent = 0
                rows = []

                for label_key, percent in indicator_values:
                    if label_key >= good_threshold:
                        total_good_percent += percent
                        rows.append(
                            [
                                "",
                                indicator_config["valueLabels"][str(label_key)],
                                "{:,}".format(percent_to_acres(percent, total_acres)),
                                "{0}%".format(percent),
                            ]
                        )
                    else:
                        total_not_good_percent += percent
                        rows.append(
                            [
                                "",
                                indicator_config["valueLabels"][str(label_key)],
                                "{:,}".format(percent_to_acres(percent, total_acres)),
                                "{0}%".format(percent),
                            ]
                        )

                # Add lows & highs
                rows[0][0] = "High"
                rows[-1][0] = "Low"

                # Add condition rows

                good_total_row = [
                    "",
                    "Total in good condition",
                    "{:,}".format(percent_to_acres(total_good_percent, total_acres)),
                    "{0}%".format(round(total_good_percent, 1)),
                ]
                not_good_total_row = [
                    "",
                    "Total not in good condition",
                    "{:,}".format(
                        percent_to_acres(total_not_good_percent, total_acres)
                    ),
                    "{0}%".format(round(total_not_good_percent, 1)),
                ]

                # Insert total_good row at threshold and total_not_good at table end
                rows.insert(int(threshold_position) + 1, good_total_row)
                rows.append(not_good_total_row)

                indicator_context["table"]["indicator_table"]["rows"] = rows

            else:
                rows = [
                    [
                        "",
                        indicator_config["valueLabels"][str(label_key)],
                        "{:,}".format(percent_to_acres(percent, total_acres)),
                        "{0}%".format(percent),
                    ]
                    for label_key, percent in indicator_values
                ]

                # Add lows & highs
                rows[0][0] = "High"
                rows[-1][0] = "Low"

                indicator_context["table"]["indicator_table"]["rows"] = rows

            ecosystem_indicators["indicators"].append(indicator_context)

        context["ecosystem_indicators"].append(ecosystem_indicators)

    # Partners list - name and url separated by categories

    context["partner_headers"] = {
        "regional": "Regional Conservation Plans",
        "state": "Statewide Conservation Plans",
        "marine": "Marine Conservation Plans",
    }

    context["partners"] = defaultdict(list)
    for plan_key in data["plans"]:
        plan = config["plans"][plan_key]
        label = plan["label"]
        url = plan.get("url", "")
        context["partners"][plan["type"]].append((label, url))

    # Counties list - name and url

    if "counties" in data:
        context["counties"] = [
            {"name": value, "url": "http://findalandtrust.org/counties/{0}".format(key)}
            for key, value in data["counties"].items()
        ]  # key is FIPS and value is county with state

    # Ownership table

    if "owner" in data:
        owners = dict(col_names=["Ownership", "Acres", "Percent of Area"])

        owners["rows"] = [
            [
                config["owners"][key]["label"],
                "{:,}".format(percent_to_acres(percent, total_acres)),
                "{0}%".format(percent),
            ]
            for key, percent in data["owner"].items()
        ]

        own_perc_sum = sum(data["owner"].values())

        if own_perc_sum < 100:
            perc_remainder = round(100 - own_perc_sum, 1)
            owners["rows"].append(
                [
                    "Not conserved",
                    "{:,}".format(percent_to_acres(perc_remainder, total_acres)),
                    "{0}%".format(perc_remainder),
                ]
            )

        context["table"]["ownership"] = owners
    else:
        context["table"]["ownership"] = context["no_info"]["no_info"]

        # Protections table

    protection = dict(col_names=["Land Protection Status", "Acres", "Percent of Area"])

    if "gap" in data:
        protection["rows"] = [
            [
                config["protection"][key]["label"],
                "{:,}".format(percent_to_acres(percent, total_acres)),
                "{0}%".format(percent),
            ]
            for key, percent in data["gap"].items()
        ]

        pro_perc_sum = sum(data["gap"].values())

        if pro_perc_sum < 100:
            perc_remainder = round(100 - pro_perc_sum, 1)
            protection["rows"].append(
                [
                    "Not conserved",
                    "{:,}".format(percent_to_acres(perc_remainder, total_acres)),
                    "{0}%".format(perc_remainder),
                ]
            )

        context["table"]["protection"] = protection
    else:
        context["table"]["protection"] = context["no_info"]["no_info"]

        # Threats

    if is_marine:
        context["chart"]["slr"] = context["no_info"]["threats_marine_blocks"]

    else:
        if "slr" in data and data["slr"]:
            chart = get_line_chart(
                config["slr"],
                data["slr"],
                x_label="Amount of sea level rise (feet)",
                y_label="Percent of area",
                color="#004da8",
                alpha=0.3,
            )
            context["chart"]["slr"] = chart
            context["caption"]["chart_slr"] = "Figure {0}: Extent of inundation by projected sea level rise within the {1} {2}.  Values from the ".format(
                figure_num, data["name"], summary_unit_type)
            figure_num += 1
        else:
            context["chart"]["slr"] = context["no_info"]["slr"]

        if "urban" in data and data["urban"]:
            chart = get_line_chart(
                config["urbanization"],
                data["urban"],
                x_label="Decade",
                y_label="Percent of area",
                color="#D90000",
                alpha=0.3,
            )
            context["chart"]["urban"] = chart
            context["caption"]["chart_urban"] = "Figure {0}: Extent of projected urbanization within the {1} {2}.  Values from the ".format(
                figure_num, data["name"], summary_unit_type
            )
            figure_num += 1
        else:
            context["chart"]["urban"] = context["no_info"]["urban"]

    # Table captions for last two tables, Ownership and Protection status

    for ecosystem in context["ecosystem_indicators"]:
        for indicator in ecosystem.get("indicators", []):
            table_num += 1

    if "rows" in context["table"]["ownership"]:
        context["caption"]["table_ownership"] = "Table {0}: Extent of ownership class within the {1} {2}.".format(
            table_num, data["name"], summary_unit_type
        )
        table_num += 1
    else:
        context["caption"]["table_ownership"] = ""

    if "rows" in context["table"]["protection"]:
        context["caption"]["table_protection"] = "Table {0}: Extent of land protection status within the {1} {2}.".format(
            table_num, data["name"], summary_unit_type
        )
        table_num += 1
    else:
        context["caption"]["table_protection"] = ""

    return context

def percent_to_acres(percentage, total_acres):
    return round(total_acres * (percentage / 100))

def create_table(doc, data, para):
    """
    Builds table at end of doc

    Parameters
    ----------
    doc: docx.Document object
    data: dict
        table data; 'data' contains a list of column names ('col_names') and a list of rows ('rows'),
        where each row is a list of values for each column
    para: paragraph object
        Where the table needs to be moved

    Returns
    -------
    end_buffer
        location that the last item was added to the document

    """
    # TODO: ensure tables don't run over a page break

    styles = doc.styles
    ncols = len(data["col_names"])

    # Create table with one row for column headings
    table = doc.add_table(rows=1, cols=ncols)
    table.style = styles[DOC_STYLE]

    for index, heading in enumerate(data["col_names"]):
        cell = table.cell(0, index)
        cell.text = heading
        # Change justification of first heading
        if ncols is 3 and index is 0:
            cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
        elif ncols is 4 and index is 1:
            cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT

    for datarow in data["rows"]:
        row = table.add_row()
        condition_style = (
            str(datarow[1]) == "Total in good condition"
            or str(datarow[1]) == "Total not in good condition"
        )
        for index, datacell in enumerate(datarow):
            row.cells[index].text = str(datacell)
            if ncols is 4 and index is 1:
                row.cells[index].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
            if condition_style is True:
                # A new shade must be generated for each cell
                new_shade = shade_generator(TOTAL_ROW_COLOR)
                row.cells[index]._tc.get_or_add_tcPr().append(new_shade)

    if "Indicator Values" in data["col_names"]:
        table_type = "indicator"
    else:
        table_type = "other"

    set_col_widths(table, table_type)

    _move_table_after(table, para)

    end_buffer = doc.add_paragraph("")
    _move_p_after_t(table, end_buffer)

    return end_buffer

def shade_generator(color):
    return docx.oxml.parse_xml(
        r'<w:shd {0} w:fill="{1}"/>'.format(docx.oxml.ns.nsdecls("w"), color)
    )

def set_col_widths(table, table_type):
    """
    Set widths for standard table columns

    Parameters
    ----------
    table: table object

    """
    if table_type is "indicator":
        widths = (Cm(1), Cm(9), Cm(3.5), Cm(3.5))
    else:
        widths = (Cm(9), Cm(4), Cm(4))

    for row in table.rows:
        for idx, width in enumerate(widths):
            row.cells[idx].width = width

def _move_p_after(p_move, p_destination):
    """
    Move one paragraph after another

    Parameters
    ----------
    heading
        heading to be moved
    paragraph
        location where table will be moved

    """
    p_destination._p.addnext(p_move._p)

def _move_table_after(table, paragraph):
    """
    Move the table after 'paragraph.'
    Normally, tables are created at the end of the document.
    This function allows you to move them to a different location in the document.
    The paragraph passed in is not modified, it is just used as a reference point for inserting the table.

    Parameters
    ----------
    table
        table to be moved
    paragraph
        location where table will be moved

    """
    paragraph._p.addnext(table._tbl)

def _move_p_after_t(table, para):
    """
    Move a paragraph after a table

    Parameters
    ----------
    table
        table to be moved
    para: paragraph object
        location where table will be moved

    """
    table._tbl.addnext(para._p)

def add_hyperlink(paragraph, url, text):
    """
    A function that places a hyperlink within a paragraph object.

    Parameters
    ----------
    paragraph: The paragraph we are adding the hyperlink to.
    url: A string containing the required url
    text: The text displayed for the url

    Returns
    -------
    hyperlink: hyperlink object

    """

    # This gets access to the document.xml.rels file and gets a new relation id value
    part = paragraph.part
    r_id = part.relate_to(
        url, docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True
    )

    # Create the w:hyperlink tag and add needed values
    hyperlink = OxmlElement("w:hyperlink")
    hyperlink.set(qn("r:id"), r_id)

    # Create a w:r element
    new_run = OxmlElement("w:r")

    # Create a new w:rPr element
    rPr = OxmlElement("w:rPr")

    # Style it
    c = OxmlElement("w:color")
    c.set(qn("w:val"), "4F81BD")
    rPr.append(c)
    u = OxmlElement("w:u")
    u.set(qn("w:val"), "none")
    rPr.append(u)

    # Join all the xml elements together add add the required text to the w:r element
    new_run.append(rPr)
    new_run.text = text
    hyperlink.append(new_run)

    paragraph._p.append(hyperlink)

    return hyperlink

def delete_paragraph(paragraph):
    p = paragraph._element
    p.getparent().remove(p)
    p._p = p._element = None

def get_map(unit_id, data, priorities):
    bounds = data["bounds"]

    # convert priorities to legend, in descending priority (dropping Not a Priority class)
    legend = [
        (int(k), v["color"], v["label"]) for k, v in priorities.items() if k != "0"
    ]
    legend = sorted(legend, key=lambda x: x[0], reverse=True)
    # strip off the value used for sorting
    legend = [entry[1:] for entry in legend]

    m = get_map_image(unit_id, bounds, legend)

    buffer = BytesIO()
    m.save(buffer, "PNG")
    return buffer

def _resolve(scope, key, context):
    """
    Resolve scope and key to a context item
    Provides very minimal validation of presence

    Parameters
    ----------
    scope: str
        singular variant of scope in context
    key: str
        key to lookup context item in context within scope
    context: dict

    Returns
    -------
    context item: str, dict, etc
    """

    if scope not in context:
        raise ValueError("Scope {0} is not found in context!".format(scope))

    if key not in context[scope]:
        raise ValueError("Key {0} is not found in context!".format(key))

    return context[scope][key]

if __name__ == "__main__":
    id = "I1"
    outpath = "/tmp/{0}.docx".format(id)
    create_report(id, outpath)


import logging
import boto3
import json
import azure.functions as func
from botocore.exceptions import ClientError
from os import environ

def main(req: func.HttpRequest) -> func.HttpResponse:

    logging.info(f'Resource Requested: {func.HttpRequest}')

    # Get AWS ID and Key
    try:
        aws_access_key_id = environ['AWSAccessKeyID']
        aws_secret_access_key = environ['AWSSecretAccessKey']
        aws_region_name = environ['AWSRegionName']

    except KeyError as ke:
        logging.error(f'Invalid Settings. {ke.args} configuration is missing.')
        return func.HttpResponse(
             'Invalid Settings. AWS Access ID/Key configuration is missing.',
             status_code=500
        )

    # Get Data Catalog Name from the request parameter
    catalog_name = req.params.get('CatalogName')

    if not catalog_name:
        try:
            req_body = req.get_json()
        except ValueError:
            pass
        else:
            catalog_name = req_body.get('CatalogName')

    if catalog_name:

        try:
            logging.info(f'Creating Boto3 Athena Client.')
            athena_client = boto3.client(
                "athena",
                region_name=aws_region_name,
                aws_access_key_id=aws_access_key_id,
                aws_secret_access_key=aws_secret_access_key
            )

            try:
                # Get the list of databases for given data catalog
                logging.info(f'Sending Request for data catalog.')
                response = athena_client.list_databases(CatalogName=catalog_name)
                dblist = response['DatabaseList']

                # Parse dblist to get the dbnames and format as json
                dbnames = []
                for db in dblist:
                    dbnames.append(db['Name'])

                results = {"Databases" : dbnames}

                return func.HttpResponse(
                    json.dumps(results),
                    headers = {"Content-Type": "application/json"},
                    status_code = 200
                )

            except athena_client.exceptions.InternalServerException as ex:
                logging.error(f"Internal Server Error: {str(ex)}")
                return func.HttpResponse("Internal Server Error", status_code=404)

            except athena_client.exceptions.InvalidRequestException as ex:
                logging.error(f"Invalid Request Exception: {str(ex)}")
                return func.HttpResponse("Invalid Request Exception: Not a valid CatalogName", status_code=400)

            except athena_client.exceptions.MetadataException as ex:
                logging.error(f"Metadata Exception: {str(ex)}")
                return func.HttpResponse("Metadata Exception", status_code=400)

        except ClientError as ex:
            logging.error(f"Athena Client Error: {str(ex)}")
            return func.HttpResponse("Athena Client Error", status_code=401)

        except Exception as ex:
            logging.error(f"Exception Occured: {str(ex)}")
            return func.HttpResponse("Internal Server Exception", status_code=500)

    else:
        return func.HttpResponse(
             "Please pass CatalogName in the query string.",
             status_code=400
        )


import os
import shutil
import subprocess
import yaml
from yacs.config import CfgNode
import json
from pathlib import Path
import soundfile as sf
from typing import List
import numpy as np
import paddle
import re

from paddlespeech.t2s.exps.syn_utils import am_to_static
from paddlespeech.t2s.exps.syn_utils import get_am_inference
from paddlespeech.t2s.exps.syn_utils import get_am_output
from paddlespeech.t2s.exps.syn_utils import get_frontend
from paddlespeech.t2s.exps.syn_utils import run_frontend
from paddlespeech.t2s.exps.syn_utils import get_predictor
from paddlespeech.t2s.exps.syn_utils import get_voc_output
from paddlespeech.t2s.models.fastspeech2 import FastSpeech2
from paddlespeech.t2s.modules.nets_utils import make_pad_mask
from paddlespeech.t2s.modules.normalizer import ZScore

# 配置路径信息
# 这里的路径是提前写好在 Aistudio 上的路径信息，如果需要迁移到其它的环境，请自行配置
cwd_path = "./"
pretrained_model_dir = "/home/aistudio/PaddleSpeech/examples/other/tts_finetune/tts3/models/fastspeech2_mix_ckpt_1.2.0"
config_path = "/home/aistudio/PaddleSpeech/examples/other/tts_finetune/tts3/conf/finetune.yaml"
inference_dir = "./inference"
exp_base = "./work/exp_"

os.makedirs(inference_dir, exist_ok=True)

pwg_inference_dir = os.path.join(cwd_path, "inference/pwgan_aishell3_static_1.1.0")
hifigan_inference_dir = os.path.join(cwd_path, "inference/hifigan_aishell3_static_1.1.0")
wavernn_inference_dir = os.path.join(cwd_path, "inference/wavernn_csmsc_static_0.2.0")

# 命令行执行函数，可以进入指定路径下执行
def run_cmd(cmd, cwd_path, message, check_result_path=None):
    p = subprocess.Popen(cmd, shell=True, cwd=cwd_path)
    res = p.wait()
    print(cmd)
    print("运行结果：", res)

    # 检查是否有对应文件生成
    if check_result_path:
        file_num = [filename for filename in os.listdir(check_result_path)]
        if file_num == 0:
            res = 1

    if res == 0:
        # 运行成功
        return True
    else:
        # 运行失败
        print(message)
        print(f"你可以新建终端，打开命令行，进入：{cwd_path}")
        print(f"执行：{cmd}")
        print("查看命令行中详细的报错信息")
        return False

def status_change(status_path, status):
    with open(status_path, "w", encoding="utf8") as f:
        status = {
            'on_status': status
        }
        json.dump(status, f, indent=4)

# 找到最新生成的模型
def find_max_ckpt(model_path):
    max_ckpt = 0
    for filename in os.listdir(model_path):
        if filename.endswith('.pdz'):
            files = filename[:-4]
            a1, a2, it = files.split("_")
            if int(it) > max_ckpt:
                max_ckpt = int(it)
    return max_ckpt

# Step1 : 生成标准数据集
def step1_generate_standard_dataset(label_path, exp_path, data_dir):
    error_code = 0

    print("Step1 开始执行: 生成标注数据集")
    # 生成 data 数据路径
    exp_data_path = data_dir
    os.makedirs(exp_data_path, exist_ok=True)
    exp_data_label_file = os.path.join(exp_data_path, "labels.txt")

    if not os.path.exists(label_path):
        print(f"标注文件: {label_path} 不存在，请检查数据预标注是否成功！")
        error_code = 1
        return None, error_code

    with open(label_path, "r", encoding='utf8') as f:
        ann_result = json.load(f)

    if len(ann_result) == 0:
        print(f"标注文件: {label_path} 内容为空，请检查数据预标注是否成功！")
        error_code = 2
        return None, error_code

    if len(ann_result) < 5:
        print(f"标注文件: {label_path} 中，音频数据小于5条，请增加上传的音频数量！")
        error_code = 3
        return None, error_code

    # 复制音频文件
    for label in ann_result:
        if os.path.exists(label['filepath']):
            shutil.copy(label['filepath'], exp_data_path)
        else:
            print(f"音频文件：{label['filepath']} 不存在，请重新检查数据是否正确")
            error_code = 4
            return None, error_code

    # 生成标准标注文件
    with open(exp_data_label_file, "w", encoding="utf8") as f:
        for ann in ann_result:
            f.write(f"{ann['filename'].split('.')[0]}|{ann['pinyin']}\n")

    # 检查 data 里面是否为空，为空则显示异常
    wav_file = [filename for filename in os.listdir(exp_data_path) if filename.endswith(".wav")]
    if len(wav_file) == 0:
        # 为空则是异常
        shutil.rmtree(exp_data_path)
        error_code = 5
        return None, error_code
    return exp_data_path, error_code

# Step2: 检查非法数据
def step2_check_oov(data_dir, new_dir, lang="zh"):
    print("Step2 开始执行: 检查数据集是否合法")
    cmd = f"""
            python3 /home/aistudio/util/check_oov.py \
                --input_dir={data_dir} \
                --pretrained_model_dir={pretrained_model_dir} \
                --newdir_name={new_dir} \
                --lang={lang}
            """
    if not run_cmd(cmd, cwd_path, message="Step2 检查非法数据 执行失败，请检查数据集是否配置正确"):
        if os.path.exists(new_dir):
            shutil.rmtree(new_dir)
        return False
    else:
        return True

# Step3: 生成 MFA 对齐结果
def step3_get_mfa(new_dir, mfa_dir, lang="zh"):
    print("Step3 开始执行: 使用 MFA 对数据进行对齐")
    cmd = f"""
        python3 /home/aistudio/util/get_mfa_result.py \
            --input_dir={new_dir} \
            --mfa_dir={mfa_dir} \
            --lang={lang}
        """
    if not run_cmd(cmd, cwd_path, message="Step3 MFA对齐 执行失败，请执行命令行，查看 MFA 详细报错", check_result_path=mfa_dir):
        if os.path.exists(mfa_dir):
            shutil.rmtree(mfa_dir)
        return False
    else:
        return True

# Step4: 生成 Duration
def step4_duration(mfa_dir):
    if not os.path.exists(mfa_dir):
        print("Step3 执行失败，未生成 MFA 对齐文件夹")
        return False
    print("Step4 开始执行: 生成 Duration 文件")
    cmd = f"""
        python3 /home/aistudio/util/generate_duration.py \
            --mfa_dir={mfa_dir}
        """
    if not run_cmd(cmd, cwd_path, message="Step4 生成 Duration 执行失败，请执行命令行"):
        if os.path.exists("./durations.txt"):
            os.remove("./durations.txt")
        return False
    else:
        return True

# Step5: 数据预处理
def step5_extract_feature(data_dir, dump_dir):
    print("Step5 开始执行: 开始数据预处理")
    cmd = f"""
        python3 /home/aistudio/util/extract_feature.py \
            --duration_file="./durations.txt" \
            --input_dir={data_dir} \
            --dump_dir={dump_dir}\
            --pretrained_model_dir={pretrained_model_dir}\
        """
    if not run_cmd(cmd, cwd_path, message="Step5 执行失败，请执行命令行"):
        if os.path.exists(dump_dir):
            shutil.rmtree(dump_dir)
        return False
    else:
        return True

# Step6: 准备微调环境
def step6_prepare_env(output_dir):
    print("Step6 开始执行: 准备微调环境")
    cmd = f"""
        python3 /home/aistudio/util/prepare_env.py \
            --pretrained_model_dir={pretrained_model_dir} \
            --output_dir={output_dir}
    """
    if not run_cmd(cmd, cwd_path, message="Step6 准备训练环境 执行失败，请执行命令行"):
        return False
    else:
        return True

# Step7: 微调训练
def step7_finetune(dump_dir, output_dir, max_finetune_step, batch_size=None, learning_rate=None):
    # # 读取默认的yaml文件
    # with open(config_path) as f:
    #     finetune_config = yaml.safe_load(f)
    # # 多少个 step 保存一次
    # finetune_config['num_snapshots'] = num_snapshots
    # # 1. 自动调整 batch ，通过 dump/train里面文件数量判断
    train_data_dir = os.path.join(dump_dir, "train/norm/data_speech")
    file_num = len([filename for filename in os.listdir(train_data_dir) if filename.endswith(".npy")])
    if not batch_size:
        if file_num <= 32:
            batch_size = file_num
        else:
            batch_size = 32
    else:
        # 指定 batch size 的情况下还是需要看音频文件的数量
        if file_num <= batch_size:
            batch_size = file_num

    # # 2. 支持调整 learning_rate
    if not learning_rate:
        learning_rate = 0.001

    print("Step7 开始执行: 微调试验开始")

    cmd = f"""
        python3 /home/aistudio/util/finetuneTrain.py \
            --dump_dir={dump_dir} \
            --output_dir={output_dir} \
            --max_step={max_finetune_step} \
            --batch_size={batch_size} \
            --learning_rate={learning_rate}
        """
    if not run_cmd(cmd, cwd_path, message="Step7 微调试验失败 执行失败，请执行命令行"):
        return False
    else:
        return True

# 导出成静态图
def step8_export_static_model(exp_name):
    exp_path = os.path.join(exp_base + exp_name)
    model_path = os.path.join(exp_path, "output/checkpoints")
    dump_dir = os.path.join(exp_path, "dump")
    output_dir = os.path.join(exp_path, "output")
    ckpt = find_max_ckpt(model_path)

    am_config = f"{cwd_path}/models/fastspeech2_mix_ckpt_1.2.0/default.yaml"

    with open(am_config) as f:
        am_config = CfgNode(yaml.safe_load(f))
    finetune_model_path = f"{output_dir}/checkpoints/snapshot_iter_{ckpt}.pdz"
    am_inference = get_am_inference(
        am="fastspeech2_mix",
        am_config=am_config,
        am_ckpt=finetune_model_path,
        am_stat=f"{cwd_path}/models/fastspeech2_mix_ckpt_1.2.0/speech_stats.npy",
        phones_dict=f"{dump_dir}/phone_id_map.txt",
        tones_dict=None,
        speaker_dict=f"{dump_dir}/speaker_id_map.txt")

    out_inference_dir = os.path.join(inference_dir, exp_name)
    # 静态图导出
    am_inference = am_to_static(
            am_inference=am_inference,
            am="fastspeech2_mix",
            inference_dir=out_inference_dir,
            speaker_dict=f"{dump_dir}/speaker_id_map.txt")

    # 把 phone_dict 也复制过去
    shutil.copy(f"{dump_dir}/phone_id_map.txt", out_inference_dir)

    # 把预训练模型中 npy 文件，yaml 文件，txt文件都放进去
    for filename in os.listdir(pretrained_model_dir):
        fileend = os.path.splitext(filename)[1]
        if fileend in [".npy", ".yaml", ".txt"]:
            shutil.copy(f"{pretrained_model_dir}/{filename}", out_inference_dir)

    # 动态图模型也放进去
    if os.path.exists(finetune_model_path):
        shutil.copy(finetune_model_path, out_inference_dir)

# 使用微调后的模型生成音频
def step9_generateTTS(text_dict, wav_output_dir, exp_name, voc="PWGan"):
    # 配置一下参数信息
    exp_path = os.path.join(exp_base + exp_name)
    model_path = os.path.join(exp_path, "output/checkpoints")
    output_dir = os.path.join(exp_path, "output")
    dump_dir = os.path.join(exp_path, "dump")

    text_file = os.path.join(exp_path, "sentence.txt")
    status_json = os.path.join(exp_path, "generate_status.json")
    # 写一个物理锁
    status_change(status_json, True)

    with open(text_file, "w", encoding="utf8") as f:
        for k,v in sorted(text_dict.items(), key=lambda x:x[0]):
            f.write(f"{k} {v}\n")

    ckpt = find_max_ckpt(model_path)

    if voc == "PWGan":
        cmd = f"""
        python3 /home/aistudio/PaddleSpeech/paddlespeech/t2s/exps/fastspeech2/../synthesize_e2e.py \
                        --am=fastspeech2_mix \
                        --am_config=models/fastspeech2_mix_ckpt_1.2.0/default.yaml \
                        --am_ckpt={output_dir}/checkpoints/snapshot_iter_{ckpt}.pdz \
                        --am_stat=models/fastspeech2_mix_ckpt_1.2.0/speech_stats.npy \
                        --voc="pwgan_aishell3" \
                        --voc_config=models/pwg_aishell3_ckpt_0.5/default.yaml \
                        --voc_ckpt=models/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \
                        --voc_stat=models/pwg_aishell3_ckpt_0.5/feats_stats.npy \
                        --lang=mix \
                        --text={text_file} \
                        --output_dir={wav_output_dir} \
                        --phones_dict={dump_dir}/phone_id_map.txt \
                        --speaker_dict={dump_dir}/speaker_id_map.txt \
                        --spk_id=0 \
                        --inference_dir
                        --ngpu=1
        """
    elif voc == "WaveRnn":
        cmd = f"""
        python3 /home/aistudio/PaddleSpeech/paddlespeech/t2s/exps/fastspeech2/../synthesize_e2e.py \
                        --am=fastspeech2_mix \
                        --am_config=models/fastspeech2_mix_ckpt_1.2.0/default.yaml \
                        --am_ckpt={output_dir}/checkpoints/snapshot_iter_{ckpt}.pdz \
                        --am_stat=models/fastspeech2_mix_ckpt_1.2.0/speech_stats.npy \
                        --voc="wavernn_csmsc" \
                        --voc_config=models/wavernn_csmsc_ckpt_0.2.0/default.yaml \
                        --voc_ckpt=models/wavernn_csmsc_ckpt_0.2.0/snapshot_iter_400000.pdz \
                        --voc_stat=models/wavernn_csmsc_ckpt_0.2.0/feats_stats.npy \
                        --lang=mix \
                        --text={text_file} \
                        --output_dir={wav_output_dir} \
                        --phones_dict={dump_dir}/phone_id_map.txt \
                        --speaker_dict={dump_dir}/speaker_id_map.txt \
                        --spk_id=0 \
                        --ngpu=1
        """
    elif voc == "HifiGan":
        cmd = f"""
        python3 /home/aistudio/PaddleSpeech/paddlespeech/t2s/exps/fastspeech2/../synthesize_e2e.py \
                        --am=fastspeech2_mix \
                        --am_config=models/fastspeech2_mix_ckpt_1.2.0/default.yaml \
                        --am_ckpt={output_dir}/checkpoints/snapshot_iter_{ckpt}.pdz \
                        --am_stat=models/fastspeech2_mix_ckpt_1.2.0/speech_stats.npy \
                        --voc="hifigan_aishell3" \
                        --voc_config=models/hifigan_aishell3_ckpt_0.2.0/default.yaml \
                        --voc_ckpt=models/hifigan_aishell3_ckpt_0.2.0/snapshot_iter_2500000.pdz \
                        --voc_stat=models/hifigan_aishell3_ckpt_0.2.0/feats_stats.npy \
                        --lang=mix \
                        --text={text_file} \
                        --output_dir={wav_output_dir} \
                        --phones_dict={dump_dir}/phone_id_map.txt \
                        --speaker_dict={dump_dir}/speaker_id_map.txt \
                        --spk_id=0 \
                        --ngpu=1
        """
    else:
        print("声码器不符合要求，请重新选择")
        status_change(status_json, False)
        return False

    if not run_cmd(cmd, cwd_path, message="Step9 生成音频 执行失败，请执行命令行"):
        status_change(status_json, False)
        return False
    else:
        status_change(status_json, False)
        return True

# 使用静态图生成
def step9_generateTTS_inference(text_dict, exp_name, voc="PWGan"):
    exp_path = exp_base + exp_name
    wav_output_dir = os.path.join(exp_path, "wav_out")
    am_inference_dir = os.path.join(inference_dir, exp_name)

    device = "gpu"

    status_json = os.path.join(exp_path, "generate_status.json")
    # 写一个物理锁
    status_change(status_json, True)

    # frontend
    frontend = get_frontend(
        lang="mix",
        phones_dict=os.path.join(am_inference_dir, "phone_id_map.txt"),
        tones_dict=None
    )

    # am_predictor
    am_predictor = get_predictor(
        model_dir=am_inference_dir,
        model_file="fastspeech2_mix" + ".pdmodel",
        params_file="fastspeech2_mix" + ".pdiparams",
        device=device)

    # voc_predictor
    if voc == "PWGan":
        voc_predictor = get_predictor(
            model_dir=pwg_inference_dir,
            model_file="pwgan_aishell3" + ".pdmodel",
            params_file="pwgan_aishell3" + ".pdiparams",
            device=device)
    elif voc == "WaveRnn":
        voc_predictor = get_predictor(
            model_dir=wavernn_inference_dir,
            model_file="wavernn_csmsc" + ".pdmodel",
            params_file="wavernn_csmsc" + ".pdiparams",
            device=device)
    elif voc == "HifiGan":
        voc_predictor = get_predictor(
            model_dir=hifigan_inference_dir,
            model_file="hifigan_aishell3" + ".pdmodel",
            params_file="hifigan_aishell3" + ".pdiparams",
            device=device)

    output_dir = Path(wav_output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    sentences = list(text_dict.items())

    merge_sentences = True
    fs = 24000
    for utt_id, sentence in sentences:
        am_output_data = get_am_output(
            input=sentence,
            am_predictor=am_predictor,
            am="fastspeech2_mix",
            frontend=frontend,
            lang="mix",
            merge_sentences=merge_sentences,
            speaker_dict=os.path.join(am_inference_dir, "phone_id_map.txt"),
            spk_id=0, )
        wav = get_voc_output(
                voc_predictor=voc_predictor, input=am_output_data)
        # 保存文件
        sf.write(output_dir / (utt_id + ".wav"), wav, samplerate=fs)

    status_change(status_json, False)
    return True

def freeze_layer(model, layers: List[str]):
    """freeze layers

    Args:
        layers (List[str]): frozen layers
    """
    for layer in layers:
        for param in eval("model." + layer + ".parameters()"):
            param.trainable = False
# 前端
def get_tts_phone_ids(sentence, exp_name, exp_fun=False):
    am_inference_dir = os.path.join(inference_dir, exp_name)
    phones_dict = f"{am_inference_dir}/phone_id_map.txt"
    # 检测有没有SSML标记
    if sentence.strip() != "" and re.match(r".*?<speak>.*?</speak>.*", sentence,
                                           re.DOTALL):
        # 有SSML标记，只支持中文修改多音字
        lang = "zh"
        merge_sentences = False
        frontend = get_frontend(lang=lang, phones_dict=phones_dict, tones_dict=None)
    else:
        # 没有 SSML 标记，按照中英双语处理
        lang = "mix"
        frontend = get_frontend(lang=lang, phones_dict=phones_dict, tones_dict=None)
        if exp_fun:
            merge_sentences = True
        else:
            merge_sentences = False
    print(f"merge_sentences: {merge_sentences}")
    frontend_dict = run_frontend(
                    frontend=frontend,
                    text=sentence,
                    merge_sentences=merge_sentences,
                    get_tone_ids=False,
                    lang=lang)
    phone_ids = frontend_dict['phone_ids']
    return phone_ids

def get_idx2ph_dict(exp_name):
    am_inference_dir = os.path.join(inference_dir, exp_name)
    phones_dict = f"{am_inference_dir}/phone_id_map.txt"
    id2phs_dict = {}

    with open(phones_dict, "r", encoding="utf8") as f:
        for line in f.readlines():
            if line.strip():
                ph, idx = line.strip().split()
                id2phs_dict[idx] = ph
    return id2phs_dict

def fs2_inference(model, text, spk_id, alpha=1.0, duration=None, return_duration=False):
    x = paddle.cast(text, 'int64')
    xs = x.unsqueeze(0)
    ilens = paddle.shape(x)[0]
    # forward encoder
    x_masks = model._source_mask(ilens)
    # (B, Tmax, adim)
    hs, _ = model.encoder(xs, x_masks)
    # spk_id 添加进去了
    spk_emb = model.spk_embedding_table(spk_id)
    hs = model._integrate_with_spk_embed(hs, spk_emb)
    # forward duration predictor and variance predictors
    d_masks = make_pad_mask(ilens)

    # True
    if model.stop_gradient_from_pitch_predictor:
        p_outs = model.pitch_predictor(hs.detach(), d_masks.unsqueeze(-1))
    else:
        p_outs = model.pitch_predictor(hs, d_masks.unsqueeze(-1))

    # False
    if model.stop_gradient_from_energy_predictor:
        e_outs = model.energy_predictor(hs.detach(), d_masks.unsqueeze(-1))
    else:
        e_outs = model.energy_predictor(hs, d_masks.unsqueeze(-1))

    # duration
    # 输出已经是整数(float32)了
    if duration is None:
        d_outs = model.duration_predictor.inference(hs, d_masks)
    else:
        d_outs = duration

    if return_duration:
        return d_outs

    # print(d_outs)

    # 往下添加 pitch, energy 信息
    p_embs = model.pitch_embed(p_outs.transpose((0, 2, 1))).transpose(
                (0, 2, 1))
    e_embs = model.energy_embed(e_outs.transpose((0, 2, 1))).transpose(
        (0, 2, 1))
    hs = hs + e_embs + p_embs

    # (B, Lmax, adim)
    hs = model.length_regulator(hs, d_outs, alpha, is_inference=True)

    # decoder
    h_masks = None
    zs, _ = model.decoder(hs, h_masks)
    # (B, Lmax, odim)
    before_outs = model.feat_out(zs).reshape(
        (paddle.shape(zs)[0], -1, model.odim))
    after_outs = before_outs + model.postnet(
                before_outs.transpose((0, 2, 1))).transpose((0, 2, 1))
    return before_outs, after_outs, d_outs, p_outs, e_outs

def duration_phones_to_list(duration, phone_ids, id2phs_dict):
    phs_lists = []
    phs_lens = phone_ids[0].shape[0]
    for i in range(phs_lens):
        ph = id2phs_dict[str(phone_ids[0].numpy()[i])]
        dur = int(duration[0].numpy()[i])
        phs_lists.append({
            "ph": ph,
            "dur": dur
        })
        # print(f"{i}\t{ph}\t{dur}")
    return phs_lists

def list_to_durations(phs_lists):
    duration = []
    for k in phs_lists:
        dur = k['dur']
        duration.append(int(dur))
    duration = paddle.to_tensor([duration], dtype=paddle.float32)
    return duration

def load_fs2_model(exp_name):
    am_inference_dir = os.path.join(inference_dir, exp_name)

    ckpt = find_max_ckpt(am_inference_dir)
    if ckpt == 0 or ckpt is None:
        print("ckpt 模型不存在，请将模型放入inference路径")
        return None
    model_path = f"{am_inference_dir}/snapshot_iter_{ckpt}.pdz"
    speaker_dict = f"{am_inference_dir}/speaker_id_map.txt"
    phones_dict = f"{am_inference_dir}/phone_id_map.txt"

    # 加载fs2模型
    default_config_file = f"{am_inference_dir}/default.yaml"
    with open(default_config_file) as f:
        config = CfgNode(yaml.safe_load(f))

    with open(phones_dict, "r") as f:
        phn_id = [line.strip().split() for line in f.readlines()]
    vocab_size = len(phn_id)
    print("vocab_size:", vocab_size)
    odim = config.n_mels

    with open(speaker_dict, 'rt') as f:
        spk_id = [line.strip().split() for line in f.readlines()]
    spk_num = len(spk_id)
    model = FastSpeech2(
        idim=vocab_size,
        odim=odim,
        spk_num=spk_num,
        **config["model"])

    archive = paddle.load(model_path)
    model.set_state_dict(archive['main_params'])
    model.eval()
    return model

def load_voc_model(voc):
    device = "gpu"
    # voc_predictor
    if voc == "PWGan":
        voc_predictor = get_predictor(
            model_dir=pwg_inference_dir,
            model_file="pwgan_aishell3" + ".pdmodel",
            params_file="pwgan_aishell3" + ".pdiparams",
            device=device)
    elif voc == "WaveRnn":
        voc_predictor = get_predictor(
            model_dir=wavernn_inference_dir,
            model_file="wavernn_csmsc" + ".pdmodel",
            params_file="wavernn_csmsc" + ".pdiparams",
            device=device)
    elif voc == "HifiGan":
        voc_predictor = get_predictor(
            model_dir=hifigan_inference_dir,
            model_file="hifigan_aishell3" + ".pdmodel",
            params_file="hifigan_aishell3" + ".pdiparams",
            device=device)
    else:
        return None

    return voc_predictor

def generateTTS_inference_adjust_duration(text_dict, exp_name, voc="PWGan", alpha=1.0, duration=None, merge_sentences=False, exp_fun=False):
    exp_path = exp_base + exp_name
    if not os.path.exists(exp_path):
        os.makedirs(exp_path, exist_ok=True)

    wav_output_dir = os.path.join(exp_path, "wav_out")
    if not os.path.exists(wav_output_dir):
        os.makedirs(wav_output_dir, exist_ok=True)
    am_inference_dir = os.path.join(inference_dir, exp_name)
    phones_dict = f"{am_inference_dir}/phone_id_map.txt"
    status_json = os.path.join(exp_path, "generate_status.json")

    # 写一个物理锁
    status_change(status_json, True)

    # 前端
    # frontend = get_frontend(
    #             lang="mix", phones_dict=phones_dict, tones_dict=None)

    # 加载fs2模型
    model = load_fs2_model(exp_name)
    if not model:
        status_change(status_json, False)
        print("微调模型加载失败！")
        return False

    fs = 24000

    # 加载声码器
    # voc_predictor
    voc_predictor = load_voc_model(voc)
    if not voc:
        status_change(status_json, False)
        print("声码器加载失败！")
        return False

    am_stat = os.path.join(am_inference_dir, "speech_stats.npy")
    am_mu, am_std = np.load(am_stat)
    am_mu = paddle.to_tensor(am_mu)
    am_std = paddle.to_tensor(am_std)
    am_normalizer = ZScore(am_mu, am_std)

    for utt_id, sentence in text_dict.items():
        phone_ids = get_tts_phone_ids(sentence, exp_name, exp_fun=exp_fun)
        wavs = []
        for phone_id in phone_ids:
            spk_id = paddle.to_tensor(0)
            before_outs, after_outs, d_outs, p_outs, e_outs = fs2_inference(model, text=phone_id, alpha=alpha, spk_id=spk_id,duration=duration)
            normalized_mel = after_outs[0]
            logmel = am_normalizer.inverse(normalized_mel)
            am_output_data = logmel.numpy()
            # 给到 vocoder 进行推理
            wav = get_voc_output(
                voc_predictor=voc_predictor, input=am_output_data)
            wavs.append(wav)
        wavs = np.concatenate(wavs)
        # 保存文件
        sf.write(f"{wav_output_dir}/{utt_id}.wav", wavs, samplerate=fs)

    status_change(status_json, False)
    return True

# 全流程微调训练
def finetuneTTS(label_path, exp_name, start_step=1, end_step=100, max_finetune_step=100, batch_size=None, learning_rate=None):
    exp_path = os.path.join(exp_base + exp_name)
    os.makedirs(exp_path, exist_ok=True)
    data_dir = os.path.join(exp_path, "data")
    finetune_status_json = os.path.join(exp_path, "finetune_status.json")
    status_change(finetune_status_json, True)

    if start_step <= 1 and end_step >= 1:
        # Step1 : 生成标准数据集
        if not os.path.exists(data_dir):
            exp_data_path, error_code = step1_generate_standard_dataset(label_path, exp_path, data_dir)
            if exp_data_path is None:
                status_change(finetune_status_json, False)
                # 步骤一出错
                return
        else:
            print(f"{data_dir} 已存在，跳过此步骤！")

    # Step2: 检查非法数据
    new_dir = os.path.join(exp_path, "new_dir")
    if start_step <= 2 and end_step >= 2:
        if not os.path.exists(new_dir):
            if not step2_check_oov(data_dir, new_dir):
                status_change(finetune_status_json, False)
                return
        else:
            print(f"{new_dir} 已存在，跳过此步骤")

    mfa_dir = os.path.join(exp_path, "mfa")
    if start_step <= 3 and end_step >= 3:
        # Step3: MFA 对齐
        if not os.path.exists(mfa_dir):
            if not step3_get_mfa(new_dir, mfa_dir):
                status_change(finetune_status_json, False)
                return
        else:
            print(f"{mfa_dir} 已存在，跳过此步骤")

    if start_step <= 4 and end_step >= 4:
        # Step4: 生成时长信息文件
        if not step4_duration(mfa_dir):
            status_change(finetune_status_json, False)
            return

    dump_dir = os.path.join(exp_path, "dump")
    if start_step <= 5 and end_step >= 5:
        # Step5: 数据预处理
        if not os.path.exists(dump_dir):
            if not step5_extract_feature(data_dir, dump_dir):
                status_change(finetune_status_json, False)
                return
        else:
            print(f"{dump_dir} 已存在，跳过此步骤")

    output_dir = os.path.join(exp_path, "output")
    if start_step <= 6 and end_step >= 6:
        # Step6: 生成训练环境
        if not os.path.exists(output_dir):
            if not step6_prepare_env(output_dir):
                status_change(finetune_status_json, False)
                return
        else:
            print(f"{output_dir} 已存在，跳过此步骤")

    if start_step <= 7 and end_step >= 7:
        # Step7: 微调训练
        if not step7_finetune(dump_dir, output_dir, max_finetune_step, batch_size=None, learning_rate=None):
            status_change(finetune_status_json, False)
            return

    if start_step <= 8 and end_step >= 8:
        # 这一步不需要运行中
        status_change(finetune_status_json, False)
        # Step8: 导出静态图模型
        if not step8_export_static_model(exp_name):
            return

    status_change(finetune_status_json, False)
    return True

# 生成音频
def generateTTS(text_dict, exp_name, voc="PWGan"):
    if not step9_generateTTS_inference(text_dict, exp_name, voc):
        print("音频生成失败，请微调模型是否成功!")
        return None
    else:
        return True


from typing import List,Callable

def cost(avg:int,x:int)->int:
    diff=x-avg
    if diff>0:
        return 2*diff
    else:
        return -diff

def fst_nonzero(x:List[int])->int:
    for idx,it in enumerate(x):
        if it!=0:
            return idx

def get_eta(hdist:List[int],MAXH:int)->Callable[[int],int]:
    def eta(pivot:int):
        return sum(map(lambda h,n:n*cost(pivot,h),range(MAXH+1),hdist))
    return eta

def main(MAXH=256):
    hdist=[0 for _ in range(MAXH+1)]
    n,m,b=map(int,input().split())

    for _ in range(n):
        for i in map(int,input().split()):
            hdist[i]+=1

    blocks=b+sum(map(lambda h,n:h*n,range(MAXH+1),hdist))
    eta=get_eta(hdist,MAXH)

    minh=fst_nonzero(hdist)
    maxh=min(blocks//(m*n),MAXH)+1

    while minh<maxh:
        middle=(minh+maxh)//2
        meta=eta(middle)
        beta=eta(middle-1)

        if beta<meta:
            maxh=middle
        else:
            minh=middle+1
    h=minh-1

    print(eta(h),h)

if __name__=='__main__':
    main()


import matplotlib.pyplot as plt
import numpy as np
from obspy import UTCDateTime
from obspy.core.stream import Stream
from obspy.signal.trigger import coincidence_trigger as ct
from reviewData import reviewData
from removeTeleseisms import removeTeleseisms, searchComCatforLandslides
from findFirstArrivals import findFirstArrivals, plotFirstArrivals

import warnings
warnings.filterwarnings("ignore")

def getStreamObject(starttime,endtime,lslat,lslon,radius=100.):
    """
    Uses seisk reviewData module to grab seismic data using FDSN webservices
    for stations within a specified radius of the landslide as a stream object.
    Increments radius until minimum number of traces or maximum search radius
    is achieved.
    INPUTS
    starttime (UTCDateTime) - start time of stream object
    endtime (UTCDateTime) - end time of stream object
    lslat (float) - latitudinal coordinate of landslide (make negative for south
        of Equator)
    lslon (float) - longitudinal coordinate of landslide (make negative for west
        of Prime Meridian)
    OUTPUT
    st - obspy stream object with seismic data
    """

    # Seismic channels to search for
    channels = 'EHZ,BHZ,HHZ'

    # Search for data within initial radius
    print('Retrieving data for radius = %i km...' % int(radius))
    st = Stream()
    st_init = reviewData.getepidata(lslat, lslon, starttime, tstart=0.,
                               tend=endtime-starttime, minradiuskm=0.,
                               maxradiuskm=radius, chanuse=channels,
                               location='*', clientnames=['IRIS'],
                               savedat=False, detrend='demean')
    for trace in st_init:
        st.append(trace)

    # Check if number of traces in stream object less than minimum; if so,
    # increment radius by 50 km and search for data again
    maxradius = 350 # maximum radius to search within (in km)
    mintraces = 5 # minimum number of traces accepted
    while (not st or len(st) < mintraces) and radius <= maxradius:
        radius += 50. # km
        print('Incrementing radius to %i km and retrieving data...' % int(radius))
        st = reviewData.getepidata(lslat, lslon, starttime, tstart=0.,
                                   tend=endtime-starttime, minradiuskm=0.,
                                   maxradiuskm=radius, chanuse=channels,
                                   location='*', clientnames=['IRIS'],
                                   savedat=False, detrend='demean')

    if st:
        print('%i stations within %i km of landslide.' % (len(st),int(radius)))

    return(st)

# Get seismic signal files from specified seismic network
def findLandslides(st,useTrig=True,plotTrig=False):
    """
    Uses seisk reviewData module to grab seismic data using FDSN webservices
    for stations within a specified radius of the landslide as a stream object.
    Increments radius until minimum number of traces or maximum search radius
    is achieved.
    INPUTS
    st - obspy stream object with seismic data
    useTrig (boolean) - optional, set to False to skip checking for landslides
                        and just plot stream object
    plotTrig (boolean) - optional, set to False to skip plotting and just check
                         for landslides in stream object
    OUTPUTS
    trigger_times (list of UTCDateTimes) - times returned from obspy.signal.trigger
        coincidence_trigger() for stream object as potential landslide events
    event_times (list of UTCDateTimes) - times from trigger_times which are
        potential landslides based on various checks
    """

    print('Modifying stream object...')
    st = reviewData.attach_distaz_IRIS(st, lslat, lslon)
    st = st.sort(keys=['rdist', 'channel'])
    st.filter('bandpass', freqmin=1.0, freqmax=5.0)

    network = st[0].stats.network
    station = st[0].stats.station # first station in stream

    trigger_times = []
    event_times = []

    print('Looking for triggers...')
    if useTrig:
        # play with these parameters
        init_trig = ct("recstalta", 2.5, 1, st, 3, sta=4., lta=20.)
        print('%i triggers initially returned.' % len(init_trig))
        if len(init_trig) > 0:
            trig, removed_triggers = removeTeleseisms(starttime,endtime,network,
                                                      station,init_trig)
            trigger_times = [t['time'] for t in trig]

            for t in range(0,len(trig)):
                # Select part of signal around trigger
                # If multiple triggers, cut signal so that two triggers aren't in same slice
                if t > 1 and ((t < (len(trig)-1)) and (trig[t+1]['time']-trig[t]['time'] < 300)):
                    temp = st.copy().trim(trig[t]['time'] - 100., trig[t+1]['time'] - 100.)
                    # If traces in temp are too short, return to original length
                    if len(temp[0]) < 10000:
                        temp = st.copy().trim(trig[t]['time'] - 100., trig[t]['time'] + 300.)
                else:
                    temp = st.copy().trim(trig[t]['time']-100., trig[t]['time']+300.)

                # Delete problematic stations (will need to add to)
                channels_to_remove = []
                for channel in temp:
                    if channel.std() < 1.0:
                        channels_to_remove.append(channel)

                for channel in channels_to_remove:
                    #if len(st.select(station = channel.stats.station)) > 0:
                    temp.remove(channel)

                # Find first arrivals in signal
                F4, arrival_times, arrival_inds = findFirstArrivals(trig[t]['time']-100., temp, plot_checkcalcs = False)

                # TO-DO: Figure out how to check if arrival time belongs to event
                diff = np.asarray(arrival_times) - arrival_times[1]
                if sum(1 for i in diff if i <= 50. and i > 0) >= 4:
                    event_times.append(arrival_times[0])
                    plotFirstArrivals(trig[t]['time']-100., temp, F4, arrival_times, arrival_inds)

                # Plot arrival times as a function of distance away from landslide
                if 1:
                    if len(arrival_times) > 0:
                        arrival_seconds = []
                        for time in arrival_times:
                            arrival_seconds.append(time.second)

                        station_dist = []
                        for tr in temp:
                            station_dist.append(tr.stats.rdist)

                        m, b = np.polyfit(station_dist, arrival_seconds, 1)
                        print('m = ', m)

                        plt.figure()
                        plt.plot(station_dist, arrival_seconds, 'ro',
                                 label='Signal arrival times')
                        plt.plot(station_dist, np.dot(m, station_dist) + b, '--b',
                                 label='Line of best fit')
                        plt.xlabel('Distance from station to landslide (km)')
                        plt.ylabel('Landslide signal arrival time (s)')
                        plt.legend()

                # Plot signal
                if plotTrig:
                    print('Plotting signal...')
                    reviewData.InteractivePlot(temp,xlim=(0,400))

    return(trigger_times, event_times)

# Input parameters

lslat = 61.226
lslon = -152.167
radius = 50.

starttime = UTCDateTime(2014,9,10,19,20,0)
endtime = UTCDateTime(2014,9,10,19,22,0)
absendtime = endtime

st = getStreamObject(starttime,endtime,lslat,lslon,radius)
network = st[0].stats.network
station = st[0].stats.station # first station in stream

trigger_times, event_times = findLandslides(st,useTrig=True,plotTrig=True)
possible_event_times = searchComCatforLandslides(starttime,absendtime,lslat,
                                                 lslon,network,station)


#Motion Detection

import cv2
import time
import  datetime
import imutils

# Using Socket Communication for interfacing of processing IDE and motion detector
import socket

HOST = ''                 # Symbolic name meaning all available interfaces
PORT = 5204           # Arbitrary  port  number that will be used to get
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print('Connected by', addr)

def motion_detection():
    video_capture = cv2.VideoCapture(0) # webcam
    time.sleep(2)

    first_frame = None
    temp =0   # used to get the previous y coordinate of the hand
    count = 0  # To calculate number of frames
    width = video_capture.get(3)
    height = video_capture.get(4)
    fps = video_capture.get(5)

    y0 = 0    # Gets current y coordinate of hand
    motion = ''
    temp = 0
    duration = 0
    t =0
    while True :

        check, frame = video_capture.read()  # getting frames from the
        text = 'Nothing'
        data = 'n'

        grayscale_frame = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY) # converts into a gray scale image

        gaussian_frame = cv2.GaussianBlur(grayscale_frame,(21,21),0) # 21x21 gaussian grid is multiplied with input image so that we can get the centre index

        blur_frame = cv2.blur(gaussian_frame, (5,5),0) #5x5 blur grid is multiplied with input image

        grayscale_img = blur_frame

        if first_frame is None :
            first_frame = grayscale_img

        else :
            pass

        frame = imutils.resize(frame, width = 500)

        '''Calculates the absolute difference between each element/pixel between
        the two images , first_frame - grayscale (on each pixel) '''

        frame_delta = cv2.absdiff(first_frame, grayscale_img)

        ''' Threshold gives two output check, threshold image, using [1]
        will give the threshold image '''

        thresh = cv2.threshold(frame_delta,100,225,cv2.THRESH_BINARY)[1]

        '''dilate gives 3 output - dilate ,grow, expand
        dilate basically enlarges the white pixels which are in the foreground in a
        binary image i.e black background and white foreground
        element = Mat() = default 3x3 kernal matrix and iteratios =2 means it will do it twice'''

        dilate_image = cv2.dilate(thresh,None,iterations = 2)

        '''contours gives 3 diff o/p - image, contours and hierarchy, so using
        [1] at the end means contours = (cnt)[1]  cv2.CHAIN_APPROX _SIMPLE
        saves memory by removing all redundant points and compressing contour,
        if u have a rectangle with 4 straight lines u don't need to plot each
        point on the line , u only need to plot the corners of the rectangle and
        the join th elines, eg - if 100 points were used before now only 4 points are being used
        '''
        cnt =cv2.findContours(dilate_image.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) [0] # Gets the image
        count =count +1
        for c in cnt :
            # if the contour area is more than 800 non -zero pixels (not black)
            if cv2.contourArea(c) > 20000 :
                # so we are getting the coordinates of the object detected
                (x,y,w,h) = cv2.boundingRect(c)

                cv2.rectangle(frame,(x,y),(x+w,y+h),(0,255,0),2) # 0,255,0 means green color box with thickness 2

                text = 'object detected'
                y0 = y + h
                temp = y
                duration = 1
                data = ' '

            elif duration == 1:

                if (temp > 150 or y0 >200) :
                    motion = 'down'
                    data = 'd'

                if (temp <100 ):
                    motion ='up'
                    data = 'u'
                temp = 0
                y0 = 0
                conn.send(data.encode()) # sending  data character to processing IDE
                t = t+1
                duration = 0

            else :
                duration = 0
                data = ' '

        font = cv2.FONT_HERSHEY_SIMPLEX

        cv2.putText(frame, f' Object Moved {text}', (10,20), font , 0.5, (0,0,255),2) # 0.5 is size, 2 is thickness

        cv2.putText(frame, f' Object Moved y-coor {y0} ,{count},{temp}, {motion}, {duration},{t}', (100,50), font , 0.5, (0,0,255),2)

        #cv2.imshow("Dilate", dilate_image)
        #cv2.imshow('Frame_delta', frame_delta)
        cv2.imshow("Object_Detection",frame)

        key = cv2.waitKey(1) & 0xFF # 0xFF = 11111111 in binary format and taking AND with the key so that we get the last 8 bits value
        if key == ord('q'):
            cv2.destroyAllWindows()
            break

motion_detection()


from django.http import HttpResponse
from django.shortcuts import render, redirect
from .models import Item

def home_page(request):
    # if request.method == 'POST':
    #     return HttpResponse(request.POST['item_text'])
    # return render(request, 'lists/home.html')
    # item = Item()
    # item.text = request.POST.get('item_text', '')
    # item.save()

    if request.method == 'POST':
        # new_item_text = request.POST['item_text']
        Item.objects.create(text=request.POST['item_text'])
        return redirect('/')
    # else:
    #     new_item_text = ''

    items = Item.objects.all()
    context = {
        'items': items,
        }
    return render(request, 'lists/home.html', context)


# Modified from https://gist.github.com/bricef/1b0389ee89bd5b55113c7f3f3d6394ae
# Found this from Armin R. on Twitter, what a beautiful gem ;)

import ctypes
import functools as fn
from types import MappingProxyType

##-- Figure out the size of size_t type --##
if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'):
    _Py_ssize_t = ctypes.c_int64
else:
    _Py_ssize_t = ctypes.c_int

##-- Define PyObject type --##
# regular python
class _PyObject(ctypes.Structure):
    pass
_PyObject._fields_ = [
    ('ob_refcnt', _Py_ssize_t),
    ('ob_type', ctypes.POINTER(_PyObject))
]

# python with trace
if object.__basicsize__ != ctypes.sizeof(_PyObject):
    class _PyObject(ctypes.Structure):
        pass
    _PyObject._fields_ = [
        ('_ob_next', ctypes.POINTER(_PyObject)),
        ('_ob_prev', ctypes.POINTER(_PyObject)),
        ('ob_refcnt', _Py_ssize_t),
        ('ob_type', ctypes.POINTER(_PyObject))
    ]

class _DictProxy(_PyObject):
    """Proxy for casting _PyObject back to object"""
    _fields_ = [('dict', ctypes.POINTER(_PyObject))]

pendingFlush = False
def needFlush(func):
    """Decorator for handling flush keyword argument"""
    @fn.wraps(func)
    def inner(*args, flush=None, **kwargs):
        global pendingFlush
        removePending = False
        if flush is None and not pendingFlush:
            flush = pendingFlush = removePending = True
        res = func(*args, **kwargs)
        if flush:
            ctypes.pythonapi.PyType_ClearCache()
        if removePending:
            pendingFlush = False
        return res
    return inner

def reveal_dict(proxy):
    """
    Get the underlying dict from MappingProxyType
    and force convert to python dict
    """
    if not isinstance(proxy, MappingProxyType):
        raise TypeError('dictproxy expected')
    dp = _DictProxy.from_address(id(proxy))
    ns = {}
    ctypes.pythonapi.PyDict_SetItem(ctypes.py_object(ns),
                                    ctypes.py_object(None),
                                    dp.dict)
    return ns[None]

def get_class_dict(cls):
    """Get the dict and convert the type if it's MappingProxyType"""
    d = getattr(cls, '__dict__', None)
    if d is None:
        raise TypeError('given class does not have a dictionary')
    if isinstance(d, MappingProxyType):
        return reveal_dict(d)
    return d

@needFlush
def patch(cls, key, value, override=None):
    """Monkey patch for both custom and builtin types"""
    if not isinstance(cls, type):
        raise TypeError('Not a type')
    if hasattr(cls, key):
        if override is None:
            raise KeyError(f'Unexpected Override: {cls}[{key}]')
        elif override is False:
            return False
    try:
        setattr(cls, key, value)
    except TypeError:
        d = get_class_dict(cls)
        d[key] = value
    return True

if __name__ == '__main__':
    class cls():
        def foo(self):
            return 'no'
    patch(cls, 'foo', lambda x: 'bar', override=True)
    if cls().foo() == 'bar':
        print('[+] Patch normal class success')

    patch(str, 'lower', lambda x: 'bar', override=True)
    if 'test'.lower() == 'bar':
        print("[+] Patch builtin class success")


#!/usr/bin/python3
"""
Simple API module
"""
from datetime import datetime, timezone
from flask import Flask, jsonify, request
from os import getenv

app = Flask(__name__)

@app.route('/api', strict_slashes=False)
def index():
    """
    Reads URL parameters and returns JSON content
    """
    days = ["Monday", "Tuesday", "Wednesday",
            "Thursday", "Friday", "Saturday", "Sunday"]

    slack_name = request.args.get('slack_name')
    track = request.args.get('track')

    print(request.query_string)

    repo_url = 'https://github.com/Michael-Maina/hng-projects'
    file_url = 'https://github.com/Michael-Maina/hng-projects/blob/master/stage_1/api.py'

    utc_time = datetime.now(timezone.utc)
    today = days[utc_time.weekday()]
    formatted_time = utc_time.strftime("%Y-%m-%dT%H:%M:%SZ")

    output = {
        'slack_name': slack_name,
        'current_day': today,
        'utc_time': formatted_time,
        'track': track,
        'github_file_url': file_url,
        'github_repo_url': repo_url,
        'status_code': 200
    }
    return jsonify(output), 200

if __name__ == "__main__":
    API_HOST = getenv('API_HOST')
    API_PORT = getenv('API_PORT')

    host = API_HOST if API_HOST else '0.0.0.0'
    port = API_PORT if API_PORT else 5000

    app.run(port=port, host=host)


from typing import Optional

from ConfigSpace.conditions import EqualsCondition, InCondition
from ConfigSpace.configuration_space import ConfigurationSpace
from ConfigSpace.hyperparameters import (
    CategoricalHyperparameter,
    UniformFloatHyperparameter,
    UnParametrizedHyperparameter,
)

from autosklearn.askl_typing import FEAT_TYPE_TYPE
from autosklearn.pipeline.components.base import (
    AutoSklearnClassificationAlgorithm,
    IterativeComponentWithSampleWeight,
)
from autosklearn.pipeline.constants import DENSE, PREDICTIONS, SPARSE, UNSIGNED_DATA
from autosklearn.pipeline.implementations.util import softmax
from autosklearn.util.common import check_for_bool

class SGD(
    IterativeComponentWithSampleWeight,
    AutoSklearnClassificationAlgorithm,
):
    def __init__(
        self,
        loss,
        penalty,
        alpha,
        fit_intercept,
        tol,
        learning_rate,
        l1_ratio=0.15,
        epsilon=0.1,
        eta0=0.01,
        power_t=0.5,
        average=False,
        random_state=None,
    ):
        self.max_iter = self.get_max_iter()
        self.loss = loss
        self.penalty = penalty
        self.alpha = alpha
        self.fit_intercept = fit_intercept
        self.tol = tol
        self.learning_rate = learning_rate
        self.l1_ratio = l1_ratio
        self.epsilon = epsilon
        self.eta0 = eta0
        self.power_t = power_t
        self.random_state = random_state
        self.average = average

        self.estimator = None
        self.n_iter_ = None

    @staticmethod
    def get_max_iter():
        return 1024

    def get_current_iter(self):
        return self.n_iter_

    def iterative_fit(self, X, y, n_iter=2, refit=False, sample_weight=None):
        from sklearn.linear_model import SGDClassifier

        # Need to fit at least two iterations, otherwise early stopping will not
        # work because we cannot determine whether the algorithm actually
        # converged. The only way of finding this out is if the sgd spends less
        # iterations than max_iter. If max_iter == 1, it has to spend at least
        # one iteration and will always spend at least one iteration, so we
        # cannot know about convergence.

        if refit:
            self.estimator = None
            self.n_iter_ = None

        if self.estimator is None:
            self.fully_fit_ = False

            self.alpha = float(self.alpha)
            self.l1_ratio = float(self.l1_ratio) if self.l1_ratio is not None else 0.15
            self.epsilon = float(self.epsilon) if self.epsilon is not None else 0.1
            self.eta0 = float(self.eta0)
            self.power_t = float(self.power_t) if self.power_t is not None else 0.5
            self.average = check_for_bool(self.average)
            self.fit_intercept = check_for_bool(self.fit_intercept)
            self.tol = float(self.tol)

            self.estimator = SGDClassifier(
                loss=self.loss,
                penalty=self.penalty,
                alpha=self.alpha,
                fit_intercept=self.fit_intercept,
                max_iter=n_iter,
                tol=self.tol,
                learning_rate=self.learning_rate,
                l1_ratio=self.l1_ratio,
                epsilon=self.epsilon,
                eta0=self.eta0,
                power_t=self.power_t,
                shuffle=True,
                average=self.average,
                random_state=self.random_state,
                warm_start=True,
            )
            self.estimator.fit(X, y, sample_weight=sample_weight)
            self.n_iter_ = self.estimator.n_iter_
        else:
            self.estimator.max_iter += n_iter
            self.estimator.max_iter = min(self.estimator.max_iter, self.max_iter)
            self.estimator._validate_params()
            self.estimator._partial_fit(
                X,
                y,
                alpha=self.estimator.alpha,
                C=1.0,
                loss=self.estimator.loss,
                learning_rate=self.estimator.learning_rate,
                max_iter=n_iter,
                sample_weight=sample_weight,
                classes=None,
                coef_init=None,
                intercept_init=None,
            )
            self.n_iter_ += self.estimator.n_iter_

        if (
            self.estimator.max_iter >= self.max_iter
            or self.estimator.max_iter > self.n_iter_
        ):
            self.fully_fit_ = True

        return self

    def configuration_fully_fitted(self):
        if self.estimator is None:
            return False
        elif not hasattr(self, "fully_fit_"):
            return False
        else:
            return self.fully_fit_

    def predict(self, X):
        if self.estimator is None:
            raise NotImplementedError()
        return self.estimator.predict(X)

    def predict_proba(self, X):
        if self.estimator is None:
            raise NotImplementedError()

        if self.loss in ["log", "modified_huber"]:
            return self.estimator.predict_proba(X)
        else:
            df = self.estimator.decision_function(X)
            return softmax(df)

    @staticmethod
    def get_properties(dataset_properties=None):
        return {
            "shortname": "SGD Classifier",
            "name": "Stochastic Gradient Descent Classifier",
            "handles_regression": False,
            "handles_classification": True,
            "handles_multiclass": True,
            "handles_multilabel": False,
            "handles_multioutput": False,
            "is_deterministic": True,
            "input": (DENSE, SPARSE, UNSIGNED_DATA),
            "output": (PREDICTIONS,),
        }

    @staticmethod
    def get_hyperparameter_search_space(
        feat_type: Optional[FEAT_TYPE_TYPE] = None, dataset_properties=None
    ):
        cs = ConfigurationSpace()

        loss = CategoricalHyperparameter(
            "loss",
            ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
            default_value="log",
        )
        penalty = CategoricalHyperparameter(
            "penalty", ["l1", "l2", "elasticnet"], default_value="l2"
        )
        alpha = UniformFloatHyperparameter(
            "alpha", 1e-7, 1e-1, log=True, default_value=0.0001
        )
        l1_ratio = UniformFloatHyperparameter(
            "l1_ratio", 1e-9, 1, log=True, default_value=0.15
        )
        fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
        tol = UniformFloatHyperparameter(
            "tol", 1e-5, 1e-1, log=True, default_value=1e-4
        )
        epsilon = UniformFloatHyperparameter(
            "epsilon", 1e-5, 1e-1, default_value=1e-4, log=True
        )
        learning_rate = CategoricalHyperparameter(
            "learning_rate",
            ["optimal", "invscaling", "constant"],
            default_value="invscaling",
        )
        eta0 = UniformFloatHyperparameter(
            "eta0", 1e-7, 1e-1, default_value=0.01, log=True
        )
        power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default_value=0.5)
        average = CategoricalHyperparameter(
            "average", ["False", "True"], default_value="False"
        )
        cs.add_hyperparameters(
            [
                loss,
                penalty,
                alpha,
                l1_ratio,
                fit_intercept,
                tol,
                epsilon,
                learning_rate,
                eta0,
                power_t,
                average,
            ]
        )

        # TODO add passive/aggressive here, although not properly documented?
        elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
        epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")

        power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling")

        # eta0 is only relevant if learning_rate!='optimal' according to code
        # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
        # linear_model/sgd_fast.pyx#L603
        eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"])
        cs.add_conditions(
            [elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]
        )

        return cs


from django.shortcuts import render,redirect
from .forms import ProductoForm,PedidoForm,Burger_BossForm,ExtraForm,IngredienteForm,HamburguesaForm
from django.views.generic import CreateView,DeleteView,ListView,UpdateView,DetailView
from .models import Producto,Pedido,Burger_Boss,Extra,Ingrediente,Hamburguesa
from django.urls import reverse_lazy
#Crea clases
class HomePage(ListView):
    model = Producto
    template_name = 'homepage3.html'

class Ubicanos(ListView):
    model = Producto
    template_name = 'ubicanos.html'
class Pagos(ListView):
    model = Producto
    template_name = 'pagos.html'
class ProductoList(ListView):
    model = Producto
    template_name = 'pedido.html'

    def get_context_data(self, **kwargs):
        context= super().get_context_data()
        context['extra_list'] = Extra.objects.all()
        return context

class PedidoCreate(CreateView):
    model = Pedido
    form_class = PedidoForm
    template_name = 'crear_pedido.html'
    success_url = reverse_lazy('pedido_completado')

class PedidoList(ListView):
    model = Pedido
    template_name = 'pedido_completado.html'

class ProductoListCambiarEstado(ListView):
    model = Pedido
    template_name = 'cambiar_estado.html'

class OtroEstado(UpdateView):
    model = Pedido
    form_class = PedidoForm
    template_name = 'otro_estado.html'
    success_url = reverse_lazy('cambiar_estado')

class ProductoCreate(CreateView):
    model = Producto
    form_class = ProductoForm
    template_name = 'crear_producto.html'
    success_url = reverse_lazy('pedido_admin')

class ExtraCreate(CreateView):
    model = Extra
    form_class = ExtraForm
    template_name = 'crear_extra.html'
    success_url = reverse_lazy('pedido_admin')

class IngredienteCreate(CreateView):
    model = Ingrediente
    form_class = IngredienteForm
    template_name = 'crear_ingrediente.html'
    success_url = reverse_lazy('pedido_admin')

class BurgerBossCreate(CreateView):
    model = Burger_Boss
    form_class = Burger_BossForm
    template_name = 'registro.html'
    success_url = reverse_lazy('crear_hamburguesa2')

class HamburguesaCreate(CreateView):
    model = Hamburguesa
    form_class = HamburguesaForm
    template_name = 'crear_hamburguesa.html'
    success_url = reverse_lazy('extra')

#class VerHamburguesas2(ListView):
 #   model = Hamburguesa
  #  template_name = 'ver_hamburugesa.html'

class Pagos(ListView):
    model = Producto
    template_name = 'pagos.html'

class ModificarProductoCarne(UpdateView):
    model = Producto
    form_class = ProductoForm
    template_name = 'modificar_producto_carne.html'
    success_url = reverse_lazy('pedido_admin')

class ModificarProductoQueso(UpdateView):
    model = Producto
    form_class = ProductoForm
    template_name = 'modificar_producto_queso.html'
    success_url = reverse_lazy('pedido_admin')

class ModificarIngrediente(UpdateView):
    model = Ingrediente
    form_class = IngredienteForm
    template_name = 'modificar_ingrediente.html'
    success_url = reverse_lazy('pedido_admin')

class ModificarExtra(UpdateView):
    model = Extra
    form_class = ExtraForm
    template_name = 'modificar_extra.html'
    success_url = reverse_lazy('pedido_admin')

class EliminarCarne(DeleteView):
    model = Producto
    template_name = 'eliminar_carne.html'
    success_url = reverse_lazy('pedido_admin')

class EliminarQueso(DeleteView):
    model = Producto
    template_name = 'eliminar_queso.html'
    success_url = reverse_lazy('pedido_admin')

class EliminarIngrediente(DeleteView):
    model = Ingrediente
    template_name = 'eliminar_ingrediente.html'
    success_url = reverse_lazy('pedido_admin')

class EliminarExtra(DeleteView):
    model = Extra
    template_name = 'eliminar_extra.html'
    success_url = reverse_lazy('pedido_admin')


import pymortar
import time
import pytz
import datetime
from rfc3339 import rfc3339
import xbos_services_getter as xbos

pymortar_client = pymortar.Client()

def get_outside_temp_data(building):

    interval = 3600

    outside_temperature_query = """SELECT ?temp WHERE {
        ?temp rdf:type brick:Weather_Temperature_Sensor .
    };"""

    weather_stations_view = pymortar.View(
        name="weather_stations_view",
        sites=[building],
        definition=outside_temperature_query,
    )

    weather_stations_stream = pymortar.DataFrame(
        name="weather_stations",
        aggregation=pymortar.MEAN,
        window=str(int(interval)) + 's',
        timeseries=[
            pymortar.Timeseries(
                view="weather_stations_view",
                dataVars=["?temp"],
            )
        ]
    )

    weather_stations_time_params = pymortar.TimeParams(
        start=rfc3339(start),
        end=rfc3339(end),
    )

    request = pymortar.FetchRequest(
        sites=[building],
        views=[
            weather_stations_view
        ],
        dataFrames=[
            weather_stations_stream
        ],
        time=weather_stations_time_params
    )

    df = outside_temperature_data = pymortar_client.fetch(request)['weather_stations']

    return df

#This query doesn't return any data for "hayward-station-1", but returns data for all other buildings
end = datetime.datetime.now().replace(tzinfo=pytz.utc) - datetime.timedelta(weeks=52)
start = end - datetime.timedelta(days=10)

for building in xbos.get_buildings(xbos.get_building_zone_names_stub()):
    #Looping through all buildings and getting response
    df = get_outside_temp_data(building)
    if df is None:
        print(building) #The only building that doesn't return data is "hayward-station-1"


import sqlite3 as sq3
conn=sq3.connect("First.db")
#==================================================
cur=conn.cursor()
cur.execute('Create table T_fish(date text,name text,nums int,price real,Explain text)')
cur.execute("insert into T_fish Values('2018-3-28','黑鱼',10,28.3,'Tom')")
conn.commit()

#==================================================
conn.close