import discord
from discord.ext import commands
import wikipedia
import asyncio
from random import randint
import traceback
from bot import guilds

class Wiki(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.slash_command(name="wikisearch", description="Returns a list of Wikipedia articles based on the request.")
    async def wikisearch(self, ctx, *, request):
        try:
            wikicontent = wikipedia.search(request, results=10, suggestion=False)  # Wikipedia search request

            wikilinks = []
            for i in wikicontent:
                wikilinks.append('https://en.wikipedia.org/wiki/Special:Search?search=' + i.replace(' ', '_'))

            embed = discord.Embed(
                title="Wikipedia search results:",
                color=int(str(ctx.author.color)[1:], 16),
            )
            if not wikicontent:
                wikicontent = f'Sorry, there are no search results for "{request}".'
            wikiresults = []
            for i in range(len(wikicontent)):
                wikiresults.append(f'[{wikicontent[i]}]({wikilinks[i]})')

            embed.description = "\n".join(wikiresults)
            embed.set_thumbnail(
                url='https://www.wikipedia.org/static/images/project-logos/enwiki.png'
            )
            await ctx.respond(embed=embed)

        except Exception as error:
            if error == wikipedia.exceptions.DisambiguationError:
                await ctx.send('There are too many search results for this query. Please be more specific.')
            await ctx.send(f'Sorry, an error occurred: \n`{error}`\n - Please report to `tylersfoot#8888`.')
            print(traceback.format_exc())

    @commands.slash_command(name="wiki", description="Returns a summary of the Wikipedia article specified.")
    async def wiki(self, ctx, *, request):
        try:
            pagecontent = wikipedia.page(request, auto_suggest=False, redirect=True, preload=False)
            pagetext = wikipedia.summary(request, auto_suggest=False, redirect=True, sentences=5)

            # Tries to set first image in article to embed thumbnail
            try:
                thumbnail = pagecontent.images[0]
            except Exception as error:
                # If there are no images, it will set it to the default wikipedia picture
                try:
                    print(f'Couldn\'t load {thumbnail}, {error}')
                    thumbnail = 'https://www.wikipedia.org/static/images/project-logos/enwiki.png'
                except:
                    print(f'Couldn\'t load thumbnail, {error}')
                    thumbnail = 'https://www.wikipedia.org/static/images/project-logos/enwiki.png'

            embed = discord.Embed(
                title=pagecontent.title,
                color=int(str(ctx.author.color)[1:], 16),
                description=f'{pagetext}\n\n[Read further]({pagecontent.url})'
            )
            embed.set_thumbnail(
                url=thumbnail
            )
            await ctx.respond(embed=embed)

        except wikipedia.PageError:
            await ctx.respond(f'Sorry, there are no results for "{request}".')

        except Exception as error:
            await ctx.respond(f'Sorry, an error occurred: \n`{str(error)[:1900]}`\n - Please report to `tylersfoot#8888`.')
            print(traceback.format_exc())

    @commands.slash_command(name="wikirandom", description="Returns the summary of a random Wikipedia article.")
    async def wikirandom(self, ctx):
        try:
            tries = 0
            # Tries to get a random article 10 times
            for i in range(10):
                tries += 1
                try:
                    random_article = wikipedia.random(pages=1)
                    pagecontent = wikipedia.page(random_article)
                    break
                except Exception as e:
                    continue

            pagetext = wikipedia.summary(random_article, sentences=5)

            # Tries to set first image in article to embed thumbnail
            try:
                thumbnail = pagecontent.images[0]
            except Exception as error:
                # If there are no images, it will set it to the default wikipedia picture
                try:
                    print(f'Couldn\'t load {thumbnail}, {error}')
                    thumbnail = 'https://www.wikipedia.org/static/images/project-logos/enwiki.png'
                except:
                    print(f'Couldn\'t load thumbnail, {error}')
                    thumbnail = 'https://www.wikipedia.org/static/images/project-logos/enwiki.png'

            embed = discord.Embed(title=random_article,
                                  color=int(str(ctx.author.color)[1:], 16),
                                  description=f'{pagetext}\n\n[Read further]({pagecontent.url})'
                                  )
            embed.set_thumbnail(url=thumbnail)
            embed.set_footer(text=f'Requested by {ctx.author.name} | {tries} attempts', icon_url=ctx.author.avatar.url)
            await ctx.respond(embed=embed)
        except Exception as error:
            await ctx.respond(f'Sorry, an error occurred: \n`{str(error)[:1900]}`\n - Please report to `tylersfoot#8888`')
            print(traceback.format_exc())

def setup(bot):
    bot.add_cog(Wiki(bot))


import shutil
import logging
import itertools
from pathlib import Path
import numpy
import pandas
import plotly
from pepclibs.helperlibs import Trivial, FSHelpers
from pepclibs.helperlibs.Exceptions import Error
from wultlibs.helperlibs import Jinja2
from wultlibs.rawresultlibs import RORawResult

_LOG = logging.getLogger()

# List of diagram markers that we use in scatter plots.
_SCATTERPLOT_MARKERS = ['circle', 'square', 'diamond', 'cross', 'triangle-up', 'pentagon']

def _colname_to_fname(colname):
    """
    Turn column name 'colname' into a file name, replacing problematic characters that browsers
    may refuse.
    """

    return colname.replace("%", "_pcnt").replace("/", "-to-")

class HTMLReportBase:
    """This is the base class for generating HTML reports for raw test results."""

    def _prepare_intro_table(self, stats_paths, logs_paths, descr_paths):
        """
        Create the intro table, which is the very first table in the report and it shortly
        summarizes the entire report. The 'stats_paths' should be a dictionary indexed by report ID
        and containing the stats directory path. Similarly, the 'logs_paths' and 'descr_paths'
        contains paths to the logs and the test result description files.
        """

        intro_tbl = {}
        intro_tbl["Title"] = {}
        for res in self.rsts:
            intro_tbl[res.reportid] = {}

        # Add tool information.
        key = "tool_info"
        intro_tbl["Title"][key] = "Data collection tool"
        for res in self.rsts:
            intro_tbl[res.reportid][key] = f"{res.info['toolname'].capitalize()} version " \
                                         f"{res.info['toolver']}"

        # Add datapoint counts.
        key = "datapoints_cnt"
        intro_tbl["Title"][key] = "Datapoints Count"
        for res in self.rsts:
            intro_tbl[res.reportid][key] = len(res.df.index)

        # Add measurement resolution.
        for res in self.rsts:
            key = "device_resolution"
            resolution = res.info.get("resolution")
            if resolution:
                intro_tbl["Title"][key] = "Device Resolution"
                intro_tbl[res.reportid][key] = f"{resolution}ns"

        # Add links to the stats directories.
        if stats_paths:
            key = "stats"
            intro_tbl["Title"][key] = "Statistics"
            for res in self.rsts:
                path = stats_paths.get(res.reportid, "Not available")
                intro_tbl[res.reportid][key] = path

        # Add links to the logs directories.
        if logs_paths:
            key = "logs"
            intro_tbl["Title"][key] = "Logs"
            for res in self.rsts:
                path = logs_paths.get(res.reportid, "Not available")
                intro_tbl[res.reportid][key] = path

        # Add links to the descriptions.
        if descr_paths:
            key = "descr"
            intro_tbl["Title"][key] = "Test description"
            for res in self.rsts:
                path = descr_paths.get(res.reportid, "Not available")
                intro_tbl[res.reportid][key] = path

        return intro_tbl

    def _prepare_links_table(self):
        """Creates the links table which refers to HTML sub-pages."""

        links_tbl = {}
        for colname in itertools.islice(self._pinfos, 1, None):
            links_tbl[colname] = {}
            links_tbl[colname]["name"] = f"{colname}"
            fname = _colname_to_fname(colname) + ".html"
            links_tbl[colname]["fname"] = fname
            links_tbl[colname]["hlink"] = f"{colname}"

        return links_tbl

    def _prepare_smrys_tables(self, pinfos):
        """
        Summaries table includes values like average and median values for a single metric (column).
        It "summarizes" the metric. This function creates summaries table for each metrics included
        in 'pinfos' list.
        """

        if not pinfos:
            return {}

        smrys_tbl = {}
        smrys_tbl["Title"] = {}
        for res in self.rsts:
            smrys_tbl[res.reportid] = {}

        for pinfo in pinfos:
            for colname in (pinfo["colname"], pinfo["xcolname"]):
                if colname in smrys_tbl["Title"]:
                    continue

                # Skip non-numeric columns, as summaries are calculated only for numeric columns.
                if not self.rsts[0].is_numeric(colname):
                    continue

                # Each column name is represented by a row in the summary table. Fill the "Title"
                # column.
                title_dict = smrys_tbl["Title"][colname] = {}
                defs = self._refdefs.info[colname]

                title_dict["colname"] = colname
                unit = defs.get("short_unit", "")
                if unit:
                    title_dict["colname"] += f", {unit}"
                title_dict["coldescr"] = defs["descr"]

                title_dict["funcs"] = {}
                for funcname in self._smry_funcs:
                    # Select only those functions that are present in all test results. For example,
                    # 'std' will not be present if the result has only one datapoint. In this case,
                    # we need to exclude the 'std' function.
                    if all(res.smrys[colname].get(funcname) for res in self.rsts):
                        title_dict["funcs"][funcname] = RORawResult.get_smry_func_descr(funcname)

                # Now fill the values for each result.
                for res in self.rsts:
                    res_dict = smrys_tbl[res.reportid][colname] = {}
                    res_dict["funcs"] = {}

                    for funcname in title_dict["funcs"]:
                        val = res.smrys[colname][funcname]
                        fmt = "{}"
                        if defs["type"] == "float":
                            fmt = "{:.2f}"

                        fdict = res_dict["funcs"][funcname] = {}
                        fdict["val"] = fmt.format(val)
                        fdict["raw_val"] = val

                        if self._refres.reportid == res.reportid:
                            fdict["hovertext"] = "This is the reference result, other results " \
                                                 "are compared to this one."
                            continue

                        ref_fdict = smrys_tbl[self._refres.reportid][colname]["funcs"][funcname]
                        change = val - ref_fdict["raw_val"]
                        if ref_fdict["raw_val"]:
                            percent = (change / ref_fdict["raw_val"]) * 100
                        else:
                            percent = change
                        change = fmt.format(change) + unit
                        percent = "{:.1f}%".format(percent)
                        fdict["hovertext"] = f"Change: {change} ({percent})"

        return smrys_tbl

    def _copy_raw_data(self):
        """Copy raw test results to the output directory."""

        if self.relocatable == "noraw":
            return (None, None, None)

        # Paths to the stats directory.
        stats_paths = {}
        # Paths to the logs directory.
        logs_paths = {}
        # Paths to test reports' description files.
        descr_paths = {}

        for res in self.rsts:
            srcpath = res.dirpath
            resrootdir = "raw-" + res.reportid
            dstpath = self.outdir.joinpath(resrootdir)

            if self.relocatable == "copy":
                FSHelpers.copy_dir(srcpath, dstpath, exist_ok=True, ignore=["html-report"])
                FSHelpers.set_default_perm(dstpath)
            else:
                FSHelpers.move_copy_link(srcpath, dstpath, action="symlink", exist_ok=True)

            if res.stats_path.is_dir():
                hlink = f"Statistics"
                stats_paths[res.reportid] = hlink
            if res.logs_path.is_dir():
                hlink = f"Logs"
                logs_paths[res.reportid] = hlink
            if res.descr_path.is_file():
                hlink = f"Test description"
                descr_paths[res.reportid] = hlink

        return stats_paths, logs_paths, descr_paths

    def _generate_report(self):
        """Put together the final HTML report."""

        _LOG.info("Generating the HTML report.")

        # Make sure the output directory exists.
        try:
            self.outdir.mkdir(parents=True, exist_ok=True)
        except OSError as err:
            raise Error(f"failed to create directory '{self.outdir}': {err}") from None

        stats_paths, logs_paths, descr_paths = self._copy_raw_data()

        # Find the styles and templates paths.
        templdir = FSHelpers.find_app_data("wult", Path("templates"),
                                           descr="HTML report Jinja2 templates")
        csspath = FSHelpers.find_app_data("wult", Path("css/style.css"),
                                          descr="HTML report CSS file")

        # Copy the styles file to the output directory.
        dstpath = self.outdir.joinpath("style.css")
        try:
            shutil.copyfile(csspath, dstpath)
        except OSError as err:
            raise Error(f"cannot copy CSS file from '{csspath}' to '{dstpath}':\n{err}") from None

        # The intro table is only included into the main HTML page.
        intro_tbl = self._prepare_intro_table(stats_paths, logs_paths, descr_paths)
        links_tbl = self._prepare_links_table()

        jenv = Jinja2.build_jenv(templdir, trim_blocks=True, lstrip_blocks=True)

        all_pinfos = self._pinfos
        if not self._pinfos:
            # This may happen if there are no diagrams to plot. In this case we still want to
            # generate an HTML report, but without diagrams.
            _LOG.warning("no diagrams to plot")
            all_pinfos = {"Dummy" : {}}

        # Each column name gets its own HTML page.
        for colname, pinfos in all_pinfos.items():
            smrys_tbl = self._prepare_smrys_tables(pinfos)

            # Render the template.
            jenv.globals["smrys_tbl"] = smrys_tbl
            jenv.globals["pinfos"] = pinfos
            jenv.globals["colname"] = colname
            jenv.globals["title_descr"] = self.title_descr
            jenv.globals["toolname"] = self._refinfo["toolname"]

            if intro_tbl:
                jenv.globals["intro_tbl"] = intro_tbl
                jenv.globals["links_tbl"] = links_tbl
                templfile = outfile = "index.html"
                intro_tbl = None
            else:
                templfile = "metric.html"
                outfile = links_tbl[colname]["fname"]

            Jinja2.render_template(jenv, Path(templfile), outfile=self.outdir.joinpath(outfile))

    def _add_pinfo(self, xcolname, ycolname, is_hist=False):
        """Add information about a plot to 'self._create_diagrams'."""

        pinfo = {}
        pinfo["xcolname"] = xcolname
        pinfo["ycolname"] = ycolname
        pinfo["fname"] = _colname_to_fname(pinfo['ycolname']) + "-vs-" + \
                         _colname_to_fname(pinfo['xcolname']) + ".html"

        if is_hist:
            colname = xcolname
        else:
            colname = ycolname

        pinfo["colname"] = colname

        if colname not in self._pinfos:
            self._pinfos[colname] = []
        self._pinfos[colname].append(pinfo)

        return pinfo

    def _base_unit(self, df, colname):
        """
        Convert columns with 'microsecond' units to seconds, and return the converted column.
        """

        # This is not generic, but today we have to deal only with microseconds, so this is good
        # enough.
        if self._refdefs.info[colname].get("unit") != "microsecond":
            return df[colname]

        base_colname = f"{colname}_base"
        if base_colname not in df:
            df[base_colname] = df[colname] / 1000000
        return df[base_colname]

    def _create_diagram_axis_info(self, axis, pinfo):
        """Configure axis information dictionary for plotly's 'Figure()' method."""

        nkey = f"{axis}colname"
        colname = pinfo[nkey]
        defs = self._refdefs.info.get(colname, {})
        title = defs.get("title", colname)

        fontfmt = {"family" : "Arial, sans-serif",
                   "size"   : 18,
                   "color"  : "black"}

        axis = {"showline"  : True,
                "showgrid"  : True,
                "title"     : title,
                "titlefont" : fontfmt,
                "ticks"     : "outside",
                "tickwidth" : 1,
                "showticklabels" : True,
                "linewidth" : 1,
                "linecolor" : "black",
                "zeroline" : True,
                "zerolinewidth" : 1,
                "zerolinecolor" : "black"}

        if defs.get("unit") == "microsecond":
            axis["tickformat"] = ".3s"
            axis["ticksuffix"] = "s"
            axis["hoverformat"] = ".4s"
        elif colname == "Percentile":
            axis["ticksuffix"] = "%"

        if defs and not self.rsts[0].is_numeric(colname):
            axis["type"] = "category"
            axis["autorange"] = False
            axis["categoryorder"] = "category ascending"

        return axis

    def _create_hover_text(self, res, df, pinfo):
        """
        Create and return a list containing hover text for every datapoint in the 'df' dataframe.
        """

        xcolname, ycolname = pinfo["xcolname"], pinfo["ycolname"]
        _LOG.debug("Preparing hover text for '%s vs %s'", xcolname, ycolname)

        # The loop below creates the following objects.
        #  o colnames - names of the columns to include to the hover text.
        #  o fmts - the hover text format.
        colnames = []
        fmts = []
        for colname in self._hov_colnames[res.reportid]:
            if colname not in df:
                continue
            if colname in (xcolname, ycolname):
                # The X and Y datapoint values will be added automatically.
                continue

            defs = res.defs.info[colname]
            fmt = f"{colname}: {{"
            if defs["type"] == "float":
                fmt += ":.2f"
            fmt += "}"
            unit = defs.get("short_unit")
            if unit and unit not in colname:
                fmt += f"{unit}"

            colnames.append(colname)
            fmts.append(fmt)

        text = []
        fmt = "
".join(fmts)

        for row in df[colnames].itertuples(index=False):
            text.append(fmt.format(*row))

        return text

    def _create_diagram(self, gobjs, pinfo):
        """Put the 'gobjs' objects to a single plot and save it in the output directory."""

        xaxis = self._create_diagram_axis_info("x", pinfo)
        yaxis = self._create_diagram_axis_info("y", pinfo)

        legend_format = {"font"    : {"size" : 14},
                         "bgcolor" : "#E2E2E2",
                         "borderwidth" : 2,
                         "bordercolor" : "#FFFFFF"}

        layout = {"showlegend"  : True,
                  "hovermode"   : "closest",
                  "xaxis"   : xaxis,
                  "yaxis"   : yaxis,
                  "barmode" : "overlay",
                  "bargap"  : 0,
                  "legend"  : legend_format}

        outdir = self.outdir.joinpath("plots")
        try:
            outdir.mkdir(parents=True, exist_ok=True)
        except OSError as err:
            raise Error(f"failed to create directory '{outdir}': {err}") from None

        fpath = outdir.joinpath(pinfo["fname"])

        try:
            fig = plotly.graph_objs.Figure(data=gobjs, layout=layout)
            if hasattr(fig, "update_layout") and fig.update_layout:
                # In plotly version 4 the default theme has changed. The old theme is called
                # 'plotly_white'. Use it to maintain consistent look for plotly v3 and v4.
                fig.update_layout(template="plotly_white")
            plotly.offline.plot(fig, filename=str(fpath), auto_open=False,
                                config={"showLink" : False})
        except Exception as err:
            raise Error(f"failed to create the '{pinfo['fname']}' diagram:\n{err}") from err

        pinfo["path"] = str(fpath.relative_to(self.outdir))

    @staticmethod
    def _reduce_df_density(res, xcolname, ycolname):
        """
        This function reduces the density of the dataframe belonging to the test result 'res'. The
        density reduction is only used for the scatter plot. The problem it solves is that that
        there are thousands and thousands of "uninteresting" datapoints per one "interesting"
        datapoint (an outlier). And if the total amount of datapoints is huge (say, 10000000), a web
        browser simply cannot show it because the plot is too large (gigabytes). So what we are
        trying to do is to:
        1. Split the scatter plot on NxN squares, where N is the bins count.
        2. Calculate how many datapoints each square contains (the 2D histogram).
        3. If a square has few datapoints, these are outliers, we leave them alone. "Few" is defined
           by the "low threshold" value.
        4. The for the squares containing many datapoints, we do the reduction. We basically drop
           the datapoints and leave maximum "high threshold" amount of datapoints. And we try to
           scale the amount of datapoints left proportionally to the original value between the
           values of ("low threshold", "high threshold").
        """

        def _map_non_numeric(colname):
            """
            In order to reduce density for a non-numeric column, we need to map that column to
            unique numbers, find datapoints to keep, and then reduce the dataframe. This function
            does exactly that - maps a non-numeric column 'colname' to unique numbers and returns
            the corresponding pandas series object.
            """

            if not res.is_numeric(colname):
                num_rmap = {name : idx for idx, name in enumerate(df[colname].unique())}
                return df[colname].map(num_rmap)

            return df[colname]

        lo_thresh = 20
        hi_thresh = 200
        bins_cnt = 100

        _LOG.info("Reducing density for report ID '%s', diagram '%s-vs-%s'",
                  res.reportid, ycolname, xcolname)

        # Create a new dataframe with just the X- and Y-columns, which we'll be reducing. It should
        # be a bit more optimal than reducing the bigger original dataframe.
        df = res.df[[xcolname, ycolname]]

        xdata = _map_non_numeric(xcolname)
        ydata = _map_non_numeric(ycolname)

        # Crete a histogram for the columns in question.
        hist, xbins, ybins = numpy.histogram2d(xdata, ydata, bins_cnt)
        # Turn the histogram into a dataframe.
        hist = pandas.DataFrame(hist, dtype=int)

        hist_max = hist.max().max()
        if hist_max <= lo_thresh:
            _LOG.debug("cancel density reduction: max frequency for '%s vs %s' is %d, but scaling "
                       "threshold is %d", xcolname, ycolname, hist_max, lo_thresh)
            return res.df

        # The histogram scaling factor.
        factor = hi_thresh / hist_max

        # Scale the histogram down. Do not change the buckets with few datapoints (< lo_thresh),
        # scale down all the other buckets so that they would have maximum 'hi_thresh' datapoints.
        scaling_func = lambda f: max(int(f * factor), lo_thresh) if f > lo_thresh else f
        hist = hist.applymap(scaling_func)

        # Create a copy of the histogram, but populate it with zeroes.
        cur_hist = pandas.DataFrame(0, columns=hist.columns, index=hist.index)

        # Calculate bin indexes for all the X and Y values in the dataframe.
        xindeces = numpy.digitize(xdata, xbins[:-1])
        yindeces = numpy.digitize(ydata, ybins[:-1])

        # This is how many datapoints we are going to have in the reduced dataframe.
        reduced_datapoints_cnt = hist.values.sum()
        _LOG.debug("reduced datapoints count is %d", reduced_datapoints_cnt)

        # Here we'll store 'df' indexes of the rows that will be included into the resulting
        # reduced dataframe.
        copy_cols = []

        for idx in range(0, len(df)):
            xidx = xindeces[idx] - 1
            yidx = yindeces[idx] - 1

            if cur_hist.at[xidx, yidx] >= hist.at[xidx, yidx]:
                continue

            cur_hist.at[xidx, yidx] += 1
            copy_cols.append(idx)

        # Include all the colums in reduced version of the dataframe.
        return res.df.loc[copy_cols]

    def _generate_scatter_plots(self):
        """Generate the scatter plots."""

        plot_axes = [(x, y) for x, y in itertools.product(self.xaxes, self.yaxes) if x != y]

        if self.exclude_xaxes and self.exclude_yaxes:
            x_axes = self._refres.find_colnames([self.exclude_xaxes])
            y_axes = self._refres.find_colnames([self.exclude_yaxes])
            exclude_axes = list(itertools.product(x_axes, y_axes))
            plot_axes = [axes for axes in plot_axes if axes not in exclude_axes]

        for xcolname, ycolname in plot_axes:
            _LOG.info("Generating scatter plot: %s vs %s.", xcolname, ycolname)

            pinfo = self._add_pinfo(xcolname, ycolname, is_hist=False)
            markers = itertools.cycle(_SCATTERPLOT_MARKERS)
            gobjs = []

            for res in self.rsts:
                df = self._reduce_df_density(res, xcolname, ycolname)

                # How many datapoints were included into the scatter plot.
                pinfo["sp_datapoints_cnt"] = len(df.index)

                text = self._create_hover_text(res, df, pinfo)

                # Non-numeric columns will have only few unique values, e.g. 'ReqState' might have
                # "C1", "C1E" and "C6". Using dotted markers for such data will have 3 thin lines
                # which is hard to see. Improve it by using line markers to turn lines into wider
                # "bars".
                if all((res.is_numeric(xcolname), res.is_numeric(ycolname))):
                    marker_size = 4
                    marker_symbol = next(markers)
                else:
                    marker_size = 30
                    marker_symbol = "line-ns"

                marker = {"size" : marker_size, "symbol" : marker_symbol, "opacity" : self._opacity}
                try:
                    gobj = plotly.graph_objs.Scattergl(x=self._base_unit(df, xcolname),
                                                       y=self._base_unit(df, ycolname),
                                                       opacity=self._opacity,
                                                       text=text, mode="markers",
                                                       name=res.reportid, marker=marker)
                except Exception as err:
                    raise Error(f"failed to create scatter plot '{ycolname}-vs-{xcolname}':\n"
                                f"{err}") from err
                gobjs.append(gobj)

            self._create_diagram(gobjs, pinfo)

    def _generate_histograms(self):
        """Generate the scatter plots."""

        def get_xbins(xcolname):
            """Returns the 'xbins' dictinary for plotly's 'Histrogram()' method."""

            xmin, xmax = (float("inf"), -float("inf"))
            for res in self.rsts:
                # In case of non-numeric column there is only one x-value per bin.
                if not res.is_numeric(xcolname):
                    return {"size" : 1}

                xdata = self._base_unit(res.df, xcolname)
                xmin = min(xmin, xdata.min())
                xmax = max(xmax, xdata.max())

            return {"size" : (xmax - xmin) / 1000}

        xcolnames = Trivial.list_dedup(self.hist + self.chist)
        hist_set = set(self.hist)
        chist_set = set(self.chist)

        for xcolname in xcolnames:
            if xcolname in hist_set:
                ycolname = "Count"
                pinfo = self._add_pinfo(xcolname, ycolname, is_hist=True)
                _LOG.info("Generating histogram: %s vs %s.", xcolname, ycolname)
                gobjs = []
                xbins = get_xbins(xcolname)
                for res in self.rsts:
                    xdata = self._base_unit(res.df, xcolname)
                    try:
                        gobj = plotly.graph_objs.Histogram(x=xdata, name=res.reportid, xbins=xbins,
                                                           opacity=self._opacity)
                    except Exception as err:
                        raise Error(f"failed to create histogram '{ycolname}-vs-{xcolname}':\n"
                                    f"{err}") from err
                    gobjs.append(gobj)

                self._create_diagram(gobjs, pinfo)

            if xcolname in chist_set:
                ycolname = "Percentile"
                _LOG.info("Generating cumulative histogram: %s vs %s.", xcolname, ycolname)
                pinfo = self._add_pinfo(xcolname, ycolname, is_hist=True)
                gobjs = []
                if xcolname not in hist_set:
                    xbins = get_xbins(xcolname)
                for res in self.rsts:
                    xdata = self._base_unit(res.df, xcolname)
                    try:
                        gobj = plotly.graph_objs.Histogram(x=xdata, name=res.reportid, xbins=xbins,
                                                           cumulative=dict(enabled=True),
                                                           histnorm="percent",
                                                           opacity=self._opacity)
                    except Exception as err:
                        raise Error(f"failed to create cumulative histogram "
                                    f"'{ycolname}-vs-{xcolname}':\n{err}") from err
                    gobjs.append(gobj)

                self._create_diagram(gobjs, pinfo)

    def _calc_smrys(self):
        """Calculate summaries that we are going to show in the summary table."""

        for res in self.rsts:
            _LOG.debug("calculate summary functions for '%s'", res.reportid)
            res.calc_smrys(regexs=self._smry_colnames, funcnames=self._smry_funcs)

    def _mangle_loaded_res(self, res): # pylint: disable=no-self-use, unused-argument
        """
        This method is called for every dataframe corresponding to the just loaded CSV file. The
        subclass can override this method to mangle the dataframe.
        """

        for colname in res.df:
            defs = res.defs.info.get(colname)
            if not defs:
                continue

            # Some columns should be dropped if they are "empty", i.e., contain only zero values.
            # For example, the C-state residency columns may be empty. This usually means that the
            # C-state was either disabled or just does not exist.
            if defs.get("drop_empty") and not res.df[colname].any():
                _LOG.debug("dropping empty column '%s'", colname)
                res.df.drop(colname, axis="columns", inplace=True)

        # Update columns lists in case some of the columns were removed from the loaded dataframe.
        for name in ("_smry_colnames", "xaxes", "yaxes", "hist", "chist"):
            colnames = []
            for colname in getattr(self, name):
                if colname in res.df:
                    colnames.append(colname)
            setattr(self, name, colnames)

        for name in ("_hov_colnames", ):
            colnames = []
            val = getattr(self, name)
            for colname in val[res.reportid]:
                if colname in res.df:
                    colnames.append(colname)
            val[res.reportid] = colnames
        return res.df

    def _load_results(self):
        """Load the test results from the CSV file and/or apply the columns selector."""

        _LOG.debug("summaries will be calculated for these columns: %s",
                   ", ".join(self._smry_colnames))
        _LOG.debug("additional colnames: %s", ", ".join(self._more_colnames))

        for res in self.rsts:
            _LOG.debug("hover colnames: %s", ", ".join(self._hov_colnames[res.reportid]))

            colnames = []
            for colname in self._hov_colnames[res.reportid] + self._more_colnames:
                if colname in res.colnames_set:
                    colnames.append(colname)

            csel = Trivial.list_dedup(self._smry_colnames + colnames)
            res.set_csel(csel)
            res.load_df()

            # We'll be dropping columns and adding temporary columns, so we'll affect the original
            # dataframe. This is more effecient than creating copies.
            self._mangle_loaded_res(res)

        # Some columns from the axes lists could have been dropped, updata the lists.
        self._drop_absent_colnames()

    def generate(self):
        """Generate the HTML report and store the result in 'self.outdir'.

        Important note: this method will modify the input test results in 'self.rsts'. This is done
        for effeciency purposes, to avoid copying the potentially large amounts of data (pandas
        dataframes).
        """

        if self.relocatable not in ("copy", "noraw", "symlink"):
            raise Error("bad 'relocatable' value, use one of: copy, noraw, symlink")

        # Load the required datapoints into memory.
        self._load_results()

        # Calculate the summaries for the datapoints, like min. and max. values.
        self._calc_smrys()

        # Generate the plots.
        self._generate_scatter_plots()
        self._generate_histograms()

        # Put together the final HTML report.
        self._generate_report()

    def set_hover_colnames(self, regexs):
        """
        This methods allows for specifying CSV file column names that have to be included to the
        hover test on the scatter plot. The 'regexs' argument should be a list of hover text colum
        name regular expressions. In other words, each element of the list will be treated as a
        regular expression. Every CSV colum name will be matched against this regular expression,
        and matched column names will be added to the hover text.
        """

        for res in self.rsts:
            self._hov_colnames[res.reportid] = res.find_colnames(regexs, must_find_any=False)

    def _drop_absent_colnames(self):
        """
        Verify that test results provide the columns in 'xaxes', 'yaxes', 'hist' and 'chist'. Drop
        the absent columns. Also drop uknown columns (those not present in the "definitions").
        """

        lists = ("xaxes", "yaxes", "hist", "chist")

        for name in lists:
            intersection = set(getattr(self, name))
            for res in self.rsts:
                intersection = intersection & res.colnames_set
            colnames = []
            for colname in getattr(self, name):
                if colname in intersection:
                    colnames.append(colname)
                else:
                    _LOG.warning("dropping column '%s' from '%s' because it is not present in one "
                                 "of the results", colname, name)
            setattr(self, name, colnames)

        for name in lists:
            for res in self.rsts:
                colnames = []
                for colname in getattr(self, name):
                    if colname in res.defs.info:
                        colnames.append(colname)
                    else:
                        _LOG.warning("dropping column '%s' from '%s' because it is not present in "
                                     "the definitions file at '%s'", colname, name, res.defs.path)
            setattr(self, name, colnames)

    def _init_colnames(self):
        """
        Assign default values to the diagram/histogram column names and remove possible
        duplication in user-provided input.
        """

        for name in ("xaxes", "yaxes", "hist", "chist"):
            if getattr(self, name):
                # Convert list of regular expressions into list of names.
                colnames = self._refres.find_colnames(getattr(self, name))
            else:
                colnames = []
            setattr(self, name, colnames)

        self._drop_absent_colnames()

        # Both X- and Y-axes are required for scatter plots.
        if not self.xaxes or not self.yaxes:
            self.xaxes = self.yaxes = []

        # Ensure '_hov_colnames' dictionary is initialized.
        self.set_hover_colnames(())

    def _validate_init_args(self):
        """Validate the class constructor input arguments."""

        if self.outdir.exists() and not self.outdir.is_dir():
            raise Error(f"path '{self.outdir}' already exists and it is not a directory")

        # Ensure that results are compatible.
        rname, rver = self._refinfo["toolname"], self._refinfo["toolver"]
        for res in self.rsts:
            name, ver = res.info["toolname"], res.info["toolver"]
            if name != rname:
                raise Error(f"the following test results are not compatible:\n"
                            f"1. {self._refres.dirpath}: created by '{rname}'\n"
                            f"2. {res.dirpath}: created by '{name}'\n"
                            f"Cannot put incompatible results to the same report")
            if ver != rver:
                _LOG.warning("the following test results may be not compatible:\n"
                             "1. %s: created by '%s' version '%s'\n"
                             "2. %s: created by '%s' version '%s'",
                             self._refres.dirpath, rname, rver, res.dirpath, name, ver)

        # Ensure the report IDs are unique.
        reportids = set()
        for res in self.rsts:
            reportid = res.reportid
            if reportid in reportids:
                # Try to construct a unique report ID.
                for idx in range(1, 20):
                    new_reportid = f"{reportid}-{idx:02}"
                    if new_reportid not in reportids:
                        _LOG.warning("duplicate reportid '%s', using '%s' instead",
                                     reportid, new_reportid)
                        res.reportid = new_reportid
                        break
                else:
                    raise Error(f"too many duplicate report IDs, e.g., '{reportid}' is problematic")

            reportids.add(res.reportid)

        if self.title_descr and Path(self.title_descr).is_file():
            try:
                with open(self.title_descr, "r") as fobj:
                    self.title_descr = fobj.read()
            except OSError as err:
                raise Error(f"failed to read the report description file {self.title_descr}:\n"
                            f"{err}") from err

        for res in self.rsts:
            if res.dirpath.resolve() == self.outdir.resolve():
                # Don't create report in results directory, use 'html-report' subdirectory instead.
                self.outdir = self.outdir.joinpath("html-report")

    def __init__(self, rsts, outdir, title_descr=None, xaxes=None, yaxes=None, hist=None,
                 chist=None, exclude_xaxes=None, exclude_yaxes=None):
        """
        The class constructor. The arguments are as follows.
          * rsts - list of 'RORawResult' objects representing the raw test results to generate the
                   HTML report for.
          * outdir - the output directory path to store the HTML report at.
          * title_descr - a string describing this report or a file path containing the description.
          *               The description will be put at the top part of the HTML report. It should
          *               describe the report in general (e.g., it compares platform A to platform
          *               B). By default no title description is added to the HTML report.
          * xaxes - list of regular expressions matching datapoints CSV file column names to use for
                    the X axis of scatter plot diagrams. A scatter plot will be generated for each
                    combination of 'xaxes' and 'yaxes' column name pair (except for the pairs in
                    'exclude_xaxes'/'exclude_yaxes'). Default is the first column in the datapoints
                    CSV file.
          * yaxes - list of regular expressions matching datapoints CSV file column names to use for
                    the Y axis of scatter plot diagrams. Default is the second column in the
                    datapoints CSV file.
          * hist - list of regular expressions matching datapoints CSV file column names to create a
                   histogram for. Default is the first column in the datapoints CSV file. An empty
                   string can be used to disable histograms.
          * chist - list of regular expressions matching datapoints CSV file column names to create
                    a cumulative histogram for. Default is he first column in the datapoints CSV
                    file. An empty string can be used to disable cumulative histograms.
          * exclude_xaxes - by default all diagrams of X- vs Y-axes combinations will be created.
                            The 'exclude_xaxes' is a list regular expressions matching datapoints
                            CSV file column names. There will be no scatter plot for each
                            combinations of 'exclude_xaxes' and 'exclude_yaxes'. In other words,
                            this argument along with 'exclude_yaxes' allows for excluding some
                            diagrams from the 'xaxes' and 'yaxes' combinations.
          * exclude_yaxes - same as 'exclude_xaxes', but for Y-axes.
        """

        self.rsts = rsts
        self.outdir = Path(outdir)
        self.title_descr = title_descr
        self.xaxes = xaxes
        self.yaxes = yaxes
        self.hist = hist
        self.chist = chist
        self.exclude_xaxes = exclude_xaxes
        self.exclude_yaxes = exclude_yaxes

        if (self.exclude_xaxes and not self.exclude_yaxes) or \
           (self.exclude_yaxes and not self.exclude_xaxes):
            raise Error("'exclude_xaxes' and 'exclude_yaxes' must both be 'None' or both not be "
                        "'None'")

        # Users can change this to 'copy' or 'noraw' to make the reports relocatable. In the former
        # case the raw results will be copied from the test result directories to the output
        # directory. In the latter case, the raw results won't be referenced at all, neither from
        # the HTML report, nor at the file-system level. The logs, statistics and other stuff from
        # the raw result will also be excluded.
        self.relocatable = "symlink"

        # The first result is the 'reference' result.
        self._refres = rsts[0]
        # The reference definitions - it contains helpful information about every CSV file column,
        # for example the title, units, and so on.
        self._refdefs = self._refres.defs
        # The raw reference result information.
        self._refinfo = self._refres.info

        # Names of columns in the datapoints CSV file to provide the summary function values for
        # (e.g., median, 99th percentile). The summaries will show up in the summary tables (one
        # table per metric).
        self._smry_colnames = None
        # List of functions to provide in the summary tables.
        self._smry_funcs = ("nzcnt", "max", "99.999%", "99.99%", "99.9%", "99%", "med", "avg",
                            "min", "std")
        # The diagram/histogram transparency level. It is helpful to have some transparency in case
        # there are several test results rendered on the same diagram.
        self._opacity = 0.8 if len(self.rsts) > 1 else 1
        # Plot information dictionaries. This is a dictionary of of lists, each list containing
        # sub-dictionaries describing a single plot. The lists of sub-dictionaries are grouped by
        # the X" and "Y" axis column names, because later plots with the same "Y" and "Y" axes will
        # go to the same HTML page.
        self._pinfos = {}
        # Per-test result list of column names to include into the hover text of the scatter plot.
        # By default only the x and y axis values are included.
        self._hov_colnames = {}
        # Additional columns to load, if they exist in the CSV file.
        self._more_colnames = []

        self._init_colnames()

        # We'll provide summaries for every column participating in at least one diagram.
        smry_colnames = Trivial.list_dedup(self.yaxes + self.xaxes + self.hist + self.chist)
        # Summary functions table includes all test results, but the results may have a bit
        # different set of column names (e.g., they were collected with different wult versions # or
        # using different methods, or on different systems). Therefore, include only common columns
        # into it.
        self._smry_colnames = []
        for colname in smry_colnames:
            for res in rsts:
                if colname not in res.colnames_set:
                    break
            else:
                self._smry_colnames.append(colname)

        self._validate_init_args()


'''
Input: s = "pwwkew"
Output: 3
Explanation: The answer is "wke", with the length of 3.
Notice that the answer must be a substring, "pwke" is a subsequence and not a substring.
'''

import collections

def lengthOfLongestSubstring_1(s: str) -> int:
    if len(s) == 0:
        return 0

    answer_table = collections.defaultdict(int)
    curr_i = 0
    ls = ''

    for i, char in enumerate(s):
        answer_table[i] += 1
        if i == 0:
            ls = char
        elif char in ls:
            # This part is wrong
            curr_i = i
            ls = char
        else:
            answer_table[curr_i] += 1
            ls += char

    return max(answer_table.values())

'''
lengthOfLongestSubstring_1()
s = "dvdf"
Output : 2
Expected : 3
'''

def lengthOfLongestSubstring_2(s: str) -> int:
    if len(s) == 0:
        return 0

    answer_table = collections.defaultdict(int)
    curr_i = 0
    ls = ''

    for i, char in enumerate(s):
        answer_table[i] += 1
        char_idx = ls.find(char)

        if char_idx == -1:
            ls += char
        else:
            # 이게 잘못되었다.
            curr_i = char_idx + 1
            ls = s[curr_i:i+1]
        answer_table[curr_i] = len(ls)

    return max(answer_table.values())

def lengthOfLongestSubstring_3(s: str) -> int:
    ans = {}
    max_length = start = 0

    for i, char in enumerate(s):
        if char in ans:
            start = ans[char] + 1
        else:
            max_length = max(max_length, i - start + 1)

        ans[char] = i

    return max_length

if __name__ == '__main__':
    # test_input_1 = "abcabcbb"
    # print(lengthOfLongestSubstring_2(test_input_1))

    # test_input_2 = "pwwkew"
    # print(lengthOfLongestSubstring_2(test_input_2))

    # test_input_3 = "bbbbb"
    # print(lengthOfLongestSubstring_2(test_input_3))

    test_input_4 = "tmmzuxt"
    print(lengthOfLongestSubstring_3(test_input_4))


import sys
from Bio import SeqIO
import argparse

parser = argparse.ArgumentParser(description="Filter FASTA data")

parser.add_argument(
    "--pattern",
    help="Only print records that contain this pattern.",
)

parser.add_argument(
    "--translate", "-t", action="store_true",
    help="Translate the sequences to AA.",
)

parser.add_argument(
    "--ignoreDuplicates", action="store_true",
    help="Ignore sequences with duplicate ids.",
)

parser.add_argument(
    "--maxLength", type=int,
    help="Only show sequences of length less than or equal to this length.",
)

args = parser.parse_args()

# print("Translate:", args.translate)
# print("Pattern:", args.pattern)
# print("Max length:", args.maxLength)

ids = set()

# Use BioPython to read the input FASTA file.

for record in SeqIO.parse(sys.stdin, "fasta"):
    seqId = record.id

    if seqId in ids:
        if args.ignoreDuplicates:
            continue

    ids.add(seqId)

    if args.translate:
        seq = record.seq.translate()
    else:
        seq = record.seq

    if args.maxLength is not None:
        if len(seq) > args.maxLength:
            continue

    if args.pattern is None:
        print(f">{seqId}\n{str(seq)}")
    else:
        if args.pattern in seq:
            print(f">{seqId}\n{str(seq)}")


# coding=utf-8
import traceback
from flask import current_app
from . import db, ContestSeries, Resource
from datetime import date

ContestLevel = [
    u'省级(赛区)',
    u'国家级'
]

ContestType = [
    u'学科竞赛',
    u'体育竞赛',
    u'创新创业竞赛',
    u'其它'
]

ContestResult = [
    u'未定档',
    u'一档',
    u'二档',
    u'三档',
    u'四档'
]

class Contest(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    contest_id = db.Column(db.String(128), unique=True,
                           nullable=False, index=True)
    name_cn = db.Column(db.Unicode(512))
    name_en = db.Column(db.String(512))
    level = db.Column(db.Unicode(32))   # 竞赛等级
    result = db.Column(db.Unicode(32))  # 竞赛审核结果
    type = db.Column(db.Unicode(32))    # 竞赛类型
    department = db.Column(db.Unicode(128))
    organizer = db.Column(db.Unicode(512))
    co_organizer = db.Column(db.Unicode(512))
    year = db.Column(db.Integer)
    start_date = db.Column(db.Date)
    end_date = db.Column(db.Date)
    place = db.Column(db.Unicode(1024))
    site = db.Column(db.String(1024))
    budget = db.Column(db.Float)
    budget_text = db.Column(db.Unicode(2048))
    student_num = db.Column(db.Integer)
    teacher_num = db.Column(db.Integer)
    subject = db.Column(db.Unicode(512))

    series_name = db.Column(db.Unicode(256),
                            db.ForeignKey('contest_series.name', ondelete="CASCADE"),
                            nullable=True)
    series = db.relationship('ContestSeries',
                             backref=db.backref('contests',
                                                cascade="all, delete-orphan",
                                                passive_deletes=True,
                                                lazy='dynamic'))

    def __repr__(self):
        return '<Contest %s %s>' % (self.contest_id, self.name_cn)

    @property
    def is_pass(self):
        return self.result and self.result != ContestResult[0]

    def save(self):
        db.session.add(self)
        db.session.commit()

    def remove(self):
        db.session.delete(self)
        db.session.commit()

def generate_next_contest_id(year):
    ''' generate next contest id for the new contest's year '''
    last_contest = Contest.query.filter(Contest.year == year)\
        .order_by(Contest.contest_id.desc())\
        .with_lockmode('update').first()
    if last_contest == None:
        return 'W%d%s' % (year, '001')
    else:
        last_contest_id = int(last_contest.contest_id[6:])
        new_id = str(last_contest_id + 1)
        return 'W%d%s' % (year, new_id.rjust(3, '0'))

def get_by_id(id):
    return Contest.query.filter(Contest.id == id).first()

def get_count(filter_pass=0, department=None):
    query = Contest.query
    if filter_pass == 1:
        query = query.filter(Contest.result == ContestResult[0])
    elif filter_pass == -1:
        query = query.filter(Contest.result != ContestResult[0])
    if department:
        query = query.filter(Contest.department == department)
    return query.count()

def get_list_pageable(page=1, per_page=20, filter_pass=0, department=None, year=None):
    query = Contest.query
    if filter_pass == 1:
        query = query.filter(Contest.result == ContestResult[0])
    elif filter_pass == -1:
        query = query.filter(Contest.result != ContestResult[0])
    if department:
        query = query.filter(Contest.department == department)
    if year:
        query = query.filter(Contest.year == year)
    if page == -1:
        return query.order_by(Contest.id).all()
    return query.order_by(Contest.id)\
        .paginate(page, per_page, error_out=False)

def create_contest(contest_form, request):
    try:
        contest = Contest()
        contest.contest_id = generate_next_contest_id(contest_form.year.data)
        contest.name_cn = contest_form.name_cn.data
        contest.name_en = contest_form.name_en.data
        contest.level = contest_form.level.data
        contest.result = ContestResult[0]
        contest.type = contest_form.type.data
        contest.department = contest_form.department.data
        contest.site = contest_form.site.data
        contest.organizer = contest_form.organizer.data
        contest.co_organizer = contest_form.co_organizer.data
        contest.year = contest_form.year.data
        contest.start_date = contest_form.date_range.data[0]
        contest.end_date = contest_form.date_range.data[1]
        contest.budget = contest_form.budget.data
        contest.budget_text = contest_form.budget_text.data
        contest.student_num = contest_form.student_num.data
        contest.teacher_num = contest_form.teacher_num.data
        contest.subject = contest_form.subject.data
        city = request.form.get('prov', '') + \
               request.form.get('city', '') + \
               request.form.get('dist', '')
        contest.place = city + contest_form.place.data
        series = ContestSeries.get_by_id(contest_form.series_id.data)
        if series:
            contest.series = series
        contest.save()
        current_app.logger.info(u'录入竞赛 %s 成功', contest.name_cn)
        return 'OK'
    except Exception:
        current_app.logger.error(u'录入竞赛 %s 失败', contest_form.name_cn.data)
        current_app.logger.error(traceback.format_exc())
        return 'FAIL'

def update_contest(contest, contest_form, request):
    try:
        contest.name_cn = contest_form.name_cn.data
        contest.name_en = contest_form.name_en.data
        contest.level = contest_form.level.data
        contest.type = contest_form.type.data
        contest.department = contest_form.department.data
        contest.site = contest_form.site.data
        contest.organizer = contest_form.organizer.data
        contest.co_organizer = contest_form.co_organizer.data
        contest.year = contest_form.year.data
        contest.start_date = contest_form.date_range.data[0]
        contest.end_date = contest_form.date_range.data[1]
        contest.budget = contest_form.budget.data
        contest.budget_text = contest_form.budget_text.data
        contest.student_num = contest_form.student_num.data
        contest.teacher_num = contest_form.teacher_num.data
        contest.subject = contest_form.subject.data
        city = request.form.get('prov', '') + \
               request.form.get('city', '') + \
               request.form.get('dist', '')
        contest.place = city + contest_form.place.data
        series = ContestSeries.get_by_id(contest_form.series_id.data)
        if series:
            contest.series = series
        contest.save()
        current_app.logger.info(u'更新竞赛 %s 成功', contest.name_cn)
        return 'OK'
    except Exception:
        current_app.logger.error(u'更新竞赛 %s 失败', contest_form.name_cn.data)
        current_app.logger.error(traceback.format_exc())
        return 'FAIL'

def delete_contest(contest):
    try:
        for awards in contest.awards.all():
            for res in awards.resources.all():
                Resource.delete_res(res)
        contest.remove()
        current_app.logger.info(u'删除竞赛成功')
        return 'OK'
    except Exception:
        current_app.logger.error(u'删除竞赛失败')
        current_app.logger.error(traceback.format_exc())
        return 'FAIL'


from sqlalchemy.schema import MetaData
import sqlalchemy as sa
import pandas as pd

engine = sa.create_engine("sqlite:///sqlalchemyCFTCDATA.sqlite")
df = pd.read_sql("EUR", engine)
df.Date = pd.to_datetime(df.Date)
meta = MetaData()
meta.reflect(bind=engine)

columnOptions, tableOptions = [], []

tables = meta.tables.keys()

for table in tables:
    tableOptions.append(table)
for col in df.head():
    columnOptions.append(col)

pie_options = [
    "All",
    "Noncommercial",
    "Commercial",
    "Nonreportable Positions",
]

columnOptions.remove("Date")


import pytest
from fparser.two.Fortran2003 import Program, Comment, Subroutine_Subprogram
from fparser.two.utils import walk
from fparser.api import get_reader

from fparser.two.parser import ParserFactory

# this is required to setup the fortran2003 classes
_ = ParserFactory().create(std="f2003")

def test_ignore_comments():
    """Check that the parser does throw away comments when requested"""
    tree = Program(
        get_reader(
            """\
PROGRAM a_prog
! A full line comment
PRINT *, "Hello" ! This block gets executed
END PROGRAM a_prog
    """,
            ignore_comments=True,
        )
    )
    gen = str(tree)
    assert "PROGRAM a_prog\n" '  PRINT *, "Hello"\n' "END PROGRAM a_prog" in gen
    assert "full line comment" not in gen
    assert "block gets executed" not in gen

    # Check that the default behaviour is to ignore comments
    tree = Program(
        get_reader(
            """\
PROGRAM a_prog
! A full line comment
PRINT *, "Hello" ! This block gets executed
END PROGRAM a_prog
    """
        )
    )
    gen = str(tree)
    assert "full line comment" not in gen
    assert "block gets executed" not in gen

def test_simple_prog():
    """Tests simplest case of comments in a program unit"""
    tree = Program(
        get_reader(
            """\
PROGRAM a_prog
! A full line comment
PRINT *, "Hello" ! This block gets executed
END PROGRAM a_prog
    """,
            ignore_comments=False,
        )
    )
    assert (
        str(tree) == "PROGRAM a_prog\n"
        "  ! A full line comment\n"
        '  PRINT *, "Hello"\n'
        "  ! This block gets executed\n"
        "END PROGRAM a_prog\n"
    )

def test_ifthen():
    """Tests for comments within an if-then block"""
    cls = Program
    tree = cls(
        get_reader(
            """\
PROGRAM a_prog
IF(.TRUE.)THEN
! A full line comment
PRINT *, "Hello"
! Another full line comment
END IF
END PROGRAM a_prog
    """,
            ignore_comments=False,
        )
    )
    assert (
        str(tree) == "PROGRAM a_prog\n"
        "  IF (.TRUE.) THEN\n"
        "    ! A full line comment\n"
        '    PRINT *, "Hello"\n'
        "    ! Another full line comment\n"
        "  END IF\n"
        "END PROGRAM a_prog\n"
    )

def test_inline_ifthen():
    """Tests for in-line comments within an if-then block"""
    tree = Program(
        get_reader(
            """\
PROGRAM a_prog
IF(.TRUE.)THEN
PRINT *, "Hello" ! An in-line comment here
END IF
! A comment after a block
END PROGRAM a_prog
    """,
            ignore_comments=False,
        )
    )
    assert isinstance(tree, Program)
    assert (
        str(tree) == "PROGRAM a_prog\n"
        "  IF (.TRUE.) THEN\n"
        '    PRINT *, "Hello"\n'
        "    ! An in-line comment here\n"
        "  END IF\n"
        "  ! A comment after a block\n"
        "END PROGRAM a_prog\n"
    )

@pytest.mark.xfail(reason="fails to preserve formatting in fixed format")
def test_fixed_fmt():
    """Test that we handle comments in fixed-format mode"""
    reader = get_reader(
        """\
      subroutine foo
c this is a subroutine
        end subroutine foo""",
        isfree=False,
        ignore_comments=True,
    )
    cls = Subroutine_Subprogram
    obj = cls(reader)
    assert isinstance(obj, cls), repr(obj)
    assert str(obj) == "SUBROUTINE foo\nEND SUBROUTINE foo"
    assert (
        repr(obj) == "Subroutine_Subprogram(Subroutine_Stmt(None, "
        "Name('foo'), None, None), End_Subroutine_Stmt('SUBROUTINE', "
        "Name('foo')))"
    )
    reader = get_reader(
        """\
      subroutine foo
c this is a subroutine
      end subroutine foo""",
        isfree=False,
        ignore_comments=False,
    )
    cls = Subroutine_Subprogram
    obj = cls(reader)
    fort = obj.tofortran(isfix=True)
    assert isinstance(obj, cls), repr(obj)
    assert (
        fort == "      SUBROUTINE foo\nc this is a subroutine\n"
        "      END SUBROUTINE foo"
    )
    assert (
        repr(obj) == "Subroutine_Subprogram(Subroutine_Stmt(None, "
        "Name('foo'), None, None), End_Subroutine_Stmt('SUBROUTINE', "
        "Name('foo')))"
    )

@pytest.mark.xfail(reason="fails to preserve formatting in fixed format")
def test_fixed_continuation():
    """Check that we handle comments that occur within fixed-format
    continued lines"""
    reader = get_reader(
        """\
      subroutine foo
c this is a subroutine
      i = 1 +
c this comment is a problem
     &2 + 3
      end subroutine foo""",
        isfree=False,
        ignore_comments=False,
    )
    cls = Subroutine_Subprogram
    obj = cls(reader)
    assert isinstance(obj, cls)
    assert "i = 1 + 2 + 3\n     c this comment is a problem" in str(obj)

def test_prog_comments():
    """Unit tests for lines in programs containing comments"""
    cls = Program
    reader = get_reader(
        """\
   ! A troublesome comment
   program foo
     ! A full comment line
     write(*,*) my_int ! An in-line comment
    end program foo
! A really problematic comment
""",
        isfree=True,
        ignore_comments=False,
    )

    obj = cls(reader)
    assert type(obj) == Program
    # Check that the AST has the expected structure:
    # Program
    #   |--> Comment
    #   |--> Main_Program
    #   .    |--> Program_Stmt
    #   .    |--> Specification_Part
    #   .    .    \--> Implicit_Part
    #   .    .         \--> Comment
    #        |--> Execution_Part
    #        |    |--> Write_Stmt
    #        |    \--> Comment
    #   .    .
    #   .
    #   |--> Comment
    from fparser.two.Fortran2003 import Main_Program, Write_Stmt, End_Program_Stmt

    walk(obj.children, Comment, debug=True)
    assert type(obj.content[0]) == Comment
    assert str(obj.content[0]) == "! A troublesome comment"
    assert type(obj.content[1]) == Main_Program
    main_prog = obj.content[1]
    assert type(main_prog.content[1].content[0].content[0]) == Comment
    assert str(main_prog.content[1].content[0].content[0]) == "! A full comment line"
    exec_part = main_prog.content[2]
    assert type(exec_part.content[0]) == Write_Stmt
    # Check that we have the in-line comment as a second statement
    assert len(exec_part.content) == 2
    assert type(exec_part.content[1]) == Comment
    assert type(main_prog.content[3]) == End_Program_Stmt
    assert "! An in-line comment" in str(obj)
    # Check that we still have the ending comment
    assert type(obj.content[-1]) == Comment
    assert str(obj).endswith("! A really problematic comment")

def test_module_comments():
    """Tests for comments in modules"""
    source = """! This is a module
      module my_mod
        implicit none
        private
      end module my_mod
"""
    # Test when the reader is explicitly set to free-form mode
    reader = get_reader(source, isfree=True, ignore_comments=False)
    prog_unit = Program(reader)
    assert type(prog_unit.content[0]) == Comment
    assert str(prog_unit.content[0]) == "! This is a module"

def test_function_comments():
    """Tests for comments in functions"""
    source = """\
function my_mod()
  ! This is a function
  implicit none
  integer my_mod
  ! Comment1
  my_mod = 1 ! Comment2
  ! That was a function
end function my_mod
"""
    from fparser.two.Fortran2003 import Function_Subprogram

    reader = get_reader(source, isfree=True, ignore_comments=False)
    fn_unit = Function_Subprogram(reader)
    # <class 'fparser.two.Fortran2003.Function_Stmt'>
    #    <type 'NoneType'>
    #    <class 'fparser.two.Fortran2003.Name'>
    #    <type 'NoneType'>
    #    <type 'NoneType'>
    # <class 'fparser.two.Fortran2003.Specification_Part'>
    #   <class 'fparser.two.Fortran2003.Implicit_Part'>
    #     <class 'fparser.two.Fortran2003.Comment'>
    #       <type 'str'>, "'! This is a function'"
    comment = fn_unit.content[1].content[0].content[0]
    assert isinstance(comment, Comment)
    assert "! This is a function" in str(comment)
    comment = fn_unit.content[1].content[2].content[0]
    assert isinstance(comment, Comment)
    assert "! Comment1" in str(comment)
    exec_part = fn_unit.content[2]
    comment = exec_part.content[1]
    assert isinstance(comment, Comment)
    assert "! Comment2" in str(comment)
    comment = exec_part.content[2]
    assert isinstance(comment, Comment)
    assert "! That was a function" in str(comment)

def test_subroutine_comments():
    """Tests for comments in subroutines"""
    source = """\
subroutine my_mod()
  ! First comment
  implicit none
  integer my_mod
  ! Body comment
  my_mod = 1 ! Inline comment
  ! Ending comment
end subroutine my_mod
"""
    reader = get_reader(source, isfree=True, ignore_comments=False)
    fn_unit = Subroutine_Subprogram(reader)
    assert isinstance(fn_unit, Subroutine_Subprogram)
    walk(fn_unit.children, Comment, debug=True)
    spec_part = fn_unit.content[1]
    comment = spec_part.content[0].content[0]
    assert isinstance(comment, Comment)
    assert "! First comment" in str(comment)
    comment = spec_part.content[2].content[0]
    assert isinstance(comment, Comment)
    assert comment.parent is spec_part.content[2]
    assert "! Body comment" in str(comment)
    exec_part = fn_unit.content[2]
    comment = exec_part.content[1]
    assert isinstance(comment, Comment)
    assert comment.parent is exec_part
    assert "! Inline comment" in str(comment)

def test_derived_type():
    """Test for comments within declaration of derived type"""
    source = """\
type my_type ! Inline comment1
  ! First comment
  integer :: start ! Inline comment2
  integer :: stop
  ! Ending comment
end type my_type
"""
    from fparser.two.Fortran2003 import Derived_Type_Def

    reader = get_reader(source, isfree=True, ignore_comments=False)
    dtype = Derived_Type_Def(reader)
    assert isinstance(dtype, Derived_Type_Def)
    comments = [
        (1, "! Inline comment1"),
        (2, "! First comment"),
        (4, "! Inline comment2"),
        (6, "! Ending comment"),
    ]
    for comment in comments:
        idx = comment[0]
        assert isinstance(dtype.content[idx], Comment)
        assert comment[1] in str(dtype.content[idx])

def test_action_stmts():
    """Tests for comments within action statements such as allocate"""
    # We have to have our allocate() within some other block otherwise
    # we lose the comment (it is deferred until after the current,
    # continuted line is complete but once that happens, the match for
    # the allocate() is done and the reader/parser returns without
    # ever dealing with the comment).
    source = """\
if(.true.)then
allocate(my_array(size), &
         ! that's a big array
         my_array2(size))
end if
"""
    from fparser.two.Fortran2003 import If_Construct, Allocate_Stmt
    from fparser.two.utils import get_child

    reader = get_reader(source, isfree=True, ignore_comments=False)
    ifstmt = If_Construct(reader)
    assert isinstance(ifstmt, If_Construct)
    assert isinstance(ifstmt.content[1], Allocate_Stmt)
    assert "a big array" in str(ifstmt)
    cmt = get_child(ifstmt, Comment)
    assert cmt.parent is ifstmt


import logging
from collections import defaultdict
from typing import Dict, List, Tuple

from glom import SKIP, GlomError, glom
from rest_framework import fields, serializers

from objects.token.constants import PermissionModes

logger = logging.getLogger(__name__)

ALL_FIELDS = ["*"]

def build_spec(fields, ui=False) -> dict:
    spec = {}
    for spec_field in fields:
        build_spec_field(spec, name=spec_field, value=spec_field, ui=ui)
    return spec

def build_spec_field(spec, name, value, ui):
    if "__" in name:
        parent, field_name = name.split("__", 1)
        spec[parent] = spec.get(parent, {})
        build_spec_field(spec[parent], field_name, value, ui)
    else:
        # SKIP data attributes which are not required
        spec_val = (
            value.replace("__", ".")
            if not value.startswith("record__data__")
            else lambda x: glom(x, value.replace("__", "."), default=SKIP)
        )
        spec[name] = value if ui else spec_val

def get_field_names(data: Dict[str, fields.Field]) -> List[str]:
    """return list of names for all serializer fields. Supports nesting"""
    names_and_sources = get_field_names_and_sources(data)
    return [name for name, source in names_and_sources]

def get_field_names_and_sources(data: Dict[str, fields.Field]) -> List[Tuple[str, str]]:
    """return list of (name, source) for all serializer fields. Supports nesting"""
    names_and_sources = []
    for key, value in data.items():
        if isinstance(value, dict):
            names_and_sources += [
                (f"{key}__{name}", source.replace(".", "__"))
                for name, source in get_field_names_and_sources(value)
            ]
        elif isinstance(value, serializers.Serializer):
            names_and_sources += [
                (f"{key}__{name}", source.replace(".", "__"))
                for name, source in get_field_names_and_sources(value.fields)
            ]
        elif isinstance(value, fields.Field):
            names_and_sources.append((key, value.source.replace(".", "__")))
        else:
            names_and_sources.append((key, key))

    return names_and_sources

class NotAllowedDict(defaultdict):
    def pretty(self):
        if len(self.keys()) == 0:
            return ""
        if len(self.keys()) == 1:
            value = list(self.values())[0]
            return ",".join(value)
        return "; ".join([f"{key}={','.join(value)}" for (key, value) in self.items()])

class DynamicFieldsMixin:
    """
    this mixin allows selecting fields for serializer in the query param
    It also supports nested fields.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.not_allowed = NotAllowedDict(set)

    def to_representation(self, instance):
        data = super().to_representation(instance)

        allowed_fields = self.get_allowed_fields(instance)
        query_fields = self.get_query_fields()

        if not query_fields and allowed_fields == ALL_FIELDS:
            return data

        #  limit all data to the allowed
        if allowed_fields == ALL_FIELDS:
            allowed_data = data
        else:
            spec_allowed = build_spec(allowed_fields)
            try:
                allowed_data = glom(data, spec_allowed)
            except GlomError as exc:
                raise serializers.ValidationError(
                    f"Fields in the configured authorization are absent in the data: {exc.args[0]}"
                )

        #  limit allowed data to requested in fields= query param
        if not query_fields:
            result_data = allowed_data
            not_allowed = set(get_field_names(data)) - set(get_field_names(result_data))
            if not_allowed:
                self.not_allowed[
                    f"{instance.object.object_type.url}({instance.version})"
                ] |= not_allowed
        else:
            spec_query = build_spec(query_fields)
            try:
                result_data = glom(allowed_data, spec_query)
            except GlomError as exc:
                raise serializers.ValidationError(
                    f"'fields' query parameter has invalid or unauthorized values: {exc.args[0]}"
                )
            not_allowed = set(get_field_names(glom(data, spec_query))) - set(
                get_field_names(result_data)
            )
            if not_allowed:
                self.not_allowed[
                    f"{instance.object.object_type.url}({instance.version})"
                ] |= not_allowed

        return result_data

    def get_query_fields(self) -> list:
        request = self.context.get("request")
        if not request:
            return []

        fields = request.query_params.get("fields")
        if not fields:
            return []

        return list(set(field.strip() for field in fields.split(",")))

    def get_allowed_fields(self, instance) -> list:
        request = self.context.get("request")

        # if not instance -> create or update -> all fields are allowed
        if not request:
            return ALL_FIELDS

        # use prefetch_related for DB optimization
        if getattr(instance.object.object_type, "token_permissions", None):
            permission = instance.object.object_type.token_permissions[0]
        else:
            permission = request.auth.get_permission_for_object_type(
                instance.object.object_type
            )
        if permission.mode == PermissionModes.read_only and permission.use_fields:
            return permission.fields.get(str(instance.version), [])

        return ALL_FIELDS


from concurrent.futures import ThreadPoolExecutor as PoolExecutor
from itertools import chain
from functools import partial
from utils import parse
from os import makedirs, environ, path

def go_to_download(session, link):
    result = session.get(link)
    page = parse(result.text)
    get_download_links(session, page)

def get_videos_links(course):
    video_tags = course.find_all("a", class_="item")

    base = "https://chinesezerotohero.teachable.com/"
    video_links = [base + tag["href"] for tag in video_tags]
    return video_links

def get_download_links(session, page):
    tag = page.find("a", class_="download")
    sideBar = page.find("div", id="courseSidebar")
    title = sideBar.find("h2").string
    title = title.replace(" ", "")
    link = tag["href"] if tag else False
    if link:
        download(session, link, title)
    return link

def videos(session, courses):
    print("finding videos")
    links = [get_videos_links(course) for course in courses]
    links = list(chain.from_iterable(links))
    get_videos = partial(go_to_download, session)
    with PoolExecutor(max_workers=32) as executor:
        executor.map(get_videos, links)

def download(session, link, title):
    print("Downloding from "+link)
    base = environ.get("DOWNLOAD_PATH", "./videos")
    folder = path.join(base, title)
    makedirs(folder, exist_ok=True)
    result = session.get(link)
    name = result.headers["X-File-Name"]
    print("download " + name)
    with open(path.join(folder, name), 'wb') as f:
        f.write(result.content)


import os
import uuid
from PIL import Image

from django.conf import settings
from django.db import models
from django.utils.text import slugify

from app.common.utils import ImageField
from app.common.format import format_price_

class Product(models.Model):
    code = models.UUIDField(editable=False, blank=True, null=True)
    name = models.CharField(max_length=255, verbose_name='Nome')
    description_short = models.TextField(max_length=255, verbose_name='Descrição curta')
    description_long = models.TextField(verbose_name='Descrição longa')
    image = ImageField(upload_to='product_images/%y/%m/', blank=True, null=True)
    slug = models.SlugField(unique=True, blank=True, null=True)
    price_marketing = models.FloatField(default=0, verbose_name='Preço')
    price_marketing_promotional = models.FloatField(default=0, verbose_name='Preço Promo.')
    type = models.CharField(default='V', max_length=1, choices=(
        ('V', 'Variável'),
        ('S', 'Simples'),
    ))

    class Meta:
        verbose_name = 'Produto'
        verbose_name_plural = 'Produtos'

    def __str__(self):
        return self.name

    @property
    def get_price_format(self):
        return format_price_(self.price_marketing)
    get_price_format.fget.short_description = 'Preço'

    @property
    def get_price_promotional_format(self):
        return format_price_(self.price_marketing_promotional)
    get_price_promotional_format.fget.short_description = 'Preço Promo.'

    @staticmethod
    def resize_image(img, new_width=800):
        img_full_path = os.path.join(settings.MEDIA_ROOT, img.name)
        img_pil = Image.open(img_full_path)

        current_width, current_height = img_pil.size
        new_height = round((new_width * current_height) / current_width)

        if new_width >= current_width:
            img_pil.close()
            return
        new_img = img_pil.resize((new_width, new_height), Image.LANCZOS)
        new_img.save(img_full_path, optimize=True, quality=65)
        img_pil.close()

    def save(self, *args, **kwargs):
        if not self.code:
            self.code = uuid.uuid4()
        if not self.slug:
            self.slug = f'{slugify(self.name[:20] + str(self.code)[:14])}'
        super().save(*args, **kwargs)

        if self.image:
            self.resize_image(self.image, 800)

class VariationProduct(models.Model):
    product = models.ForeignKey(Product, on_delete=models.CASCADE)
    name = models.CharField(max_length=60, blank=True, null=True)
    price = models.FloatField()
    price_promotional = models.FloatField(default=0)
    stock = models.PositiveIntegerField(default=1)

    class Meta:
        verbose_name = 'Variação do produto'
        verbose_name_plural = 'Variações dos produtos'

    def __str__(self):
        return self.name or self.product.name


import numpy as np
import matplotlib.pyplot as plt
import mplhep as hep
import math
import os

import llama

plt.style.use(hep.style.ROOT)

def histplot1d(
    plottable,
    ax=None,
    hist=True,
    errors=True,
    color="blue",
    hist_kwargs={"lw": 2},
    errors_kwargs=dict(),
):

    bins = plottable.axes[0].edges
    x = plottable.axes[0].centers
    y = plottable.values()
    yerr = np.sqrt(plottable.variances())
    xerr = np.diff(bins) / 2 if errors and not hist else None
    if ax is None:
        ax = plt.gca()

    if hist:
        ax.hist(x, bins=bins, weights=y, histtype="step", color=color, **hist_kwargs)

    if errors:
        ax.errorbar(x, y, yerr=yerr, xerr=xerr, ls="none", color=color, **errors_kwargs)

    ax.set_xlim((bins[0], bins[-1]))
    ax.set_xlabel(plottable.axes[0].metadata)

    if isinstance(hist, llama.Spectrum):
        ax.set_ylabel("Events")
    return ax

def savefig(name, directory="./"):
    name = os.path.join(directory, name)
    plt.savefig(name)
    print(f"Wrote {name}")
    plt.close()


import matplotlib.pyplot as plt
from pwn import *
import string
import pickle
import numpy as np

def get():
    conn = connect("mercury.picoctf.net", 58251)
    alphabet = string.ascii_lowercase
    variantss = []
    for i in range(1, 6):
        variants = []
        print(i)
        for j in range(100):
            try:
                conn.sendafter(b"I will encrypt whatever you give me: ", alphabet[:i] + "\n")
                conn.recvuntil(b"Here you go: ", drop=True)
                ans = conn.recvline()
                if ans not in variants:
                    variants.append(ans)
            except Exception as error:
                print(error)
                pass
        variantss.append(variants)
    conn.close()

    with open("variants.txt", "wb") as doc:
        pickle.dump(variantss, doc)

def load():
    with open("variants.txt", "rb") as doc:
        variants = pickle.load(doc)
    return variants

if __name__ == "__main__":
    # get()
    variants = load()

    f = lambda x: np.prod([i for i in range(1, x + 1)])
    plt.plot(list(range(len(variants))), [len(e) for e in variants], "x", label="data")
    plt.plot(list(range(len(variants))), [f(i + 1) for i in range(len(variants))], label="Faculty")
    plt.xlabel("letters")
    plt.ylabel("variants")
    plt.legend()
    plt.grid()
    plt.show()

    lengths = [all([len(var) == len(var_letter[0]) for var in var_letter]) for var_letter in variants]
    print(f"same length: {all(lengths)}")

    lengths = [len(var_letter[0]) for var_letter in variants]
    print(f"lengths: {lengths}")

    diffs = [lengths[i + 1] - lengths[i] for i in range(len(lengths) - 1)]
    print(f"diffs: {diffs}")

    last_variants = [variants[0][0].replace(b"\n", b"")]
    equals = []
    states = []
    for i in range(1, len(variants)):
        states.append(all(last_variants[-1] in variant for variant in variants[i]))
        candidates = []
        for variant in variants[i]:
            for kill in last_variants:
                variant = variant.replace(kill, b"")
            candidates.append(variant)

        equals.append(all([candidates[0] == candidate for candidate in candidates]))
        last_variants.append(candidates[0].replace(b"\n", b""))
    print(states)
    print(equals)


from datetime import datetime
import logging
import socket
import json

from django.core.serializers.json import DjangoJSONEncoder
from django.conf import settings

class ELKFormatter(logging.Formatter):

    def format(self, record):
        assert isinstance(record.msg, dict)

        user = record.__dict__.get('user')
        message = record.msg.copy()
        data = dict(
            event=message.pop('event', ''),
            eventdate=datetime.utcnow().isoformat(timespec='seconds'),
            host=dict(
                name=socket.gethostname(),
                ip=socket.gethostbyname(socket.gethostname())
            ),
            subsystem=settings.APP_NAME,
            format_version=1,
            username=(user.username if user else None),
            **message
        )

        return json.dumps(data, cls=DjangoJSONEncoder, ensure_ascii=False)


# -*- coding: utf-8 -*-
"""
Created on Sat Sep 17 17:34:12 2016

@author: user

2D convolutional classifier for mnist data set
unsupervised learning
"""

import os
from sh import mkdir
import configparser

import theano
from theano import tensor as T
from theano.tensor.nnet import conv2d

import numpy as np

theano.config.floatX = 'float32'
theano.config.optimizer = 'fast_compile'
theano.config.exception_verbosity = 'high'

class CNN2D(object):

    def __init__(self, **kwargs):

        self.name = kwargs.get('name', 'CNN2D')
        self.rng = kwargs.get('rng', np.random.RandomState(42))
        load_folder = kwargs.get('network_folder', None)

        #Prepare the network weights
        self.configData = "network.cfg"
        self.network_folder = None
        if load_folder is not None:
            try:
                self.__load_config(load_folder)
                if not self.__load_weights(load_folder):
                    return None
                self.network_folder = load_folder
            except:
                load_folder=None

        if self.network_folder is None:
            self.network_folder = self.name
            mkdir('-p', self.name)

            self.nConvLayers = kwargs.get('convolutional_layers', 1)
            self.nConvFilters = kwargs.get('convolutional_filters', 20)
            self.convFilterSize = kwargs.get('filter_size', 7)

            self.nRegLayers = kwargs.get('regression_layers', 1)
            self.regLayerSize = kwargs.get('layer_size', 50)
            self.regLayers = ()
            for i in range(self.nRegLayers):
                self.regLayers += (self.regLayerSize ,)

            self.nClasses = kwargs.get('number_classes',10)
            self.regLayers += (self.nClasses ,)

            self.activation = kwargs.get('activation', 'tanh')

            self.imageShape = kwargs.get('image_size', (28, 28))
            self.device = kwargs.get('device', 'cpu')

            self.sample_size = self.nConvLayers*(self.convFilterSize -1) +1

            self.__save_config(self.network_folder)

            self.__define_network()
            self.__init_weights()

        #symbolic variable for import to network
        self.X = T.tensor3('X')

        #create symbolic representation for network
        self.__create_model()

        #Currently, cpu only
        self.forward = theano.function(inputs=[self.X], outputs=self.out, allow_input_downcast=True)

    """ Make predictions on some input x """
    def run(self, x, **kwargs):
        return self.forward(x)

    """ Store weights and meta-data to folder """
    def save(self, folder):
        if not folder.endswith("/"):
            folder += "/"
        if not os.path.exists(folder+"weights"):
            mkdir('-p', folder+"weights/conv")
            mkdir('-p', folder+"weights/reg")

        for i in range(0, self.nConvLayers):
            self.cw[i].get_value().tofile(folder+"weights/conv/layer_"+str(i)+"_weights.csv", sep=',')
            self.cb[i].get_value().tofile(folder+"weights/conv/layer_"+str(i)+"_bias.csv", sep=',')
        for i in range(0, self.nRegLayers):
            self.rw[i].get_value().tofile(folder+"weights/reg/layer_"+str(i)+"_weights.csv", sep=',')
            self.rb[i].get_value().tofile(folder+"weights/reg/layer_"+str(i)+"_bias.csv", sep=',')

    """ Generate Definition of Network Structure """
    def __define_network(self):
        ## Define convolutional network component
        self.postConvImageShape = np.ndarray(2)
        self.postConvImageShape[0] = self.imageShape[0] - self.nConvLayers*(self.convFilterSize -1)
        self.postConvImageShape[1] = self.imageShape[1] - self.nConvLayers*(self.convFilterSize -1)
        self.convnet_shape = np.ndarray([self.nConvLayers, 4])
        self.convnet_shape[0,:] = [self.nConvFilters, 1, self.convFilterSize, self.convFilterSize]
        #self.convnet_shape[0,:] = [1, self.nConvFilters, self.convFilterSize, self.convFilterSize]
        for i in range(1, self.nConvLayers):
            self.convnet_shape[i,:] = [self.nConvFilters, self.nConvFilters, self.convFilterSize, self.convFilterSize]
        ## Define regression network component
        if self.regLayers is not None:
            self.regnet_shape = np.ndarray([len(self.regLayers), 2])
            self.regnet_shape[0, :] = \
                [self.nConvFilters*self.postConvImageShape[0]*self.postConvImageShape[1], self.regLayers[0]]
            for i in range(1, len(self.regLayers)):
                self.regnet_shape[i,:] = [self.regLayers[i-1], self.regLayers[i]]
        else:
            self.regnet_shape = None

    """ Randomly Initialize Network Weights """
    def __init_weights(self):
        self.cw = () #Convolution weights
        self.cb = () #Convolution biases
        for layer in range(0, self.nConvLayers):
            #Initialize within optimum range for tanh activation]
            #Initialize convoluional layer weights
            fan_in = self.convnet_shape[layer, 1] * (self.sample_size**3)
            fan_out = self.convnet_shape[layer, 0] * (self.convFilterSize**3)
            bound = np.sqrt(6.0/(fan_in+fan_out))
            self.cw += (theano.shared(np.asarray(self.rng.uniform(low= -bound,
                                                                  high= bound,
                                                                  size = self.convnet_shape[layer, :]),
                                                dtype=theano.config.floatX), name='cw-'+str(layer)) ,)
            self.cb += (theano.shared(np.asarray(np.ones(self.convnet_shape[layer, 0]),
                                                dtype=theano.config.floatX), name='cb-'+str(layer)) ,)
        self.rw = () #Regression weights
        self.rb = () #Regression biases
        for layer in range(0, len(self.regLayers)):
            #Initialize regression layer weights
            bound = 0.75
            self.rw += (theano.shared(np.asarray(self.rng.uniform(low= -bound,
                                                                  high= bound,
                                                                  size = self.regnet_shape[layer, :]),
                                                dtype=theano.config.floatX), name='rw-'+str(layer)) ,)
            self.rb += (theano.shared(np.asarray(np.ones(self.regnet_shape[layer, 1]),
                                                dtype=theano.config.floatX), name='rb-'+str(layer)) ,)

    """ Load network meta-data from folder """
    def __save_config(self, folder):
        try:
            if not folder.endswith("/"):
                folder += "/"
            file = folder+self.configData
            config = configparser.ConfigParser()
            if os.path.isfile(file):
                config.read(file)
            if not config.has_section('General'):
                config.add_section('General')
                config.set('General', 'device', str(self.device))
            if not config.has_section('Network'):
                config.add_section('Network')
            config.set('Network', 'network_folder', folder)
            config.set('Network', 'name', str(self.name))
            config.set('Network', 'image_width', str(self.imageShape[0]))
            config.set('Network', 'image_height', str(self.imageShape[1]))

            config.set('Network', 'convolution_layers', str(self.nConvLayers))
            config.set('Network', 'convolution_filters', str(self.nConvFilters))
            config.set('Network', 'filter_size', str(self.convFilterSize))

            config.set('Network', 'regression_layers', str(self.nRegLayers))
            config.set('Network', 'layer_size', str(self.regLayerSize))
            config.set('Network', 'number_classes', str(self.nClasses))

            config.set('Network', 'activation', str(self.activation))

            with open(file, 'w') as f:
                config.write(f)

            return True
        except Exception as e:
            print("Unable to save network configuration data: ",e)
            return False

    """ Load network meta-data from folder """
    def __load_config(self, folder):
        if not folder.endswith("/"):
            folder += "/"
        file = folder+self.configData
        if os.path.isfile(file):
            config = configparser.ConfigParser(file)
            self.name=config.get('Network', 'name')
            self.imageShape=( config.getint('Network', 'image_width'), config.getint('Network', 'image_height') )

            self.nConvLayers=config.getint('Network', 'convolutional_layers')
            self.nConvFilters=config.getint('Network', 'convolutional_filters')
            self.convFilterSize=config.getint('Network', 'filter_size')

            self.nRegLayers=config.getint('Network', 'regression_layers')
            self.regLayerSize=config.getint('Network', 'layer_size')
            self.nClasses=config.getint('Network', 'number_classes')
            self.regLayers = ()
            for i in range(self.nRegLayers):
                self.regLayers += (self.regLayerSize ,)
            self.regLayers += (self.nClasses ,)

            self.activation=config.get('Network', 'activation')
            self.sample_size = self.nConvLayers*(self.convFilterSize -1) +1

            return True
        else:
            print("Unable to load network configuration data.")
            return False

    """ Load weights from folder """
    def __load_weights(self, folder):
        if not folder.endswith("/"):
            folder += "/"
        if not os.path.exists(folder+"weights"):
            return False
        try:
            self.cw = () #Convolution weights
            self.cb = () #Convolution biases
            for layer in range(0, self.nConvLayers):
                self.cw += (theano.shared(
                            np.genfromtxt(folder+"weights/conv/layer_"+str(layer)+"_weights.csv",
                                          delimiter=',')) ,)
                self.cb += (theano.shared(
                            np.genfromtxt(folder+"weights/conv/layer_"+str(layer)+"_bias.csv",
                                          delimiter=',')) ,)
            self.rw = () #Regression weights
            self.rb = () #Regression biases
            for layer in range(0, len(self.regLayers)):
                self.rw += (theano.shared(
                            np.genfromtxt(folder+"weights/reg/layer_"+str(layer)+"_weights.csv",
                                          delimiter=',')) ,)
                self.rb += (theano.shared(
                            np.genfromtxt(folder+"weights/reg/layer_"+str(layer)+"_bias.csv",
                                          delimiter=',')) ,)
        except:
            return False

    """ Create Symbolic Theano definition for network """
    def __create_model(self):

        #Prepare input tensor
        Xin = self.X.dimshuffle(0, 'x', 1, 2)

        #Set up convolutional layers
        out = T.tanh(conv2d(Xin, self.cw[0], border_mode='valid') + self.cb[0].dimshuffle('x',0,'x','x'))

        for layer in range(1, self.nConvLayers):
            out = T.tanh(conv2d(out, self.cw[layer], border_mode='valid') + self.cb[layer].dimshuffle('x',0,'x','x'))

        #Set up regression layers
        out = T.tanh(T.dot(out.flatten(2), self.rw[0]) + self.rb[0])

        for layer in range(1, len(self.regLayers)-1):
            out = T.tanh(T.dot(out, self.rw[layer]) + self.rb[layer])

        #Set up output layer
        out = T.nnet.sigmoid(T.dot(out, self.rw[-1]) + self.rb[-1])

        self.out = out


'''jinja_us_presidents.py'''
from flask import Flask, render_template
import pandas as pd

app = Flask(__name__)

@app.route("/")
def template_presidents():
    '''template_presidents'''
    return render_template('presidents.html', my_string="US presidents", presidents=get_presidents()
)

def get_presidents():
    '''Cleans Presidents Text using Pandas'''
    #Read
    df = pd.read_csv('US_Presidents_unordered.txt')

    #Number
    df['Number'] = df["'Name'"].map(lambda x : int(x.replace("'", '').replace('.', '').split()[0]))

    #Name
    df["'Name'"] = df["'Name'"].map(lambda x : ' '.join(x.replace("'", '').replace('.', '').split()[1:]))

    #Term Start
    df["'Term Start'"] = df["'Term Start'"].map(lambda x : x.replace("'", ''))

    #Term End
    df["'Term End'"] = df["'Term End'"].map(lambda x : x.replace("'", ''))

    #Sort
    df = df.sort_values(by=['Number'])

    # drop
    df.drop(columns = ["Number"], inplace=True)
    # print(df.dtypes)
    solution = df
    return solution

# for key,value in get_presidents().iterrows():
#     print(key,type(key))
#     print(value,type(value))

if __name__ == '__main__':
    app.run(host='localhost',debug=True, port=4000)


from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils import simplejson

import oembed
from oembed.models import StoredOEmbed, StoredProvider
from oembed.tests.tests.base import BaseOEmbedTestCase

class OEmbedViewTestCase(BaseOEmbedTestCase):
    def test_missing_endpoint(self):
        response = self.client.get('/oembed/json/?url=http://www.nothere.com/asdf/')
        self.assertEqual(response.status_code, 404)

    def test_bad_request(self):
        # no url provided raises bad request (400)
        response = self.client.get('/oembed/json/')
        self.assertEqual(response.status_code, 400)

    def test_basic_handling(self):
        response = self.client.get('/oembed/json/?url=%s' % self.category_url)
        self.assertEqual(response.status_code, 200)
        response_json = simplejson.loads(response.content)

        stored_oembed = StoredOEmbed.objects.get(match=self.category_url)
        self.assertEqual(response_json, stored_oembed.response)

    def test_stored_provider_signals(self):
        response = self.client.get('/oembed/json/?url=%s' % self.youtube_url)

        # Check some response details - this provider doesn't provide upstream
        self.assertEqual(response.status_code, 404)

        # re-register the youtube provider to provide upstream
        yt = StoredProvider.objects.get(endpoint_url='http://www.youtube.com/oembed')
        yt.provides = True
        yt.save()

        # now we should be able to get the youtube object through the endpoint
        response = self.client.get('/oembed/json/?url=%s' % self.youtube_url)
        self.assertEqual(response.status_code, 200)

        stored = StoredOEmbed.objects.get(match=self.youtube_url)
        self.assertEqual(simplejson.loads(response.content), stored.response)

    def test_oembed_schema(self):
        response = self.client.get('/oembed/')
        self.assertEqual(response.status_code, 200)

        json_data = simplejson.loads(response.content)
        self.assertEqual(json_data, [
            {
                "matches": "http://example.com/testapp/blog/*/*/*/*/",
                "endpoint": "/oembed/json/",
                "type": "link"
            },
            {
                "matches": "http://example.com/testapp/category/*/",
                "endpoint": "/oembed/json/",
                "type": "photo"
            },
            {
                "matches": "http://example.com/testapp/rich/*/",
                "endpoint": "/oembed/json/",
                "type": "rich"
            }
        ])

        stored_provider = StoredProvider.objects.get(pk=100)
        stored_provider.provides = True
        stored_provider.save()

        expected = {
            'matches': 'http://www.active.com/*',
            'endpoint': '/oembed/json/',
            'type': 'photo'
        }

        response = self.client.get('/oembed/')
        self.assertEqual(response.status_code, 200)

        json_data = simplejson.loads(response.content)
        self.assertTrue(expected in json_data)