from typing import Set, Any, Dict, List
import collections

import data_algebra.expr_rep
import data_algebra.pipe
import data_algebra.env
import data_algebra.pending_eval

have_black = False
try:
    # noinspection PyUnresolvedReferences
    import black
    have_black = True
except ImportError:
    pass

have_sqlparse = False
try:
    # noinspection PyUnresolvedReferences
    import sqlparse
    have_sqlparse = True
except ImportError:
    pass

class ViewRepresentation(data_algebra.pipe.PipeValue):
    """Structure to represent the columns of a query or a table.
       Abstract base class."""

    column_names: List[str]
    column_set: Set[str]
    column_map: data_algebra.env.SimpleNamespaceDict
    sources: List[Any]  # actually ViewRepresentation

    def __init__(self, column_names, *, sources=None):
        self.column_names = [c for c in column_names]
        for ci in self.column_names:
            if not isinstance(ci, str):
                raise Exception("non-string column name(s)")
        if len(self.column_names) < 1:
            raise Exception("no column names")
        self.column_set = set(self.column_names)
        if not len(self.column_names) == len(self.column_set):
            raise Exception("duplicate column name(s)")
        column_dict = {
            ci: data_algebra.expr_rep.ColumnReference(self, ci)
            for ci in self.column_names
        }
        self.column_map = data_algebra.env.SimpleNamespaceDict(**column_dict)
        if sources is None:
            sources = []
        for si in sources:
            if not isinstance(si, ViewRepresentation):
                raise Exception("all sources must be of class ViewRepresentation")
        self.sources = [si for si in sources]
        data_algebra.pipe.PipeValue.__init__(self)

    # characterization

    def get_tables(self, tables=None):
        """get a dictionary of all tables used in an operator DAG,
        raise an exception if the values are not consistent"""
        if tables is None:
            tables = {}
        for s in self.sources:
            tables = s.get_tables(tables)
        return tables

    def columns_used_from_sources(self, using=None):
        """Give column names used from source nodes when this node is exececuted
        with the using columns (None means all)."""
        raise Exception("base method called")

    def columns_used_implementation(self, *, columns_used, using=None):
        cu_list = self.columns_used_from_sources(using)
        for i in range(len(self.sources)):
            self.sources[i].columns_used_implementation(
                columns_used=columns_used, using=cu_list[i]
            )

    def columns_used(self):
        """Determine which columns are used from source tables."""
        tables = self.get_tables()
        columns_used = {ki: set() for ki in tables.keys()}
        self.columns_used_implementation(columns_used=columns_used, using=None)
        return columns_used

    # collect as simple structures for YAML I/O and other generic tasks

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        raise Exception("base method called")

    def collect_representation(self, *, pipeline=None, dialect='Python'):
        """Collect a representation of the operator DAG as simple serializable objects.
                   These objects can be saved/loaded in YAML format and also can rebuild the
                   pipeline via data_algebra.yaml.to_pipeline()."""
        self.get_tables()  # for table consistency check/raise
        return self.collect_representation_implementation(pipeline=pipeline, dialect=dialect)

    # printing

    def to_python_implementation(self, *, indent=0, strict=True):
        return "ViewRepresentation(" + self.column_names.__repr__() + ")"

    def to_python(self, *, indent=0, strict=True, pretty=False, black_mode=None):
        global have_black
        self.get_tables()  # for table consistency check/raise
        if pretty:
            strict = True
        python_str = self.to_python_implementation(indent=indent, strict=strict)
        if pretty and have_black:
            if black_mode is None:
                black_mode = black.FileMode()
            python_str = black.format_str(python_str, mode=black_mode)
        return python_str

    def __repr__(self):
        return self.to_python(strict=True)

    def __str__(self):
        return self.to_python(strict=True)

    # query generation

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        raise Exception("base method called")

    def to_sql(self, db_model,
               *,
               pretty=False,
               encoding=None,
               sqlparse_options=None):
        global have_sqlparse
        if sqlparse_options is None:
            sqlparse_options = {'reindent': True,
                                'keyword_case': 'upper'}
        self.get_tables()  # for table consistency check/raise
        temp_id_source = [0]
        sql_str = self.to_sql_implementation(
            db_model=db_model, using=None, temp_id_source=temp_id_source
        )
        if pretty and have_sqlparse:
            sql_str = sqlparse.format(sql_str, encoding=encoding, **sqlparse_options)
        return sql_str

    # Pandas realization

    def eval_pandas(self, data_map):
        """
        Evaluate pipeline taking tables by name from data_map
        :param data_map: Dict[str, pandas.DataFrame]
        :return: pandas.DataFrame
        """
        raise Exception("base method called")

    # define builders for all non-leaf node types on base class

    def extend(self, ops, *, partition_by=None, order_by=None, reverse=None):
        return ExtendNode(
            source=self,
            ops=ops,
            partition_by=partition_by,
            order_by=order_by,
            reverse=reverse,
        )

    def project(self, ops, *, group_by=None, order_by=None, reverse=None):
        raise Exception("not implmented yet")
        # return ProjectNode(
        #     source=self,
        #     ops=ops,
        #     group_by=group_by,
        #     order_by=order_by,
        #     reverse=reverse,
        # )

    def natural_join(self, b, *, by=None, jointype="INNER"):
        if not isinstance(b, ViewRepresentation):
            raise Exception(
                "expected b to be a data_algebra.dat_ops.ViewRepresentation"
            )
        return NaturalJoinNode(a=self, b=b, by=by, jointype=jointype)

    def select_rows(self, expr):
        return SelectRowsNode(source=self, expr=expr)

    def drop_columns(self, column_deletions):
        return DropColumnsNode(source=self, column_deletions=column_deletions)

    def select_columns(self, columns):
        return SelectColumnsNode(source=self, columns=columns)

    def rename_columns(self, column_remapping):
        return RenameColumnsNode(source=self, column_remapping=column_remapping)

    def order_rows(self, columns, *, reverse=None, limit=None):
        return OrderRowsNode(source=self, columns=columns, reverse=reverse, limit=limit)

class TableDescription(ViewRepresentation):
    """Describe columns, and qualifiers, of a table.

       If outer namespace is set user values are visible and
       _-side effects can be written back.

       Example:
           from data_algebra.data_ops import *
           import data_algebra.env
           with data_algebra.env.Env(globals()) as env:
               d = TableDescription('d', ['x', 'y'])
           print(_) # should be a SimpleNamespaceDict, not d/ViewRepresentation
           print(d)
    """

    table_name: str
    qualifiers: Dict[str, str]
    key: str

    def __init__(self, table_name, column_names, *, qualifiers=None):
        ViewRepresentation.__init__(self, column_names=column_names)
        if (table_name is not None) and (not isinstance(table_name, str)):
            raise Exception("table_name must be a string")
        self.table_name = table_name
        self.column_names = column_names.copy()
        if qualifiers is None:
            qualifiers = {}
        if not isinstance(qualifiers, dict):
            raise Exception("qualifiers must be a dictionary")
        self.qualifiers = qualifiers.copy()
        key = ""
        if len(self.qualifiers) > 0:
            keys = [k for k in self.qualifiers.keys()]
            keys.sort()
            key = "{"
            for k in keys:
                key = key + "(" + k + ", " + str(self.qualifiers[k]) + ")"
            key = key + "}."
        self.key = key + self.table_name

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        if pipeline is None:
            pipeline = []
        od = collections.OrderedDict()
        od["op"] = "TableDescription"
        od["table_name"] = self.table_name
        od["qualifiers"] = self.qualifiers.copy()
        od["column_names"] = self.column_names
        od["key"] = self.key
        pipeline.insert(0, od)
        return pipeline

    def to_python_implementation(self, *, indent=0, strict=True):
        nc = min(len(self.column_names), 20)
        if (not strict) and (nc < len(self.column_names)):
            cols_str = (
                "["
                + ", ".join([self.column_names[i].__repr__() for i in range(nc)])
                + ", + "
                + str(len(self.column_names) - nc)
                + " more]"
            )
        else:
            cols_str = self.column_names.__repr__()
        s = (
            "TableDescription("
            + "table_name="
            + self.table_name.__repr__()
            + ", column_names="
            + cols_str
        )
        if len(self.qualifiers) > 0:
            s = s + ", qualifiers=" + self.qualifiers.__repr__()
        s = s + ")"
        return s

    def get_tables(self, tables=None):
        """get a dictionary of all tables used in an operator DAG,
        raise an exception if the values are not consistent"""
        if tables is None:
            tables = {}
        if self.key in tables.keys():
            other = tables[self.key]
            if self.column_set != other.column_set:
                raise Exception(
                    "Two tables with key " + self.key + " have different column sets."
                )
        else:
            tables[self.key] = self
        return tables

    def eval_pandas(self, data_map):
        if len(self.qualifiers) > 0:
            raise Exception(
                "table descriptions used with eval_pandas() must not have qualifiers"
            )
        # make an index-free copy of the data to isolate side-effects and not deal with indices
        res = data_map[self.table_name]
        res = res.copy()
        res.reset_index(drop=True, inplace=True)
        return res

    def columns_used_from_sources(self, using=None):
        return []  # no inputs to table description

    def columns_used_implementation(self, *, columns_used, using=None):
        cset = columns_used[self.key]
        if using is None:
            using = self.column_set
        unexpected = using - self.column_set
        if len(unexpected) > 0:
            raise Exception("asked for undefined columns: " + str(unexpected))
        cset.update(using)

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        return db_model.table_def_to_sql(self, using=using)

    # comparable to other table descriptions
    def __lt__(self, other):
        if not isinstance(other, TableDescription):
            return True
        return self.key.__lt__(other.key)

    def __eq__(self, other):
        if not isinstance(other, TableDescription):
            return False
        return self.key.__eq__(other.key)

    def __hash__(self):
        return self.key.__hash__()

def describe_pandas_table(d, table_name):
    return TableDescription(table_name, [c for c in d.columns])

class ExtendNode(ViewRepresentation):
    ops: Dict[str, data_algebra.expr_rep.Expression]

    def __init__(self, source, ops, *, partition_by=None, order_by=None, reverse=None):
        ops = data_algebra.expr_rep.check_convert_op_dictionary(
            ops, source.column_map.__dict__
        )
        if len(ops) < 1:
            raise Exception("no ops")
        self.ops = ops
        if partition_by is None:
            partition_by = []
        if isinstance(partition_by, str):
            partition_by = [partition_by]
        self.partition_by = partition_by
        if order_by is None:
            order_by = []
        if isinstance(order_by, str):
            order_by = [order_by]
        self.order_by = order_by
        if reverse is None:
            reverse = []
        if isinstance(reverse, str):
            reverse = [reverse]
        self.reverse = reverse
        column_names = source.column_names.copy()
        consumed_cols = set()
        for (k, o) in ops.items():
            o.get_column_names(consumed_cols)
        unknown_cols = consumed_cols - source.column_set
        if len(unknown_cols) > 0:
            raise Exception("referred to unknown columns: " + str(unknown_cols))
        known_cols = set(column_names)
        for ci in ops.keys():
            if ci not in known_cols:
                column_names.append(ci)
        if len(partition_by) != len(set(partition_by)):
            raise Exception("Duplicate name in partition_by")
        if len(order_by) != len(set(order_by)):
            raise Exception("Duplicate name in order_by")
        if len(reverse) != len(set(reverse)):
            raise Exception("Duplicate name in reverse")
        unknown = set(partition_by) - known_cols
        if len(unknown) > 0:
            raise Exception("unknown partition_by columns: " + str(unknown))
        unknown = set(order_by) - known_cols
        if len(unknown) > 0:
            raise Exception("unknown order_by columns: " + str(unknown))
        unknown = set(reverse) - set(order_by)
        if len(unknown) > 0:
            raise Exception("reverse columns not in order_by: " + str(unknown))
        bad_overwrite = set(ops.keys()).intersection(
            set(partition_by).union(order_by, reverse)
        )
        if len(bad_overwrite) > 0:
            raise Exception("tried to change: " + str(bad_overwrite))
        ViewRepresentation.__init__(self, column_names=column_names, sources=[source])

    def columns_used_from_sources(self, using=None):
        columns_we_take = self.sources[0].column_set.copy()
        if using is None:
            return [columns_we_take]
        subops = {k: op for (k, op) in self.ops.items() if k in using}
        if len(subops) <= 0:
            return [columns_we_take]
        columns_we_take = using.union(self.partition_by, self.order_by, self.reverse)
        columns_we_take = columns_we_take - subops.keys()
        for (k, o) in subops.items():
            o.get_column_names(columns_we_take)
        return [columns_we_take]

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        if pipeline is None:
            pipeline = []
        od = collections.OrderedDict()
        od["op"] = "Extend"
        od["ops"] = {ci: vi.to_source(dialect=dialect) for (ci, vi) in self.ops.items()}
        od["partition_by"] = self.partition_by
        od["order_by"] = self.order_by
        od["reverse"] = self.reverse
        pipeline.insert(0, od)
        return self.sources[0].collect_representation_implementation(pipeline=pipeline, dialect=dialect)

    def to_python_implementation(self, *, indent=0, strict=True):
        s = (
            self.sources[0].to_python_implementation(indent=indent, strict=strict)
            + " .\\\n"
            + " " * (indent + 3)
            + "extend({"
            + ", ".join(
                [
                    k.__repr__() + ": " + opi.to_python().__repr__()
                    for (k, opi) in self.ops.items()
                ]
            )
            + "}"
        )
        if len(self.partition_by) > 0:
            s = s + ", partition_by=" + self.partition_by.__repr__()
        if len(self.order_by) > 0:
            s = s + ", order_by=" + self.order_by.__repr__()
        if len(self.reverse) > 0:
            s = s + ", reverse=" + self.reverse.__repr__()
        s = s + ")"
        return s

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        return db_model.extend_to_sql(self, using=using, temp_id_source=temp_id_source)

    def eval_pandas(self, data_map):
        window_situation = (len(self.partition_by) > 0) or (len(self.order_by) > 0)
        if window_situation:
            # check these are forms we are prepared to work with
            for (k, op) in self.ops.items():
                if len(op.args) > 1:
                    raise Exception(
                        "non-trivial windows expression: " + str(k) + ": " + str(op)
                    )
                if len(op.args) == 1:
                    if not isinstance(
                        op.args[0], data_algebra.expr_rep.ColumnReference
                    ):
                        raise Exception(
                            "windows expression argument must be a column: "
                            + str(k)
                            + ": "
                            + str(op)
                        )
        res = self.sources[0].eval_pandas(data_map)
        res.reset_index(inplace=True, drop=True)
        if not window_situation:
            for (k, op) in self.ops.items():
                res[k] = res.eval(op.to_pandas())
        else:
            for (k, op) in self.ops.items():
                # work on a slice of the data frame
                col_list = [c for c in set(self.partition_by)]
                for c in self.order_by:
                    if c not in col_list:
                        col_list = col_list + [c]
                value_name = None
                if len(op.args) > 0:
                    value_name = op.args[0].to_pandas()
                    if value_name not in set(col_list):
                        col_list = col_list + [value_name]
                ascending = [c not in set(self.reverse) for c in col_list]
                subframe = res[col_list].copy()
                subframe.reset_index(inplace=True, drop=True)
                subframe['_data_algebra_orig_index'] = [i for i in range(subframe.shape[0])]
                subframe.sort_values(by=col_list, ascending=ascending, inplace=True)
                subframe.reset_index(inplace=True, drop=True)
                if len(self.partition_by) > 0:
                    opframe = subframe.groupby(self.partition_by)
                    #  Groupby preserves the order of rows within each group.
                    # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.groupby.html
                else:
                    opframe = subframe
                if len(op.args) == 0:
                    if op.op == "row_number":
                        subframe[k] = opframe.cumcount() + 1
                    else:  # TODO: more of these
                        raise Exception("not implemented: " + str(k) + ": " + str(op))
                else:
                    # len(op.args) == 1
                    subframe[k] = opframe[value_name].transform(op.op)
                subframe.reset_index(inplace=True, drop=True)
                subframe.sort_values(by=['_data_algebra_orig_index'], inplace=True)
                subframe.reset_index(inplace=True, drop=True)
                res[k] = subframe[k]
        return res

class SelectRowsNode(ViewRepresentation):
    expr: data_algebra.expr_rep.Expression
    decision_columns: Set[str]

    def __init__(self, source, expr):
        ops = data_algebra.expr_rep.check_convert_op_dictionary(
            {"expr": expr}, source.column_map.__dict__
        )
        if len(ops) < 1:
            raise Exception("no ops")
        self.expr = ops["expr"]
        self.decision_columns = set()
        self.expr.get_column_names(self.decision_columns)
        ViewRepresentation.__init__(
            self, column_names=source.column_names, sources=[source]
        )

    def columns_used_from_sources(self, using=None):
        columns_we_take = self.sources[0].column_set.copy()
        if using is None:
            return [columns_we_take]
        columns_we_take = columns_we_take.intersection(using)
        columns_we_take = columns_we_take.union(self.decision_columns)
        return [columns_we_take]

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        if pipeline is None:
            pipeline = []
        od = collections.OrderedDict()
        od["op"] = "SelectRows"
        od["expr"] = self.expr.to_source(dialect=dialect)
        pipeline.insert(0, od)
        return self.sources[0].collect_representation_implementation(pipeline=pipeline, dialect=dialect)

    def to_python_implementation(self, *, indent=0, strict=True):
        s = (
            self.sources[0].to_python_implementation(indent=indent, strict=strict)
            + " .\\\n"
            + " " * (indent + 3)
            + "select_rows("
            + self.expr.to_python().__repr__()
            + ")"
        )
        return s

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        return db_model.select_rows_to_sql(
            self, using=using, temp_id_source=temp_id_source
        )

    def eval_pandas(self, data_map):
        res = self.sources[0].eval_pandas(data_map)
        res = res.query(self.expr.to_pandas())
        res.reset_index(inplace=True, drop=True)
        return res

class SelectColumnsNode(ViewRepresentation):
    column_selection: List[str]

    def __init__(self, source, columns):
        column_selection = [c for c in columns]
        self.column_selection = column_selection
        # TODO: check column conditions
        ViewRepresentation.__init__(
            self, column_names=column_selection, sources=[source]
        )

    def columns_used_from_sources(self, using=None):
        cols = set(self.column_selection.copy())
        if using is None:
            return [cols]
        return [cols.intersection(using)]

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        if pipeline is None:
            pipeline = []
        od = collections.OrderedDict()
        od["op"] = "SelectColumns"
        od["columns"] = self.column_selection
        pipeline.insert(0, od)
        return self.sources[0].collect_representation_implementation(pipeline=pipeline, dialect=dialect)

    def to_python_implementation(self, *, indent=0, strict=True):
        s = (
            self.sources[0].to_python_implementation(indent=indent, strict=strict)
            + " .\\\n"
            + " " * (indent + 3)
            + "select_columns("
            + self.column_selection.__repr__()
            + ")"
        )
        return s

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        return db_model.select_columns_to_sql(
            self, using=using, temp_id_source=temp_id_source
        )

    def eval_pandas(self, data_map):
        res = self.sources[0].eval_pandas(data_map)
        return res[self.column_selection]

class DropColumnsNode(ViewRepresentation):
    column_deletions: List[str]

    def __init__(self, source, column_deletions):
        column_deletions = [c for c in column_deletions]
        self.column_deletions = column_deletions
        remaining_columns = [c for c in source.column_names if c not in column_deletions]
        # TODO: check column conditions
        ViewRepresentation.__init__(
            self, column_names=remaining_columns, sources=[source]
        )

    def columns_used_from_sources(self, using=None):
        if using is None:
            using = set(self.sources[0].column_names)
        return [set([c for c in using if c not in self.column_deletions])]

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        if pipeline is None:
            pipeline = []
        od = collections.OrderedDict()
        od["op"] = "DropColumns"
        od["column_deletions"] = self.column_deletions
        pipeline.insert(0, od)
        return self.sources[0].collect_representation_implementation(pipeline=pipeline, dialect=dialect)

    def to_python_implementation(self, *, indent=0, strict=True):
        s = (
            self.sources[0].to_python_implementation(indent=indent, strict=strict)
            + " .\\\n"
            + " " * (indent + 3)
            + "drop_columns("
            + self.column_deletions.__repr__()
            + ")"
        )
        return s

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        return db_model.drop_columns_to_sql(
            self, using=using, temp_id_source=temp_id_source
        )

    def eval_pandas(self, data_map):
        res = self.sources[0].eval_pandas(data_map)
        column_selection = [c for c in res.columns if c not in self.column_deletions]
        return res[column_selection]

class OrderRowsNode(ViewRepresentation):
    order_columns: List[str]
    reverse: List[str]

    def __init__(self, source, columns, *, reverse=None, limit=None):
        self.order_columns = [c for c in columns]
        if reverse is None:
            reverse = []
        self.reverse = [c for c in reverse]
        self.limit = limit
        # TODO: check column conditions
        ViewRepresentation.__init__(
            self, column_names=source.column_names, sources=[source]
        )

    def columns_used_from_sources(self, using=None):
        cols = set(self.column_names.copy())
        if using is None:
            return [cols]
        cols = cols.intersection(using).union(self.order_columns)
        return [cols]

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        if pipeline is None:
            pipeline = []
        od = collections.OrderedDict()
        od["op"] = "Order"
        od["order_columns"] = self.order_columns
        od["reverse"] = self.reverse
        od["limit"] = self.limit
        pipeline.insert(0, od)
        return self.sources[0].collect_representation_implementation(pipeline=pipeline, dialect=dialect)

    def to_python_implementation(self, *, indent=0, strict=True):
        s = (
            self.sources[0].to_python_implementation(indent=indent, strict=strict)
            + " .\\\n"
            + " " * (indent + 3)
            + "order_rows("
            + self.order_columns.__repr__()
        )
        if len(self.reverse) > 0:
            s = s + ", reverse=" + self.reverse.__repr__()
        if self.limit is not None:
            s = s + ", limit=" + self.limit.__repr__()
        s = s + ")"
        return s

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        return db_model.order_to_sql(self, using=using, temp_id_source=temp_id_source)

    def eval_pandas(self, data_map):
        res = self.sources[0].eval_pandas(data_map)
        ascending = [False if ci in set(self.reverse) else True for ci in self.order_columns]
        res.sort_values(by=self.order_columns, ascending=ascending)
        return res

class RenameColumnsNode(ViewRepresentation):
    column_remapping: Dict[str, str]
    reverse_mapping: Dict[str, str]
    mapped_columns: Set[str]

    def __init__(self, source, column_remapping):
        self.column_remapping = column_remapping.copy()
        self.reverse_mapping = {v: k for (k, v) in self.column_remapping.items()}
        self.mapped_columns = set(self.column_remapping.keys()).union(
            set(self.reverse_mapping.keys())
        )
        column_names = [
            (k if k not in self.reverse_mapping.keys() else self.reverse_mapping[k])
            for k in source.column_names
        ]
        # TODO: check column conditions, don't allow name collisions
        ViewRepresentation.__init__(self, column_names=column_names, sources=[source])

    def columns_used_from_sources(self, using=None):
        if using is None:
            using = self.column_names
        cols = [
            (k if k not in self.column_remapping.keys() else self.column_remapping[k])
            for k in using
        ]
        return [set(cols)]

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        if pipeline is None:
            pipeline = []
        od = collections.OrderedDict()
        od["op"] = "Rename"
        od["column_remapping"] = self.column_remapping
        pipeline.insert(0, od)
        return self.sources[0].collect_representation_implementation(pipeline=pipeline, dialect=dialect)

    def to_python_implementation(self, *, indent=0, strict=True):
        s = (
            self.sources[0].to_python_implementation(indent=indent, strict=strict)
            + " .\\\n"
            + " " * (indent + 3)
            + "rename_columns("
            + self.column_remapping.__repr__()
            + ")"
        )
        return s

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        return db_model.rename_to_sql(self, using=using, temp_id_source=temp_id_source)

    def eval_pandas(self, data_map):
        res = self.sources[0].eval_pandas(data_map)
        return res.rename(columns=self.reverse_mapping)

class NaturalJoinNode(ViewRepresentation):
    by: List[str]
    jointype: str

    def __init__(self, a, b, *, by=None, jointype="INNER"):
        sources = [a, b]
        column_names = sources[0].column_names.copy()
        for ci in sources[1].column_names:
            if ci not in sources[0].column_set:
                column_names.append(ci)
        if isinstance(by, str):
            by = [by]
        by_set = set(by)
        if len(by) != len(by_set):
            raise Exception("duplicate column names in by")
        missing_left = by_set - a.column_set
        if len(missing_left) > 0:
            raise Exception("left table missing join keys: " + str(missing_left))
        missing_right = by_set - b.column_set
        if len(missing_right) > 0:
            raise Exception("right table missing join keys: " + str(missing_right))
        self.by = by
        self.jointype = jointype
        ViewRepresentation.__init__(self, column_names=column_names, sources=sources)

    def columns_used_from_sources(self, using=None):
        if using is None:
            return [self.sources[i].column_set.copy() for i in range(2)]
        using = using.union(self.by)
        return [self.sources[i].column_set.intersection(using) for i in range(2)]

    def collect_representation_implementation(self, *, pipeline=None, dialect='Python'):
        if pipeline is None:
            pipeline = []
        od = collections.OrderedDict()
        od["op"] = "NaturalJoin"
        od["by"] = self.by
        od["jointype"] = self.jointype
        od["b"] = self.sources[1].collect_representation_implementation(dialect=dialect)
        pipeline.insert(0, od)
        return self.sources[0].collect_representation_implementation(pipeline=pipeline, dialect=dialect)

    def to_python_implementation(self, *, indent=0, strict=True):
        return (
            self.sources[0].to_python_implementation(indent=indent, strict=strict)
            + " .\\\n"
            + " " * (indent + 3)
            + "natural_join(b=\n"
            + " " * (indent + 6)
            + self.sources[1].to_python_implementation(indent=indent + 6, strict=strict)
            + ",\n"
            + " " * (indent + 6)
            + "by="
            + self.by.__repr__()
            + ", jointype="
            + self.jointype.__repr__()
            + ")"
        )

    def to_sql_implementation(self, db_model, *, using, temp_id_source):
        return db_model.natural_join_to_sql(
            self, using=using, temp_id_source=temp_id_source
        )


#!/usr/bin/python3

# Lance en // le serveur et affiche le flux d'erreur
##Code serveur
import http.server
PORT = 8888
server_address = ("", PORT)
server = http.server.HTTPServer
handler = http.server.CGIHTTPRequestHandler
handler.cgi_directories = ["/"]
##Code serveur - fin

import sys
from threading import Thread
#import accel
import time
import subprocess

x = subprocess.Popen(['python3','test3.py'], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) #-u','E:\Mario family\Multimonde\serveur\

class GetAccel(Thread):
    def __init__(self):
        Thread.__init__(self)
##
    def run(self): #Code à exécuter pendant l'exécution du thread. Lance le serveur.
##        print("Serveur actif sur le port :", PORT)
##        httpd = server(server_address, handler)
##        httpd.serve_forever()
        #subprocess.run("python3 serveur.py",  stdout=PIPE)
        global x
        print('run thread')
        x=subprocess.Popen(['python3','-u','Serveur.py'],stdout=subprocess.PIPE,stderr=subprocess.PIPE, universal_newlines=True) #text=True

thread_1 = GetAccel() # Cree le thread
thread_1.start() # Lancement des threads - lance la methode run

##b=accel.accel
##while(True):
##    a=accel.accel
##    print(a)
##    if a!=b: time.sleep(2)
##    b=a
##    time.sleep(0.5)
##x = subprocess.Popen(['python3','Serveur.py'],
##                      stdout=subprocess.PIPE,
##                      stderr=subprocess.PIPE )
print("run main")
line = ''#x.stdout.readline()
i=0
for j in range(50):
    i+=1
    line = x.stderr.readline()
    if line!='': print('main ',i,': ', line)
    time.sleep(0.05)
    #sys.stdout.flush()

    #(out, err) = x.communicate()
    #line=out+err

# Attend que les threads se terminent

thread_1.join()


# Dit script maakt een aantal contourplots van de vectorpotentiaal ten gevolge van geladen deeltjes en slaat deze op in diverse plaatjes
# dit script is geschreven door Tijs Aarts en Victor Schyns op 22-11-2019

import numpy as np
import matplotlib.pyplot as plt

# deze functie neemt een verschil in x,y,z en berekent de coulombpotentiaal in dat punt
def V(deltax, deltay=0, deltaz=0, q=1):
    return q / (np.sqrt(deltax**2 + deltay**2 + deltaz**2))

# deze functie kan heel veel inputs ontvangen en maakt hiermee een plot

def maakplot(deeltjesx=np.array([]), deeltjesy=np.array([]), deeltjesz=np.array([]), deeltjesq=np.array([]), domein=[-1, 1, -1, 1], z=[0], naam='maakplot', titel='titel', contourhoogtes=15, nrows=1, ncols=1):
    x = np.linspace(domein[0], domein[1], 400)
    y = np.linspace(domein[2], domein[3], 400)
    X, Y = np.meshgrid(x, y)

    fig = plt.figure(naam, figsize=(6*ncols, 4*nrows))
    # ik zorg dat er een lege lijst is met een lang genoege lengte om alle subplots te gaan bevatten zodat mijn for loop goed werkt
    ax = [0 for i in range(nrows*ncols)]
    # print(ax)

    # hier tel ik de waarden van V voor elk punt voor elk deeltje op waardoor ik voor elk punt de coulombpotentiaal krijg in een 2D numpy array
    for p, j in enumerate(z):
        v = sum([V(X - deeltjesx[i], Y - deeltjesy[i], j - deeltjesz[i], deeltjesq[i])
                 for i in range(len(deeltjesx))])
        #print(p, ax[p])
        ax[p] = fig.add_subplot(nrows, ncols, p+1)
        cmap = ax[p].contour(X, Y, v, levels=contourhoogtes)
        ax[p].axis('scaled')
        ax[p].set_ylabel('$y$')
        ax[p].set_xlabel('$x$')
        ax[p].set_title('z = '+str(j), loc='left')
        cbar = fig.colorbar(cmap)
        cbar.set_label('kleuren bij V')

    plt.axis(domein)
    plt.suptitle(titel)
    plt.savefig(naam, bbox_inches='tight', format='png')

############## opdracht 1 ##############
q = 1
r = np.linspace(0, 1, 1000)[1:]
# ik pak r in de x-richting om de funcie V te kunnen gebruiken

fig1, [ax1, ax2] = plt.subplots(
    1, 2, False, False, num='opdracht 1', figsize=(16, 9), dpi=100)

color = 'tab:red'
ax1.set_xlabel('$r$')
ax1.set_ylabel('$V/q$', color=color)
# ik maak de y-as hier lineair (dit is niet nodig maar wel leesbaarder)
ax1.set_yscale('linear')
ax1.tick_params(axis='y', labelcolor=color)
ax1.tick_params(axis='x', labelcolor=color)
ax1.set_title('lineaire grafiek')
# ik roep hier de functie aan die ik in r5 heb gedefinieerd voor consistency
ax1.plot(r, V(r, 0, 0, 1), color=color)

color = 'tab:blue'
ax2.set_xlabel('$r$')
ax2.set_yscale('log')  # hier maak ik de as logaritmisch
ax2.tick_params(axis="y", labelcolor=color)
ax2.tick_params(axis="x", labelcolor=color)
ax2.set_title('logaritmische grafiek')
# hier doe ik hetzelfde als in regel 20
ax2.plot(r, V(r, 0, 0, 1), color=color)

plt.savefig('opdracht 1')
print('opdracht 1 klaar')

############## opdracht 2 ##############

deeltjex = [0]
deeltjey = [0]
deeltjez = [1]
deeltjeq = [1]
domein = [-1, 1, -1, 1]
z = [0]
naam = 'opdracht 2'
titel = 'coulombpotentiaal met een geladen deeltje op z = 1'
maakplot(deeltjex, deeltjey, deeltjez, deeltjeq, domein, z, naam, titel)
print('opdracht 2 klaar')

############## opdracht 3 ##############
# ik heb een library gemaakt die het coulombpotentiaal van elk aantal deeltjes in 3D kan uitrekenen,
# hier zet ik de waarden van de deeltjes van de kubus in variabelen voor overzicht voordat ik het in de functie doe
deeltjesx = [-0.5, -0.5, 0.5, 0.5, -0.5, -0.5, 0.5, 0.5]
deeltjesy = [-0.5, 0.5, 0.5, -0.5, -0.5, 0.5, 0.5, -0.5]
deeltjesz = [0.5, 0.5, 0.5, 0.5, -0.5, -0.5, -0.5, -0.5]
deeltjesq = [1, 1, 1, 1, 1, 1, 1, 1]
domein = [-1, 1, -1, 1]
z = [0]
naam = 'opdracht 3'
titel = 'plot van V op xy-vlak met kubus van deeltjes'
maakplot(deeltjesx, deeltjesy, deeltjesz, deeltjesq, domein, z, naam, titel)
print('opdracht 3 klaar')

############## opdracht 4 ##############
# om nu meerdere plaatjes op z = [0,0.25,0.5,0.75,1] te maken gebruik ik weer de functie, de kubus laat ik gelijk dus die variablen blijven staan
z = [0, 0.25, 0.5, 0.75, 1]
naam = 'opdracht 4'
titel = 'plots van V op diverse z \n met kubus van deeltjes'
# hier maak ik de contourplots duidelijk dat elke subplot dezelfde hoogtelijnen heeft
contourlines = [5, 5.25, 5.5, 5.75, 6, 6.25, 6.5, 6.75, 7,
                7.25, 7.5, 7.75, 8, 8.25, 8.5, 8.75, 9, 9.25, 9.5, 9.75, 10]
# om de z-waarde van onder naar boven te laten groeien draai ik de lijst om met [::-1]
maakplot(deeltjesx, deeltjesy, deeltjesz, deeltjesq,
         domein, z[::-1], naam, titel, nrows=len(z), contourhoogtes=contourlines)
print('opdracht 4 klaar')


# -*- encoding: utf-8 -*-
import pymysql
import re

conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='', db='bar_poem3', charset='utf8', autocommit=True)
curDB = conn.cursor()
curDB.execute("SET NAMES utf8")

#set to zero before counting match
curDB.execute("UPDATE `academ16` SET `metr_id`=0")
#curDB.execute("UPDATE `academ16` SET `metr_id`=0 WHERE `metr_id` IN (147,153,208,309,361,421,437,508,786)")
#exit()

#replace everything, except russian letters on space
def delete_punctutaion(name):
    name = name.lower()
    letters = u'йцукенгшщзхъэждлорпавыфячсмитьбю1234567890'
    new_name = ''
    for l in name:
        if l in letters:
            new_name += l
        else:
            new_name += u' '
    return new_name
#deletes all words of one letter and also deletes double spaces
def delete_small_words(text):
    text = text.replace(u" гр ", ' ')
    new_text = []
    text = text.split()
    for word in text:
        if len(word) > 1:
            new_text.append(word)
    new_text = " ".join(new_text)
    return new_text

#extract name and first line from brackets
def get_from_brackets(ll):
    bracket_1 = ll.find(u'(')
    bracket_2 = ll.find(u')')
    if bracket_1 < 0 or bracket_2 < 0:
        return ll, u''
    name = ll[:bracket_1]
    first_line = ll[(bracket_1+1):bracket_2]
    return name, first_line

def clean_me(tt):
    tt = delete_punctutaion(tt)
    #tt = delete_small_words(tt)
    tt = " ".join(tt.split())
    return tt

def compare_names(n1, line1, n2):
    n2_origin = n2
    #delete name in parentheses
    # n2 = re.sub(r'\([^)]*\)', '', n2)

    n1 = delete_punctutaion(n1)
    n2 = delete_punctutaion(n2)
    line1 = delete_punctutaion(line1)

    #delete small words5
    n1 = delete_small_words(n1)
    n2_test = delete_small_words(n2)
    line1 = delete_small_words(line1)
    if len(n2_test)>8:
        n1 = delete_small_words(n1)
        n2 = n2_test
        line1 = delete_small_words(line1)

    #delete multiple spaces
    n1 = " ".join(n1.split())
    n2 = " ".join(n2.split())
    line1 = " ".join(line1.split())

    if len(n2) < 3:
        print(n2)
        print(n2_origin)
        return False

    if n1 == n2 or line1 == n2:
        return True
    # get first chars
    n2_length = len(n2)
    n1_cutted = n1[:n2_length]
    if n1_cutted == n2 and n2_length>12:
        print(n2)
        return True

    return False

#with small words
def compare_names2(n1, line1, y1, n2, line2, y2):
    #names in french are empty
    if n1=='' or line1=='':
        return 0
    cut_length = min(len(line1), len(line2))
    line1_cutted = line1[:cut_length]
    line2_cutted = line2[:cut_length]
    n1_small = delete_small_words(n1)
    n2_small = delete_small_words(n2)
    # if line1 == line2 and (line1.find(' ')>-1):
    year_diff = abs(y2-y1)
    #RUL 1
    # if line1 == line2 and (y1==y2) and y1!=0 and n1!=u'ода':
    #     return True
    #RUL 2
    # if line1 == line2 and year_diff<3 and y1!=0 and n1!=u'ода':
    #     return True
    #RUL 3
    # if n1 == n2 and year_diff<3 and (n1.find(' ')>-1):
    #     return True
    #RUL 4 - very cool rule!
    if n1 == n2 and line2=='' and (year_diff<3 or year_diff>1800):
        return 0.7
    #RUL 5
    elif line2 != '' and line1_cutted==line2_cutted and (line2_cutted.count(' ')>0) and (year_diff<3 or year_diff>1800):
        return 0.9
    elif n1_small==n2_small and (len(n1_small)>3):
        return 0.2

    # if n2 == u'сестра братья' and n1 == u'сестра братья':
    #     print(n1+u' <> '+line1+u' <> '+n2+u' <> '+str(line2))
    # print(n1, n2)
    # exit()
    # print(n1+u' <> '+line1+u' <> '+n2+u' <> '+line2)
    # if line2 != '':
    #     if line1==line2 and len(line1)>1:
    #         #print(n1+u' <> '+line1+u' <> '+n2+u' <> '+line2)
    #         return True
    #     cut_length = min(len(line1), len(line2))
    #     line1_cutted = line1[:cut_length]
    #     line2_cutted = line2[:cut_length]
    #
    # else:
    #     if n1==n2 and len(n1)>1:
    #         #print(n1+u' <> '+line1+u' <> '+n2+u' <> '+line2)
    #         return True
    return 0

#get all poems from academ
curDB.execute("SELECT * FROM academ16")
academ16 = []
for r in curDB.fetchall():
    # print( r[2], delete_punctutaion(r[2]) )
    # print( r[2] )
    # print delete_punctutaion(r[2])
    # print('==========')
    rr = dict()
    rr['id'] = r[0]
    rr['a_title'] = clean_me(r[2])
    rr['a_line'] = clean_me(r[3])
    rr['year'] = int('0'+str(r[1]))
    academ16.append(rr)

#get all poems from konk
curDB.execute("SELECT * FROM poems_info")
metr_spr = []
for r in curDB.fetchall():
    rr = dict()
    new_name, new_line = get_from_brackets(r[1])
    rr['id'] = r[0]
    rr['m_title'] = clean_me(new_name)
    rr['m_line'] = clean_me(new_line)
    rr['year'] = int('0'+str(r[3]))
    metr_spr.append(rr)

num_found = 0
already_filled = set()
weights = dict()
for academ in academ16:
    nothing_found = True
    for mm in metr_spr:
        new_weight = compare_names2(academ['a_title'], academ['a_line'], academ['year'], mm['m_title'], mm['m_line'], mm['year'])
        #let's remember all weight
        if new_weight>0:
            add_weight = dict()
            add_weight['id'] = mm['id']
            add_weight['weight'] = new_weight
            if academ['id'] not in already_filled:
                weights[academ['id']] = list()
            weights[academ['id']].append(add_weight)
            nothing_found = False

        continue
    if nothing_found:
        #print(academ['a_title']+' ! '+academ['a_line']+' ! '+str(academ['year']))
        continue

        # if compare_names(academ[2], academ[3], mm[1]):
        if compare_names2(academ['a_title'], academ['a_line'], academ['year'], mm['m_title'], mm['m_line'], mm['year']):
            # print(str(mm[3])+' ,,'+mm[1])
            # print(academ[1]+' ,,'+academ[2])
            # print(academ[1]+' ,,'+academ[3])
            # print('=======')
            if academ['id'] in already_filled:
                print(str(academ['id'])+'!'+str(mm['id'])+'!'
                      +academ['a_title']+'!'+academ['a_line']+'!'
                      +mm['m_title']+'!'+mm['m_line'])
            else:
                already_filled.add(academ['id'])
                num_found+=1
            #update DB
            #curDB.execute("""UPDATE `academ16` SET `metr_id` = %s WHERE  `id` = %s """, (str(mm[0]), str(academ[0])))

#Ok, we have all possible combinations, now let's find with the most weight
#print(weights)
#{2: [{'id': 399, 'weight': 0.7}], 4: [{'id': 288, 'weight': 0.7}]}
for key in weights:
    biggest = 0
    id_now = 0
    for m in weights[key]: #m - dict
        if m['weight']>biggest:
            id_now = m['id']
    if id_now>0:
        curDB.execute("""UPDATE `academ16` SET `metr_id` = %s WHERE  `id` = %s """, (str(id_now), str(key)))

print(len(weights))

print(num_found)


#!/usr/bin/env python

import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import numpy as np

def plot(lonGrid, latGrid, data, show=True):

  lllat = np.nanmin(latGrid)
  urlat = np.nanmax(latGrid)
  lllon = np.nanmin(lonGrid)
  urlon = np.nanmax(lonGrid)

  m = Basemap(projection='cyl', urcrnrlat=urlat, urcrnrlon=urlon, llcrnrlat=lllat, llcrnrlon=lllon)
  m.drawcoastlines(linewidth=.5)
  cnt = m.contourf(lonGrid, latGrid, data, cmap='jet')
  # for c in cnt.collections:
    # c.set_edgecolor('k')
    # c.set_linewidth(0.01)
  m.colorbar()
  m.drawparallels(np.arange(-90, 90, 25), labels=[True, False, False, False])
  m.drawmeridians(np.arange(-180, 180, 50), labels=[False, False, False, True])

  if (show):
    plt.show()


"""Tests for cx_Freeze.command.build_exe."""

from __future__ import annotations

import shutil
import sys
from pathlib import Path
from sysconfig import get_platform, get_python_version

from cx_Freeze.sandbox import run_setup

PLATFORM = get_platform()
PYTHON_VERSION = get_python_version()
BUILD_EXE_DIR = f"build/exe.{PLATFORM}-{PYTHON_VERSION}"

def test_build_exe(fix_main_samples_path: Path):
    """Test the simple sample."""
    setup_path = fix_main_samples_path / "simple"
    dist_created = setup_path / BUILD_EXE_DIR
    dist_already_exists = dist_created.exists()

    run_setup(setup_path / "setup.py", ["build_exe", "--silent"])

    suffix = ".exe" if sys.platform == "win32" else ""
    file_created = dist_created / f"hello{suffix}"
    assert file_created.is_file()
    file_created.unlink()

    if not dist_already_exists:
        shutil.rmtree(dist_created, ignore_errors=True)


# Cannon Fight

import random

#  00000000011111111112222222222333333333344444444445
#  12345678901234567890123456789012345678901234567890
# +==================================================+
#1|A                       O                        F|1
#2|                                                  |2
#3|B                       0                        G|3
#4|                                                  |4
#5|C                       O                        H|5
#6|                                                  |6
#7|D                       0                        I|7
#8|                                                  |8
#9|E                       O                        J|9
# +==================================================+
#  00000000011111111112222222222333333333344444444445
#  12345678901234567890123456789012345678901234567890
#Turn: Player 1
#Wind Speed: 5 mph
#Wind Driection: N
#You have 3 actions left

def printBoard(board):
    print('  00000000011111111112222222222333333333344444444445  ')
    print('  12345678901234567890123456789012345678901234567890  ')
    print(' +==================================================+ ')
    i = 0
    for y in range(9):
        print(str(y + 1) + '|', end='')
        for x in range(50):
            print(board[i], end='')
            i += 1
        print('|' + str(y + 1))
    print(' +==================================================+ ')
    print('  00000000011111111112222222222333333333344444444445  ')
    print('  12345678901234567890123456789012345678901234567890  ')

def isOnTopEdge(playerLoc):
    return playerLoc in list(range(50))

def isOnBottomEdge(playerLoc):
    return playerLoc in list(range(400, 450))

def isOnLeftEdge(playerLoc):
    return playerLoc in [0, 50, 150, 200, 250, 300, 350, 400]

def isOnRightEdge(playerLoc):
    return playerLoc in [49, 149, 199, 249, 299, 349, 399, 449]

def getCannonMove(playerLoc, turn, trueboard):
    while True:
        if turn == 0:
            print('Enter Direction: (W = up, S = Down, A = Left, D = Right)')
        elif turn == 2:
            print('Enter Direction: (I = up, K = Down, J = Left, L = Right)')
        direction = input().upper()
        if (direction in ['W', 'A', 'S', 'D'] and turn == 0) or (direction in ['I', 'J', 'K', 'L'] and turn == 2):
          if ((direction == 'W' and turn == 0) or (direction == 'I' and turn == 2)) and trueboard[playerLoc - 50] in [' ', '#'] and not isOnTopEdge(playerLoc):
            playerLoc -= 50
            return playerLoc
          elif ((direction == 'S' and turn == 0) or (direction == 'K' and turn == 2)) and trueboard[playerLoc + 50] in [' ', '#'] and not isOnBottomEdge(playerLoc):
            playerLoc += 50
            return playerLoc
          elif ((direction == 'A' and turn == 0) or (direction == 'J' and turn == 2)) and trueboard[playerLoc - 1] in [' ', '#'] and not isOnLeftEdge(playerLoc):
            playerLoc -= 1
            return playerLoc
          elif ((direction == 'D' and turn == 0) or (direction == 'L' and turn == 2)) and trueboard[playerLoc + 1] in [' ', '#'] and not isOnRightEdge(playerLoc):
            playerLoc += 1
            return playerLoc
          else:
            print('Invalid.', end=' ')
        else:
          print('Invaild.', end=' ')

def moveCannon(playerLoc, playerSprite, turn, board):
  board[playerLoc] = ' '
  playerLoc = getCannonMove(playerLoc, turn, board)
  board[playerLoc] = playerSprite
  print()
  return playerLoc

def getPlayerAction():
    while True:
        print('Enter action: (1. Move; 2. Fire; 3. Check Cannon; 4. Pass)')
        action = input()
        if action in ['1', '2', '3', '4']:
            return action
        else:
            print('Invalid.', end=' ')

player1Cannons = [['A', 0, 5.0, 25, 1], ['B', 100, 5.0, 25, 2], ['C', 200, 5.0, 25, 3], ['D', 300, 5.0, 25, 4], ['E', 400, 5.0, 25, 5]]
player2Cannons = [['F', 49, 5.0, 25, 1], ['G', 149, 5.0, 25, 2], ['H', 249, 5.0, 25, 3], ['I', 349, 5.0, 25, 4], ['J', 449, 5.0, 25, 5]]
mainBoard = [' '] * 450

for r in range(len(mainBoard)):
    if random.randint(0, 50) == 0:
        mainBoard[r] = '0'

for r in range(5):
    mainBoard[25 + (100 * r)] = '0'

for c1 in range(len(player1Cannons)):
    mainBoard[c1 * 100] = player1Cannons[c1][0]

for c2 in range(len(player2Cannons)):
    mainBoard[49 + (100 * c2)] = player2Cannons[c2][0]

cardinalDir = ['N', 'S', 'E', 'W']

turn = 0
multiplayer = True
thereIsWar = True
destroyed1 = []
destroyed2 = []
print('WELCOME TO THE CANNON FIGHT SIMULATOR')
print('============ (EST. 1324) ============')
print('Do you want any instructions? (y/n)')
ins = input().lower()
if ins == 'y':
    print('''
You have been at war. But you are using some of the most primitive fighting methods yet: Cannons. Well, ya gotta love the good old days.
Here is what the board may look like:
======================================================
  00000000011111111112222222222333333333344444444445
  12345678901234567890123456789012345678901234567890
 +==================================================+
1|A                       0     0                  F|1
2|            0                                     |2
3|B                 0     0          0    0        G|3
4|           0                                      |4
5|C                       0    0                   H|5
6|               0                                  |6
7|D                       0         0              I|7
8|             0                                    |8
9|E                       0               00       J|9
 +==================================================+
  00000000011111111112222222222333333333344444444445
  12345678901234567890123456789012345678901234567890
Turn: Player 1
Wind Speed: 5 mph
Wind Driection: N
You have 3 actions left
======================================================
You can do 4 things: Move, Fight, Check, or Pass.
Moving is simple: You move the "Cannons" (The uppercase letters on the board.) by 1 unit every turn. You cannot move into another cannon, or an obstacle. But you can move under scorch marks ('#')

Fighting is a bit more complicated. 4 factors come into play: Supplies, Wind, Obstacles, Range, and Direction. (SWORD)
SWORD is complicated since you have to deal with many things.
For the first part, Supplies. Each cannon is equipped with 5 lbs of gunpowder. And each cannon has an enormous arsenal of 100 cannonballs. You cannont replenish these things. Every time you fire, you use some gunpowder, and 1 cannonball. Here is the formula for calculating gunpowder use:
G = P / 100 (G is Gunpowder use, and P is Power)
For example, shooting 30 units (The numbers on the side) will use 0.3 lbs of gunpowder.
The next part is wind. Depending on how strong the wind is, your shot may veer to the side, get pushed back, or pushed forward! Be careful.
Then there are obstacles. Those little "0"s in the center are obstacles, which you cannot shoot, nor move past. These can serve as a friend, (Hiding behind a rock) or an enemy (Blocks your path / shots)
The last playing factors of SWORD are range and direction. Range is how far you can shoot, and direction is which way you can shoot.
No pressure, right?

The third action is checking a cannon. If you seem to fail to shoot a cannonball, that may be because you have run out of supplies. You can check using this action. And this is also the only thing you can do without using up actions.

The last thing you can do is pass, and move onto the next player's turn immediately.

Whoever destroys all of the opponent's cannons first wins. Good luck... You'll need it.
(Press enter/return to continue...)''')
    input()

while True:
    winner = None
    while thereIsWar:
        actionsLeft = 5
        while actionsLeft >= 1:
            printBoard(mainBoard)

            windSpeed = random.randint(0, 20)
            windDir = cardinalDir[random.randint(0, 3)]

            if turn == 0:
                print('Turn: Player 1')
            elif turn == 2:
                print('Turn: Player 2')
            print('Wind Speed: %s mph' % (windSpeed))
            print('Wind Direction: %s' % (windDir))
            print('You have %s action(s) left.' % (actionsLeft))
            action = getPlayerAction()
            if action == '1':
                while True:
                    if turn == 0:
                        print('Select Cannon To Move: (1 = A; 2 = B; 3 = C; 4 = D; 5 = E)')
                        cannon = input()
                        if cannon.isdigit() and int(cannon) in [1, 2, 3, 4, 5]:
                            break
                        else:
                            print('Invalid.', end=' ')
                    elif turn == 2:
                        print('Select Cannon To Move: (1 = F; 2 = G; 3 = H; 4 = I; 5 = J)')
                        cannon = input()
                        if cannon.isdigit() and int(cannon) in [1, 2, 3, 4, 5]:
                            break
                        else:
                            print('Invalid.', end=' ')
                cannon = int(cannon) - 1
                if turn == 0 and player1Cannons[cannon][0] != '#':
                    actionsLeft -= 1
                    player1Cannons[cannon][1] = moveCannon(player1Cannons[cannon][1], player1Cannons[cannon][0], 0, mainBoard)
                elif turn == 2 and player2Cannons[cannon][0] != '#':
                    actionsLeft -= 1
                    player2Cannons[cannon][1] = moveCannon(player2Cannons[cannon][1], player2Cannons[cannon][0], 2, mainBoard)
                else:
                    print('Invalid.')
            if action == '2':
                while True:
                    if turn == 0:
                        print('Select Cannon To Fire: (1 = A; 2 = B; 3 = C; 4 = D; 5 = E)')
                    elif turn == 2:
                        print('Select Cannon To Fire: (1 = F; 2 = G; 3 = H; 4 = I; 5 = J)')
                    cannon = input()
                    if cannon in ['1', '2', '3', '4', '5']:
                        break
                cannon = int(cannon) - 1
                if turn == 0 and player1Cannons[cannon][0] != '#':
                    actionsLeft -= 1
                    while True:
                        print('Set Power: (Remember: Range is preportional to gunpowder use. Shooting all the way across a field (50 units) will use about 0.5 lbs of gunpowder. But shooting only 10 units will use 0.1 lbs of gunpowder.)')
                        power = input()
                        if power.isdigit() and (int(power) >= 1 and int(power) <= 50):
                            power = int(power)
                            break
                    while True:
                        print('Set Direction: (N = Up; S = Down; E = Right; W = Left)')
                        direction = input().upper()
                        if direction in cardinalDir:
                            break
                    if player1Cannons[cannon][2] >= (power / 100) and player1Cannons[cannon][3] > 0:
                        player1Cannons[cannon][2] -= power / 100
                        player1Cannons[cannon][3] -= 1
                        cannonLoc = player1Cannons[cannon][1]
                        j = 0
                        if direction in cardinalDir:
                            if direction == 'N':
                                while not isOnTopEdge(cannonLoc) and mainBoard[cannonLoc - 50] != '0' and j < power:
                                    cannonLoc -= 50
                                    j += 1
                            elif direction == 'S':
                                while not isOnBottomEdge(cannonLoc) and mainBoard[cannonLoc + 50] != '0' and j < power:
                                    cannonLoc += 50
                                    j += 1
                            elif direction == 'E':
                                while not isOnLeftEdge(cannonLoc) and mainBoard[cannonLoc - 1] != '0' and j < power:
                                    cannonLoc -= 1
                                    j += 1
                            elif direction == 'W':
                                while not isOnRightEdge(cannonLoc) and mainBoard[cannonLoc + 1] != '0' and j < power:
                                    cannonLoc += 1
                                    j += 1
                        j = 0
                        if windDir == 'N': # Wind
                            while not isOnTopEdge(cannonLoc) and mainBoard[cannonLoc - 50] != '0' and j < random.randint(0, windSpeed):
                                cannonLoc -= 50
                                j += 1
                        elif windDir == 'S':
                            while not isOnBottomEdge(cannonLoc) and mainBoard[cannonLoc + 50] != '0' and j < random.randint(0, windSpeed):
                                cannonLoc += 50
                                j += 1
                        elif windDir == 'E':
                            while not isOnLeftEdge(cannonLoc) and mainBoard[cannonLoc - 1] != '0' and j < random.randint(0, windSpeed):
                                cannonLoc -= 1
                                j += 1
                        elif windDir == 'W':
                            while not isOnRightEdge(cannonLoc) and mainBoard[cannonLoc + 1] != '0' and j < random.randint(0, windSpeed):
                                cannonLoc += 1
                                j += 1
                        for c2 in range(len(player2Cannons) - 1):
                            if mainBoard[cannonLoc] == player2Cannons[c2][0]:
                                player2Cannons[c2][0] = '#'
                        mainBoard[cannonLoc] = '#'
                elif turn == 2 and player2Cannons[cannon][0] != '#':
                    actionsLeft -= 1
                    while True:
                        print('Set Power: (Remember: Range is preportional to gunpowder use. Shooting all the way across a field (50 units) will use about 0.5 lbs of gunpowder. But shooting only 10 units will use 0.1 lbs of gunpowder.)')
                        power = input()
                        if power.isdigit() and (int(power) >= 1 and int(power) <= 50):
                            power = int(power)
                            break
                    while True:
                        print('Set Direction: (N = Up; S = Down; E = Right; W = Left)')
                        direction = input().upper()
                        if direction in cardinalDir:
                            break
                    if player2Cannons[cannon][2] >= (power / 10) and player2Cannons[cannon][3] > 0:
                        cannonLoc = player2Cannons[cannon][1]
                        j = 0
                        if direction in cardinalDir:
                            if direction == 'N':
                                while not isOnTopEdge(cannonLoc) and mainBoard[cannonLoc - 50] != '0' and j < power:
                                    cannonLoc -= 50
                                    j += 1
                            elif direction == 'S':
                                while not isOnBottomEdge(cannonLoc) and mainBoard[cannonLoc + 50] != '0' and j < power:
                                    cannonLoc += 50
                                    j += 1
                            elif direction == 'E':
                                while not isOnLeftEdge(cannonLoc) and mainBoard[cannonLoc - 1] != '0' and j < power:
                                    cannonLoc -= 1
                                    j += 1
                            elif direction == 'W':
                                while not isOnRightEdge(cannonLoc) and mainBoard[cannonLoc + 1] != '0' and j < power:
                                    cannonLoc += 1
                                    j += 1
                        j = 0
                        if windDir == 'N': # Wind
                            while not isOnTopEdge(cannonLoc) and mainBoard[cannonLoc - 50] != '0' and j < random.randint(0, windSpeed):
                                cannonLoc -= 50
                                j += 1
                        elif windDir == 'S':
                            while not isOnBottomEdge(cannonLoc) and mainBoard[cannonLoc + 50] != '0' and j < random.randint(0, windSpeed):
                                cannonLoc += 50
                                j += 1
                        elif windDir == 'E':
                            while not isOnLeftEdge(cannonLoc) and mainBoard[cannonLoc - 1] != '0' and j < random.randint(0, windSpeed):
                                cannonLoc -= 1
                                j += 1
                        elif windDir == 'W':
                            while not isOnRightEdge(cannonLoc) and mainBoard[cannonLoc + 1] != '0' and j < random.randint(0, windSpeed):
                                cannonLoc += 1
                                j += 1
                        for c1 in range(len(player1Cannons) - 1):
                            if mainBoard[cannonLoc] == player1Cannons[c1][0]:
                                player1Cannons[c1][0] = '#'
                        mainBoard[cannonLoc] = '#'
                else:
                    print('Invalid.')
            elif action == '3':
                while True:
                    if turn == 0:
                        print('Select Cannon To Check: (1 = A; 2 = B; 3 = C; 4 = D; 5 = E)')
                    elif turn == 2:
                        print('Select Cannon To Check: (1 = F; 2 = G; 3 = H; 4 = I; 5 = J)')
                    cannon = input()
                    if cannon in ['1', '2', '3', '4', '5']:
                        cannon = int(cannon) - 1
                        break
                print('============================')
                print('''                    +
                  +
             _,--\\ +
       _   _- |   \\
      *_\\--   | _-/'
     -- **   _|_^  *
    /    _-/*-|-*\\ *
    |  _-  |->O<-| *
    \\--____\\~-|-~/_*____
        ''')
                if turn == 0 and player1Cannons[cannon][0] != '#':
                    print('Cannon: %s' % (player1Cannons[cannon][0]))
                    print('Gunpowder: %s lbs' % (round(player1Cannons[cannon][2], 2)))
                    print('Ammunition: %s cannonballs' % (player1Cannons[cannon][3]))
                elif turn == 2 and player2Cannons[cannon][0] != '#':
                    print('Cannon: %s' % (player2Cannons[cannon][0]))
                    print('Gunpowder: %s' % (player2Cannons[cannon][2]))
                    print('Ammunition: %s' % (player2Cannons[cannon][3]))
                else:
                    print('     CANNON DESTROYED')
                print('============================')
                print('(Press enter/return to continue...)')
                input()
            elif action == '4':
                actionsLeft = 0
                break
        for c1 in player1Cannons:
            if c1[0] == '#':
                destroyed1.append(c1[4])
        for c2 in player2Cannons:
            if c2[0] == '#':
                destroyed2.append(c2[4])

        if 1 in destroyed1 and 2 in destroyed1 and 3 in destroyed1 and 4 in destroyed1 and 5 in destroyed1:
            thereIsWar = False
            winner = 2
            break
        if 1 in destroyed2 and 2 in destroyed2 and 3 in destroyed2 and 4 in destroyed2 and 5 in destroyed2:
            thereIsWar = False
            winner = 0
            break

        if turn == 0:
            if multiplayer:
                turn = 2
            else:
                turn = 1
        elif turn == 2 or turn == 1:
            turn = 0
    if winner == 0:
        print('Player 1 Wins! Good Job.')
    if winner == 2:
        print('Player 2 Wins! Good Job.')
    print('Play Again? (y/n)')
    playAgain = input().lower()
    if playAgain == 'n':
        break
    else:
        winner = None
        thereIsWar = True


from django.shortcuts import render

from django.http import HttpResponse
from django.template import loader,RequestContext
# Create your views here.

from models import Entry

def index(request):
#	return HttpResponse('Hello Django')
	contexts=RequestContext(request,{
		'message': 'OK',
		})

	template = loader.get_template('polls/index.html')

	return HttpResponse(template.render(contexts))

def createrecord(request):

	en = Entry()
	en.title = 'hogehoge'
	en.body = 'fuga'

	en.save()

	contexts=RequestContext(request,{
		'message': 'createrecord',
		})

	template = loader.get_template('polls/index.html')

	return HttpResponse(template.render(contexts))

def showrecord(request):

	count = Entry.objects.count()

#	return HttpResponse('Hello Django')
	contexts=RequestContext(request,{
		'message': count,
		})

	template = loader.get_template('polls/index.html')

	return HttpResponse(template.render(contexts))

     # url(r'createrecord', createrecord ),
     # url(r'showrecord', showrecord ),


__author__ = 'panderson'
plus = '+'
minus = '-'
bang = '|'
space = '    '
grid = 2
wgrid = grid
grid_counter = 1
linout = ''

def border():
	linout = (plus+minus*4)
	return linout

def filler():
	linout = (bang+space)
	return linout

def fifth(n):
	if n%5 == 0:
		return True
	else:
		return False

while (grid_counter <= grid):
	i=0
	while i<=4:
		if fifth(i):
			linout = border()*wgrid+plus
		else:
			linout = filler()*wgrid+bang
		print (linout)
		i = i+1
	grid_counter+=1
linout = border()*wgrid+plus
print (linout)

print ('finished')


from subprocess import Popen, PIPE
import matplotlib.pyplot as plt
import csv

class geneticAlgorithm():

    def __init__(self):
        pass

    def fitness(self, wekaConfig):
        print("Procesando {0}".format(wekaConfig))
        learning_rate =  float(wekaConfig[4]) / 100
        momentum = float(wekaConfig[3]) / 100
        hidden_layers_string = ",".join([ str(wekaConfig[0]) for index in range(0, wekaConfig[1])])
        multilayerPerceptron = "java -cp weka.jar weka.classifiers.functions.MultilayerPerceptron -L {0} -M {1} -N {2} -V 0 -S 0 -E 20 -H {3} -t physhing.arff".format(learning_rate, momentum, wekaConfig[2],hidden_layers_string).split(" ")
        result = Popen(multilayerPerceptron,stdout= PIPE)
        output = result.stdout.read().decode("utf-8")
        wekaResult = [ item for item in output.split("\n") if "Correctly Classified Instances" in item ][1].split()
        print("Procesado por weka")
        return wekaResult[-2]

    def sortData(self):
        with open("resultado.csv", 'r', newline='') as f_input:
            csv_input = csv.reader(f_input)
            next(csv_input)
            sortedData = sorted(csv_input, key=lambda row: (row[-1]), reverse=False)
        return sortedData

    def showGraph(self, data):
        x = []
        y = []
        for index in range(0, len(data)):
            y.append(index)
            x.append(data[index][-1])

        plt.plot(x,y, label='Resultados')
        plt.xlabel('x')
        plt.ylabel('y')
        plt.title('Porcentaje de instancias correctamente clasificadas')
        plt.legend()
        plt.show()

    def convertToBinaryRepresentation(self):
        data = self.sortData()
        binaryRepresentation = []
        for row in data[:50]:
            layers_to_bin = '{0:02b}'.format(int(row[0]))
            neurons_to_bin = '{0:04b}'.format(int(row[1]))
            epochs_to_bin = '{0:012b}'.format(int(row[2]))
            momentum_to_bin = '{0:06b}'.format(int(row[3]))
            learning_date_to_bin = '{0:06b}'.format(int(row[4]))
            binaryRepresentation.append([layers_to_bin, neurons_to_bin, epochs_to_bin, momentum_to_bin, learning_date_to_bin])
        return binaryRepresentation

    def generateDecentens(self):
        binaryData = self.convertToBinaryRepresentation()
        dataPairs = [ binaryData[i:i+2] for i in range(0, 50, 2) ]
        decendents = [ self.crossGenes(row[0], row[1]) for row in dataPairs ]
        result = []
        for binaryString in decendents:
            layers_to_dec = int(binaryString[0], 2)
            if layers_to_dec < 1:
                layers_to_dec = 1
            neurons_to_dec = int(binaryString[1], 2)
            epochs_to_dec = int(binaryString[2], 2)
            if epochs_to_dec < 100:
                epochs_to_dec = 100
            momentum_to_dec = int(binaryString[3], 2)
            learning_to_dec = int(binaryString[4], 2)
            clasified_instances = self.fitness([layers_to_dec, neurons_to_dec, epochs_to_dec, momentum_to_dec, learning_to_dec])
            result.append([layers_to_dec, neurons_to_dec, epochs_to_dec, momentum_to_dec, learning_to_dec, clasified_instances])
        return result

    def crossGenes(self, x, y):
        return [
            self.stepOperator(x[0], y[0]),
            self.crossOperator(x[1], y[1]),
            self.crossOperator(x[2], y[2]),
            self.stepOperator(x[3], y[3]),
            self.stepOperator(x[4], y[4])
        ]

    def stepOperator(self, x, y):
        half = round(len(x)/2)
        return str(x[:half] + y[half:])

    def crossOperator(self, x, y):
        half = round(len(x)/2)
        xHalf = x[:half]
        yHalf = y[half:]
        newChild = "".join([ str(xHalf[index]) + str(yHalf[index]) for index in range(0, half) ])
        return newChild

def generations():
    number_of_generations = int(input("Numero de generaciones: "))
    genetic = geneticAlgorithm()
    new_generations = []
    for generation in range(0, number_of_generations):
        new_generations.append({
            'generation_number': generation,
            'descendents': genetic.generateDecentens()
        })
    for row in new_generations:
        print("Generacion {0} : {1}".format(row['generation_number'], row['descendents']))

    option = input("Desea guardar el resultado en el archivo resultado.csv ? S/N")
    if option in ['s', 'S']:
        with open("resultado.csv", "a") as csvfile:
            result = csv.writer(csvfile, delimiter=',')
            for row in new_generations:
                result.writerows(row['descendents'])

def switch(option):
    genetic = geneticAlgorithm()
    if option == "1":
        generations()
    if option == "2":
        genetic.showGraph(genetic.sortData())
    else:
        print("Saliendo")

if __name__ == "__main__":

    print("Que desea hacer ? \n 1.- Generar individuos  \n 2.- Mostrar grafica \n 3.- Salir")
    option = input("Ingrese opcion: ")
    switch(option)


from numpy import (mean, std, zeros, max, min, exp, tanh, log, sum, cumsum,
        append)

from numpy.random import RandomState
from numpy.linalg import norm

def sech2(data):
    return 1 - tanh(data)**2

def sign(value):
    return -1. if value < 0 else 1.

def sharpe(returns):
    s = zeros(len(returns))
    s[0] = returns[0]

    for i in range(1, len(returns)):
        mu = mean(returns[:i+1])
        sigma = std(returns[:i+1])

        if sigma == 0:
           continue

        s[i] = mu / sigma

    return s

def wealth(returns):
    return exp(cumsum(log(1 + returns)))

def synthetic(n, phi=0.9, k=3, var=1, seed=1):
    """Auto-regressive test data generator."""
    prng = RandomState(seed)
    P = zeros((n,))
    X = zeros((n,))

    # Initial values for P, X.
    P[0] = prng.normal(0,var)
    X[0] = prng.normal(0, var)

    for i in xrange(1, n):
        P[i] = P[i-1] + X[i-1] + k*prng.normal(0, var)
        X[i] = phi*X[i-1] + prng.normal(0, var)

    R = max(P) - min(P)
    Z = exp(P/R)

    return Z


# -*- coding: utf-8 -*-

import os
import sys
import json
import hashlib
import argparse
import requests

from transformers import BertConfig, BertModel

HUGGINGFACE_VOCAB_FILE = "kobert_vocab_huggingface_format.txt"
NEW_HUGGINGFACE_VOCAB_FILE = "new_kobert_vocab_huggingface_format.txt"
KOBERT_CONFIG_FILE = "kobert-8002-config.json"
DISTILKOBERT_CONFIG_FILE = "distilkobert_student_config.json"
KOBERT_TOKENIZER_CONFIG_FILE = "kobert_tokenizer_config.json"

kobert_models = {
    'pytorch_kobert': {
        'url':
        'https://kobert.blob.core.windows.net/models/kobert/pytorch/pytorch_kobert_2439f391a6.params',
        'fname': 'pytorch_kobert_2439f391a6.params',
        'chksum': '2439f391a6'
    },
    'vocab': {
        'url':
        'https://kobert.blob.core.windows.net/models/kobert/vocab/kobertvocab_f38b8a4d6d.json',
        'fname': 'kobertvocab_f38b8a4d6d.json',
        'chksum': 'f38b8a4d6d'
    }
}

kobert_config = {
    'attention_probs_dropout_prob': 0.1,
    'hidden_act': 'gelu',
    'hidden_dropout_prob': 0.1,
    'hidden_size': 768,
    'initializer_range': 0.02,
    'intermediate_size': 3072,
    'max_position_embeddings': 512,
    'num_attention_heads': 12,
    'num_hidden_layers': 12,
    'type_vocab_size': 2,
    'vocab_size': 8002
}

distilkobert_config = {
    "activation": "gelu",
    "attention_dropout": 0.1,
    "dim": 768,
    "dropout": 0.1,
    "finetuning_task": None,
    "hidden_dim": 3072,
    "initializer_range": 0.02,
    "max_position_embeddings": 512,
    "n_heads": 12,
    "n_layers": 6,
    "num_labels": 2,
    "output_attentions": False,
    "output_hidden_states": False,
    "pruned_heads": {},
    "qa_dropout": 0.1,
    "seq_classif_dropout": 0.2,
    "sinusoidal_pos_embds": False,
    "tie_weights_": True,
    "torchscript": False,
    "vocab_size": 8002
}

kobert_tokenizer_config = {
    "vocab_file": None,
    "do_lower_case": False,
    "do_basic_tokenize": True,
    "never_split": None,
    "unk_token": "[UNK]",
    "sep_token": "[SEP]",
    "pad_token": "[PAD]",
    "cls_token": "[CLS]",
    "mask_token": "[MASK]",
    "tokenize_chinese_chars": False
}

def download(url, filename, chksum, cachedir='.kobert/'):
    f_cachedir = os.path.expanduser(cachedir)
    os.makedirs(f_cachedir, exist_ok=True)
    file_path = os.path.join(f_cachedir, filename)
    if os.path.isfile(file_path):
        if hashlib.md5(open(file_path,
                            'rb').read()).hexdigest()[:10] == chksum:
            print('using cached model')
            return file_path
    with open(file_path, 'wb') as f:
        response = requests.get(url, stream=True)
        total = response.headers.get('content-length')

        if total is None:
            f.write(response.content)
        else:
            downloaded = 0
            total = int(total)
            for data in response.iter_content(
                    chunk_size=max(int(total / 1000), 1024 * 1024)):
                downloaded += len(data)
                f.write(data)
                done = int(50 * downloaded / total)
                sys.stdout.write('\r[{}{}]'.format('█' * done,
                                                   '.' * (50 - done)))
                sys.stdout.flush()
    sys.stdout.write('\n')
    assert chksum == hashlib.md5(open(
        file_path, 'rb').read()).hexdigest()[:10], 'corrupted file!'
    return file_path

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Download KoBERT weights and vocabulary.')
    parser.add_argument('--cache_dir', type=str, default='.kobert/')
    args = parser.parse_args()

    # Download model
    model_info = kobert_models['pytorch_kobert']
    model_path = download(
        url=model_info['url'],
        filename=model_info['fname'],
        chksum=model_info['chksum'],
        cachedir=args.cache_dir,
    )
    print(f"Downloaded model to `{model_path}`")

    # Download vocabulary
    vocab_info = kobert_models['vocab']
    vocab_path = download(
        url=vocab_info['url'],
        filename=vocab_info['fname'],
        chksum=vocab_info['chksum'],
        cachedir=args.cache_dir,
    )
    print(f"Downloaded vocab to `{vocab_path}`")

    # Load vocab
    with open(vocab_path, 'rt') as f:
        vocab = json.load(f)
    print("Loaded vocabulary file.")

    # Save vocab in huggingface format
    vocab_file_huggingface = os.path.join(args.cache_dir, HUGGINGFACE_VOCAB_FILE)
    with open(vocab_file_huggingface, 'wt', encoding='utf-8') as f:
        f.write('\n'.join(vocab.get('idx_to_token')))

    # Preprocess vocabulary file (_ -> ##)
    tokens = []
    for token in vocab['idx_to_token']:
        if token == '▁':
            tokens.append(token)
        else:
            token = token.replace('▁', '##')
            tokens.append(token)

    # Save new vocab in huggingface format
    new_vocab_file_huggingface = os.path.join(args.cache_dir, NEW_HUGGINGFACE_VOCAB_FILE)
    with open(new_vocab_file_huggingface, 'wt', encoding='utf-8') as f:
        f.write('\n'.join(tokens))
    print("Wrote vocab for huggingface's BertTokenizer class.")

    # Save KoBERT model using `.save_pretrained`
    kobert_model = BertModel(config=BertConfig.from_dict(kobert_config))
    pretrained_dir = os.path.join(args.cache_dir, 'pretrained/')
    os.makedirs(pretrained_dir, exist_ok=True)
    kobert_model.save_pretrained(save_directory=pretrained_dir)
    print(f"Saved KoBERT model using `.save_pretrained.` to {pretrained_dir}")

    # Save KoBERT configurations
    with open(os.path.join(args.cache_dir, KOBERT_CONFIG_FILE), 'w') as f:
        json.dump(kobert_config, f, indent=4)
    print("Saved KoBERT configurations.")

    # Save DistilKoBERT configurations
    with open(os.path.join(args.cache_dir, DISTILKOBERT_CONFIG_FILE), 'w') as f:
        json.dump(distilkobert_config, f, indent=4)

    # Save KoBERT tokenizer configurations
    kobert_tokenizer_config['vocab_file'] = os.path.join(args.cache_dir, NEW_HUGGINGFACE_VOCAB_FILE)
    with open(os.path.join(args.cache_dir, KOBERT_TOKENIZER_CONFIG_FILE), 'w') as f:
        json.dump(kobert_tokenizer_config, f, indent=4)


# reference video for calculating simple linear regression https://www.youtube.com/watch?v=JvS2triCgOY
#====to calculate h(xi)= b0 + b1xi====#
#b0 is y-intercept
#b1 is slope

'''
xbar = mean of x = mean_x
ybar = mean of y = mean_y

b1 = sum(x-xbar).(y-ybar) / sum((x-xbar)square)

therefore for b1 we have to calculate
(x-xbar), (y-ybar),  sum(x-xbar),  sum((x-xbar)square)
(x-meanx),(y-meany), sum(x-meanx), sum((x-meanx)square)

'''

#import
import numpy as np

# Dataset
x = np.array([1,2,3,4,5])
y = np.array([2,4,5,4,5])

#calculating using normal method
'''

# mean of x and y vector
meanx = np.mean(x)
meany = np.mean(y)

# calculating sum(x-xbar).(y-ybar)
x_min_xbar_into_y_min_ybar = 0
for i in x:
    x_min_xbar_into_y_min_ybar = x_min_xbar_into_y_min_ybar + ( (x[i-1] - meanx) * (y[i-1] - meany) )
    print(x_min_xbar_into_y_min_ybar)
'''

#calculating sum((x-xbar).(y-ybar)) using numpy
sum_of_x_min_xbar_into_y_min_ybar = 0
sum_of_x_min_xbar_into_y_min_ybar = np.sum((x - x.mean())*(y-y.mean()))

#calculating sum((x-xbar)square) using numpy
sum_of_x_min_xbar_square =np.sum(np.square(x-x.mean()))

#calculating b1 formula : b1 = sum( (x-xbar).(y-ybar) ) / sum( (x-xbar)square) )
b1 = sum_of_x_min_xbar_into_y_min_ybar / sum_of_x_min_xbar_square

#calculating b0 :
#   will substitute in formula y = b0 + b1 as from calculating the mean we have got base case values fro x and y
#   which are x = 3 , y = 4 (we got it from calcuating the mean)
#formula    y = b0 + b1.x
# therefore b0 = b1.x - y
b0 = b1*meanx - meany

#Training model


from urllib import request,parse
import threading
import re
import os
import sys

class InfoFinder:

    IDs = set()

    # Update crawled movies from file. This method should be run only once
    @staticmethod
    def update_file(filename):
        if os.path.isfile(filename):
            f = open(filename,'r',encoding='utf-8')
            lines = f.readlines()
            f.close()
            for line in lines[1:]:
                ID = line.split(',')[0][2:]
                InfoFinder.IDs.add(ID)
        else:
            f = open(filename,'w')
            f.write('IMDBid,Name,Year,Rating,Genres,Runtime,Outline\n')
            f.close()

    def __init__(self):
        self.movies = dict()

    # Main function: parse a page of movies searching results
    def parse_page(self,page_url):
        self.movies = dict()
        raw_data = self.get_raw_data(page_url)
        Partition = re.findall(r'detailed">((.|\s|\n)*?)
',raw_data)
        for part in Partition:
            if type(part) is tuple:
                part = ''.join(part)
            ID = (re.findall(r'title/tt(.*?)/',part)+[''])[0]
            if ID and ID not in InfoFinder.IDs:
                InfoFinder.IDs.add(ID)
                Info = self.find_movie_info(ID,part)
                self.movies[ID] = Info

    # Parse the part of raw data of one movie
    @staticmethod
    def find_movie_info(ID,data):
        Name = (re.findall(ID+r'/">(.*?)',data)+[' '])[0]
        Year = (re.findall(r'"year_type">\((.*?)\)',data)+[' '])[0]
        Rating = (re.findall(r'Users rated this (.*?)/10',data)+[' '])[0]
        Outline = (re.findall(r'"outline">(.*?)',data)+[' '])[0]
        Genres = re.findall(r'genre/(.*?)">',data)+[' ']
        Runtime = (re.findall(r'"runtime">(.*?)',data)+[' '])[0]

        info = {'Name':Name,'Year':Year,'Rating':Rating,'Outline':Outline,
                'Genres':'|'+''.join([g+'|' for g in Genres[:-1]]),'Runtime':Runtime}

        return info

    # Getting data from movie's page
    # This function is for backup only
    def get_info_by_page(self,ID):
        url = 'http://www.imdb.com/title/tt'+ID+'/'
        data = self.get_raw_data(url)

        string1 = re.findall(r'<title>(.*?)</title>',data)[0]
        name,year = self.get_name_year(string1)
        self.movies[ID]['Name'] = name
        self.movies[ID]['Year'] = year

        res = re.findall(r'"(.*?)\sbased\son.*?user\sratings',data)
        if res:
            self.movies[ID]['Rating'] = res[0]

    # Write all info of one instance to file
    def info_to_file(self,filename):
        f = open(filename,'ab')
        for ID,info in self.movies.items():
            text_to_write = self.info_to_text(ID,info)
            try:
                f.write(text_to_write.encode('utf-8'))
            except:
                print("Codec Error: IMDBid =",ID)
        f.close()

    # Convert info to a line of text
    @staticmethod
    def info_to_text(ID,info):
        Name = ',"'+info['Name']+'"'
        Year = ','+info['Year']
        Rating = ','+info['Rating']
        Outline = ',"'+info['Outline']+'"'
        Genres = ','+info['Genres']
        Runtime = ','+info['Runtime']

        return 'tt'+ID+Name+Year+Rating+Genres+Runtime+Outline+'\n'

    # Generate raw data
    @staticmethod
    def get_raw_data(url):
        try:
            resp = request.urlopen(url)
            data = resp.read().decode('utf-8')
            return data
        except Exception as e:
            print(str(e))
            return ''

# Threading class
class T(threading.Thread):

    def __init__(self,values,startpoints):
        super().__init__()
        self.values = values
        self.startpoints = startpoints
        self.MIF = InfoFinder()

    def run(self):
        for sp in self.startpoints:
            self.values['start'] = str(sp)
            current_url = 'http://www.imdb.com/search/title?'+parse.urlencode(self.values)
            print("Parsing:",sp,'-',min(N,sp+49),'  /',N)
            self.MIF.parse_page(current_url)
            self.MIF.info_to_file(FILENAME)

def main():
    global FILENAME,N
    FILENAME = 'movies_info.csv'

    genres = {'action','adventure','animation','biography','comedy',
            'crime','documentary','drama','family','fantasy','film_noir',
            'history','horror','music','musical','mystery','romance',
            'sci_fi','sport','thriller','war','western'}

    VALUES = {'start':'1','title_type':'feature'}

    for argv in sys.argv[1:]:
        var_val = argv.split('=')
        if len(var_val) == 2:
            var = var_val[0]
            val = var_val[1]
            VALUES[var] = val
            FILENAME = val+'_'+FILENAME
            print(var,':',val)

    InfoFinder.update_file(FILENAME)
    N = init(VALUES)
    numb_of_T = min(N//50+1,16)
    Ts = []

    print("="*8+'START PARSING'+"="*8)

    for i in range(numb_of_T):
        StartPoints = range(i*50+1,N+1,50*numb_of_T)
        Ts.append(T(VALUES,StartPoints))
        Ts[i].start()

def init(values):
    url = 'http://www.imdb.com/search/title?'+parse.urlencode(values)
    print("Initialized url:",url,"\n\nConnecting to IMDB...")
    try:
        data = request.urlopen(url).read().decode()
        res = re.findall(r'1-50\sof\s(.*)',data)
        N = int(''.join(res[0].strip().split(',')))
        print("\nTotal",N,"titles\n")
        return N
    except Exception as e:
        print(str(e)+"\nUnable to proceed")
        quit()

main()