# Copyright (c) 2022-2023, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import annotations

import warnings

import numpy
from dask import delayed
from dask.distributed import Lock, get_client, wait

import dask_cudf
import cudf
import cupy as cp

from pylibcugraph import ResourceHandle

from pylibcugraph import uniform_neighbor_sample as pylibcugraph_uniform_neighbor_sample
from pylibcugraph.utilities.api_tools import deprecated_warning_wrapper

from cugraph.dask.comms import comms as Comms
from cugraph.dask.common.input_utils import get_distributed_data
from cugraph.dask import get_n_workers

from typing import Sequence, List, Union, Tuple
from typing import TYPE_CHECKING

from cugraph.dask.common.part_utils import (
    get_persisted_df_worker_map,
    persist_dask_df_equal_parts_per_worker,
)

if TYPE_CHECKING:
    from cugraph import Graph

src_n = "sources"
dst_n = "destinations"
indices_n = "indices"
weight_n = "weight"
edge_id_n = "edge_id"
edge_type_n = "edge_type"
batch_id_n = "batch_id"
offsets_n = "offsets"
hop_id_n = "hop_id"

map_n = "map"
map_offsets_n = "renumber_map_offsets"

start_col_name = "_START_"
batch_col_name = "_BATCH_"

def create_empty_df(indices_t, weight_t):
    df = cudf.DataFrame(
        {
            src_n: numpy.empty(shape=0, dtype=indices_t),
            dst_n: numpy.empty(shape=0, dtype=indices_t),
            indices_n: numpy.empty(shape=0, dtype=weight_t),
        }
    )
    return df

def create_empty_df_with_edge_props(
    indices_t, weight_t, return_offsets=False, renumber=False
):
    if renumber:
        empty_df_renumber = cudf.DataFrame(
            {
                map_n: numpy.empty(shape=0, dtype=indices_t),
                map_offsets_n: numpy.empty(shape=0, dtype="int32"),
            }
        )

    if return_offsets:
        df = cudf.DataFrame(
            {
                src_n: numpy.empty(shape=0, dtype=indices_t),
                dst_n: numpy.empty(shape=0, dtype=indices_t),
                weight_n: numpy.empty(shape=0, dtype=weight_t),
                edge_id_n: numpy.empty(shape=0, dtype=indices_t),
                edge_type_n: numpy.empty(shape=0, dtype="int32"),
                hop_id_n: numpy.empty(shape=0, dtype="int32"),
            }
        )
        empty_df_offsets = cudf.DataFrame(
            {
                offsets_n: numpy.empty(shape=0, dtype="int32"),
                batch_id_n: numpy.empty(shape=0, dtype="int32"),
            }
        )

        if renumber:
            return df, empty_df_offsets, empty_df_renumber
        else:
            return df, empty_df_offsets
    else:
        df = cudf.DataFrame(
            {
                src_n: numpy.empty(shape=0, dtype=indices_t),
                dst_n: numpy.empty(shape=0, dtype=indices_t),
                weight_n: numpy.empty(shape=0, dtype=weight_t),
                edge_id_n: numpy.empty(shape=0, dtype=indices_t),
                edge_type_n: numpy.empty(shape=0, dtype="int32"),
                hop_id_n: numpy.empty(shape=0, dtype="int32"),
                batch_id_n: numpy.empty(shape=0, dtype="int32"),
            }
        )
        if renumber:
            return df, empty_df_renumber
        else:
            return df

def convert_to_cudf(
    cp_arrays, weight_t, with_edge_properties, return_offsets=False, renumber=False
):
    """
    Creates a cudf DataFrame from cupy arrays from pylibcugraph wrapper
    """
    df = cudf.DataFrame()

    if with_edge_properties:
        if renumber:
            (
                sources,
                destinations,
                weights,
                edge_ids,
                edge_types,
                batch_ids,
                offsets,
                hop_ids,
                renumber_map,
                renumber_map_offsets,
            ) = cp_arrays
        else:
            (
                sources,
                destinations,
                weights,
                edge_ids,
                edge_types,
                batch_ids,
                offsets,
                hop_ids,
            ) = cp_arrays

        df[src_n] = sources
        df[dst_n] = destinations
        df[weight_n] = weights
        df[edge_id_n] = edge_ids
        df[edge_type_n] = edge_types
        df[hop_id_n] = hop_ids

        return_dfs = [df]

        if return_offsets:
            offsets_df = cudf.DataFrame(
                {
                    batch_id_n: batch_ids,
                    offsets_n: offsets[:-1],
                }
            )

            if renumber:
                offsets_df[map_offsets_n] = renumber_map_offsets[:-1]

            return_dfs.append(offsets_df)
        else:
            batch_ids_b = batch_ids
            if len(batch_ids_b) > 0:
                batch_ids_b = cudf.Series(batch_ids_b).repeat(cp.diff(offsets))
                batch_ids_b.reset_index(drop=True, inplace=True)

            df[batch_id_n] = batch_ids_b

        if renumber:
            renumber_df = cudf.DataFrame(
                {
                    "map": renumber_map,
                }
            )

            if not return_offsets:
                batch_ids_r = cudf.Series(batch_ids).repeat(
                    cp.diff(renumber_map_offsets)
                )
                batch_ids_r.reset_index(drop=True, inplace=True)
                renumber_df["batch_id"] = batch_ids_r

            return_dfs.append(renumber_df)

        return tuple(return_dfs)
    else:
        cupy_sources, cupy_destinations, cupy_indices = cp_arrays

        df[src_n] = cupy_sources
        df[dst_n] = cupy_destinations
        df[indices_n] = cupy_indices

        if cupy_indices is not None:
            if weight_t == "int32":
                df.indices = df.indices.astype("int32")
            elif weight_t == "int64":
                df.indices = df.indices.astype("int64")

        return (df,)

def __get_label_to_output_comm_rank(min_batch_id, max_batch_id, n_workers):
    num_batches = max_batch_id - min_batch_id + 1
    num_batches = int(num_batches)
    z = cp.zeros(num_batches, dtype="int32")
    s = cp.array_split(cp.arange(num_batches), n_workers)
    for i, t in enumerate(s):
        z[t] = i

    return z

def _call_plc_uniform_neighbor_sample(
    sID,
    mg_graph_x,
    st_x,
    keep_batches_together,
    n_workers,
    min_batch_id,
    max_batch_id,
    fanout_vals,
    with_replacement,
    weight_t,
    with_edge_properties,
    random_state=None,
    return_offsets=False,
    return_hops=True,
    prior_sources_behavior=None,
    deduplicate_sources=False,
    renumber=False,
):
    st_x = st_x[0]
    start_list_x = st_x[start_col_name]
    batch_id_list_x = st_x[batch_col_name] if batch_col_name in st_x else None

    label_list = None
    label_to_output_comm_rank = None
    if keep_batches_together:
        label_list = cp.arange(min_batch_id, max_batch_id + 1, dtype="int32")
        label_to_output_comm_rank = __get_label_to_output_comm_rank(
            min_batch_id, max_batch_id, n_workers
        )

    cp_arrays = pylibcugraph_uniform_neighbor_sample(
        resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()),
        input_graph=mg_graph_x,
        start_list=start_list_x,
        label_list=label_list,
        label_to_output_comm_rank=label_to_output_comm_rank,
        h_fan_out=fanout_vals,
        with_replacement=with_replacement,
        do_expensive_check=False,
        with_edge_properties=with_edge_properties,
        batch_id_list=batch_id_list_x,
        random_state=random_state,
        prior_sources_behavior=prior_sources_behavior,
        deduplicate_sources=deduplicate_sources,
        return_hops=return_hops,
        renumber=renumber,
    )
    return convert_to_cudf(
        cp_arrays,
        weight_t,
        with_edge_properties,
        return_offsets=return_offsets,
        renumber=renumber,
    )

def _call_plc_uniform_neighbor_sample_legacy(
    sID,
    mg_graph_x,
    st_x,
    label_list,
    label_to_output_comm_rank,
    fanout_vals,
    with_replacement,
    weight_t,
    with_edge_properties,
    random_state=None,
    return_offsets=False,
    return_hops=True,
):
    start_list_x = st_x[start_col_name]
    batch_id_list_x = st_x[batch_col_name] if batch_col_name in st_x else None
    cp_arrays = pylibcugraph_uniform_neighbor_sample(
        resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()),
        input_graph=mg_graph_x,
        start_list=start_list_x,
        label_list=label_list,
        label_to_output_comm_rank=label_to_output_comm_rank,
        h_fan_out=fanout_vals,
        with_replacement=with_replacement,
        do_expensive_check=False,
        with_edge_properties=with_edge_properties,
        batch_id_list=batch_id_list_x,
        random_state=random_state,
        return_hops=return_hops,
    )

    output = convert_to_cudf(
        cp_arrays, weight_t, with_edge_properties, return_offsets=return_offsets
    )

    if isinstance(output, (list, tuple)) and len(output) == 1:
        return output[0]
    return output

def _mg_call_plc_uniform_neighbor_sample_legacy(
    client,
    session_id,
    input_graph,
    ddf,
    label_list,
    label_to_output_comm_rank,
    fanout_vals,
    with_replacement,
    weight_t,
    indices_t,
    with_edge_properties,
    random_state,
    return_offsets=False,
    return_hops=True,
):
    result = [
        client.submit(
            _call_plc_uniform_neighbor_sample_legacy,
            session_id,
            input_graph._plc_graph[w],
            ddf[w][0],
            label_list,
            label_to_output_comm_rank,
            fanout_vals,
            with_replacement,
            weight_t=weight_t,
            with_edge_properties=with_edge_properties,
            # FIXME accept and properly transmute a numpy/cupy random state.
            random_state=hash((random_state, i)),
            workers=[w],
            allow_other_workers=False,
            pure=False,
            return_offsets=return_offsets,
            return_hops=return_hops,
        )
        for i, w in enumerate(Comms.get_workers())
    ]

    empty_df = (
        create_empty_df_with_edge_props(
            indices_t, weight_t, return_offsets=return_offsets
        )
        if with_edge_properties
        else create_empty_df(indices_t, weight_t)
    )

    if return_offsets:
        result = [delayed(lambda x: x, nout=2)(r) for r in result]
        ddf = dask_cudf.from_delayed(
            [r[0] for r in result], meta=empty_df[0], verify_meta=False
        ).persist()
        ddf_offsets = dask_cudf.from_delayed(
            [r[1] for r in result], meta=empty_df[1], verify_meta=False
        ).persist()
        wait(ddf)
        wait(ddf_offsets)
        wait([r.release() for r in result])
        return ddf, ddf_offsets
    else:
        ddf = dask_cudf.from_delayed(result, meta=empty_df, verify_meta=False).persist()
        wait(ddf)
        wait([r.release() for r in result])
        return ddf

def _mg_call_plc_uniform_neighbor_sample(
    client,
    session_id,
    input_graph,
    ddf,
    keep_batches_together,
    min_batch_id,
    max_batch_id,
    fanout_vals,
    with_replacement,
    weight_t,
    indices_t,
    with_edge_properties,
    random_state,
    return_offsets=False,
    return_hops=True,
    prior_sources_behavior=None,
    deduplicate_sources=False,
    renumber=False,
):
    n_workers = None
    if keep_batches_together:
        n_workers = get_n_workers()

        if hasattr(min_batch_id, "compute"):
            min_batch_id = min_batch_id.compute()
        if hasattr(max_batch_id, "compute"):
            max_batch_id = max_batch_id.compute()

    result = [
        client.submit(
            _call_plc_uniform_neighbor_sample,
            session_id,
            input_graph._plc_graph[w],
            starts,
            keep_batches_together,
            n_workers,
            min_batch_id,
            max_batch_id,
            fanout_vals,
            with_replacement,
            weight_t=weight_t,
            with_edge_properties=with_edge_properties,
            # FIXME accept and properly transmute a numpy/cupy random state.
            random_state=hash((random_state, w)),
            return_offsets=return_offsets,
            return_hops=return_hops,
            prior_sources_behavior=prior_sources_behavior,
            deduplicate_sources=deduplicate_sources,
            renumber=renumber,
            allow_other_workers=False,
            pure=False,
        )
        for w, starts in ddf.items()
    ]
    del ddf

    empty_df = (
        create_empty_df_with_edge_props(
            indices_t,
            weight_t,
            return_offsets=return_offsets,
            renumber=renumber,
        )
        if with_edge_properties
        else create_empty_df(indices_t, weight_t)
    )
    if not isinstance(empty_df, (list, tuple)):
        empty_df = [empty_df]

    wait(result)

    nout = 1
    if return_offsets:
        nout += 1
    if renumber:
        nout += 1

    result_split = [delayed(lambda x: x, nout=nout)(r) for r in result]

    ddf = dask_cudf.from_delayed(
        [r[0] for r in result_split], meta=empty_df[0], verify_meta=False
    ).persist()
    return_dfs = [ddf]

    if return_offsets:
        ddf_offsets = dask_cudf.from_delayed(
            [r[1] for r in result_split], meta=empty_df[1], verify_meta=False
        ).persist()
        return_dfs.append(ddf_offsets)

    if renumber:
        ddf_renumber = dask_cudf.from_delayed(
            [r[-1] for r in result_split], meta=empty_df[-1], verify_meta=False
        ).persist()
        return_dfs.append(ddf_renumber)

    wait(return_dfs)
    wait([r.release() for r in result_split])
    wait([r.release() for r in result])
    del result

    if len(return_dfs) == 1:
        return return_dfs[0]
    else:
        return tuple(return_dfs)

def _uniform_neighbor_sample_legacy(
    input_graph: Graph,
    start_list: Sequence,
    fanout_vals: List[int],
    with_replacement: bool = True,
    with_edge_properties: bool = False,
    batch_id_list: Sequence = None,
    label_list: Sequence = None,
    label_to_output_comm_rank: bool = None,
    random_state: int = None,
    return_offsets: bool = False,
    return_hops: bool = False,
    _multiple_clients: bool = False,
) -> Union[dask_cudf.DataFrame, Tuple[dask_cudf.DataFrame, dask_cudf.DataFrame]]:
    warnings.warn(
        "The batch_id_list, label_list, and label_to_output_comm_rank "
        "parameters are deprecated.  Consider using with_batch_ids, "
        "keep_batches_together, min_batch_id, and max_batch_id instead."
    )

    if isinstance(start_list, int):
        start_list = [start_list]

    if isinstance(start_list, list):
        start_list = cudf.Series(
            start_list,
            dtype=input_graph.edgelist.edgelist_df[
                input_graph.renumber_map.renumbered_src_col_name
            ].dtype,
        )

    elif with_edge_properties and batch_id_list is None:
        batch_id_list = cudf.Series(cp.zeros(len(start_list), dtype="int32"))

    # fanout_vals must be a host array!
    # FIXME: ensure other sequence types (eg. cudf Series) can be handled.
    if isinstance(fanout_vals, list):
        fanout_vals = numpy.asarray(fanout_vals, dtype="int32")
    else:
        raise TypeError("fanout_vals must be a list, " f"got: {type(fanout_vals)}")

    if "value" in input_graph.edgelist.edgelist_df:
        weight_t = input_graph.edgelist.edgelist_df["value"].dtype
    else:
        weight_t = "float32"

    if "_SRC_" in input_graph.edgelist.edgelist_df:
        indices_t = input_graph.edgelist.edgelist_df["_SRC_"].dtype
    elif src_n in input_graph.edgelist.edgelist_df:
        indices_t = input_graph.edgelist.edgelist_df[src_n].dtype
    else:
        indices_t = numpy.int32

    start_list = start_list.rename(start_col_name)
    if batch_id_list is not None:
        batch_id_list = batch_id_list.rename(batch_col_name)
        if hasattr(start_list, "compute"):
            # mg input
            start_list = start_list.to_frame()
            batch_id_list = batch_id_list.to_frame()
            ddf = start_list.merge(
                batch_id_list,
                how="left",
                left_index=True,
                right_index=True,
            )
        else:
            # sg input
            ddf = cudf.concat(
                [
                    start_list,
                    batch_id_list,
                ],
                axis=1,
            )
    else:
        ddf = start_list.to_frame()

    if input_graph.renumbered:
        ddf = input_graph.lookup_internal_vertex_id(ddf, column_name=start_col_name)

    if hasattr(ddf, "compute"):
        ddf = get_distributed_data(ddf)
        wait(ddf)
        ddf = ddf.worker_to_parts
    else:
        splits = cp.array_split(cp.arange(len(ddf)), len(Comms.get_workers()))
        ddf = {w: [ddf.iloc[splits[i]]] for i, w in enumerate(Comms.get_workers())}

    client = get_client()
    session_id = Comms.get_session_id()
    if _multiple_clients:
        # Distributed centralized lock to allow
        # two disconnected processes (clients) to coordinate a lock
        # https://docs.dask.org/en/stable/futures.html?highlight=lock#distributed.Lock
        lock = Lock("plc_graph_access")
        if lock.acquire(timeout=100):
            try:
                ddf = _mg_call_plc_uniform_neighbor_sample_legacy(
                    client=client,
                    session_id=session_id,
                    input_graph=input_graph,
                    ddf=ddf,
                    label_list=label_list,
                    label_to_output_comm_rank=label_to_output_comm_rank,
                    fanout_vals=fanout_vals,
                    with_replacement=with_replacement,
                    weight_t=weight_t,
                    indices_t=indices_t,
                    with_edge_properties=with_edge_properties,
                    random_state=random_state,
                    return_offsets=return_offsets,
                    return_hops=return_hops,
                )
            finally:
                lock.release()
        else:
            raise RuntimeError(
                "Failed to acquire lock(plc_graph_access) while trying to sampling"
            )
    else:
        ddf = _mg_call_plc_uniform_neighbor_sample_legacy(
            client=client,
            session_id=session_id,
            input_graph=input_graph,
            ddf=ddf,
            label_list=label_list,
            label_to_output_comm_rank=label_to_output_comm_rank,
            fanout_vals=fanout_vals,
            with_replacement=with_replacement,
            weight_t=weight_t,
            indices_t=indices_t,
            with_edge_properties=with_edge_properties,
            random_state=random_state,
            return_offsets=return_offsets,
            return_hops=return_hops,
        )

    if return_offsets:
        ddf, offsets_ddf = ddf
    if input_graph.renumbered:
        ddf = input_graph.unrenumber(ddf, "sources", preserve_order=True)
        ddf = input_graph.unrenumber(ddf, "destinations", preserve_order=True)

    if return_offsets:
        return ddf, offsets_ddf

    return ddf

uniform_neighbor_sample_legacy = deprecated_warning_wrapper(
    _uniform_neighbor_sample_legacy
)

def uniform_neighbor_sample(
    input_graph: Graph,
    start_list: Sequence,
    fanout_vals: List[int],
    with_replacement: bool = True,
    with_edge_properties: bool = False,
    batch_id_list: Sequence = None,  # deprecated
    label_list: Sequence = None,  # deprecated
    label_to_output_comm_rank: bool = None,  # deprecated
    with_batch_ids: bool = False,
    keep_batches_together=False,
    min_batch_id=None,
    max_batch_id=None,
    random_state: int = None,
    return_offsets: bool = False,
    return_hops: bool = True,
    prior_sources_behavior: str = None,
    deduplicate_sources: bool = False,
    renumber: bool = False,
    _multiple_clients: bool = False,
) -> Union[dask_cudf.DataFrame, Tuple[dask_cudf.DataFrame, dask_cudf.DataFrame]]:
    """
    Does neighborhood sampling, which samples nodes from a graph based on the
    current node's neighbors, with a corresponding fanout value at each hop.

    Parameters
    ----------
    input_graph : cugraph.Graph
        cuGraph graph, which contains connectivity information as dask cudf
        edge list dataframe

    start_list : int, list, cudf.Series, or dask_cudf.Series (int32 or int64)
        a list of starting vertices for sampling

    fanout_vals : list
        List of branching out (fan-out) degrees per starting vertex for each
        hop level.

    with_replacement: bool, optional (default=True)
        Flag to specify if the random sampling is done with replacement

    with_edge_properties: bool, optional (default=False)
        Flag to specify whether to return edge properties (weight, edge id,
        edge type, batch id, hop id) with the sampled edges.

    batch_id_list: cudf.Series or dask_cudf.Series (int32), optional (default=None)
        Deprecated.
        List of batch ids that will be returned with the sampled edges if
        with_edge_properties is set to True.

    label_list: cudf.Series or dask_cudf.Series (int32), optional (default=None)
        Deprecated.
        List of unique batch id labels.  Used along with
        label_to_output_comm_rank to assign batch ids to GPUs.

    label_to_out_comm_rank: cudf.Series or dask_cudf.Series (int32),
    optional (default=None)
        Deprecated.
        List of output GPUs (by rank) corresponding to batch
        id labels in the label list.  Used to assign each batch
        id to a GPU.
        Must be in ascending order (i.e. [0, 0, 1, 2]).

    with_batch_ids: bool, optional (default=False)
        Flag to specify whether batch ids are present in the start_list

    keep_batches_together: bool (optional, default=False)
        If True, will ensure that the returned samples for each batch are on the
        same partition.

    min_batch_id: int (optional, default=None)
        Required for the keep_batches_together option.  The minimum batch id.

    max_batch_id: int (optional, default=None)
        Required for the keep_batches_together option.  The maximum batch id.

    random_state: int, optional
        Random seed to use when making sampling calls.

    return_offsets: bool, optional (default=False)
        Whether to return the sampling results with batch ids
        included as one dataframe, or to instead return two
        dataframes, one with sampling results and one with
        batch ids and their start offsets per rank.

    return_hops: bool, optional (default=True)
        Whether to return the sampling results with hop ids
        corresponding to the hop where the edge appeared.
        Defaults to True.

    prior_sources_behavior: str (Optional)
        Options are "carryover", and "exclude".
        Default will leave the source list as-is.
        Carryover will carry over sources from previous hops to the
        current hop.
        Exclude will exclude sources from previous hops from reappearing
        as sources in future hops.

    deduplicate_sources: bool, optional (default=False)
        Whether to first deduplicate the list of possible sources
        from the previous destinations before performing next
        hop.

    renumber: bool, optional (default=False)
        Whether to renumber on a per-batch basis.  If True,
        will return the renumber map and renumber map offsets
        as an additional dataframe.

    _multiple_clients: bool, optional (default=False)
        internal flag to ensure sampling works with multiple dask clients
        set to True to prevent hangs in multi-client environment

    Returns
    -------
    result : dask_cudf.DataFrame or Tuple[dask_cudf.DataFrame, dask_cudf.DataFrame]
        GPU distributed data frame containing several dask_cudf.Series

        If with_edge_properties=True:
            ddf['sources']: dask_cudf.Series
                Contains the source vertices from the sampling result
            ddf['destinations']: dask_cudf.Series
                Contains the destination vertices from the sampling result
            ddf['indices']: dask_cudf.Series
                Contains the indices from the sampling result for path
                reconstruction

        If with_edge_properties=False:
            If return_offsets=False:
                df['sources']: dask_cudf.Series
                    Contains the source vertices from the sampling result
                df['destinations']: dask_cudf.Series
                    Contains the destination vertices from the sampling result
                df['edge_weight']: dask_cudf.Series
                    Contains the edge weights from the sampling result
                df['edge_id']: dask_cudf.Series
                    Contains the edge ids from the sampling result
                df['edge_type']: dask_cudf.Series
                    Contains the edge types from the sampling result
                df['batch_id']: dask_cudf.Series
                    Contains the batch ids from the sampling result
                df['hop_id']: dask_cudf.Series
                    Contains the hop ids from the sampling result
                If renumber=True:
                    (adds the following dataframe)
                    renumber_df['map']: dask_cudf.Series
                        Contains the renumber maps for each batch
                    renumber_df['offsets']: dask_cudf.Series
                        Contains the batch offsets for the renumber maps

            If return_offsets=True:
                df['sources']: dask_cudf.Series
                    Contains the source vertices from the sampling result
                df['destinations']: dask_cudf.Series
                    Contains the destination vertices from the sampling result
                df['edge_weight']: dask_cudf.Series
                    Contains the edge weights from the sampling result
                df['edge_id']: dask_cudf.Series
                    Contains the edge ids from the sampling result
                df['edge_type']: dask_cudf.Series
                    Contains the edge types from the sampling result
                df['hop_id']: dask_cudf.Series
                    Contains the hop ids from the sampling result

                offsets_df['batch_id']: dask_cudf.Series
                    Contains the batch ids from the sampling result
                offsets_df['offsets']: dask_cudf.Series
                    Contains the offsets of each batch in the sampling result
                If renumber=True:
                    (adds the following dataframe)
                    renumber_df['map']: dask_cudf.Series
                        Contains the renumber maps for each batch
                    renumber_df['offsets']: dask_cudf.Series
                        Contains the batch offsets for the renumber maps
    """

    if (
        batch_id_list is not None
        or label_list is not None
        or label_to_output_comm_rank is not None
    ):
        if prior_sources_behavior or deduplicate_sources:
            raise ValueError(
                "unique sources, carry_over_sources, and deduplicate_sources"
                " are not supported with batch_id_list, label_list, and"
                " label_to_output_comm_rank.  Consider using with_batch_ids"
                " and keep_batches_together instead."
            )

        if renumber:
            raise ValueError(
                "renumber is not supported with batch_id_list, label_list, "
                "and label_to_output_comm_rank.  Consider using "
                "with_batch_ids and keep_batches_together instead."
            )

        return uniform_neighbor_sample_legacy(
            input_graph,
            start_list,
            fanout_vals,
            with_replacement=with_replacement,
            with_edge_properties=with_edge_properties,
            batch_id_list=batch_id_list,
            label_list=label_list,
            label_to_output_comm_rank=label_to_output_comm_rank,
            random_state=random_state,
            return_offsets=return_offsets,
            return_hops=return_hops,
            _multiple_clients=_multiple_clients,
        )

    if isinstance(start_list, int):
        start_list = [start_list]

    if isinstance(start_list, list):
        start_list = cudf.Series(
            start_list,
            dtype=input_graph.edgelist.edgelist_df[
                input_graph.renumber_map.renumbered_src_col_name
            ].dtype,
        )
    elif with_edge_properties and not with_batch_ids:
        if isinstance(start_list, (cudf.DataFrame, dask_cudf.DataFrame)):
            raise ValueError("expected 1d input for start list without batch ids")

        start_list = start_list.to_frame()
        if isinstance(start_list, dask_cudf.DataFrame):
            start_list = start_list.map_partitions(
                lambda df: df.assign(
                    **{batch_id_n: cudf.Series(cp.zeros(len(df), dtype="int32"))}
                )
            ).persist()
        else:
            start_list = start_list.reset_index(drop=True).assign(
                **{batch_id_n: cudf.Series(cp.zeros(len(start_list), dtype="int32"))}
            )

    if keep_batches_together and min_batch_id is None:
        raise ValueError(
            "must provide min_batch_id if using keep_batches_together option"
        )
    if keep_batches_together and max_batch_id is None:
        raise ValueError(
            "must provide max_batch_id if using keep_batches_together option"
        )
    if renumber and not keep_batches_together:
        raise ValueError(
            "mg uniform_neighbor_sample requires that keep_batches_together=True "
            "when performing renumbering."
        )

    # fanout_vals must be a host array!
    # FIXME: ensure other sequence types (eg. cudf Series) can be handled.
    if isinstance(fanout_vals, list):
        fanout_vals = numpy.asarray(fanout_vals, dtype="int32")
    else:
        raise TypeError("fanout_vals must be a list, " f"got: {type(fanout_vals)}")

    if "value" in input_graph.edgelist.edgelist_df:
        weight_t = input_graph.edgelist.edgelist_df["value"].dtype
    else:
        weight_t = "float32"

    if "_SRC_" in input_graph.edgelist.edgelist_df:
        indices_t = input_graph.edgelist.edgelist_df["_SRC_"].dtype
    elif src_n in input_graph.edgelist.edgelist_df:
        indices_t = input_graph.edgelist.edgelist_df[src_n].dtype
    else:
        indices_t = numpy.int32

    if isinstance(start_list, (cudf.Series, dask_cudf.Series)):
        start_list = start_list.rename(start_col_name)
        ddf = start_list.to_frame()
    else:
        ddf = start_list
        columns = ddf.columns
        ddf = ddf.rename(
            columns={columns[0]: start_col_name, columns[-1]: batch_col_name}
        )

    if input_graph.renumbered:
        ddf = input_graph.lookup_internal_vertex_id(ddf, column_name=start_col_name)

    client = get_client()
    session_id = Comms.get_session_id()
    n_workers = get_n_workers()

    if isinstance(ddf, cudf.DataFrame):
        ddf = dask_cudf.from_cudf(ddf, npartitions=n_workers)

    ddf = ddf.repartition(npartitions=n_workers)
    ddf = persist_dask_df_equal_parts_per_worker(ddf, client)
    ddf = get_persisted_df_worker_map(ddf, client)

    if _multiple_clients:
        # Distributed centralized lock to allow
        # two disconnected processes (clients) to coordinate a lock
        # https://docs.dask.org/en/stable/futures.html?highlight=lock#distributed.Lock
        lock = Lock("plc_graph_access")
        if lock.acquire(timeout=100):
            try:
                ddf = _mg_call_plc_uniform_neighbor_sample(
                    client=client,
                    session_id=session_id,
                    input_graph=input_graph,
                    ddf=ddf,
                    keep_batches_together=keep_batches_together,
                    min_batch_id=min_batch_id,
                    max_batch_id=max_batch_id,
                    fanout_vals=fanout_vals,
                    with_replacement=with_replacement,
                    weight_t=weight_t,
                    indices_t=indices_t,
                    with_edge_properties=with_edge_properties,
                    random_state=random_state,
                    return_offsets=return_offsets,
                    return_hops=return_hops,
                    prior_sources_behavior=prior_sources_behavior,
                    deduplicate_sources=deduplicate_sources,
                    renumber=renumber,
                )
            finally:
                lock.release()
        else:
            raise RuntimeError(
                "Failed to acquire lock(plc_graph_access) while trying to sampling"
            )
    else:
        ddf = _mg_call_plc_uniform_neighbor_sample(
            client=client,
            session_id=session_id,
            input_graph=input_graph,
            ddf=ddf,
            keep_batches_together=keep_batches_together,
            min_batch_id=min_batch_id,
            max_batch_id=max_batch_id,
            fanout_vals=fanout_vals,
            with_replacement=with_replacement,
            weight_t=weight_t,
            indices_t=indices_t,
            with_edge_properties=with_edge_properties,
            random_state=random_state,
            return_offsets=return_offsets,
            return_hops=return_hops,
            prior_sources_behavior=prior_sources_behavior,
            deduplicate_sources=deduplicate_sources,
            renumber=renumber,
        )

    if return_offsets:
        if renumber:
            ddf, offsets_df, renumber_df = ddf
        else:
            ddf, offsets_ddf = ddf
    else:
        if renumber:
            ddf, renumber_df = ddf

    if input_graph.renumbered and not renumber:
        ddf = input_graph.unrenumber(ddf, "sources", preserve_order=True)
        ddf = input_graph.unrenumber(ddf, "destinations", preserve_order=True)

    if return_offsets:
        if renumber:
            return ddf, offsets_df, renumber_df
        else:
            return ddf, offsets_ddf

    if renumber:
        return ddf, renumber_df

    return ddf


import io
import os
from settings.data_norm_constants import MIN_WORD_FREQUENCY, SEQUENCE_LEN
import keras
import numpy as np
import csv

def corpus_to_dictionary(path: str):
    print('corupus to dict')
    with io.open(path) as f:
        # Get words from corpus file
        text = f.read().lower().replace('\n', ' \n ').replace('\\', ' \n ')
        text_in_words = keras.preprocessing.text.text_to_word_sequence(text, filters='"#$%&()*+,-./:;<=>@[\\]^_`{|}~\t',
                                                                       lower=True,
                                                                       split=' ')
        print('Corpus length in words:', len(text_in_words))
        if ('\n' in text_in_words):
            index = text_in_words.index('\n')
            test = text_in_words[index]

        # Count how many times word appears in text_in_words
        word_freq = {}
        for word in text_in_words:
            word_freq[word] = word_freq.get(word, 0) + 1

        # Get ignored words and add them to ignored_words_set
        ignored = set()
        for k, v in word_freq.items():
            if word_freq[k] < MIN_WORD_FREQUENCY:
                ignored.add(k)

        words = set(text_in_words)
        print('Unique words:', len(words))

        # Remove ignored words from set
        words = sorted(set(words) - ignored)
        print('Unique words after removing ignored words:', len(words))

        # Create two dictionaries. One with word as a key and index as value. One with index as key and word as a value
        word_indices = dict((c, i) for i, c in enumerate(words))
        indices_word = dict((i, c) for i, c in enumerate(words))

        print('EOF: corpus_to_dictionary()')
        return text_in_words, ignored, word_indices, indices_word

def create_and_filter_sequences(text_in_words, ignored_words):
    print('start: create_and_filter_sequences')
    STEP = 1
    sentences = []
    next_words = []
    ignored = 0

    # Loop original corpus. Add SEQUENCES_LEN long sentences to sentences and SEQUENCES_LEN next words to next_words
    # Only add sentences that don't contain ignored words
    for i in range(0, len(text_in_words) - SEQUENCE_LEN, STEP):
        # Only add sequences where no word is in ignored_words
        if len(set(text_in_words[i: i + SEQUENCE_LEN + 1]).intersection(ignored_words)) == 0:
            sentences.append(text_in_words[i: i + SEQUENCE_LEN])
            next_words.append(text_in_words[i + SEQUENCE_LEN])
        else:
            ignored = ignored + 1
    print('Ignored sequences:', ignored)
    print('Remaining sequences:', len(sentences))

    return sentences, next_words

def shuffle_and_split_training_set(sentences_original, next_original, percentage_test=2):
    # shuffle at unison
    print('Shuffling sentences')

    tmp_sentences = []
    tmp_next_word = []

    for i in np.random.RandomState(seed=42).permutation(len(sentences_original)):
        tmp_sentences.append(sentences_original[i])
        tmp_next_word.append(next_original[i])

    cut_index = int(len(sentences_original) * (1. - (percentage_test / 100.)))
    x_train, x_test = tmp_sentences[:cut_index], tmp_sentences[cut_index:]
    y_train, y_test = tmp_next_word[:cut_index], tmp_next_word[cut_index:]

    print("Size of training set = %d" % len(x_train))
    print("Size of test set = %d" % len(y_test))

    print('end: shuffle_and_split_training_set')

    return (x_train, y_train), (x_test, y_test)


from selenium import webdriver

import urllib.request  # temp for images
import os

from other import common_tools

MIN_FOLL = 12  # avoid annoying suggestions section in short lists

"""
private methods generally assume that bot is already on user's page
"""

class InstaBot:
    def __init__(self, username, pw, sleeper):
        self.username = username
        self.s = sleeper
        self.driver = webdriver.Chrome()
        self.driver.get("https://instagram.com")
        sleeper.random_delay(2)

        # accept cookies
        try:
            self.driver.find_element_by_xpath("/html/body/div[2]/div/div/div/div[2]/button[1]").click()
            sleeper.random_delay(1)
        except Exception as e:
            pass

        # Log in
        self.driver.find_element_by_xpath("//input[@name=\"username\"]").send_keys(username)
        self.driver.find_element_by_xpath("//input[@name=\"password\"]").send_keys(pw)
        self.driver.find_element_by_xpath('//button[@type="submit"]').click()
        sleeper.random_delay(3)

        # Click not now on enable notifications
        try:
            self.driver.find_element_by_xpath("//button[contains(text(), 'Not Now')]").click()
            sleeper.random_delay(2)
            self.driver.find_element_by_xpath("//button[contains(text(), 'Not Now')]").click()
            sleeper.random_delay(2)
        except Exception as e:
            pass

        self.scroll_retries = 15
        self.scroll_delay = 0.4
        self.scroll_retry_delay = 1.5

    def find_account(self, username):
        self.driver.get("https://instagram.com/{}".format(username))
        self.s.random_delay(1.5)
        not_found = "Sorry, this page isn't available." in self.driver.page_source
        if not_found:
            raise NameError("{} not found".format(username))

    def _get_followers(self):
        return self._get_list("followers")

    def _get_following(self):
        return self._get_list("following")

    def _get_list(self, foll):
        """Starts from user page, assumes that not private to self.username.
        Lists where suggestions may appear fail."""
        assert foll in ["followers", "following"], "Invalid foll: {}" % foll

        try:
            # Click followers / following button
            foll_btn = self.driver.find_element_by_xpath("//a[contains(@href,'/{}')]".format(foll))
            foll_btn.click()
            self.s.random_delay(3)

            # scroll_box = self.driver.find_element_by_xpath("/html/body/div[4]/div/div/div[2]")
            scroll_box = self.driver.find_element_by_class_name("isgrP")

            last_ht, ht = 0, 1
            retries_left = self.scroll_retries
            retries_used = 0
            current_retries = 0
            while last_ht != ht or retries_left > 0:
                last_ht = ht
                self.s.random_delay(self.scroll_delay)
                ht = self.driver.execute_script("""
                                    arguments[0].scrollTo(0, arguments[0].scrollHeight);
                                    return arguments[0].scrollHeight;
                                    """, scroll_box)
                if last_ht == ht:
                    retries_left -= 1
                    current_retries += 1
                    self.s.random_delay(self.scroll_retry_delay)
                else:
                    # if managed to move, adds the amount of retries to the retries used counter
                    # this won't be called when the list actually ends
                    retries_used += current_retries
                    current_retries = 0

            links = scroll_box.find_elements_by_tag_name('a')

            names = [name.text for name in links if name.text != '']
            if "See All Suggestions" in names:
                print("Error: 'See All Suggestions' in names")
            print("-- scraped: {} with {}/{} retries used, retry delay {}s, scroll delay {:.2}s--"
                  .format(len(names), retries_used, self.scroll_retries,
                          self.scroll_retry_delay, self.scroll_delay))
            if retries_used > self.scroll_retries / 2:
                self.scroll_delay += 0.03
            else:
                self.scroll_delay -= 0.02
                self.scroll_delay = max(self.scroll_delay, 0.32)

            # Close the list
            # self.driver.find_element_by_xpath("/html/body/div[4]/div/div/div[1]/div/div[2]/button").click()
            # self.driver.find_element_by_class_name("wpO6b ")
            self.driver.refresh()
            self.s.random_delay(2)
        except Exception as e:
            print(e)
            return None

        return names

    def _get_info(self):
        """
        Private method returning is_private and counts
        Can be used on any accounts that are private or public
        """
        private = "This Account is Private" in self.driver.page_source
        # n. of posts, followers, following
        counts = self.driver.find_elements_by_class_name("g47SY ")
        counts = tuple([count.text for count in counts])
        return (private, *counts)

    def _get_profile_pic(self, user, private):
        """
        Private method for getting profile pic, and converting jpg to png with alpha channel
        """
        if private:
            img = self.driver.find_element_by_class_name("be6sR")
        else:
            img = self.driver.find_element_by_class_name("_6q-tv")

        src = img.get_attribute('src')
        path = os.getcwd() + "/img/"
        urllib.request.urlretrieve(src, "{}{}.jpg".format(path, user))
        self.s.random_delay(2)

        # Convert to PNG
        if os.path.isfile(path + user + ".jpg"):
            common_tools.add_alpha(path + user)
        else:
            print("{}: could not find image".format(user))

    def _get_post_descriptions(self):
        # TODO scroll to the bottom to load all images (if under some max amount)

        # image url provides a unique identifier
        tile_elements = self.driver.find_elements_by_css_selector(
            "div[class='v1Nh3 kIKUG  _bz0w']")
        self.s.random_delay(1)
        ids = [image.find_element_by_tag_name("a").get_attribute("href")
               for image in tile_elements]
        self.s.random_delay(1)
        # https://www.instagram.com/p/CBoaLX4p8Kk/ --> CBoaLX4p8Kk
        ids = [_id.split("/")[-2] for _id in ids]

        # get instagram generated descriptions
        image_elements = self.driver.find_elements_by_class_name("KL4Bh")
        self.s.random_delay(1)
        descriptions = [image.find_element_by_tag_name("img").get_attribute("alt")
                        for image in image_elements]
        self.s.random_delay(1)

        return list(zip(ids, descriptions))

    def _get_bio(self):
        name, bio = None, None
        try:
            name_element = self.driver.find_element_by_class_name("rhpdm")
            name = name_element.text
            bio = self.driver.find_element_by_class_name("-vDIg").text.split('\n')[1:]
            bio = '\n'.join(bio)
        except Exception as _:
            bio = self.driver.find_element_by_class_name("-vDIg").text

        return name, bio

    def get_user_data(self, username, cap=1000,
                      get_followers=True, get_following=True, get_image=True):
        """
        Public method for scraping username
        """
        # self.find_account(username)
        private, n_post, n_followers, n_following = self._get_info()

        followers, following, post_descriptions = None, None, None

        if get_image:
            self._get_profile_pic(username, private)

        name, bio = self._get_bio()

        if not private:
            post_descriptions = self._get_post_descriptions()

            n_followers_int = common_tools.string_to_int(n_followers)
            n_following_int = common_tools.string_to_int(n_following)

            if MIN_FOLL < n_followers_int < cap and MIN_FOLL < n_following_int < cap:
                if get_followers:
                    followers = self._get_followers()

                if get_following:
                    following = self._get_following()

        return private, n_post, n_followers, n_following, followers, following, name, bio, post_descriptions


import gspread
from pprint import pprint
from matplotlib import colors
import math
import numpy as np
from value_types import *
import re
# from sympy import *

def add(sheet, arg0, arg1):
    # num and entry
    if(isinstance(arg0, Value) and isinstance(arg1, Entry)):
        addNumEntry(sheet, arg0, arg1)
    # entry and entry
    if(isinstance(arg0, Entry) and isinstance(arg1, Entry)):
        addEntryEntry(sheet, arg0, arg1)
    # num and cell
    if(isinstance(arg0, Value) and isinstance(arg1, Cell)):
        print("Reached here")
        addNumCell(sheet, int(arg0.value), arg1.cell_str)

# works
def addNumCell(sheet, num, cell):
    try:
        val = sheet.acell(cell).value
        cell_update(sheet, cell, num + float(val))
    except TypeError:
        return

def addNumRange(sheet, num, range):
    values_list = sheet.range(range)
    for cell in values_list:
      if len(cell.value) > 0:
        cell.value = str(int(cell.value) + num)
    sheet.update_cells(values_list)

def addNumEntry(sheet, num, entry):
    try:
      if entry.rowOrCol == "row":
        range1 = str(gspread.utils.rowcol_to_a1(entry, 1))
        addNumRange(sheet, num, str(entry) + ":BA" + str(entry))
      else:
        range1 = str(gspread.utils.rowcol_to_a1(1, entry))
        addNumRange(sheet, num, range1+':'+range1[0]+"1000")
    except TypeError:
        return

def addEntryEntry(sheet, entry, entry2):
    try:
      if entry.type == 'row':
        range1 = str(gspread.utils.rowcol_to_a1(entry, 1))
        range2 = str(gspread.utils.rowcol_to_a1(entry2, 1))
        values_list = sheet.range(range1 + ":BA" + str(entry))
        values2_list = sheet.range(range2 + ":BA" + str(entry2))
        for i in range(len(values_list)):
          cell = values_list[i]
          cell2 = values2_list[i]
          if len(cell.value) > 0 and len(cell2.value):
            cell2.value = str(int(cell2.value) + int(cell.value))
        sheet.update_cells(values2_list)
      else:
        range1 = str(gspread.utils.rowcol_to_a1(1, entry))
        range2 = str(gspread.utils.rowcol_to_a1(1, entry2))
        values_list = sheet.range(range1 + ":" + str(entry) + "1000")
        values2_list = sheet.range(range2 + ":" + str(entry2) + "1000")
        for i in range(len(values_list)):
          cell = values_list[i]
          cell2 = values2_list[i]
          if len(cell.value) > 0 and len(cell2.value):
            cell2.value = str(int(cell2.value) + int(cell.value))
        sheet.update_cells(values2_list)
    except TypeError:
        return

def insert_entry(sheet, index, type):
    new_list = []
    if (type == 'col'):
        sheet.insert_cols(ord(index) - ord("A") + 1, 1)
    else:
        sheet.insert_row(new_list, index)

# works
def multiply_cell(sheet, num, cell):
    val = sheet.acell(cell).value
    cell_update(sheet, cell, num * int(val))

# works
def multiply_range(sheet, num, range):
    values_list = sheet.range(range)
    for cell in values_list:
      if len(cell.value) > 0:
        cell.value = str(int(cell.value) * num)
    sheet.update_cells(values_list)

# works
def multiply_entry(sheet, num, entry, type):
    if type == 0:
      num_elements = len(sheet.col_values(entry))
      range1 = str(gspread.utils.rowcol_to_a1(entry, 1))
      range2 = str(gspread.utils.rowcol_to_a1(entry, num_elements))
      multiply_range(sheet, num, range1+':'+range2)
    else:
      num_elements = len(sheet.row_values(entry))
      range1 = str(gspread.utils.rowcol_to_a1(1, entry))
      range2 = str(gspread.utils.rowcol_to_a1(num_elements, entry))
      multiply_range(sheet, num, range1+':'+range2)

# works
def divide_cell(sheet, num, cell):
    val = sheet.acell(cell).value
    cell_update(sheet, cell, int(val) / num)

# works
def divide_range(sheet, num, range):
    values_list = sheet.range(range)
    for cell in values_list:
      cell.value = str(int(cell.value) / num)
    sheet.update_cells(values_list)

# works
def divide_entry(sheet, num, entry, type):
    if type == 0:
      num_elements = len(sheet.col_values(entry))
      range1 = str(gspread.utils.rowcol_to_a1(entry, 1))
      range2 = str(gspread.utils.rowcol_to_a1(entry, num_elements))
      divide_range(sheet, num, range1+':'+range2)
    else:
      num_elements = len(sheet.row_values(entry))
      range1 = str(gspread.utils.rowcol_to_a1(1, entry))
      range2 = str(gspread.utils.rowcol_to_a1(num_elements, entry))
      divide_range(sheet, num, range1+':'+range2)

# works
def cell_update(sheet, cell, data):
    # Cell needs to be in A1 format
    sheet.update(cell, data)

# works
def update_cell_literal(sheet, row, col, data):
    # Ensure we have valid row and col int values
    sheet.update_cell(row, col, data)

def update_range(sheet, range, arr):
    # Range needs to be in "A1:B2" format
    values_list = sheet.range(range)
    for i in range(len(values_list)):
      values_list[i].value = str(arr[i])
    sheet.update_cells(values_list)

# works
def average_entry(sheet, entry, type):
    # type - 0 for row and 1 for column
    values_list = sheet.row_values(entry) if type == 'row' else sheet.col_values(ord(entry) - ord("A") + 1)
    sumRes = 0
    lenRes = 0
    for i in values_list:
        if i != '':
            sumRes += int(i)
            lenRes += 1
    average = sumRes / lenRes
    return average

# works
def format_bold(sheet, range):
    sheet.format(range, {'textFormat': {'bold': True}})

def format_bold_entry(sheet, entry):
    if(entry.type == 'col'):
        format_bold(entry.value + '0:' + entry.value + "1000")
    else:
        format_bold('A' + entry.value + ":Z" + entry.value )

# works
def sort(sheet, spec):
    #spec: (col : 'asc' or 'des')
    #example: (2: 'asc)
    #sorts based on second column in increasing order
    sheet.sort(spec, range='A2:G1000')

# works
def sin_cell(sheet, cell):
    val = sheet.acell(cell).value
    cell_update(sheet, cell, math.sin(int(val)))

# works
def cos_cell(sheet, cell):
    val = sheet.acell(cell).value
    cell_update(sheet, cell, math.cos(int(val)))

# works
def sin_range(sheet, range):
    values_list = sheet.range(range)
    for cell in values_list:
      cell.value = str(math.sin(int(cell.value)))
    sheet.update_cells(values_list)

# works
def cos_range(sheet, range):
    values_list = sheet.range(range)
    values_list = np.cos(values_list)
    update_range(range, values_list)

def sin_entry(sheet, entry, type):
    values_list = sheet.row_values(entry) if type == 0 else sheet.col_values(entry)
    new_list = []
    REPLACE_NO_SPACE = re.compile('[^A-Za-z0-9]+')
    values_list = [REPLACE_NO_SPACE.sub("", line.lower()) for line in values_list]
    for i in values_list: # RIGHT HERE
        if i != '':
            new_list.append(float(i))
    values_list = list(np.sin(new_list))
    cell_list = None
    pprint(entry)
    if(type == 'row'):
        cell_list = sheet.range("A" + entry + ":" + chr(len(values_list) + ord("A")) + entry)
    else:
        cell_list = sheet.range(chr(entry + ord("A") - 1) + str(1) + ":" + chr(entry + ord("A") - 1) + str(len(values_list) + 1))
    for i in range(len(values_list)):
        cell_list[i].value = values_list[i]
    sheet.update_cells(cell_list) # idk if this works

def cos_entry(sheet, entry, type):
    values_list = sheet.row_values(entry) if type == 0 else sheet.col_values(entry)
    new_list = []
    REPLACE_NO_SPACE = re.compile('[^A-Za-z0-9]+')
    values_list = [REPLACE_NO_SPACE.sub("", line.lower()) for line in values_list]
    for i in values_list: # RIGHT HERE
        if i != '':
            new_list.append(float(i))
    values_list = list(np.cos(new_list))
    cell_list = None
    pprint(entry)
    if(type == 'row'):
        cell_list = sheet.range("A" + entry + ":" + chr(len(values_list) + ord("A")) + entry)
    else:
        cell_list = sheet.range(chr(entry + ord("A") - 1) + str(1) + ":" + chr(entry + ord("A") - 1) + str(len(values_list) + 1))
    for i in range(len(values_list)):
        cell_list[i].value = values_list[i]
    sheet.update_cells(cell_list) # idk if this works

# works
def set_background(sheet, range, color):
    # color will be an actual color name
    arr = colors.to_rgb(color)
    sheet.format(range, {
        "backgroundColor": {
            "red": arr[0],
            "green": arr[1],
            "blue": arr[2]
        }
    })

def set_background_entry(sheet, entry, color):
    if(entry.rowOrCol == 'col'):
        set_background(sheet, entry.value + '0:' + entry.value + "1000", color)
    else:
        set_background(sheet, 'A' + entry.value + ":Z" + entry.value, color)

def filter_by_prime(sheet):
    vals = sheet.get_all_values()
    result = ()
    for i in vals:
        if(type(vals[i]) == int and isPrime(vals[i])):
            result.__add__(i)
    return result

def isPrime(n):
   if n <= 1 or n % 1 > 0:
      return False
   for i in range(2, n//2):
      if n % i == 0:
         return False
   return True

# def graph(r):
#     dataframeObj=GET.read_sheet_to_dataframe(connection_url,driveouth_json)

#     plot=r.plot_chart(g,X,Y,'line',imagename='test')


import numpy as np
class pretrained:
    def __init__(self, max_len=300, padding_item=[300*[0]], lang='en'):
        if lang == 'en':
            from gensim.models import KeyedVectors
            self.model = KeyedVectors.load_word2vec_format('../model.bin', binary=True)
        else:
            from gensim.models import KeyedVectors, Word2Vec
            self.model = Word2Vec.load('../zh/zh.bin')

        self.max_len = max_len
        self.padding_item = padding_item

    def load_LM(self):
        return self.LM

    def word2vec(self, w):
        try:
            return np.array(self.model[w])
        except KeyError:
            return np.array([0])

    def preprocess(self, corpus):
        zero = 0
        non_zero = 0

        embeddings = [map(self.word2vec, c) for c in corpus]

        for i in range(len(embeddings)):
            _ = list(embeddings[i])

            length = len(_)
            for j in range(length):
                if len(_[j]) < self.max_len:
                    lower = j-1 if j-1 > -1 else 0
                    higher = j+1 if j+1 < length else length-1

                    if len(_[lower]) < 300 and len(_[higher]) < 300:
                        _[j] = np.array(300*[0])
                        zero += 1
                    else:
                        _[j] = (_[lower]+_[higher])/2
                        non_zero += 1
            if length < self.max_len:
                embeddings[i] = _ + (self.max_len-length)*self.padding_item
            else:
                embeddings[i] = _[:self.max_len]

        #print(zero, non_zero)

        return embeddings


from django.contrib import admin
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static
from . import views

urlpatterns = [
    path('tag_search/', views.tag_search, name='tag_search'),
    path('create/', views.create, name='create'), #경로공유 등록(위도/경도 있는 모델)
    path('board_create/', views.board_create, name='board_create'), #일반 게시물
    path('profile/<str:user>',views.profile, name='profile'), #메인에서 프로필화면 가기
    path('detail/<int:post_id>',views.detail,name='detail'), # 게시글 읽기
    path('board_detail/<int:post_id>',views.board_detail,name='board_detail'), # 지도없는 게시글 읽기
    path('update_profile/<str:user>', views.update_profile, name="update_profile"),
    path('create_comment/<int:post_id>', views.create_comment, name="create_comment"), #댓글생성
    path('delete_comment/<int:post_id>/<int:comment_id>', views.delete_comment, name="delete_comment"),
     path('create_bcomment/<int:post_id>', views.create_bcomment, name="create_bcomment"), #댓글생성
    path('delete_bcomment/<int:post_id>/<int:comment_id>', views.delete_bcomment, name="delete_bcomment"),
    path('likes/', views.likes, name="likes"),
    path('board_likes/', views.board_likes, name="board_likes"),
    path('follow/', views.follow, name="follow"),
]
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from django.shortcuts import get_object_or_404, redirect
from it_bloggers.models import Entry
from django.contrib.auth.decorators import login_required
from comments.forms import CommentForm
from django.http.response import HttpResponse
from .models import Comment
from django.views.decorators.csrf import csrf_exempt
from notifications.signals import notify
from django.contrib.auth.models import User

# Create your views here.

@csrf_exempt
@login_required
def new_comment(request, entry_id, parent_comment_id=None):
    entry = get_object_or_404(Entry, id=entry_id)

    if request.method != 'POST':
        return HttpResponse("不接受非POST请求")

    else:
        comment_form = CommentForm(data=request.POST)
        if comment_form.is_valid():
            new_comment = comment_form.save(commit=False)
            new_comment.entry = entry
            new_comment.owner = request.user
            # 二级回复
            if parent_comment_id:
                parent_comment = Comment.objects.get(id=parent_comment_id)
                # 若回复层级超过二级，则转换为二级
                new_comment.parent_id = parent_comment.get_root().id
                # 被回复人
                new_comment.reply_to = parent_comment.owner
                new_comment.save()
                #  给目标用户发送通知
                if not parent_comment.owner.is_superuser:
                    notify.send(
                        request.user,
                        recipient=parent_comment.owner,
                        verb='回复了你',
                        target=entry,
                        action_object=new_comment,
                        )
                # 锚定位
                redirect_url = entry.get_absolute_url() + '#comment_elem_' + str(new_comment.id)
                return redirect(redirect_url)

            new_comment.save()
            #  给管理员发送通知
            if not request.user.is_superuser:
                notify.send(
                    request.user,
                    recipient=User.objects.filter(is_superuser=1),
                    verb='回复了你',
                    target=entry,
                    action_object=new_comment,
                    )
            redirect_url = entry.get_absolute_url() + '#comment_elem_' + str(new_comment.id)
            return redirect(redirect_url)
        else:
            return HttpResponse("输入不合法，请重新输入")


# 导入users蓝图程序用于声明路由
from . import users
from flask import render_template,session,redirect,request
from .. import db
from ..models import *

@users.route("/login",methods = ['GET','POST'])
def login():
    if request.method == 'GET':
        # 判断已经处于登录状态上(去session中有没有id和loginname)
        if 'id' in session and 'loginname' in session:
            return redirect('/')
        # 获取请求源地址,保存进session,用于登录成功后的时候
        # 去使用,如果没有请求源地址的话则将/保存进session
        url = request.headers.get('Referer','/')
        session['url'] = url
        return render_template('login.html')
    else:
        # 1.接收前端传递过来的usernane
        loginname = request.form['username']
        upwd = request.form['password']
        # 2.验证用户名称和密码是否正确
        user = User.query.filter_by(loginname=loginname,upwd = upwd).first()
        # 3.正确：则从哪来回哪去,不正确则回登录页
        if user:
            # 正确:存session,从哪来回哪去
            session['id'] = user.ID
            session['loginname'] = loginname
            # 获取请求源地址
            url = session['url']
            return redirect(url)
        else:
            return render_template('login.html')


import json
import pandas as pd

def read(filename, file_format, json_obj_top_level_id = None):
	'''
	inputs:
		filename = name of ifle
		file_format = type of format, currently can be 'csv' or 'json objects'
	outputs:
		dataset in pandas dataframe format
	'''
	df = []
	if file_format == 'csv':
		df = pd.read_csv(filename)
	elif file_format == 'json objects':

		with open(filename) as data_file:
			data_dict = {}

			for line in data_file:
				row = json.loads(line)
				top_id_val = row.pop(json_obj_top_level_id, None)
				data_dict[top_id_val] = row
		df = pd.DataFrame.from_dict(data_dict, orient = "index")

		# resets df index to integers, instead of the top_id_val
		df = df.reset_index().rename(columns = {"index": json_obj_top_level_id})

	else:
		print('the current file format is not supported')

	return df


import logging

from django.core.exceptions import PermissionDenied

from wagtail.log_actions import log
from wagtail.signals import page_unpublished

logger = logging.getLogger("wagtail")

class UnpublishPagePermissionError(PermissionDenied):
    """
    Raised when the page unpublish cannot be performed due to insufficient permissions.
    """

    pass

class UnpublishPageAction:
    def __init__(
        self,
        page,
        set_expired=False,
        commit=True,
        user=None,
        log_action=True,
        include_descendants=False,
    ):
        self.page = page
        self.set_expired = set_expired
        self.commit = commit
        self.user = user
        self.log_action = log_action
        self.include_descendants = include_descendants

    def check(self, skip_permission_checks=False):
        if (
            self.user
            and not skip_permission_checks
            and not self.page.permissions_for_user(self.user).can_unpublish()
        ):
            raise UnpublishPagePermissionError(
                "You do not have permission to unpublish this page"
            )

    def _unpublish_page(self, page, set_expired, commit, user, log_action):
        """
        Unpublish the page by setting ``live`` to ``False``. Does nothing if ``live`` is already ``False``
        :param log_action: flag for logging the action. Pass False to skip logging. Can be passed an action string.
            Defaults to 'wagtail.unpublish'
        """
        if page.live:
            page.live = False
            page.has_unpublished_changes = True
            page.live_revision = None

            if set_expired:
                page.expired = True

            if commit:
                # using clean=False to bypass validation
                page.save(clean=False)

            page_unpublished.send(sender=page.specific_class, instance=page.specific)

            if log_action:
                log(
                    instance=page,
                    action=log_action
                    if isinstance(log_action, str)
                    else "wagtail.unpublish",
                    user=user,
                )

            logger.info('Page unpublished: "%s" id=%d', page.title, page.id)

            page.revisions.update(approved_go_live_at=None)

            # Unpublish aliases
            for alias in page.aliases.all():
                alias.unpublish()

    def execute(self, skip_permission_checks=False):
        self.check(skip_permission_checks=skip_permission_checks)

        self._unpublish_page(
            self.page,
            set_expired=self.set_expired,
            commit=self.commit,
            user=self.user,
            log_action=self.log_action,
        )

        if self.include_descendants:
            from wagtail.models import UserPagePermissionsProxy

            user_perms = UserPagePermissionsProxy(self.user)
            for live_descendant_page in (
                self.page.get_descendants().live().defer_streamfields().specific()
            ):
                action = UnpublishPageAction(live_descendant_page)
                if user_perms.for_page(live_descendant_page).can_unpublish():
                    action.execute(skip_permission_checks=True)


import tkinter as tk #Module imported for Graphical user interface
from tkinter import ttk
import turtle as t #Module imported to draw Ghana Flag on tkinter canvas
from datetime import date #Module imported to validate age of users

#Password for Registrar is vtRs_20_regstr!$

V={} #Initialize Voter's List as empty dictionary

Vread=open("Voters_Register.txt","r") #Open's text file to read Voter's information
#Loop to append each line in text file as a list in the Voters dictionary, Voter's ID is the key
for i in Vread:
    d=i.split("\t")
    d.remove('\n')
    if len(d)>0:
        V[int(d[0])]=d
    else:
        V={}
Vread.close()

#Function to delete information from entries
def clear_entries():
    ent_first_name.delete(0,tk.END)
    ent_middle_name.delete(0,tk.END)
    ent_last_name.delete(0,tk.END)
    ent_DOB.delete(0,tk.END)
    gnd.set("Please select your Gender")
    var.set("Please Select your region")
    ent_country.delete(0,tk.END)
    ent_vo_id.delete(0,tk.END)

#Function to recieve data from entries and write unto file
def update_list():
    from datetime import date
    global V
    days = 365.2425
    f_name=ent_first_name.get()
    m_name=ent_middle_name.get()
    l_name=ent_last_name.get()
    DOB=ent_DOB.get()
    gender=gnd.get()
    region=var.get()
    country=ent_country.get()
    card_id=ent_vo_id.get()

    Vs=open("Voters_Register.txt","a+") #Open text file to read and write
    #Check age of Voter
    try:
        birthDate=date(int(DOB.split("/")[2]),int(DOB.split("/")[1]),int(DOB.split("/")[0]))
        age = int((date.today() - birthDate).days / days)
    except IndexError or ValueError:
        tk.messagebox.showwarning("Warning", "Inappropriate date format")

    #Conditional statements to ensure form is complete
    if (len(card_id) and len(f_name) and len (m_name)
        and len (l_name) and len(DOB) and len(gender)
        and len(region) and len(country)!=0
        and region!="Please Select your region"
        and gender!="Please select your Gender"):

        #Conditional statements to ensure user is an Eligible voter
        if age>=18 and country=="Ghanaian":

            if int(card_id)not in V.keys():
                Vs.write(card_id+'\t' )
                Vs.write(f_name+'\t' )
                Vs.write(m_name+'\t' )
                Vs.write(l_name+'\t' )
                Vs.write(DOB+'\t')
                Vs.write(gender+'\t')
                Vs.write(region+'\t')
                Vs.write(country+'\t'+'\n')
                Vs.close()
                Vread=open("Voters_Register.txt","r")
                tk.messagebox.showinfo("Success!","Form has been submitted succesfully")
                for i in Vread:
                    d=i.split("\t")
                    d.remove('\n')
                    V[int(card_id)]=d
            else:
                tk.messagebox.showerror("Error", "Voters ID already in system")
        else:
            tk.messagebox.showerror("Error", "User is not an eligible voter")
    else:
        tk.messagebox.showerror("Error","Incomplete Registration")

#Funfunction to open voters list
def openlist():
    global window2

    #Creation of 2nd Window, for the list of voters
    window2=tk.Tk()
    window2.title("Voters_list")

    vtrs_title = tk.Frame(master=window2,relief=tk.SUNKEN, borderwidth=3, height=20)
    vtrs_title.pack(fill=tk.BOTH, expand=False)

    lbl_title = tk.Label(master=vtrs_title, text="Voters Registration List:")
    lbl_title.pack()

    ttle= ["Voters ID Number",
           "First Name",
           "Middle Name",
           "Last Name",
           "Date of Birth(dd/mm/yyyy)",
           "Gender",
           "Nationality",
           "Region"
           ]

    vtrs_row2=tk.Frame(master=window2,relief=tk.SUNKEN, borderwidth=3,height=150)
    vtrs_row2.pack(fill=tk.BOTH,expand=True)

    a=0
    for i in ttle:
        vtrs_col=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=3,height=40)
        vtrs_col.grid(column=a,row=0,sticky="new")

        a=a+1

        vtrs_ttle=tk.Label(master=vtrs_col, text=i)
        vtrs_ttle.grid(sticky="nsew")

    Vread=open("Voters_Register.txt","r")
    b=0

    #Creation of frames and Listboxes for Voters list window

    vtrs_list0=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=0.5)
    vtrs_list0.grid(column=0,row=1,sticky="nsew")
    vtrs_row2.rowconfigure(1,weight=1)
    vtrs_row2.columnconfigure(0,weight=1)
    vtrs_list0.rowconfigure(0,weight=1)
    vtrs_list0.columnconfigure(0,weight=1)
    listbox0 = tk.Listbox(master=vtrs_list0)
    listbox0.grid(sticky="nsew")

    vtrs_list1=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=0.5)
    vtrs_list1.grid(column=1,row=1,sticky="nsew")
    vtrs_row2.rowconfigure(1,weight=1)
    vtrs_row2.columnconfigure(1,weight=1)
    vtrs_list1.rowconfigure(0,weight=1)
    vtrs_list1.columnconfigure(0,weight=1)
    listbox1 = tk.Listbox(master=vtrs_list1)
    listbox1.grid(sticky="nsew")

    vtrs_list2=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=0.5)
    vtrs_list2.grid(column=2,row=1,sticky="nsew")
    vtrs_row2.rowconfigure(1,weight=1)
    vtrs_row2.columnconfigure(2,weight=1)
    vtrs_list2.rowconfigure(0,weight=1)
    vtrs_list2.columnconfigure(0,weight=1)
    listbox2 = tk.Listbox(master=vtrs_list2)
    listbox2.grid(sticky="nsew")

    vtrs_list3=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=0.5)
    vtrs_list3.grid(column=3,row=1,sticky="nsew")
    vtrs_row2.rowconfigure(1,weight=1)
    vtrs_row2.columnconfigure(3,weight=1)
    vtrs_list3.rowconfigure(0,weight=1)
    vtrs_list3.columnconfigure(0,weight=1)
    listbox3 = tk.Listbox(master=vtrs_list3)
    listbox3.grid(sticky="nsew")

    vtrs_list4=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=0.5)
    vtrs_list4.grid(column=4,row=1,sticky="nsew")
    vtrs_row2.rowconfigure(1,weight=1)
    vtrs_row2.columnconfigure(4,weight=1)
    vtrs_list4.rowconfigure(0,weight=1)
    vtrs_list4.columnconfigure(0,weight=1)
    listbox4 = tk.Listbox(master=vtrs_list4)
    listbox4.grid(sticky="nsew")

    vtrs_list5=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=0.5)
    vtrs_list5.grid(column=5,row=1,sticky="nsew")
    vtrs_row2.rowconfigure(1,weight=1)
    vtrs_row2.columnconfigure(5,weight=1)
    vtrs_list5.rowconfigure(0,weight=1)
    vtrs_list5.columnconfigure(0,weight=1)
    listbox5= tk.Listbox(master=vtrs_list5)
    listbox5.grid(sticky="nsew")

    vtrs_list6=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=0.5)
    vtrs_list6.grid(column=6,row=1,sticky="nsew")
    vtrs_row2.rowconfigure(1,weight=1)
    vtrs_row2.columnconfigure(6,weight=1)
    vtrs_list6.rowconfigure(0,weight=1)
    vtrs_list6.columnconfigure(0,weight=1)
    listbox6= tk.Listbox(master=vtrs_list6)
    listbox6.grid(sticky="nsew")

    vtrs_list7=tk.Frame(master=vtrs_row2,relief=tk.SUNKEN, borderwidth=0.5)
    vtrs_list7.grid(column=7,row=1,sticky="nsew")
    vtrs_row2.rowconfigure(1,weight=1)
    vtrs_row2.columnconfigure(7,weight=1)
    vtrs_list7.rowconfigure(0,weight=1)
    vtrs_list7.columnconfigure(0,weight=1)
    listbox7= tk.Listbox(master=vtrs_list7)
    listbox7.grid(sticky="nsew")

    scrollbar = tk.Scrollbar(vtrs_list7, orient=tk.VERTICAL)
    #Function for scrollbar control
    def yview(*args):
        listbox0.yview(*args)
        listbox1.yview(*args)
        listbox2.yview(*args)
        listbox3.yview(*args)
        listbox4.yview(*args)
        listbox5.yview(*args)
        listbox6.yview(*args)
        listbox7.yview(*args)

    scrollbar.config(command=yview)
    scrollbar.grid(row=0,column=len(ttle),sticky="nse")
    lbox=[listbox0,listbox1,listbox2,listbox3,listbox4,listbox5,listbox6,listbox7]

    for i in range(len(lbox)):                                                      #Loop to append data into listboxes
        x=0

        for key in V:
            lbox[i].insert(x,V[key][i])
            x=x+1
                                                                                #Function to delete selected user
    def del_user():
        Vread1=open("Voters_Register.txt","r")                                      #Open file to read
        pos = 0
        for line in Vread1:
                                                                                # For each line, check if line contains the string
            if ent_search.get() in line:
                print(pos)
                break
            pos += 1
        Vread1.close()
        try:
            del V[int(ent_search.get())]                                            #Delete ID number stated from Dictionary

        except KeyError or ValueError:
            tk.messagebox.showinfo("my message","User not found in Voter's List")
        Vread2 = open("Voters_Register.txt", "r")                                   #Open file to read
        lines = Vread2.readlines()
        Vread2.close()
        del lines[pos]                                                              #Delete line that contains ID number stated from text file
        Vwrite = open("Voters_Register.txt", "w+")                                  #Open file to write
        for line in lines:                                                          #Write remaining files to text file
            Vwrite.write(line)

        Vwrite.close()
        window2.withdraw()
        openlist()

    def reset():
        Vread3 = open("Voters_Register.txt", "r")                                   #Open file to read
        backup_lines = Vread3.readlines()
        Vread3.close()
        Vbackup = open("Voters_Register_backup.txt", "w+")                          #Open back up file to write
        for line in backup_lines:                                                   #Write remaining files to text file
            Vbackup.write(line)
        Vbackup.close()
        V.clear()                                                                   #Clear Voters Dictionary
        Vs=open("Voters_Register.txt","w")                                          #Open File to write
        window2.withdraw()                                                          #Temporarily close Window
        openlist()                                                                  #Refresh page to display new list

    #Frame to search for Users
    vtrs_search = tk.Frame(master=window2,relief=tk.SUNKEN, borderwidth=3, height=20)
    vtrs_search.pack(fill=tk.BOTH, expand=False)
    btn_delete = tk.Button(master=vtrs_search, text="Delete",command=del_user)
    btn_delete.pack(side=tk.RIGHT, padx=10,pady=10, ipadx=10)

    btn_reset = tk.Button(master=vtrs_search, text="Reset",command=reset)
    btn_reset.pack(side=tk.LEFT, padx=10,pady=10, ipadx=10)

    ent_search = tk.Entry(master=vtrs_search)
    ent_search.pack(side=tk.RIGHT,padx=10,pady=10)
    lbl_search = tk.Label(master=vtrs_search, text="Search for Voter's ID")
    lbl_search.pack(side=tk.RIGHT,padx=10,pady=10)

    window2.mainloop()

def confirm2():
    if entry2.get()=="vtRs_20_regstr!$":
        check.withdraw()
        window.deiconify()
        entry2.delete(0,tk.END)
    else:
       tk.messagebox.showerror("error", "Incorrect password, check code")

def confirm():

    global response
    global btn_dup
    response=choice.get()
    if response==0:
        login.withdraw()
        window.deiconify()
        btn_dup.pack_forget()

        return response
    else:
        login.withdraw()
        check.deiconify()
        return response

def log_out():
    window.withdraw() or check.withdraw()
    login.deiconify()
    entry2.delete(0,tk.END)
    btn_dup.pack(side=tk.RIGHT,padx=10,pady=10, ipadx=10)

#Creation of 1st Window, for data entry
window = tk.Tk()
window.withdraw()
window.title("Voter's Registration")
##login=tk.Tk()
login = tk.Toplevel(window)
login.geometry("700x300")
login_heading=tk.Frame(master=login,relief=tk.SUNKEN,borderwidth=3,height=3)
login_heading.pack(fill=tk.BOTH,side=tk.LEFT, expand=True,pady=30,padx=(20,10))
lbl_login= tk.Label(master=login_heading, text="Welcome to Ghana's \n\nVoters Registeration form")
lbl_login.config(font=("Courier", 20))
lbl_login.pack(expand=False,side=tk.TOP,pady=10)

question = tk.Label(master=login_heading, text="Please indicate your Status:")
question.config(font=("Courier", 14))
question.pack(anchor=tk.CENTER,expand=True)
choice=tk.IntVar()
choice.set(0)
voter=tk.Radiobutton(login, text="Voter",padx = 20,variable=choice,value=0)
voter.config(font=("Times", 14))
voter.pack(anchor=tk.W, expand=True)
registrar=tk.Radiobutton(login, text="Registrar",padx = 20,variable=choice,value=1)
registrar.config(font=("Times", 14))
registrar.pack(anchor=tk.W,expand=True)
ans = tk.Button (login, text = 'Submit', command = confirm,bg="green",fg="white")
ans.pack (pady=10)
check = tk.Toplevel(window)
check.geometry("700x250")

check.withdraw()
heading=tk.Frame(master=check,height=3)
label2 = tk.Label(heading, text="Enter Registrar's passcode")
label2.config(font=("Courier", 18))
heading.pack(pady=(30,20),anchor=tk.CENTER)
entry2 = tk.Entry(check,show="*",width=30)
button2 = tk.Button(check, text="Verify", command=confirm2,bg="green",fg="white",width=7)
button2.config(font=("Times", 14))
btn_logout1 = tk.Button(master=check, text="Logout", command=log_out)

label2.pack(anchor=tk.CENTER)
entry2.pack(anchor=tk.CENTER)

button2.pack(pady=20,ipadx=10,anchor=tk.CENTER)
btn_logout1.pack(padx=10,pady=(30,5), ipadx=10,anchor=tk.CENTER)

vtrs_top=tk.Frame(master=window,borderwidth=1,height=3)
vtrs_top.pack(fill=tk.BOTH,padx=40)

vtrs_heading=tk.Frame(master=vtrs_top,relief=tk.SUNKEN,borderwidth=3,height=3)
vtrs_heading.pack(fill=tk.BOTH,side=tk.LEFT, expand=True,pady=(20,40))
lbl_heading= tk.Label(master=vtrs_heading, text="Welcome to Ghana's Voters Registeration form")
lbl_heading.config(font=("Courier", 14))
lbl_heading.pack(expand=False,side=tk.TOP)
lbl_warning= tk.Label(master=vtrs_heading, text="Please fill this form honestly and to the best of your knowledge ",height=1)
lbl_warning.pack(expand=False)
lbl_disclaimer= tk.Label(master=vtrs_heading, text="AN INACCURATE REPRESENATION OF YOUR TRUE IDENTITY IS A CRIME! ",fg="red")
lbl_disclaimer.pack(expand=False)
canvas = tk.Canvas(master=vtrs_top,width=151,height=121)
canvas.pack(expand=False,side=tk.LEFT,padx=10,pady=10)

#Ghana Flag
Kwad=t.RawTurtle(canvas)
Kwad.speed(speed="fastest")
Kwad.hideturtle()
x=-76
y=61
color=["red","yellow","green"]
for i in color:
    Kwad.penup()
    Kwad.goto(x,y)
    Kwad.pendown()
    Kwad.fillcolor(i)
    Kwad.begin_fill()
    for i in range(2):
        Kwad.forward(150)
        Kwad.right(90)
        Kwad.forward(40)
        Kwad.right(90)
    Kwad.end_fill()
    y=y-40

Kwad.penup()
Kwad.goto(-16,7)
Kwad.pendown()
Kwad.fillcolor('black')
Kwad.begin_fill()
for i in range(5):
    Kwad.forward(30)
    Kwad.right(144)
Kwad.end_fill()

#Creation of the forms frame and entries
vtrs_form = tk.Frame(master=window,relief=tk.SUNKEN, borderwidth=3)
vtrs_form.pack(fill=tk.BOTH, expand=True)

vtrs_form.grid_columnconfigure(0, weight=1)
vtrs_form.grid_columnconfigure(1, weight=1)
for i in range(9):
    vtrs_form.grid_rowconfigure(i, weight=1)

#Function to validate Voters ID entry
def votID(no,P):
    if len(P)<=10:
        if no.isdigit():
            return True
        elif no ==" ":
            return True
        else:
            return False
    else:
        return False

vi=window.register(votID)

lbl_gh_no = tk.Label(master=vtrs_form, text="Voter's ID Number:\n(10 digits)")
ent_vo_id = tk.Entry(master=vtrs_form, width=50)
lbl_gh_no.grid(row=0, column=0, sticky="ew")
ent_vo_id.grid(row=0, column=1,sticky="ew",padx=10,pady=5)
ent_vo_id.config(validate="key",validatecommand=(vi,"%S","%P"))

lbl_first_name = tk.Label(master=vtrs_form, text="First Name:")
ent_first_name = tk.Entry(master=vtrs_form, width=50)
lbl_first_name.grid(row=1, column=0, sticky="ew")
ent_first_name.grid(row=1, column=1,sticky="ew",padx=10,pady=5)

lbl_middle_name = tk.Label(master=vtrs_form, text="Middle Name:")
ent_middle_name = tk.Entry(master=vtrs_form, width=50)
lbl_middle_name.grid(row=2, column=0, sticky="ew")
ent_middle_name.grid(row=2, column=1,sticky="ew",padx=10,pady=5)

lbl_last_name = tk.Label(master=vtrs_form, text="Last Name:")
ent_last_name = tk.Entry(master=vtrs_form, width=50)
lbl_last_name.grid(row=3, column=0, sticky="ew")
ent_last_name.grid(row=3, column=1,sticky="ew",padx=10,pady=5)

#Function to validate Date of Birth entry
def DOB(ent,P):
    if len(P)<=10:
        if ent.isdigit():
            return True
        elif ent == " ":
            return True
        elif "/" in ent:
            return True

        else:
            return False
    else:
        return False
db=window.register(DOB)

lbl_DOB = tk.Label(master=vtrs_form, text="Date of Birth:\n (dd/mm/yyyy)")
ent_DOB = tk.Entry(master=vtrs_form, width=50)
lbl_DOB.grid(row=4, column=0, sticky="ew")
ent_DOB.grid(row=4, column=1,sticky="ew",padx=10,pady=5)
ent_DOB.config(validate="key",validatecommand=(db,"%S","%P"))

lbl_gender = tk.Label(master=vtrs_form, text="Gender:")
lbl_gender.grid(row=5, column=0, sticky="ew")

Options = [
    "Male",
    "Female"
    ]
gnd = tk.StringVar()
gnd.set("Please select your Gender")

opt = tk.OptionMenu(vtrs_form, gnd, *Options)
opt.grid(row=5,column=1,sticky="ew",padx=10,pady=5)

#Function to validate Nationality entry
def cntry(cnt,P):
    if len(P)<=56:
        if cnt.isalpha():
            return True

        elif "-" in cnt:
            return True
        elif cnt == "":
            return True
        else:
            return False
    else:
        return False
ct=window.register(cntry)
lbl_country = tk.Label(master=vtrs_form, text="Nationality:")
ent_country = tk.Entry(master=vtrs_form, width=50)
lbl_country.grid(row=6, column=0, sticky="ew")
ent_country.grid(row=6, column=1,sticky="ew",padx=10,pady=5)
ent_country.config(validate="key",validatecommand=(ct,"%S","%P"))

OptionList = [
"Ahafo",
"Ashanti",
"Brong Ahafo",
"Bono East",
"Central",
"Eastern",
"Greater Accra",
"North East",
"Northern",
"Oti",
"Savannah",
"Upper East",
"Upper West",
"Volta",
"Western",
"Western North"
]

lbl_region = tk.Label(master=vtrs_form, text="Region:")
lbl_region.grid(row=8, column=0, sticky="new")

var = tk.StringVar()

opt = ttk.Combobox(vtrs_form,textvariable = var)
opt['values'] =(
"Ahafo",
"Ashanti",
"Brong Ahafo",
"Bono East",
"Central",
"Eastern",
"Greater Accra",
"North East",
"Northern",
"Oti",
"Savannah",
"Upper East",
"Upper West",
"Volta",
"Western",
"Western North"
)

opt.set("Please Select your region")

opt.grid(row=8,column=1,sticky="new",padx=10,pady=5)

##Buttons
frm_buttons = tk.Frame(master=window)
frm_buttons.pack(fill=tk.BOTH,expand=False)

btn_submit = tk.Button(master=frm_buttons, text="Submit",command=lambda:[update_list(),clear_entries()])
btn_submit.pack(side=tk.RIGHT, padx=10,pady=10, ipadx=10)

btn_clear = tk.Button(master=frm_buttons, text="Clear", command=clear_entries)
btn_clear.pack(side=tk.RIGHT,padx=10,pady=10, ipadx=10)

btn_dup = tk.Button(master=frm_buttons, text="Open List", command=openlist)
btn_dup.pack(side=tk.RIGHT,padx=10,pady=10, ipadx=10)

btn_logout = tk.Button(master=frm_buttons, text="Logout", command=log_out)
btn_logout.pack(side=tk.LEFT,padx=10,pady=10, ipadx=10)

# Start the form

window.mainloop()


from __future__ import print_function
import matplotlib.pyplot as plt
import numpy as np
import os
import sys
import tarfile
from IPython.display import display, Image
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
from scipy import ndimage
from six.moves import cPickle as pickle
import tensorflow as tf

train_filename = 'train256classSimple10000'
test_filename = 'test256classSimple1000'

# extracting the datasets
num_classes = 256
np.random.seed(133)

def maybe_extract(filename, force=False):
  root = os.path.splitext(os.path.splitext(filename)[0])[0]  # remove .tar.gz
  if os.path.isdir(root) and not force:
    # You may override by setting force=True.
    print('%s already present - Skipping extraction of %s.' % (root, filename))
  else:
    print('Extracting data for %s. This may take a while. Please wait.' % root)
    tar = tarfile.open(filename)
    sys.stdout.flush()
    tar.extractall()
    tar.close()
  data_folders = [
    os.path.join(root, d) for d in sorted(os.listdir(root))
    if os.path.isdir(os.path.join(root, d))]
  if len(data_folders) != num_classes:
    raise Exception(
      'Expected %d folders, one per class. Found %d instead.' % (
        num_classes, len(data_folders)))
  print(data_folders)
  return data_folders

train_folders = maybe_extract(train_filename)
test_folders = maybe_extract(test_filename)

#Merging the data

image_size = 28  # Pixel width and height.
pixel_depth = 255.0  # Number of levels per pixel.
num_of_classes = 256

float_formatter = lambda x: "%.2f" % x

def Merge_folders(data_folders, size_per_class):
  dataset_names = []
  start_t = 0
  end_t = size_per_class
  required_size = size_per_class * num_of_classes
  trainDataset = np.ndarray((required_size, image_size, image_size), dtype=np.float32)
  labelsDataset = np.ndarray((required_size, 1), dtype=np.float32)
  for folder in data_folders:
    dataset_names.append(folder)

    print('Merging %s.' % folder)
    image_files = os.listdir(folder)
    dataset = np.ndarray(shape=(len(image_files), image_size, image_size),
                         dtype=np.float32)
    image_index = 0
    for image in os.listdir(folder):
        if image_index < size_per_class:
            image_file = os.path.join(folder, image)
            image_data = (ndimage.imread(image_file).astype(int))
            dataset[image_index, :, :] = image_data
            image_index += 1
    num_images = image_index
    dataset = dataset[0:num_images, :, :]
    trainDataset[start_t:end_t, :, :] = dataset
    x = float(folder.split("/")[1]) / 255.0
    #y = float_formatter(x)
    labelsDataset[start_t:end_t] = x
    start_t += size_per_class
    end_t += size_per_class

  return dataset_names, trainDataset, labelsDataset

train_datasets, trainDataset, labelsTrainDataset = Merge_folders(train_folders, 10000)
test_datasets, testDataset, labelsTestDataset = Merge_folders(test_folders, 1000)

print(trainDataset)
print(labelsTrainDataset)

print(testDataset)
print(labelsTestDataset)

print('trainDataset.shape' , trainDataset.shape)
print('labelsTrainDataset.shape' , labelsTrainDataset.shape)

print('testDataset.shape' , testDataset.shape)
print('labelsTestDataset.shape' , labelsTestDataset.shape)

# randomize the data
def randomize(dataset, labels):
  permutation = np.random.permutation(labels.shape[0])
  shuffled_dataset = dataset[permutation,:,:]
  shuffled_labels = labels[permutation]
  return shuffled_dataset, shuffled_labels
train_dataset, train_labels = randomize(trainDataset, labelsTrainDataset)
test_dataset, test_labels = randomize(testDataset, labelsTestDataset)
print('Randomizing is done')

# Reformat the data: image data as a cube (width by height by number
# of channels) and labels as float 1-hot encodings.
image_size = 28
num_labels = 1
num_channels = 1 # grayscale

def reformat(dataset, labels):
  dataset = dataset.reshape(
    (-1, image_size, image_size, num_channels)).astype(np.float32)
  #labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)
  return dataset, labels
train_dataset, train_labels = reformat(train_dataset, train_labels)
test_dataset, test_labels = reformat(test_dataset, test_labels)
print('Reformatted training set', train_dataset.shape, train_labels.shape)
print('Reformatted test set', test_dataset.shape, test_labels.shape)

def accuracy(predictions, labels):
  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
          / predictions.shape[0])

def medianSqr(predictions, labels):
  print (labels.shape)
  print (predictions.shape)
  return (np.median(np.square(predictions - labels)))

def meanSqr(predictions, labels):
  print (labels.shape)
  print (predictions.shape)
  return (np.mean(np.square(predictions - labels)))

def weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1)
  return tf.Variable(initial)

def bias_variable(shape):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial)

def conv2d(x, W):
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def max_pool_2x2(x):
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
                        strides=[1, 2, 2, 1], padding='SAME')

def drop_out(x, keep_prob):
  return tf.nn.dropout(x, keep_prob)

batch_size = 100

graph = tf.Graph()

with graph.as_default():

  # Input data.
  tf_train_dataset = tf.placeholder(
    tf.float32, shape=(batch_size, image_size, image_size, num_channels))
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
  tf_test_dataset = tf.constant(test_dataset)

  # Variables.
  W_conv1 = weight_variable([5, 5, 1, 6])
  b_conv1 = bias_variable([6])

  W_conv2 = weight_variable([5, 5, 6, 16])
  b_conv2 = bias_variable([16])

  W_fc1 = weight_variable([7 * 7 * 16, 120])
  b_fc1 = bias_variable([120])

  W_fc2 = weight_variable([120, 84])
  b_fc2 = bias_variable([84])

  W_fc3 = weight_variable([84, num_labels])
  b_fc3 = bias_variable([num_labels])

  # Model.
  def model(data):
    h_conv1 = tf.nn.relu(conv2d(data, W_conv1) + b_conv1)
    h_pool1 = max_pool_2x2(h_conv1)
    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
    h_pool2 = max_pool_2x2(h_conv2)
    shape = h_pool2.get_shape().as_list()
    reshape = tf.reshape(h_pool2, [shape[0], shape[1] * shape[2] * shape[3]])
    h_fc1 = tf.nn.tanh(tf.matmul(reshape, W_fc1) + b_fc1)
    h_fc2 = tf.nn.tanh(tf.matmul(h_fc1, W_fc2) + b_fc2)
    #keep_prob = tf.placeholder(tf.float32)
    #h_fc2_drop = tf.nn.dropout(h_fc2, keep_prob)
    return tf.matmul(h_fc2, W_fc3) + b_fc3

  # Training computation.
  logits = model(tf_train_dataset)
  #loss = tf.reduce_mean(
    #tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))
  loss = tf.reduce_mean(tf.square(logits - tf_train_labels))

  # Optimizer.
  optimizer = tf.train.AdamOptimizer(0.005).minimize(loss)

  # Predictions for the training, validation, and test data.
  #train_prediction = tf.nn.softmax(logits)
  train_prediction = logits
  #test_prediction = tf.nn.softmax(model(tf_test_dataset))
  test_prediction = model(tf_test_dataset)

num_steps = 25601
step_plot = []
loss_plot = []
accuracy_plot = []

with tf.Session(graph=graph) as session:
  tf.global_variables_initializer().run()

  print('Initialized')
  for step in range(num_steps):
    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]
    batch_labels = train_labels[offset:(offset + batch_size), :]
    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
    _, l, predictions = session.run(
      [optimizer, loss, train_prediction], feed_dict=feed_dict)
    if (step % 50 == 0):
      print('Minibatch loss at step %d: %f' % (step, l))
      #acc = accuracy(predictions, batch_labels)
      #print('Minibatch accuracy: %.1f%%' % acc)
      step_plot.append(step)
      loss_plot.append(l)
      #accuracy_plot.append(acc)
  print('Test dataset median regression error: %.5f' % medianSqr(test_prediction.eval(), test_labels))
  print('Test dataset medan regression error: %.5f' % meanSqr(test_prediction.eval(), test_labels))
  plt.figure(facecolor='white')
  plt.plot(step_plot, loss_plot, 'rs--')
  plt.xlabel('Steps')
  plt.ylabel('Loss')
  plt.title('Loss plot')
  plt.grid(True)
  plt.show()
  #plt.figure(facecolor='white')
  #plt.plot(step_plot, accuracy_plot, '--bs')
  #plt.xlabel('Steps')
  #plt.ylabel('Accuracy')
  #plt.title('Accuracy plot')
  #plt.axis([0, 10000, 0, 110])
  #plt.grid(True)
  #plt.show()
  #print(accuracy_plot)