import numpy as np
import pytest
from numpy import linalg
from conftest import unit_box, points
from devito import clear_cache, Operator, configuration
from devito.logger import info
from examples.seismic import demo_model, Receiver
from examples.seismic.acoustic import acoustic_setup

pytestmark = pytest.mark.skipif(configuration['backend'] == 'yask' or
                                configuration['backend'] == 'ops',
                                reason="testing is currently restricted")

presets = {
    'constant': {'preset': 'constant-isotropic'},
    'layers': {'preset': 'layers-isotropic', 'ratio': 3},
}

class TestAdjoint(object):

    def setup_method(self, method):
        # Some of these tests are memory intensive as it requires to store the entire
        # forward wavefield to compute the gradient (nx.ny.nz.nt). We therefore call
        # 'clear_cache()' to release any remaining memory from the previous tests or
        # previous instances (different parametrizations) of these tests
        clear_cache()

    @pytest.mark.parametrize('mkey, shape, kernel, space_order, nbpml', [
        # 1 tests with varying time and space orders
        ('layers', (60, ), 'OT2', 4, 10), ('layers', (60, ), 'OT2', 8, 10),
        ('layers', (60, ), 'OT4', 4, 10), ('layers', (60, ), 'OT4', 8, 10),
        # 2D tests with varying time and space orders
        ('layers', (60, 70), 'OT2', 4, 10), ('layers', (60, 70), 'OT2', 8, 10),
        ('layers', (60, 70), 'OT2', 12, 10), ('layers', (60, 70), 'OT4', 4, 10),
        ('layers', (60, 70), 'OT4', 8, 10), ('layers', (60, 70), 'OT4', 12, 10),
        # 3D tests with varying time and space orders
        ('layers', (60, 70, 80), 'OT2', 4, 10), ('layers', (60, 70, 80), 'OT2', 8, 10),
        ('layers', (60, 70, 80), 'OT2', 12, 10), ('layers', (60, 70, 80), 'OT4', 4, 10),
        ('layers', (60, 70, 80), 'OT4', 8, 10), ('layers', (60, 70, 80), 'OT4', 12, 10),
        # Constant model in 2D and 3D
        ('constant', (60, 70), 'OT2', 8, 14), ('constant', (60, 70, 80), 'OT2', 8, 14),
    ])
    def test_adjoint_F(self, mkey, shape, kernel, space_order, nbpml):
        """
        Adjoint test for the forward modeling operator.
        The forward modeling operator F generates a shot record (measurements)
        from a source while the adjoint of F generates measurments at the source
        location from data. This test uses the conventional dot test:
        < Fx, y> = <x, F^T y>
        """
        tn = 500.  # Final time

        # Create solver from preset
        solver = acoustic_setup(shape=shape, spacing=[15. for _ in shape], kernel=kernel,
                                nbpml=nbpml, tn=tn, space_order=space_order,
                                **(presets[mkey]), dtype=np.float64)

        # Create adjoint receiver symbol
        srca = Receiver(name='srca', grid=solver.model.grid,
                        time_range=solver.geometry.time_axis,
                        coordinates=solver.geometry.src_positions)

        # Run forward and adjoint operators
        rec, _, _ = solver.forward(save=False)
        solver.adjoint(rec=rec, srca=srca)

        # Adjoint test: Verify <Ax,y> matches  <x, A^Ty> closely
        term1 = np.dot(srca.data.reshape(-1), solver.geometry.src.data)
        term2 = linalg.norm(rec.data.reshape(-1)) ** 2
        info('<Ax,y>: %f, <x, A^Ty>: %f, difference: %4.4e, ratio: %f'
             % (term1, term2, (term1 - term2)/term1, term1 / term2))
        assert np.isclose((term1 - term2)/term1, 0., rtol=1.e-10)

    @pytest.mark.parametrize('space_order', [4, 8, 12])
    @pytest.mark.parametrize('shape', [(60,), (60, 70), (40, 50, 30)])
    def test_adjoint_J(self, shape, space_order):
        """
        Adjoint test for the FWI Jacobian operator.
        The Jacobian operator J generates a linearized shot record (measurements)
        from a model perturbation dm while the adjoint of J generates the FWI gradient
        from an adjoint source (usually data residual). This test uses the conventional
        dot test:
        < Jx, y> = <x ,J^T y>
        """
        tn = 500.  # Final time
        nbpml = 10 + space_order / 2
        spacing = tuple([10.]*len(shape))
        # Create solver from preset
        solver = acoustic_setup(shape=shape, spacing=spacing,
                                nbpml=nbpml, tn=tn, space_order=space_order,
                                preset='layers-isotropic', dtype=np.float64)

        # Create initial model (m0) with a constant velocity throughout
        model0 = demo_model('layers-isotropic', ratio=3, vp_top=1.5, vp_bottom=1.5,
                            spacing=spacing, space_order=space_order, shape=shape,
                            nbpml=nbpml, dtype=np.float64, grid=solver.model.grid)

        # Compute the full wavefield u0
        _, u0, _ = solver.forward(save=True, m=model0.m)

        # Compute initial born perturbation from m - m0
        dm = (solver.model.m.data - model0.m.data)

        du, _, _, _ = solver.born(dm, m=model0.m)

        # Compute gradientfrom initial perturbation
        im, _ = solver.gradient(du, u0, m=model0.m)

        # Adjoint test: Verify <Ax,y> matches  <x, A^Ty> closely
        term1 = np.dot(im.data.reshape(-1), dm.reshape(-1))
        term2 = linalg.norm(du.data.reshape(-1))**2
        info('<Jx,y>: %f, <x, J^Ty>: %f, difference: %4.4e, ratio: %f'
             % (term1, term2, (term1 - term2)/term1, term1 / term2))
        assert np.isclose((term1 - term2)/term1, 0., rtol=1.e-10)

    @pytest.mark.parametrize('shape, coords', [
        ((11, 11), [(.05, .9), (.01, .8)]),
        ((11, 11, 11), [(.05, .9), (.01, .8), (0.07, 0.84)])
    ])
    def test_adjoint_inject_interpolate(self, shape, coords, npoints=19):
        """
        Verify that p.inject is the adjoint of p.interpolate for a
        devito SparseFunction p
        """
        a = unit_box(shape=shape)
        a.data[:] = 0.
        c = unit_box(shape=shape, name='c', grid=a.grid)
        c.data[:] = 27.

        assert a.grid == c.grid
        # Inject receiver
        p = points(a.grid, ranges=coords, npoints=npoints)
        p.data[:] = 1.2
        expr = p.inject(field=a, expr=p)
        # Read receiver
        p2 = points(a.grid, name='points2', ranges=coords, npoints=npoints)
        expr2 = p2.interpolate(expr=c)
        Operator(expr + expr2)(a=a, c=c)
        # < P x, y > - < x, P^T y>
        # Px => p2
        # y => p
        # x => c
        # P^T y => a
        term1 = np.dot(p2.data.reshape(-1), p.data.reshape(-1))
        term2 = np.dot(c.data.reshape(-1), a.data.reshape(-1))
        assert np.isclose((term1-term2) / term1, 0., atol=1.e-6)


"""
Module to compute centroidal voronoi tesselation mesh from an existing mesh of
convex cells.

In principle, this is easy to compute:

* Compute the centroids of the cells.
* Invert the face_node_connectivity index array.
* For every node, find the connected faces.
* Use the connected faces to find the centroids.
* Order the centroids around the vertex in a counter-clockwise manner.

Dealing with the mesh exterior (beyond which no centroids are located) is the
tricky part. Refer to the docstrings and especially the visual examples in the
developer documentation.
"""
from typing import Tuple

import numpy as np
import pandas as pd
from scipy import sparse

from xugrid.constants import X_EPSILON, FloatArray, IntArray
from xugrid.ugrid.connectivity import renumber

def dot_product2d(U: FloatArray, V: FloatArray):
    return U[:, 1] * V[:, 1] + U[:, 0] * V[:, 0]

def _centroid_pandas(i: IntArray, x: FloatArray, y: FloatArray):
    grouped = pd.DataFrame({"i": i, "x": x, "y": y}).groupby("i").mean()
    x_centroid = grouped["x"].values
    y_centroid = grouped["y"].values
    return x_centroid, y_centroid

def _centroid_scipy(i: IntArray, x: FloatArray, y: FloatArray):
    j = np.arange(len(i))
    coo_content = (x, (i, j))
    mat = sparse.coo_matrix(coo_content)
    n = mat.getnnz(axis=1)
    x_centroid = mat.sum(axis=1).flat / n
    mat.data = y
    y_centroid = mat.sum(axis=1).flat / n
    return x_centroid, y_centroid

def compute_centroid(i: IntArray, x: FloatArray, y: FloatArray):
    return _centroid_pandas(i, x, y)

def exterior_centroids(node_face_connectivity: sparse.csr_matrix):
    n, _ = node_face_connectivity.shape
    # Find exterior nodes NOT associated with any interior edge
    is_exterior_only = node_face_connectivity.getnnz(axis=1) == 1
    j = node_face_connectivity[is_exterior_only].indices
    i = np.arange(n)[is_exterior_only]
    return i, j

def interior_centroids(
    node_face_connectivity: sparse.csr_matrix,
    edge_face_connectivity: IntArray,
    edge_node_connectivity: IntArray,
    fill_value: int,
):
    # Find exterior nodes associated with interior edges
    is_exterior = edge_face_connectivity[:, 1] == fill_value
    exterior_nodes = np.unique(edge_node_connectivity[is_exterior].ravel())
    m_per_node = node_face_connectivity.getnnz(axis=1)
    is_interior_only = m_per_node > 1

    selected_nodes = exterior_nodes[is_interior_only[exterior_nodes]]
    selection = node_face_connectivity[selected_nodes]
    m_per_selected_node = selection.getnnz(axis=1)

    j = selection.indices
    i = np.repeat(selected_nodes, repeats=m_per_selected_node)
    return i, j

def exterior_vertices(
    edge_face_connectivity: IntArray,
    edge_node_connectivity: IntArray,
    fill_value: int,
    vertices: FloatArray,
    centroids: FloatArray,
    add_vertices: bool,
):
    is_exterior = edge_face_connectivity[:, 1] == fill_value
    exterior_nodes = edge_node_connectivity[is_exterior]
    # For every exterior node, project the centroids to exterior edges
    edge_vertices = vertices[exterior_nodes]
    face_i = edge_face_connectivity[is_exterior, 0]
    centroid_vertices = centroids[face_i]
    a = edge_vertices[:, 0, :]
    b = edge_vertices[:, 1, :]
    V = b - a
    U = centroid_vertices - a
    # np.dot(U, V) doesn't do the desired thing here
    projected_vertices = a + ((dot_product2d(U, V) / dot_product2d(V, V)) * V.T).T

    # Create the numbering pointing to these new vertices.
    # Discard vertices that overlap with e.g. circumcenters.
    keep = np.linalg.norm(projected_vertices - centroid_vertices, axis=1) > (
        X_EPSILON * X_EPSILON
    )
    face_i = face_i[keep]
    vertices_keep = projected_vertices[keep]
    n_centroid = len(centroids)
    i_keep = exterior_nodes[keep].ravel()
    n = n_centroid + len(vertices_keep)
    j_keep = np.repeat(np.arange(n_centroid, n), 2)
    n_interpolated = 0

    # We add a substitution value for the actual vertex
    if add_vertices:
        n_new = len(projected_vertices)
        i = exterior_nodes.ravel()
        order = np.argsort(i)
        jj = np.repeat(np.arange(n_new), 2)[order]
        to_interpolate = projected_vertices[jj]
        interpolated = 0.5 * (to_interpolate[::2] + to_interpolate[1::2])
        n_interpolated = len(interpolated)
        i_keep = np.concatenate([i_keep, i[order][::2]])
        j_keep = np.concatenate([j_keep, np.arange(n, n + n_interpolated)])
        vertices_keep = np.concatenate([vertices_keep, interpolated])
        # Create face index: the face of the original mesh associated with every
        # voronoi vertex is a centroid. The exterior vertices are an exception to
        # this: these are associated with two faces. So we set a value of -1 here.
        face_i = np.concatenate([face_i, np.full(n_interpolated, -1)])

    return i_keep, j_keep, vertices_keep, face_i, n_interpolated

def exterior_topology(
    edge_face_connectivity: IntArray,
    edge_node_connectivity: IntArray,
    node_face_connectivity: sparse.csr_matrix,
    fill_value: int,
    vertices: FloatArray,
    centroids: FloatArray,
    add_vertices: bool,
):
    """
    The exterior topology of this voronoi tesselation consists of three kinds
    of vertices:

    * Centroids of faces that have a vertex located on the exterior, but have
      no exterior edges. These centroids are used twice in the resulting
      voronoi mesh topology.
    * Centroids of faces that have an exterior edge. These centroids are used
      once.
    * Vertices of the intersection of (infinite) rays with the mesh exterior.
      These vertices are used twice; these intersections are the orthogonal
      projection of the centroid of the face on its exterior edges.
    * Original vertices of the mesh exterior. These vertices are used once, and
      are always located in between the projected vertices.

    The last ones are trickiest, as they may create concave angles; with a
    concave angle in a polygon, we cannot easily compute the counter clockwise
    order of the vertices to form a polygon. However, since these vertices are
    always located in between the projected vertices, we can introduce a
    subsitute: a linear interpolation of the two projected vertices. We
    can then utilize this interpolated vertex to compute the counter clockwise
    order, and then replace it by the original vertex -- which then may
    introduce a concave angle.

    A note on the index arrays:

    * i refers to the node number of the original mesh.
    * In principle, j refers to the face number (and associated centroids) of
    the original mesh; exterior vertices require new numbers.

    Finally, in the resulting new (voronoi) face_node_connectivity, i becomes
    the face number, and j becomes the node number.
    """
    i0, j0 = interior_centroids(
        node_face_connectivity,
        edge_face_connectivity,
        edge_node_connectivity,
        fill_value,
    )
    i1, j1 = exterior_centroids(node_face_connectivity)
    i2, j2, projected_vertices, face_i, n_interpolated = exterior_vertices(
        edge_face_connectivity,
        edge_node_connectivity,
        fill_value,
        vertices,
        centroids,
        add_vertices,
    )

    i = np.concatenate([i0, i1, i2])
    j = np.concatenate([j0, j1, j2])
    _, n_face = node_face_connectivity.shape
    vor_vertices = np.concatenate([centroids, projected_vertices])
    face_i = np.concatenate([np.arange(n_face), face_i])
    orig_vertices = vertices[i][-n_interpolated:]

    # Now form valid counter-clockwise polygons
    voronoi_vertices = vor_vertices[j]
    x = voronoi_vertices[:, 0]
    y = voronoi_vertices[:, 1]
    centroid_x, centroid_y = compute_centroid(i, x, y)
    renumbered_i = renumber(i)
    dx = x - centroid_x[renumbered_i]
    dy = y - centroid_y[renumbered_i]
    angle = np.arctan2(dy, dx)
    # Sort to create a counter clockwise oriented polygon.
    # Use a lexsort to make sure nodes are grouped together.
    order = np.lexsort((angle, i))
    i = i[order]
    j = j[order]

    # If add_vertices is True, we have substituted interpolated points before
    # to generate the proper ordering. We overwrite those substituted points
    # here by their possibly concave true vertices.
    if add_vertices:
        vor_vertices[-n_interpolated:] = orig_vertices

    return vor_vertices, i, j, face_i

def voronoi_topology(
    node_face_connectivity: sparse.csr_matrix,
    vertices: FloatArray,
    centroids: FloatArray,
    edge_face_connectivity: IntArray = None,
    edge_node_connectivity: IntArray = None,
    fill_value: int = None,
    add_exterior: bool = False,
    add_vertices: bool = False,
) -> Tuple[FloatArray, sparse.csr_matrix]:
    """
    Compute the centroidal voronoi tesslation (CVT) of an existing mesh of
    (convex!) cells using connectivity index arrays.

    If the exterior boundary of the mesh forms a concave polygon, this will
    result in some concave voronoi cells as well. Since concave cells are often
    undesirable, there are three options for dealing with the exterior
    boundary:

    * if ``add_exterior=True`` and ``add_vertices=True``, infinite voronoi rays
      (projections) are intersected at exterior edges and all exterior vertices
      are included. If the mesh exterior forms a concave polygon, the resulting
      tesselation will contain concave cells as well.
    * if ``add_exterior=True`` and ``add_vertices=False``, the infinite voronoi
      rays are intersected, but no exterior vertices are included. This will
      always result in convex voronoi cells if the original cells are convex.
    * if ``add_exterior=False`` and ``add_vertices=False``, no vertices on the
      exterior edges are considered. Only centroids of the original mesh are
      considered. This will always result in convex voronoi cells if the
      original cells are convex.

    A direct correspondence exists between the voronoi vertices and the
    original faces: the vertices of the voronoi polygons are the centroids of
    the original faces; and the intersected rays "belong" to a single face too.
    This is not the case when all exterior vertices are included: these may
    correspond to multiple faces of the original mesh. These vertices are
    marked by a face_index of -1.

    Parameters
    ----------
    node_face_connectivity : csr matrix
    vertices: ndarray of floats with shape ``(n_vertex, 2)``
    centroids: ndarray of floats with shape ``(n_centroid, 2)``
    edge_face_connectivity: ndarray of integers with shape ``(n_edge, 2)``, optional
    edge_node_connectivity: ndarray of integers with shape ``(n_edge, 2)``, optional
    fill_value: int, optional
        Fill value for edge_face_connectivity.
    add_exterior: bool, optional
        Whether to consider exterior edges of the original mesh, or to consider
        exclusively centroids.
    add_vertices: bool, optional
        Whether to use existing exterior vertices.

    Returns
    -------
    nodes: ndarray of floats with shape ``(n_vertex, 2)``
    face_node_connectivity: scipy.sparse.csr_matrix
    face_index: ndarray of ints with shape ``(n_vertex,)``
        Connects the nodes of the voronoi topology to the faces of the original
        grid. Exterior vertices (when ``add_vertices=True``) are given an index
        of -1.
    """
    if add_exterior:
        if any(
            arg is None
            for arg in [edge_face_connectivity, edge_node_connectivity, fill_value]
        ):
            raise ValueError(
                "edge_face_connectivity, edge_node_connectivity, fill_value "
                "must be provided if add_exterior is True."
            )

    # Avoid overlapping polygons: if exterior is included, the exterior
    # algorithm will construct those polygons. If exterior is not included, we
    # take any valid internal polygon we can construct: at least a triangle.
    ncol_per_row = node_face_connectivity.getnnz(axis=1)
    if add_exterior:
        is_exterior = edge_face_connectivity[:, 1] == fill_value
        exterior_nodes = edge_node_connectivity[is_exterior]
        valid = np.full(len(vertices), True)
        valid[exterior_nodes.ravel()] = False
        valid = np.repeat(valid, ncol_per_row)
    else:
        valid = np.repeat(ncol_per_row >= 3, ncol_per_row)

    # Grab the centroids for all these faces.
    coo = node_face_connectivity.tocoo()
    node_i = coo.row[valid]
    face_i = coo.col[valid]
    voronoi_centroids = vertices[node_i]
    voronoi_vertices = centroids[face_i]
    # Compute the angle between voronoi centroids and vertices.
    x = voronoi_vertices[:, 0] - voronoi_centroids[:, 0]
    y = voronoi_vertices[:, 1] - voronoi_centroids[:, 1]
    angle = np.arctan2(y, x)
    # Now sort to create a counter clockwise oriented polygon.
    # Use a lexsort to make sure nodes are grouped together.
    order = np.lexsort((angle, node_i))
    j = face_i[order]

    if add_exterior:
        vor_vertices, exterior_i, exterior_j, face_i = exterior_topology(
            edge_face_connectivity,
            edge_node_connectivity,
            node_face_connectivity,
            fill_value,
            vertices,
            centroids,
            add_vertices,
        )
        offset = node_i.max() + 1 if len(node_i > 0) else 0
        i = np.concatenate([node_i, exterior_i + offset])
        j = np.concatenate([j, exterior_j])
    else:
        vor_vertices = centroids[np.unique(face_i)]
        face_i = np.arange(face_i.max() + 1)
        i = node_i
        j = renumber(j)

    i = renumber(i)
    coo_content = (j, (i, j))
    face_node_connectivity = sparse.coo_matrix(coo_content)
    return vor_vertices, face_node_connectivity, face_i


from django import forms
from django.core.validators import FileExtensionValidator
from crispy_forms.helper import FormHelper
from .models import ReportSet, Template, Field, RuleSet, Report, Submission
from account.models import Group, Profile

def form_horizontal():
    helper = FormHelper()
    helper.form_tag = False
    helper.form_class = 'form-horizontal'
    helper.label_class = 'col-lg-2'
    helper.field_class = 'col-lg-8'
    return helper

class ReportSetForm(forms.ModelForm):
    class Meta:
        model = ReportSet
        fields = ['name', 'group']

    def __init__(self, user, *args, **kwargs):
        super(ReportSetForm, self).__init__(*args, **kwargs)
        # limit group options based on user
        self.fields['group'].queryset = Group.objects.filter(profiles__user=user)
        # If you pass FormHelper constructor a form instance
        # It builds a default layout with all its fields
        self.helper = form_horizontal()

class TemplateForm(forms.ModelForm):
    class Meta:
        model = Template
        fields = ['name']

    def __init__(self, *args, **kwargs):
        super(TemplateForm, self).__init__(*args, **kwargs)
        self.helper = form_horizontal()

class FieldForm(forms.ModelForm):
    class Meta:
        model = Field
        fields = ['name', 'dtype']

    def __init__(self, *args, **kwargs):
        super(FieldForm, self).__init__(*args, **kwargs)
        self.helper = form_horizontal()

class FieldImportForm(forms.Form):
    docfile = forms.FileField(
        widget=forms.FileInput(attrs={'accept': ".csv"}),
        label='Upload',
        help_text='max. 42 meabytes',
        validators = [FileExtensionValidator(allowed_extensions=['csv'])]
    )

    def __init__(self, *args, **kwargs):
        super(FieldImportForm, self).__init__(*args, **kwargs)
        self.helper = form_horizontal()

class RuleSetForm(forms.ModelForm):
    class Meta:
        model = RuleSet
        fields = ['field', 'rule', 'argument', 'action', 'error_message']
        help_texts = {
            'argument': 'if multiple arguments, seperated by comma(,); date argument format: 2019-01-01',
            'action': 'Warning: flagged but submission will be accepted; Error: submission will be rejected',
        }

    def __init__(self, tpk, *args, **kwargs):
        super(RuleSetForm, self).__init__(*args, **kwargs)
        self.fields['field'].queryset = Field.objects.filter(template=tpk)
        self.helper = form_horizontal()

class ReportForm(forms.ModelForm):
    class Meta:
        model = Report
        fields = ['name', 'method', 'templates']

    def __init__(self, rspk, *args, **kwargs):
        super(ReportForm, self).__init__(*args, **kwargs)
        self.fields['templates'].queryset = Template.objects.filter(report_set=rspk)
        self.helper = form_horizontal()

class SubmissionForm(forms.ModelForm):
    class Meta:
        model = Submission
        fields = ['template', 'upload']

    def __init__(self, rpk, *args, **kwargs):
        super(SubmissionForm, self).__init__(*args, **kwargs)
        self.fields['template'].queryset = Template.objects.filter(reports__id=rpk)
        self.fields['upload'].widget = forms.FileInput(attrs={'accept': ".csv, application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"})
        self.fields['upload'].help_text = 'csv/xlsx, one sheet only'
        self.fields['upload'].validators = [FileExtensionValidator(allowed_extensions=['csv', 'xlsx'])]
        self.helper = form_horizontal()


# -*- coding: utf-8 -*-

#Step1匯入自備餐具優惠餐廳csv檔
import csv
print("""
全台各地自備餐具與飲料袋享優惠之餐廳統計表

輸入0離開程式
輸入1查看餐廳優惠方式比例
輸入2查看優惠方式為集點之餐廳
輸入3查看優惠方式為折價之餐廳
輸入4查看優惠方式為贈送餐飲之餐廳
選擇查詢內容:
""")
#Step2讀取資料
#fn1 = r'C:\Allie\FinalProject\OwnedCutlery.csv'
fn1 = r'/Users/aichung/Desktop/final_project/OwnedCutlery.csv'
with open(fn1,encoding='utf8') as csvFile:
    csvReader = csv.reader(csvFile)
    cutleryData = list(csvReader)
    #print(cutleryData)
#Step3撈出資料並統計
#Step4 import plt以圖示比例

import matplotlib.pyplot as plt

GBType = list(set([cutleryData[i][7] for i in range(1,len(cutleryData))]))
count = [0]*len(GBType)#list
supplierName = list(set(cutleryData[i][1] for i in range(1,len(cutleryData))))
#整理完的labels

for i in range(1,len(cutleryData)):
    count[GBType.index(cutleryData[i][7])]+=1
#GBType = [[GBType[i],count[i]]for i in range(len(GBType))]

'''
while True:
    keyin =eval(input())
    if keyin ==0:
        print('Bye')
        break;
    elif keyin ==1:
        x = GBType#x = labels
        y = count
        plt.title('餐廳優惠方式統計表')
        plt.pie(y,labels=x, autopct="%1.1f%%")
        plt.show()
    elif keyin ==2:
        for i in range(1,len(cutleryData)):
            if '集點' in cutleryData[i][7]:
                print(cutleryData[i][1])
    elif keyin ==3:
        for i in range(1,len(cutleryData)):
            if '折價' in cutleryData[i][7]:
                print(cutleryData[i][1])
    elif keyin ==4:
        for i in range(1,len(cutleryData)):
            if '贈送餐飲' in cutleryData[i][7]:
                print(cutleryData[i][1])
    else:
        print('輸入錯誤')
'''

while True:
    keyin =eval(input())
    if keyin ==0:
        print('Bye')
        break;
    elif keyin ==1:
        x = GBType#x = labels
        y = count
        plt.title('餐廳優惠方式統計表')
        plt.pie(y,labels=x, autopct="%1.1f%%")
        plt.show()
    elif keyin ==2:
        for i in range(1,len(cutleryData)):
            if '集點' in cutleryData[i][7]:
                print(cutleryData[i][1])
    elif keyin ==3:
        for i in range(1,len(cutleryData)):
            if '折價' in cutleryData[i][7]:
                print(cutleryData[i][1])
    elif keyin ==4:
        for i in range(1,len(cutleryData)):
            if '贈送餐飲' in cutleryData[i][7]:
                print(cutleryData[i][1])
    else:
        print('輸入錯誤')


import random
import pandas as pd
from sklearn.datasets import make_classification

def create_fake_table_dataframe(
    n_samples: int,
    n_features: int,
    n_classes: int,
    n_repeated: int = 0,
) -> pd.DataFrame:
    X, y = make_classification(
        n_samples=n_samples,
        n_features=n_features,
        n_classes=n_classes,
        n_repeated=n_repeated,
        n_clusters_per_class=1,
        n_informative=5,
        scale=[
            random.choice([random.randint(50, 1000), random.randint(1, 5)])
            for _ in range(n_features)
        ],
    )

    dataset = {}
    for i in range(n_features):
        dataset[f"feature_{i+1}"] = X[:, i]
    dataset["label"] = y
    return pd.DataFrame(dataset)

if __name__ == "__main__":
    create_fake_table_dataframe(10000, 10, 5, 0).to_csv(
        "ten_thousand_samples.csv", index=False
    )


import streamlit as st
import pandas as pd
from DBconnect import get_database_session
from json import loads
from pathlib import Path

st.write("Test File")
df=pd.read_csv(Path("./extras/stock_info.csv"), encoding='utf8')
st.table(df)
result = df.to_json(orient="records")
st.write(result)

# test = poms_db()
# client=test.get_client()
# st.write("Hi : ",client.list_database_names())

test = get_database_session()
st.write("Databases details: ",test.list_database_names())

pims_db = test["pims_db"]
st.write("POMS collection details: ",pims_db.list_collection_names())

stock_table = pims_db['stock_table']
if not stock_table.count_documents({}):
    stock_table.insert_many(loads(result))
    # st.write(type(loads(result)))
st.write("Stock table row count: ",stock_table.count_documents({}))

test_frommongo = pd.DataFrame(list(stock_table.find({},{"_id": 0})))
st.table(test_frommongo)

# Update a value

# if stock_table.count_documents({'stock':'Bread'}):
#     st.write(stock_table.find({'stock':'Bread'})[0])

stock_table.update_one(
    { 'stock' : 'Bread'},
    {
      '$set' : { 'quantity' : 7 }
    }
)


import clr
clr.AddReference('RevitAPI')
from Autodesk.Revit.DB import *

clr.AddReference("RevitNodes")
import Revit
clr.ImportExtensions(Revit.Elements)

wallinstances = UnwrapElement(IN[0])
methodlist = list()
for item in wallinstances:
	typename = item.GetType().Name
	if typename == 'Wall':
		methodlist.append('Standard')
	elif typename == 'FaceWall':
		methodlist.append('By Face')
	elif typename == 'FamilyInstance':
		methodlist.append('In-Place')
	else: methodlist.append(None)
OUT = methodlist


from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import inspect

import numpy
import matplotlib
import matplotlib.pyplot as plt

import PythonTools.Debug as DEBUG

cdict = {'red':   [(0.0,  0.0, 0.0),(0.475,  1.0, 1.0),(0.525,  1.0, 1.0),
            (1.0,  1.0, 1.0)],
         'green': [(0.0,  0.0, 0.0),(0.475,  1.0, 1.0),(0.525,  1.0, 1.0),
            (1.0,  0.0, 0.0)],
         'blue':  [(0.0,  1.0, 1.0),(0.475,  1.0, 1.0),(0.525,  1.0, 1.0),
            (1.0,  0.0, 0.0)]
        }
rwb_cmap = matplotlib.colors.LinearSegmentedColormap('rwb_colormap', cdict, 256)

def find_axes(x_axis, y_axis, x_range, y_range, flag_verbose = False):
    """
    croc.Plotting.find_axis

    INPUT:
    - x_axis, y_axis (ndarray): the axes
    - x_range, y_range (array): the ranges to be plotted
        Possible cases:
        - [min, max]: plot range min to max
        - [0, 0]: plot the whole range
        - [0, -1]: use the range from the other axis. If both have this, it will plot both axes complete. (ie. it is identical to both having [0,0])

    OUTPUT:
    - xmin, xmax, ymin, ymax: a value

    CHANGELOG
    - 20110910 RB: init

    """
    DEBUG.verbose("find axes", flag_verbose)

    # x_min and x_max are determined by y
    if x_range[0] == 0 and x_range[1] == -1:

        # they are both dependent on each other, plot everything
        if y_range[0] == 0 and y_range[1] == -1:
            x_min = x_axis[0]
            x_max = x_axis[-1]
            y_min = y_axis[0]
            y_max = y_axis[-1]

        # take the whole y-axis, the same for the x-axis
        # it may extend beyond the range of the data
        elif y_range[0] == 0 and y_range[1] == 0:
            x_min = y_axis[0]
            x_max = y_axis[-1]
            y_min = y_axis[0]
            y_max = y_axis[-1]

        # take part of the y-axis, the same for the x-axis
        # it may extend beyond the range of the data
        else:
            x_min = y_range[0]
            x_max = y_range[1]
            y_min = y_range[0]
            y_max = y_range[1]

    else:
        # plot the whole x-axis
        if x_range[0] == 0 and x_range[1] == 0:
            x_min = x_axis[0]
            x_max = x_axis[-1]
        # plot only part of the x-axis
        else:
            x_min = x_range[0]
            x_max = x_range[1]

        # plot the whole y-axis
        if y_range[0] == 0 and y_range[1] == 0:
            y_min = y_axis[0]
            y_max = y_axis[-1]
        # use the range of the x-axis
        elif y_range[0] == 0 and y_range[1] == -1:
            y_min = x_min
            y_max = x_max
        # use the specified range
        else:
            y_min = y_range[0]
            y_max = y_range[1]
    if flag_verbose:
        DEBUG.verbose("  axes found: x_min: " + str(x_min) + ", x_max: " + str(x_max) + ", y_min: " + str(y_min) + ", y_max: " + str(y_max), flag_verbose)

    return x_min, x_max, y_min, y_max

def find_axes_indices(axis, val_min, val_max, flag_verbose = False):
    """
    Find the indices of values val_min and val_max in axis, such that val_min and val_max are included.
    Because the intended use is to slice an array data[val_min_i:val_max_i], 1 is added to val_max_i.
    If the val_min or val_max exceed the axis, then the index is 0 or -1, respectively.

    BEHAVIOR:
    axis = [3,4,5,6]
    if val_min == 4.5: val_min_i = 1
    if val_min == 5: val_min_i = 1
    if val_min == 1: val_min_i = 0

    if val_max = 4.5: val_max_i = 3: axis[0:3] = [3,4,5]
    if val_max = 4: val_max_i = 2: axis[0:2] = [3,4]
    if val_max = 10: val_max_i = -1: axis[0:-1] = [3,4,5]

    CHANGELOG:
    201108xx/RB: originated in contourplot
    20130213/RB: moved to separate function. Better handling of edge cases.

    """
    if val_min > val_max:
        DEBUG.printWarning("val_min > val_max ({v1} and {v2}), will give strange result.".format(v1 = val_min, v2 = val_max), inspect.stack())

    temp = numpy.where(axis < val_min)
    if len(temp[0]) == 0:
        val_min_i = 0
    else:
        val_min_i = temp[0][-1]

    temp = numpy.where(axis > val_max)
    if len(temp[0]) == 0:
        val_max_i = -1
    else:
        val_max_i = temp[0][0] + 1
        if val_max_i == len(axis):
            val_max_i = -1

    return val_min_i, val_max_i

def make_contours_2d(data, zlimit = 0, contours = 21, flag_verbose = False):
    """
    zlimit = 0, show all, not don't care about centering around zero
    zlimit = -1, show all, centered around zero
    zlimit = all else, use that, centered around zero
    zlimit = [a,b], plot from a to b

    CHANGELOG:
    201108xx/RB: started in Plotting module
    20130213/RB: moved to Functions module
    """

    DEBUG.verbose("make contours 2D", flag_verbose)

    if zlimit == 0:
        ma = numpy.amax(data)
        mi = numpy.amin(data)
        if flag_verbose:
            DEBUG.verbose("  zlimit = 0, min: " + str(mi) + ", max: " + str(ma), flag_verbose)
        return numpy.linspace(mi, ma, num=contours)

    elif zlimit == -1:
        ma = numpy.amax(data)
        mi = numpy.amin(data)
        if flag_verbose:
            DEBUG.verbose("  zlimit = 0, min: " + str(mi) + ", max: " + str(ma), flag_verbose)

        if abs(mi) > abs(ma):
            ma = abs(mi)
        else:
            ma = abs(ma)
        return numpy.linspace(-ma, ma, num=contours)

    elif type(zlimit) == list:
        DEBUG.verbose("  zlimit == list", flag_verbose)
        return numpy.linspace(zlimit[0], zlimit[1], num=contours)

    else:
        DEBUG.verbose("  zlimit is linspace", flag_verbose)
        return numpy.linspace(-abs(zlimit), abs(zlimit), num=contours)

def truncate_data(data, x_axis, y_axis, x_min_i, x_max_i, y_min_i, y_max_i, flag_verbose = False):
    """
    Truncate data in a non-default way.
    x_axis = [1,2,3,4,5]
    x_min_i = 0
    if x_max_i == 2: x_axis = [1,2]
    if x_max_i == 4: x_axis = [1,2,3,4]
    if x_max_i == -1: x_axis = [1,2,3,4,5]

    CHANGELOG:
    20130213: was in Plotting.contourplot function, moved to Functions module, now handles x_max_i == -1
    """

    DEBUG.verbose("Truncate data", flag_verbose)

    if x_max_i == -1:
        DEBUG.verbose("  x_max == -1", flag_verbose)
        data = data[:,x_min_i:]
        x_axis = x_axis[x_min_i:]
    else:
        DEBUG.verbose("  x_max != -1", flag_verbose)
        data = data[:,x_min_i:x_max_i]
        x_axis = x_axis[x_min_i:x_max_i]

    if y_max_i == -1:
        DEBUG.verbose("  y_max == -1", flag_verbose)
        data = data[y_min_i:,:]
        y_axis = y_axis[y_min_i:]
    else:
        DEBUG.verbose("  y_max != -1", flag_verbose)
        data = data[y_min_i:y_max_i,:]
        y_axis = y_axis[y_min_i:y_max_i]

    return data, x_axis, y_axis

def find_subplots(n_plots, flag_verbose = 0):
    """
    Crocodile.Resources.Functions.find_subplots()

    Take the required number of subplots and find how many plots there should be horizontally and vertically. It is either NxN plots, or (N+1)xN plots.

    INPUT:
    - n_plots (int, >0): the number of plots

    OUTPUT:
    - x, y (int): the number of plots horizontally and vertically

    CHANGELOG
    - 20160413-RB: started function

    """
    DEBUG.verbose("find_subplots({n_plots})".format( n_plots = n_plots), flag_verbose)

    if n_plots < 1:
        return 0, 0

    t = numpy.sqrt(n_plots)

    x = numpy.floor(t)
    y = numpy.floor(t)

    if x * y < n_plots:
        x += 1

    if x * y < n_plots:
        y += 1

    DEBUG.verbose("n_plots: {n_plots}, x: {x}, y: {y}".format( n_plots = n_plots, x = x, y = y), flag_verbose)

    return int(x), int(y)

### SPECTROMETER FUNCTIONS ###

def spectrometer_axis(**kwargs):
    """

    """

    settings = {
        "grating_blaze_l_mm": 150,
        "center_wl_nm": 1e7/1603,
        "f_mm": 257,
        "spacing_mm": 0.525,
        "n_pixels": 32,
        "center_pixel": 16,
        "gamma_deg": 15,
        "half_angle_deg": 11.83,
        "order": 1
    }

    if "center_wl_nm" in kwargs:
        settings["center_wl_nm"] = kwargs["center_wl_nm"]
    elif "center_wl_cm" in kwargs:
        settings["center_wl_nm"] = 1e7 / kwargs["center_wl_cm"]

    if "grating_blaze_l_mm" in kwargs:
        settings["grating_blaze_l_mm"] = kwargs["grating_blaze_l_mm"]

    if "f_mm" in kwargs:
        settings["f_mm"] = kwargs["f_mm"]

    if "spacing_mm" in kwargs:
        settings["spacing_mm"] = kwargs["spacing_mm"]

    if "n_pixels" in kwargs:
        settings["n_pixels"] = kwargs["n_pixels"]

    if "gamma_deg" in kwargs:
        settings["gamma_deg"] = kwargs["gamma_deg"]

    if "half_angle_deg" in kwargs:
        settings["half_angle_deg"] = kwargs["half_angle_deg"]

    if "order" in kwargs:
        settings["order"] = kwargs["order"]

    a, alpha, beta, theta, phi = calculate_angels(settings)

    array = numpy.zeros(settings["n_pixels"])

    for pixel in range(settings["n_pixels"]):

        angle = -numpy.arctan( (pixel - settings["center_pixel"]) * settings["spacing_mm"] / settings["f_mm"])
        array[pixel] = a * (numpy.sin(alpha) + numpy.sin(beta + angle)) / settings["order"]

    return array, 1e7 / array

def calculate_angels(settings):

    a = 1e6 / settings["grating_blaze_l_mm"] # nm per line
    phi = settings["half_angle_deg"] * numpy.pi / 180

    theta = numpy.arcsin(
        settings["center_wl_nm"] * settings["order"] / (2 * a * numpy.cos(phi))
    )

    alpha = theta + phi
    beta = theta - phi

    return a, alpha, beta, theta, phi


from typing import Callable

import pytest
from alembic import command
from alembic.config import Config
from mixer.backend.sqlalchemy import mixer
from sqlalchemy.future import Connection
from sqlalchemy.orm import Session
from starlette.testclient import TestClient

from config.settings import settings
from database.db import SessionFactory, engine_factory
from database.models import Product
from main import create_app

@pytest.fixture(scope='session')
def engine():
    return engine_factory()

@pytest.fixture(scope='session', autouse=True)
def apply_migrations():
    config = Config(settings.BASE_DIR / 'alembic.ini')
    config.set_section_option('alembic', 'script_location', str(settings.BASE_DIR / 'migrations'))
    command.downgrade(config, 'base')
    command.upgrade(config, 'head')
    yield
    command.downgrade(config, 'base')
    command.upgrade(config, 'head')

@pytest.fixture(scope='session')
def test_app():
    app = create_app()
    return TestClient(app)

@pytest.fixture(scope='session')
def url_for(test_app) -> Callable:
    app = create_app()

    def inner(url_path: str) -> str:
        return app.url_path_for(url_path)

    return inner

@pytest.fixture
def xlsx_link():
    return 'https://docs.google.com/spreadsheets/d/11hqrBYKDtxTe7-NbPGDYMSVks-SO6f9b/export?format=xlsx&gid=1225459015'

@pytest.fixture
def db_session(engine) -> Session:
    with SessionFactory(bind=engine) as session:
        session.begin()
        try:
            yield session
        finally:
            session.rollback()

@pytest.fixture
def conn(engine) -> Connection:
    with engine.connect() as conn:
        yield conn

@pytest.fixture
def get_product():
    def inner(**kwargs):
        kwargs.setdefault('quantity', 1)
        kwargs.setdefault('price', 100)
        kwargs.setdefault('offer_id', 1)
        kwargs.setdefault('available', True)
        return mixer.blend(Product, **kwargs)

    return inner


from .models import Category, Blog, Event
import datetime

def post_context(request):
    latest_blogs = Blog.objects.filter(is_archived=False)
    categories = Category.objects.all()
    upcoming_events = Event.objects.filter(
        event_date__gte=datetime.date.today())

    return {'categories': categories, 'upcoming_events': upcoming_events, 'latest_blogs': latest_blogs}


#!/usr/bin/env python

def take_photo():

	"""take_photo: Turns on IR flash and take the photo. Saving the photo under the date and time"""

	import picamera
	import datetime
	import RPi.GPIO as GPIO

	camera = picamera.PiCamera()

	camera.hflip = True
	camera.vflip = True

	GPIO.setmode(GPIO.BCM)
	GPIO.setwarnings(False)
	GPIO.setup(18,GPIO.OUT)

	GPIO.output(18,GPIO.HIGH)

	name = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + ".jpg"
	camera.capture(name)

	GPIO.output(18,GPIO.LOW)

	return name


from argparse import ArgumentParser
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
from torchvision.datasets import FashionMNIST
import pytorch_lightning as pl

class FashionMnistData(pl.LightningDataModule):
    def __init__(self, root = '/home/bitwiz/codeden/data/mnist', batch_size = 512,
            workers = 8, debug = False, expt = 'cnn', **kw):
        super().__init__()
        self.batch_size = batch_size
        self.workers = workers
        self.debug = debug
        self.root = root

        if expt == 'cnn':
            self.train_transform = transforms.Compose([
                transforms.ToTensor(),
                transforms.Normalize((0.2859,), (0.3202,))
            ])

            self.val_transform = transforms.Compose([
                transforms.ToTensor(),
                transforms.Normalize((0.2859,), (0.3202,))
            ])
        else:
            self.train_transform = transforms.Compose([
                transforms.ToTensor()
            ])

            self.val_transform = transforms.Compose([
                transforms.ToTensor()
            ])

    def trainset(self):
        return FashionMNIST(root=self.root, train=True, download = True, transform = self.train_transform)

    def valset(self):
        return FashionMNIST(root=self.root, train=False, download = True, transform = self.val_transform)

    @staticmethod
    def add_loader_specific_args(parent_parser):
        parent_parser.add_argument('-b', '--batch_size', type = int, default = 512)
        parent_parser.add_argument('--workers', type = int, default = 8)
        parent_parser.add_argument('--debug', type = int, default = 0)
        parent_parser.add_argument('--mode', type = int, default = 10)

        return parent_parser

    def make_loader(self, split):
        if split == 'train':
            transform = self.train_transform
            dataset = self.trainset()
        elif split == 'val':
            transform = self.val_transform
            dataset = self.valset()
        else:
            transform = self.val_transform
            dataset = self.valset()

        loader = DataLoader(dataset, batch_size = self.batch_size, num_workers = self.workers)

        return loader

    def train_dataloader(self):
        return self.make_loader(split='train')

    def val_dataloader(self):
        return self.make_loader(split='val')

    def test_dataloader(self):
        return self.make_loader(split='val')


import ROOT
from collections import OrderedDict
pi=ROOT.TMath.Pi() #3.141593

# helpers
def book(h,name,n,a,b,title=""):
    h[name]=ROOT.TH1F(name,title,n,a,b)
def book2(h,name,nx,ax,bx,ny,ay,by,title=""):
    h[name]=ROOT.TH2F(name,title,nx,ax,bx,ny,ay,by)
def bookp(h,name,nx,ax,bx,ay,by,title="",err="s"):
    h[name]=ROOT.TProfile(name,title,nx,ax,bx,ay,by,err)
def draw(t,var,hname):
    t.Draw(var+">>"+hname)

# tree
fname="results.txt"
fname="full_results.txt"
for npart in [1,10,50,100,150]:
    t = ROOT.TTree("t","")
    fname="output/n{}_10k.txt".format(npart)
    t.ReadFile(fname,"ref:refphi:refx:refy:refrat:hw:hwphi:hwx:hwy:hwrat:hwratNoLUT")
    # if "full" in fname: t.ReadFile(fname,"ref:refphi:refx:refy:refrat:hw:hwphi:hwx:hwy:hwrat:hwratNoLUT")
    # else: t.ReadFile(fname,"ref:refphi:hw:hwphi")

    #hists
    h=OrderedDict()

    # basic
    book(h,"met_ref",40,0,1600,";reference MET [GeV]")
    book(h,"met_hw" ,40,0,1600,";HW MET [GeV]")
    book(h,"met_ref_fine",1024,0,1024,";[GeV]")
    book(h,"met_hw_fine" ,1024,0,1024,";[GeV]")
    book(h,"metphi_ref",40,-pi,pi)
    book(h,"metphi_hw" ,40,-pi,pi)
    book(h,"metphi_hw_int" ,1025,-512.5,512.5)
    book(h,"metphi_ref_fine",540,-pi/256*270,pi/256*270) # keep bin sizes
    book(h,"metphi_hw_fine" ,540,-pi/256*270,pi/256*270)

    #differences
    book(h,"met_diff",80,-200,200,";MET(HW)-MET(ref) [GeV]")
    book(h,"met_rel",80,0,2,";MET(HW)/MET(ref)")
    book(h,"metphi_diff",80,-1,1,";phi(HW)-phi(ref)")
    book(h,"met_diff_zoom",80,-20,20,";MET(HW)-MET(ref) [GeV]")
    book(h,"met_rel_zoom",80,0.9,1.1,";MET(HW)/MET(ref)")
    book(h,"metphi_diff_zoom",80,-0.1,0.1,";phi(HW)-phi(ref)")
    # versus vals
    book2(h,"2d_met_diff",10,0,1000,40,-200,200,";MET(ref) [GeV];MET(HW)-MET(ref) [GeV]")
    book2(h,"2d_met_rel",10,0,1000,40,0,2,";MET(ref) [GeV];MET(HW)/MET(ref)")
    book2(h,"2d_metphi_diff",12,-pi/5*6,pi/5*6, 40,-1,1,";phi(ref) ;phi(HW)-phi(ref)")
    bookp(h,"p_met_diff",10,0,1000,-200,200,";MET(ref) [GeV];MET(HW)-MET(ref) [GeV]")
    bookp(h,"p_met_rel",10,0,1000,0,2,";MET(ref) [GeV];MET(HW)/MET(ref)")
    bookp(h,"p_metphi_diff",48,-pi/5*6,pi/5*6,-1,1,";phi(ref) ;phi(HW)-phi(ref)")

    #extras
    if "full" in fname:
        book(h,"metx_rel",80,-2,2,";METX(HW)/METX(ref)")
        book(h,"mety_rel",80,-2,2,";METY(HW)/METY(ref)")
        book(h,"ratio_rel",80,-2,2,";tanPhi(HW)/tanPhi(ref)")
        book(h,"ratioNoLUT_rel",80,-2,2,";tanPhi(HW)/tanPhi(ref)")

    # reference met, phi
    draw(t,"ref","met_ref")
    draw(t,"ref","met_ref_fine")
    draw(t,"refphi","metphi_ref")
    draw(t,"refphi","metphi_ref_fine")

    # hardware met, phi
    draw(t,"hw","met_hw")
    draw(t,"hw","met_hw_fine")
    draw(t,"hwphi","metphi_hw")
    draw(t,"hwphi","metphi_hw_int")
    draw(t,"hwphi","metphi_hw_fine")

    # differences
    for s in ["","_zoom"]:
        draw(t,"hw-ref","met_diff"+s)
        draw(t,"hw/ref","met_rel"+s)
        draw(t,"hwphi-refphi","metphi_diff"+s)
    #2d
    draw(t,"hw-ref:ref"         ,"2d_met_diff")
    draw(t,"hw/ref:ref"         ,"2d_met_rel")
    draw(t,"hwphi-refphi:refphi","2d_metphi_diff")
    #profile
    draw(t,"hw-ref:ref"         ,"p_met_diff")
    draw(t,"hw/ref:ref"         ,"p_met_rel")
    draw(t,"hwphi-refphi:refphi","p_metphi_diff")

    #extras
    if "full" in fname:
        draw(t,"hwx/refx","metx_rel")
        draw(t,"hwy/refy","mety_rel")
        draw(t,"refrat/hwrat","ratio_rel")
        draw(t,"hwratNoLUT/hwrat","ratioNoLUT_rel")

    outname="output/hists_n{}_10k.root".format(npart)
    f = ROOT.TFile(outname,"recreate")
    for x in h: h[x].Write()
    f.Close()


from django.contrib import admin

from .models import Article, Tag, TagArticle
from django.core.exceptions import ValidationError
from django.forms import BaseInlineFormSet

class TagArticleInlineInlineFormset(BaseInlineFormSet):
    def clean(self):
        is_main = False
        for form in self.forms:
            data = form.cleaned_data
            if not is_main and 'is_main' in data.keys() and data['is_main']:
                is_main = True
            elif is_main and 'is_main' in data.keys() and data['is_main']:
                raise ValidationError('Должен быть только один основной раздел')
        if not is_main:
            raise ValidationError('Выбирете 1 основной раздел')
        return super().clean()  # вызываем базовый код переопределяемого метода

class TagArticleInline(admin.TabularInline):
    model = TagArticle
    formset = TagArticleInlineInlineFormset

@admin.register(Article)
class ArticleAdmin(admin.ModelAdmin):
    inlines = [
        TagArticleInline
        ]

@admin.register(Tag)
class TagAdmin(admin.ModelAdmin):
    pass


import torch
import warnings
import datetime
from pathlib import Path
from argparse import ArgumentParser
from pydatagrand.train.ner_seq_trainer import Trainer
from torch.utils.data import DataLoader
from pydatagrand.io.bert_seq_processor import BertProcessor
from pydatagrand.common.tools import init_logger, logger
from pydatagrand.common.tools import seed_everything
from pydatagrand.configs.base import config
from pydatagrand.model.nn.bert_lstm_crf import BERTLSTMCRF
from pydatagrand.model.nn.bert_lstm_crf_mdp import BERTLSTMCRFMDP
from pydatagrand.callback import ModelCheckpoint
from pydatagrand.callback import TrainingMonitor
from pydatagrand.callback import BertAdam
from pydatagrand.callback import BERTReduceLROnPlateau
from torch.utils.data import RandomSampler, SequentialSampler
from pydatagrand.callback import Lookahead
warnings.filterwarnings("ignore")

def run_train(args):
    processor = BertProcessor(vocab_path=args.pretrain_model / 'vocab.txt', do_lower_case=args.do_lower_case)
    processor.tokenizer.save_vocabulary(str(args.model_path))
    label_list = processor.get_labels()
    label2id = {label: i for i, label in enumerate(label_list)}

    train_data = processor.get_train(config['data_dir'] / f"{args.data_name}_train_fold_{args.fold}.pkl")
    train_examples = processor.create_examples(lines=train_data,
                                               example_type='train',
                                               cached_file=config[
                                                'data_dir'] / f"cached_train_seq_examples")
    train_features = processor.create_features(examples=train_examples,
                                               max_seq_len=args.train_max_seq_len,
                                               cached_file=config[
                                                'data_dir'] / "cached_train_seq_features_{}".format(
                                                args.train_max_seq_len))
    train_dataset = processor.create_dataset(train_features, is_sorted=args.sorted)
    if args.sorted:
        train_sampler = SequentialSampler(train_dataset)
    else:
        train_sampler = RandomSampler(train_dataset)
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)

    valid_data = processor.get_dev(config['data_dir'] / f'{args.data_name}_valid_fold_{args.fold}.pkl')
    valid_examples = processor.create_examples(lines=valid_data,
                                               example_type='valid',
                                               cached_file=config[
                                            'data_dir'] / f"cached_valid_seq_examples")
    valid_features = processor.create_features(examples=valid_examples,
                                               max_seq_len=args.eval_max_seq_len,
                                               cached_file=config[
                                                'data_dir'] / "cached_valid_seq_features_{}".format(
                                                args.eval_max_seq_len))
    valid_dataset = processor.create_dataset(valid_features)
    valid_sampler = SequentialSampler(valid_dataset)
    valid_dataloader = DataLoader(valid_dataset, sampler=valid_sampler, batch_size=args.eval_batch_size)
    logger.info("initializing model")
    if args.do_mdp:
        model = BERTLSTMCRFMDP
    else:
        model = BERTLSTMCRF
    if args.resume_path:
        args.resume_path = Path(args.resume_path)
        model = model.from_pretrained(args.resume_path, label2id=label2id, device=args.device)
    else:
        model = model.from_pretrained(args.pretrain_model,label2id=label2id,device=args.device)
    model = model.to(args.device)
    t_total = int(len(train_dataloader) / args.gradient_accumulation_steps * args.epochs)

    bert_param_optimizer = list(model.bert.named_parameters())
    lstm_param_optimizer = list(model.bilstm.named_parameters())
    crf_param_optimizer = list(model.crf.named_parameters())
    linear_param_optimizer = list(model.classifier.named_parameters())
    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in bert_param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01,
         'lr': args.learning_rate},
        {'params': [p for n, p in bert_param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0,
         'lr': args.learning_rate},
        {'params': [p for n, p in lstm_param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01,
         'lr': 0.001},
        {'params': [p for n, p in lstm_param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0,
         'lr': 0.001},
        {'params': [p for n, p in crf_param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01,
         'lr': 0.001},
        {'params': [p for n, p in crf_param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0,
         'lr': 0.001},
        {'params': [p for n, p in linear_param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01,
         'lr': 0.001},
        {'params': [p for n, p in linear_param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0,
         'lr': 0.001}
    ]
    if args.optimizer == 'adam':
        optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate,
                             warmup=args.warmup_proportion, t_total=t_total)
    elif args.optimizer == 'lookahead':
        base_optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate,
                             warmup=args.warmup_proportion, t_total=t_total)
        optimizer = Lookahead(base_optimizer,k=5,alpha=0.5)
    else:
        raise ValueError("unknown optimizer")
    lr_scheduler = BERTReduceLROnPlateau(optimizer, lr=args.learning_rate, mode=args.mode, factor=0.5, patience=5,
                                         verbose=1, epsilon=1e-8, cooldown=0, min_lr=0, eps=1e-8)
    if args.fp16:
        try:
            from apex import amp
        except ImportError:
            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)

    logger.info("initializing callbacks")
    train_monitor = TrainingMonitor(file_dir=config['figure_dir'], arch=args.arch)
    model_checkpoint = ModelCheckpoint(checkpoint_dir=args.model_path,mode=args.mode,monitor=args.monitor,
                                       arch=args.arch,save_best_only=args.save_best)

    # **************************** training model ***********************
    logger.info("***** Running training *****")
    logger.info("  Num examples = %d", len(train_examples))
    logger.info("  Num Epochs = %d", args.epochs)
    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
                args.train_batch_size * args.gradient_accumulation_steps * (
                    torch.distributed.get_world_size() if args.local_rank != -1 else 1))
    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
    logger.info("  Total optimization steps = %d", t_total)
    trainer = Trainer(n_gpu=args.n_gpu,
                      model=model,
                      logger=logger,
                      optimizer=optimizer,
                      lr_scheduler=lr_scheduler,
                      label2id = label2id,
                      training_monitor=train_monitor,
                      fp16=args.fp16,
                      resume_path=args.resume_path,
                      grad_clip=args.grad_clip,
                      model_checkpoint=model_checkpoint,
                      gradient_accumulation_steps=args.gradient_accumulation_steps)
    trainer.train(train_data=train_dataloader, valid_data=valid_dataloader, epochs = args.epochs,seed=args.seed)

def run_test(args):
    from pydatagrand.callback import ProgressBar
    from pydatagrand.train.ner_utils import get_entities
    from pydatagrand.common.tools import save_pickle

    processor = BertProcessor(args.model_path / 'vocab.txt', args.do_lower_case)
    label_list = processor.get_labels()
    label2id = {label: i for i, label in enumerate(label_list)}
    id2label = {i: label for i, label in enumerate(label_list)}
    if args.use_mdp:
        model = BERTLSTMCRFMDP
    else:
        model = BERTLSTMCRF
    model = model.from_pretrained(args.resume_path, label2id=label2id, device=args.device)
    model.to(args.device)
    max_seq_len = args.eval_max_seq_len
    tokenizer = processor.tokenizer
    test_data = []
    with open(str(config['data_dir'] / 'test.txt'), 'r') as fr:
        for line in fr:
            line = line.strip("\n")
            test_data.append(line)
    test_result_path = config['result_dir'] / f'{args.arch}_test_submit_{str(datetime.date.today())}.txt'
    fw = open(str(test_result_path), 'w')
    pbar = ProgressBar(n_total=len(test_data),desc='Testing')
    pred_tags = []
    for step, line in enumerate(test_data):
        token_a = line.split("_")
        tokens = tokenizer.tokenize(token_a)
        valid = [1] * len(tokens)
        if len(tokens) >= max_seq_len - 1:
            tokens = tokens[0:(max_seq_len - 2)]
            valid = valid[0:(max_seq_len - 2)]
        ntokens = []
        segment_ids = []
        ntokens.append("[CLS]")
        segment_ids.append(0)
        valid.insert(0, 1)
        for i, token in enumerate(tokens):
            ntokens.append(token)
            segment_ids.append(0)
        ntokens.append("[SEP]")
        segment_ids.append(0)
        valid.append(1)
        input_ids = tokenizer.convert_tokens_to_ids(ntokens)
        input_mask = [1] * len(input_ids)
        input_lens = len(input_ids)
        input_ids = torch.tensor([input_ids], dtype=torch.long)
        segment_ids = torch.tensor([segment_ids], dtype=torch.long)
        input_mask = torch.tensor([input_mask], dtype=torch.long)
        input_lens = torch.tensor([input_lens], dtype=torch.long)
        input_ids = input_ids.to(args.device)
        input_mask = input_mask.to(args.device)
        segment_ids = segment_ids.to(args.device)
        input_lens = input_lens.cpu().detach().numpy().tolist()
        model.eval()
        with torch.no_grad():
            features = model(input_ids=input_ids, token_type_ids=segment_ids, attention_mask=input_mask)
            tags, _ = model.crf._obtain_labels(features, id2label, input_lens)
        tags = tags[0][1:-1]
        pred_tags.append(tags)
        label_entities = get_entities(tags, id2label,args.markup)
        if len(label_entities) == 0:
            record = "_".join(token_a) + "/o"
        else:
            labels = []
            label_entities = sorted(label_entities, key=lambda x: x[1])
            o_s = 0
            for i, entity in enumerate(label_entities):
                begin = entity[1]
                end = entity[2]
                tag = entity[0]
                if begin != o_s:
                    labels.append("_".join(token_a[o_s:begin]) + "/o")
                labels.append("_".join(token_a[begin:end + 1]) + f"/{tag}")
                o_s = end + 1
                if i == len(label_entities) - 1:
                    if o_s <= len(token_a) - 1:
                        labels.append("_".join(token_a[o_s:]) + "/o")
            record = "  ".join(labels)
        fw.write(record + "\n")
        pbar(step=step)
    fw.close()
    save_pickle(pred_tags, file_path=str(config['result_dir'] / f'{args.arch}_test_tags.pkl'))

def main():
    parser = ArgumentParser()
    parser.add_argument("--arch", default='bert_lstm_crf', type=str)
    parser.add_argument("--do_train", action='store_true')
    parser.add_argument("--do_test", action='store_true')
    parser.add_argument("--save_best", action='store_true')
    parser.add_argument("--do_lower_case", action='store_true')
    parser.add_argument('--do_mdp',action='store_true',help='whether use multi-sample dropout')
    parser.add_argument('--data_name', default='datagrand', type=str)
    parser.add_argument('--optimizer',default='adam',type=str,choices=['adam','lookahead'])
    parser.add_argument('--markup',default='bios',type=str,choices=['bio','bios'])
    parser.add_argument('--checkpoint', default=900000, type=int)
    parser.add_argument("--epochs", default=30.0, type=int)
    parser.add_argument('--fold',default=0,type=int)
    parser.add_argument("--resume_path", default='', type=str)
    parser.add_argument("--mode", default='max', type=str)
    parser.add_argument("--monitor", default='valid_f1', type=str)
    parser.add_argument("--local_rank", type=int, default=-1)
    parser.add_argument("--sorted", default=1, type=int, help='1:True  0:False ')
    parser.add_argument("--n_gpu", type=str, default='0', help='"0,1,.." or "0" or "" ')
    parser.add_argument('--gradient_accumulation_steps', type=int, default=1)
    parser.add_argument("--train_batch_size", default=24, type=int)
    parser.add_argument('--eval_batch_size', default=48, type=int)
    parser.add_argument("--train_max_seq_len", default=128, type=int)
    parser.add_argument("--eval_max_seq_len", default=512, type=int)
    parser.add_argument('--loss_scale', type=float, default=0)
    parser.add_argument("--warmup_proportion", default=0.05, type=float)
    parser.add_argument("--weight_decay", default=0.01, type=float)
    parser.add_argument("--adam_epsilon", default=1e-8, type=float)
    parser.add_argument("--grad_clip", default=5.0, type=float)
    parser.add_argument("--learning_rate", default=1e-4, type=float)
    parser.add_argument('--seed', type=int, default=42)
    parser.add_argument("--no_cuda", action='store_true')
    parser.add_argument('--fp16', action='store_true')
    parser.add_argument('--fp16_opt_level', type=str, default='O1')
    args = parser.parse_args()

    args.pretrain_model = config['checkpoint_dir'] / f'lm-checkpoint-{args.checkpoint}'
    args.device = torch.device(f"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
    args.arch = args.arch + f"_{args.markup}_fold_{args.fold}"
    if args.do_mdp:
        args.arch += "_mdp"
    if args.optimizer == 'lookahead':
        args.arch += "_lah"
    args.model_path = config['checkpoint_dir'] / args.arch
    args.model_path.mkdir(exist_ok=True)
    # Good practice: save your training arguments together with the trained model
    torch.save(args, args.model_path / 'training_args.bin')
    seed_everything(args.seed)
    init_logger(log_file=config['log_dir'] / f"{args.arch}.log")
    logger.info("Training/evaluation parameters %s", args)

    if args.do_train:
        run_train(args)

    if args.do_test:
        run_test(args)

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
import os
import re
import sys
from colorama import Fore, Back, Style
import argparse

EmbyLibrary = './EmbyLibrary'
ConvertDir = './convert'

CATEGORY = {
    'Movie': {
        '1': 'Movies',
        '2': 'Animation Movies',
    },
    'Series': {
        '1': 'TV',
        '2': 'Animation Series',
    }
}

CATEGORY_INPUT = 0
TEST_MODE = True
# https://psarips.top/
listOfWordsPSA = '66CH|2160p|HDR|imax|10bit|BluRay|6CH|x265|HEVC-PSA|1080p|WEBRip|8CH|2CH|720p|web-dl|remastered|REAL|REPACK|BrRip'
listOfWordsPSA = listOfWordsPSA.split('|')

def is_linux():
    """
    is_linux()
    Returns True if running on Linux, False otherwise
    """
    return os.name == 'posix'

def set_folder_user_group(folder, user, group):
    """
    set_folder_user_group(folder, user, group)
    Sets the group of the folder
    """
    if is_linux():
        os.system(f'chown -R {user}:{group} {folder}')

def get_dirs(path):
    """
    get_dirs(path)
    Returns a list of all directories in the path
    """
    dirs = [path]
    for i in os.listdir(path):
        if os.path.isdir(os.path.join(path, i)):
            dirs.extend(get_dirs(os.path.join(path, i)))
    return dirs

def combine_path_from_category(category):
    """
    combine_path_from_category(category)
    Combines the path from the category
    """
    path = []
    for value in CATEGORY[category].values():
        path += get_dirs(os.path.join(EmbyLibrary, value))
    return path

def get_media_files_in_dir(path):
    """
    get_media_files_in_dir(path)
    Returns a list of all media files in the path
    """
    media_files = []
    for i in os.listdir(path):
        if os.path.isfile(os.path.join(path, i)):
            if i.endswith('.mkv') or i.endswith('.mp4') or i.endswith('.txt'):
                media_files.append(i)
    return media_files

def find_file_type(file):
    """
    find_file_type(file)
    Returns the file type as a string
    """
    for i in range(len(file)):
        if file[i] == '.':
            return file[i+1:]
    return ''

def remove_word_from_list(list):
    """
    remove_word_from_list(list, words)
    Removes all elements from list containing the word
    """
    temp = []
    for i in range(len(list)):
        if 'Season' not in list[i]:
            if ord(list[i][-1]) < ord('A') or ord(list[i][-1]) > ord('Z'):
                if ord(list[i][-1]) != ord('#'):
                    temp.append(list[i])
    return temp

def find_season(file):
    """
    find_season(file)
    Returns the season number as a string
    """
    for i in range(len(file)):
        if file[i] == 'S':
            if file[i+1].isdigit():
                season = file[i+1]
                if file[i+2].isdigit():
                    season += file[i+2]
                    return season
                else:
                    return season
    return '-1'

def find_episode(file):
    """
    find_episode(file)
    Returns the episode number as a string
    """
    for i in range(len(file)):
        if file[i] == 'E':
            if file[i+1].isdigit():
                episode = file[i+1]
                if file[i+2].isdigit():
                    episode += file[i+2]
                    return episode
                else:
                    return episode
    return '-1'

def find_title(file):
    """
    find_title(file)
    Returns the title as a string
    """

    if find_episode(file) != '-1':
        return file[file.index(f'E{find_episode(file)}')+4:file.index(".")]
    elif find_year(file) != '-1':
        return file[file.index(find_year(file))+5:file.index(".")]
    return ''

def remove_words(file, words):
    """
    remove_words(file, words)
    Removes words from file name
    """
    for word in words:
        file = re.sub(word, '', file, flags=re.I)
    return file

def remove_dot(file):
    """
    remove_dot(file)
    Removes the dot from the file name
    """
    temp = os.path.splitext(file)
    file = re.sub('\.', ' ', temp[0])
    file = file.strip()
    return file + temp[1]

def find_movie_name(file):
    """
    find_movie_name_in_season(file)
    Returns the movie name as a string
    """
    for i in range(len(file)):
        if file[i] == 'S':
            if file[i+1].isdigit():
                movie = file[:i]
                movie = movie.strip()
                if find_year(movie) != '-1':
                    return movie[:movie.index(find_year(movie))-1]
                else:
                    return movie
    return file[:file.index(find_year(file))-1]

def find_year(file):
    """
    find_year(file)
    Returns the year as a string
    """
    for i in range(len(file)):
        if file[i].isdigit():
            if file[i+1].isdigit():
                if file[i+2].isdigit():
                    if file[i+3].isdigit():
                        year = file[i:i+4]
                        return year
    return '-1'

def find_movie_name_with_year(file):
    """
    find_movie_name_with_year(file)
    Returns the movie name with year as a string
    """
    if find_year(file) != '-1':
        return f'{find_movie_name(file)} ({find_year(file)})'
    else:
        return find_movie_name(file)

def movie_path_in_list(file) -> str:
    """
    movie_path_in_list(file)
    Returns path if movie in th list, '' otherwise
    """
    global CATEGORY_INPUT
    origin_file = os.path.join(ConvertDir, file)
    file = remove_words(file, listOfWordsPSA)
    file = remove_dot(file)

    if find_season(file) != '-1':
        category_str = 'Series'
    else:
        category_str = 'Movie'
    combine_path = remove_word_from_list(
        combine_path_from_category(category_str))
    for i in range(len(combine_path)):
        if find_movie_name(file) in combine_path[i]:
            value = os.path.normpath(combine_path[i]).split(os.path.sep)[1]
            CATEGORY_INPUT = get_key(value, CATEGORY[category_str])
            move_file_into_folder(
                origin_file, file, category_str, combine_path[i])
            return combine_path[i]
    create_folder_movie(file, category_str)
    move_file_into_folder(origin_file, file, category_str, '')
    return ''

# get key from value
def get_key(value, dict):
    for key, val in dict.items():
        if val == value:
            return key
    return '-1'

def create_folder_movie(file, category_str):
    """
    create_folder_movie(file)
    Creates a folder for the movie
    """
    global CATEGORY_INPUT

    temp = 'Create folder for ' + Fore.GREEN + file + Style.RESET_ALL + ' in ?🧐\n'

    for key, value in CATEGORY[category_str].items():
        temp += f"{key}. {value}\n"

    CATEGORY_INPUT = input(temp)
    create_folder_helper(file, category_str)

def create_folder_helper(file, category_str):
    """
    create_folder_helper(category_str, category, file)
    Function helper for create_folder_movie
    """
    alphabet_path = os.path.join(
        EmbyLibrary, CATEGORY[category_str][CATEGORY_INPUT], file[0])
    if not os.path.exists(alphabet_path):
        print_create_folder_with_color(alphabet_path)
        if not TEST_MODE:
            os.makedirs(alphabet_path)

    name_path = os.path.join(alphabet_path, find_movie_name_with_year(file))
    if not os.path.exists(name_path):
        print_create_folder_with_color(name_path)
        if not TEST_MODE:
            os.makedirs(name_path)

    if find_season(file) != '-1':
        season_path = os.path.join(
            name_path, 'Season ' + str(int(find_season(file))))
        if not os.path.exists(season_path):
            print_create_folder_with_color(season_path)
            if not TEST_MODE:
                os.makedirs(season_path)

def print_create_folder_with_color(path):
    """
    print_create_folder_with_color(path)
    Prints the folder to create with color
    """
    print(f'🏠 Create folder {Fore.GREEN}{path}{Style.RESET_ALL}')

def print_move_file_with_color(origin_file, new_name):
    """
    print_move_file_with_color(origin_file, file, category_str, path)
    Prints the file to move with color
    """
    print(
        f'💔 Move file {Fore.GREEN}{origin_file}{Style.RESET_ALL} to {Fore.MAGENTA}{new_name}{Style.RESET_ALL}')

def move_file_into_folder(origin_file, file, category_str, des_path):
    """
    move_file_into_folder(file)
    Moves the file into the folder
    """
    if des_path == '':
        des_path = os.path.join(
            EmbyLibrary, CATEGORY[category_str][CATEGORY_INPUT], file[0], find_movie_name_with_year(file))

    title = find_title(file)
    if title != '':
        title = ' '+title
    else:
        title = ''
    if find_season(file) != '-1':
        season_path = os.path.join(
            des_path, 'Season ' + str(int(find_season(file))))
        # move file to season folder if file not in season folder
        if not os.path.exists(os.path.join(season_path, file)):
            new_name = os.path.join(
                season_path, f'{find_movie_name_with_year(file)} S{find_season(file)}E{find_episode(file)}{title}.{find_file_type(file)}')
            print_move_file_with_color(origin_file, new_name)
            if not TEST_MODE:
                os.rename(origin_file, new_name)
        else:
            print(f'- {origin_file} already in {season_path}')
    else:
        # move file to movie folder if file not in movie folder
        if not os.path.exists(os.path.join(des_path, file)):
            new_name = os.path.join(
                des_path, f'{find_movie_name_with_year(file)}{title}.{find_file_type(file)}')
            print_move_file_with_color(origin_file, new_name)
            if not TEST_MODE:
                os.rename(origin_file, new_name)
        else:
            print(f'- {origin_file} already in {des_path}')

def less_indent_formatter(prog): return argparse.RawTextHelpFormatter(
    prog, max_help_position=5)

def interactive():
    parser = argparse.ArgumentParser(
        description=f'Easy app to move file to Emby library',
        formatter_class=less_indent_formatter)
    parser.add_argument('-m', '--mode', metavar='MODE', choices=['t', 'm'],  help='''
t - Dry run without moving files
m - Move files to Emby library
    ''', required=True)
    args = parser.parse_args()
    return args

def main():
    global TEST_MODE
    args = interactive()
    print(Fore.RED + 'This script is only for Linux')
    print(Style.RESET_ALL)
    args = interactive()
    if args.mode == 't':
        TEST_MODE = True
    else:
        TEST_MODE = False
    if TEST_MODE:
        print(Fore.RED + 'Running in Test Mode')
        print(Style.RESET_ALL)

    media_files = get_media_files_in_dir(ConvertDir)
    set_folder_user_group(ConvertDir, 'emby', 'dietpi')
    for file in media_files:
        movie_path_in_list(file)

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        print('\n\nExiting')
        sys.exit(0)
    print('Done')


import fnmatch
import os
import shutil
import csv
import sqlite3

def collectFiles(path, extention='csv'):
     matches = []

     for root, dirnames, filenames in os.walk(path):
         for filename in fnmatch.filter(filenames,'*.{}'.format(extention)):
             matches.append(os.path.join(root, filename))
     return matches

def CsvtoSqlite(dbloc):
    con = sqlite3.connect(dbloc)
    cur = con.cursor()
    files = collectFiles("data/")
    for eachFile in files:
        with open(eachFile,'rb') as fin:
            dr = csv.DictReader(fin)
            to_db = [(i['datetime'], i['host'],i['guid'],i['timings']) for i in dr]
            cur.executemany("INSERT INTO performance_runs (datetime, host, guid, timings) VALUES (?, ?, ?, ?);", to_db)
    con.commit()

def main():
    # print(collectFiles("data/"))
    CsvtoSqlite("db/performancedb.db")

if __name__ == '__main__':
    main()


from django.shortcuts import render
from django.contrib.auth.decorators import login_required

from poemese.models import Poema, Usuario, RelacoesAmizade

@login_required(login_url="poemese:sign_in")
def profile(request, pseudonimo):
    profile_user = Usuario.objects.get(pk=pseudonimo)
    poemas = Poema.objects.filter(autor_poema=pseudonimo)

    # Update the likes in profile
    update_likes(profile_user, poemas)

    # Verify if the user that requested the profile is following the profile's user
    is_follower = is_following(request.user, profile_user)

    followers = RelacoesAmizade.objects.filter(destino=profile_user).count()
    following = RelacoesAmizade.objects.filter(origem=profile_user).count()

    return render(request, 'poemese/perfil.html',
                  {'profile_user': profile_user, 'following': following,
                   'followers': followers, 'is_follower': is_follower, 'poem_list': poemas})

def update_likes(user, poemas):
    """ Update the poems' number of likes of the profile that has been open """
    likes = 0
    for poema in poemas:
        likes += poema.qtd_gostei
    user.number_of_likes = likes
    user.save()

def is_following(request_user, profile_user):
    """ Return True in case the user that requested follow the profile's user """
    relationship = RelacoesAmizade.objects.filter(origem=request_user, destino=profile_user)
    if request_user != profile_user:
        if relationship is not None:
            return True
        return False


import os
import sys
import warnings
import time
import logging
import matplotlib.pyplot as plt
import numpy as np
import datetime
import traceback
import argparse

from custom_metrics import f2, average_precision, average_recall, hamming_score

sys.path.insert(0, '/home/reflex/refleX/lib/fastai')
from fastai.conv_learner import *

def save_lr_chart(lrs, losses, filename, n_skip=10, n_skip_end=5):
    plt.ylabel("validation loss")
    plt.xlabel("learning rate (log scale)")
    plt.plot(lrs[n_skip:-(n_skip_end+1)], losses[n_skip:-(n_skip_end+1)])
    plt.xscale('log')
    plt.savefig(filename)
    plt.clf()

def get_data (model, PATHS, img_size, custom_transformations, batch_size, val_idxs):
    tfms = tfms_from_model(model, img_size, aug_tfms=custom_transformations, max_zoom=1.1)
    return ImageClassifierData.from_csv(PATHS['DATA_PATH'], PATHS['DATA_FOLDER'], PATHS['CSV_PATH'], tfms=tfms,
                    suffix='.png', val_idxs=val_idxs, bs=batch_size, num_workers=2)

def grid_search(PATHS, CONFIG, experiment_name, PARAMS, output_csv='results.csv'):

    val_idxs = PARAMS['val_idxs']

    # TODO: Czy LR wyznaczamy przed na samym początku czy przed kazdym uczeniem (tj. przed zmiana rozdzielczosci?)

    # Logger
    logger = logging.getLogger(f'{experiment_name}')
    logging.basicConfig(filename=f'{PATHS["EXP_PATH"]}/{experiment_name}/{experiment_name}.log', format='%(asctime)s - %(levelname)s - %(message)s',\
        filemode='w', datefmt='%d-%m-%Y %I:%M:%S %p', level=logging.INFO)

    # Dodatowe parametry
    dropout = PARAMS['dropout']
    earlystoping = PARAMS['earlystopping']
    log_step = PARAMS['log_step']
    custom_aug_tfms = PARAMS['transformations']
    gpu = PARAMS['gpu_id']
    torch.cuda.set_device(gpu)

    # 1. INFO
    logging.info(f" [ ------ EXPERIMENT: {experiment_name.upper()} ------ ] \n")

    # Log
    logging.info(' --- PARAMETERS: --- ')
    logging.info(f"Predefined metrics: [val_loss], {', '.join([metric.__name__ for metric in CONFIG['predefined_metrics']])}")
    logging.info(f"Predefined architecutres: {', '.join([architecture[0] for architecture in CONFIG['architectures'].items()])}")
    logging.info(f"Predefined batch sizes: {', '.join([str(batch_size) for batch_size in CONFIG['batch_sizes']])}")
    logging.info(f"Predefined image sizes: {', '.join([str(img_size) for img_size in CONFIG['image_sizes']])}")
    logging.info(f"Dropout: {dropout}")
    logging.info(f"Earlystopping after: {earlystoping} epochs.")
    logging.info(f"Log every {log_step} epochs.")
    logging.info(f'GPU: {torch.cuda.current_device()} | Available: {torch.cuda.device_count()} \n')

    for model_name, model in CONFIG['architectures'].items():

        logging.info(f"    ----- *** {model_name.upper()} *** ----- \n")

        # Definiujemy obiekt learner'a
        data = get_data(model, PATHS, CONFIG['image_sizes'][0], custom_aug_tfms, CONFIG['batch_sizes'][0], val_idxs)
        learn = ConvLearner.pretrained(model, data, ps=dropout, metrics=CONFIG['predefined_metrics'])

        # Obliczanie LR - wybieranie najmniejszego loss i dzielenie przez 10
        learn.lr_find()
        save_lr_chart(learn.sched.lrs, learn.sched.losses, f'{PATHS["EXP_PATH"]}/{experiment_name}/{model_name}_lr_sched.png')

        lr = sorted(list(zip(learn.sched.lrs, learn.sched.losses)), key=lambda x: x[1])[0][0] / 10
        lrs = [lr/8, lr/4, lr/2, lr]

        # Log
        logging.info(f' Learning rate = {round(lr, 6)}')
        logging.info(f' Learning rates = {[round(lr, 6) for lr in lrs]} \n')

        for batch_size in CONFIG['batch_sizes']:

            for img_size in CONFIG['image_sizes']:

                try:
                    logging.info(f" --- Batch: {batch_size} | Img size: {img_size} --- \n ")

                    # Preparing data
                    best_model_name = f'{PATHS["EXP_PATH"]}/{experiment_name}/models/best_{model_name}_imgsize{img_size}_batch{batch_size}'
                    data = get_data(model, PATHS, img_size, custom_aug_tfms, batch_size, val_idxs)
                    learn.set_data(data)

                    # Freeze
                    logging.info(f" --- Type: FREEZE --- ")
                    learn.freeze()
                    learn.fit(lr, 12, cycle_len=1, cycle_mult=2, best_save_name=f'{best_model_name}_freeze', earlystopping=earlystoping, logger_name=experiment_name, log_step=log_step)

                    # Print results
                    results = open(f'{best_model_name}_freeze_metrics.log', 'r').read()
                    logging.info(f"[ Saved model: best_{model_name}_imgsize{img_size}_batch{batch_size}_freeze.h5 ] ")
                    logging.info(f'[ Results: {results} ]\n')

                    # Unfreeze
                    logging.info(f" --- Type: UNFREEZE --- ")
                    learn.unfreeze()
                    learn.fit(lr, 12, cycle_len=1, cycle_mult=2, best_save_name=f'{best_model_name}_unfreeze', earlystopping=earlystoping, logger_name=experiment_name, log_step=log_step)

                    # Print results
                    results = open(f'{best_model_name}_unfreeze_metrics.log', 'r').read()
                    logging.info(f"[ Saved model: best_{model_name}_imgsize{img_size}_batch{batch_size}_unfreeze.h5 ]")
                    logging.info(f'[ Results: {results} ]\n')

                    if img_size == CONFIG['image_sizes'][-1]:
                        logging.info(" ************************** ")
                        logging.info(f'COPYPASTE: {model_name} | IMS: {CONFIG["image_sizes"][0]} -> {img_size} | BS: {batch_size}')
                        logging.info(f'COPYPASTE: {results} ')
                        logging.info(" ************************** \n")

                except Exception as e:
                    logging.error(f'ERROR: A: {model_name} | IS: {img_size} | BS: {batch_size}')
                    logging.error(traceback.format_exc())

            if batch_size != CONFIG['batch_sizes'][-1]:
                logging.info(' --- INCREMENTING BATCH SIZE --- \n')

        logging.info(f"    ----- *** END: {model_name} *** ----- \n")

    print("[ --- DONE! --- ]")

def ac_polar_max_configuration():

    paths = {
        'DATA_FOLDER' : 'polar512_max'
    }

    params = {
        'transformations': [RandomLighting(0.1, 0.1)],
        'gpu_id': 0
    }

    config = {
        'image_sizes': [64, 128, 256, 512],
        'predefined_metrics': [f2, average_precision, average_recall, hamming_score],
        'batch_sizes': [8, 16, 32, 64], # PARAMETER
        'architectures' : {
            'resnet34' : resnet34 # PARAMETER
        }
    }

    return params, paths, config

def ac_polar_min_configuration():

    paths = {
        'DATA_FOLDER' : 'polar512_min'
    }

    params = {
        'transformations': [RandomLighting(0.1, 0.1)],
        'gpu_id': 1
    }

    config = {
        'image_sizes': [64, 128, 256, 512],
        'predefined_metrics': [f2, average_precision, average_recall, hamming_score],
        'batch_sizes': [8, 16, 32, 64], # PARAMETER
        'architectures' : {
            'resnet34' : resnet34 # PARAMETER
        }
    }

    return params, paths, config

def ac_resnet34_configuration():
    paths = {
        'DATA_FOLDER': 'original512'
    }

    params = {
        'gpu_id': 2,
        'transformations' : [RandomLighting(0.1, 0.1), RandomDihedral()]
    }

    config = {
        'image_sizes': [64, 128, 256, 512],
        'predefined_metrics': [f2, average_precision, average_recall, hamming_score],
        'batch_sizes': [8, 16, 32],
        'architectures' : {
            'resnet34' : resnet34
        }
    }

    return params, paths, config

def ac_resnet18_configuration():
    paths = {
        'DATA_FOLDER': 'original512'
    }

    params = {
        'gpu_id': 3,
        'transformations' : [RandomLighting(0.1, 0.1), RandomDihedral()]
    }

    config = {
        'image_sizes': [64, 128, 256, 512],
        'predefined_metrics': [f2, average_precision, average_recall, hamming_score],
        'batch_sizes': [64, 128, 256],
        'architectures' : {
            'resnet18' : resnet18
        }
    }

    return params, paths, config

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('working_title')
    parser.add_argument('-g', '--gpu_id', default=0, help="GPU ID")
    parser.add_argument('-cc', '--custom_config', default=None, help="Path to custom configuration file.")
    parser.add_argument('-cn', '--config_name', default='test', help="Name of hardcoded configuration")
    args = parser.parse_args()

    CUSTOM_PARAMS, CUSTOM_PATHS, CONFIG = globals()[args.config_name + "_configuration"]()

    #  --- DEFAULT PATHS ---
    PATHS = {
        'CSV_PATH': '/home/reflex/refleX/metadata/6K/csv/fastai_labels_val_train.csv',
        'DATA_PATH': '/home/reflex/refleX/metadata/6K/',
        'DATA_FOLDER': 'original512',
        'EXP_PATH': '/home/reflex/refleX/results/fastai_experiments'
    }

    # --- APPLY CUSTOM PATHS  ---
    for custom_path in CUSTOM_PATHS.keys():
        PATHS[custom_path] = CUSTOM_PATHS[custom_path]

    # --- DEFAULT PARAMS ---
    PARAMS = {
        'val_idxs': range(631),
        'dropout': 0.5,
        'earlystopping': 12,
        'log_step': 5,
        'gpu_id': args.gpu_id,
        'transformations' : [RandomLighting(0.1, 0.1), RandomDihedral()]
    }

    # --- APPLY CUSTOM PARAMETERS ---
    for custom_param in CUSTOM_PARAMS.keys():
        PARAMS[custom_param] = CUSTOM_PARAMS[custom_param]

    # Create dirs
    current_instance_dir = f'{PATHS["EXP_PATH"]}/{args.working_title}'
    models_dir = f'{PATHS["EXP_PATH"]}/{args.working_title}/models/'
    tmp_dir = f'{PATHS["EXP_PATH"]}/{args.working_title}/tmp/'

    os.makedirs(current_instance_dir, exist_ok=True)
    os.makedirs(models_dir, exist_ok=True)
    os.makedirs(tmp_dir, exist_ok=True)

    # Set validation set
    # convert_csv_to_fastai_format.py

    # Run grid search
    grid_search(PATHS, CONFIG, args.working_title, PARAMS)


import requests, json
import argparse
import hashlib
import os, sys
from environs import Env

env = Env()
env.read_env()  # read .env file, if it exists

METADEFENDER_URL = env('METADEFENDER_URL')
METADEFENDER_API_KEY = env('METADEFENDER_API_KEY')
META_HASH = 'sha256'
META_METADATA = '0'
META_WORKFLOW = None

def _scan_file(api_key, file_stream, file_name=None, user_agent=None):
    url = "{}/file".format(METADEFENDER_URL)
    headers = {'apikey': api_key, 'user_agent': user_agent, 'Content-Type': "application/octet-stream"}
    try:
        print('url - {}'.format(url))
        response = requests.post(url=url, data=file_stream, headers=headers)
        output_data = response.json()
        print('results = {}'.format(output_data))
        return output_data['data_id'], 'Success'
    except requests.exceptions.RequestException as err:
        print("Request Error:", err)
        return '', 'Request Error'
    except:
        print("Unable to scan file.")
        return '', 'Scan error'

def _retrieve_scan(url, api_key, metadata='0'):
    headers = {'apikey': api_key, 'file-metadata': metadata}
    try:
        print('retrieval scan - {}'.format(url))
        response = requests.get(url=url, headers=headers)
        output_data = response.json()
        # print('retrieval scan result - {}'.format(output_data))
    except requests.exceptions.RequestException as err:
        print("Request Error:", err)
        return '', 'Request Error'
    except:
        print("Unable to retrieve file status.")
        sys.exit(0)

    return output_data

def _calculate_hash(hash_type, file_stream, chunk_size=65536):
    try:
        if hash_type == "md5":
            hash = hashlib.md5()
        elif hash_type == "sha1":
            hash = hashlib.sha1()
        elif hash_type == "sha256":
            hash = hashlib.sha256()
        for chunk in iter(lambda: file_stream.read(chunk_size), b""):
            hash.update(chunk)
    except:
        print("Unable to hash file.")
        return ''
    return hash.hexdigest()

def scan_file_sync(file_name, file_stream):
    # Calculate hash of file
    file_hash = _calculate_hash(META_HASH, file_stream).upper()
    if file_hash == '':
        return file_hash, 'Unable to has file'

    # Perform hash lookup for cached results
    url = "{}/hash/{}".format(METADEFENDER_URL, file_hash)
    scan_result = _retrieve_scan(url=url, api_key=METADEFENDER_API_KEY, metadata=META_METADATA)

    # Skip if cache found
    if scan_result == '' or ('error' not in scan_result):
        return scan_result, 'Success'

    # Initiate scan and get data id
    data_id, error_msg = _scan_file(api_key=METADEFENDER_API_KEY, file_stream=file_stream, file_name=file_name,
                                    user_agent=META_WORKFLOW)
    if data_id == '':
        return data_id, error_msg

    # Loop through and check for results because scan_file is async
    url = "{}/file/{}".format(METADEFENDER_URL, data_id)
    scan_result = _retrieve_scan(url=url, api_key=METADEFENDER_API_KEY, metadata=META_METADATA)
    while scan_result['scan_results']['progress_percentage'] != 100:
        scan_result = _retrieve_scan(url=url, api_key=METADEFENDER_API_KEY, metadata=META_METADATA)

    return scan_result, 'Success'


from django.db import migrations, models
import django.db.models.deletion
import uuid

class Migration(migrations.Migration):

    initial = True

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='User',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)),
                ('email', models.EmailField(max_length=191, unique=True)),
            ],
        ),
        migrations.CreateModel(
            name='Domain',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)),
                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='desecapi.User')),
            ],
        ),
        migrations.CreateModel(
            name='RRset',
            fields=[
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('subname', models.CharField(blank=True, max_length=178)),
                ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='desecapi.Domain')),
            ],
        ),
        migrations.CreateModel(
            name='Token',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)),
                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='desecapi.User')),
            ],
        ),
        migrations.AlterField(
            model_name='rrset',
            name='subname',
            field=models.CharField(blank=False, max_length=178),
        ),
    ]


"""
                                Predicting with a Language Model

This script predicts a word-level
language model, implementation with Tensorflow 1.13, tf.Estimator API

Usage:
  train.py  <log_dir>  <sentence> <k>
  [--gpu_train=<gpu>] [--mode=<md>]
  [--embedding_size=<ems>] [--char_emb_size=<em>]
  [--num_LSTM_layers=<nmlstm>] [--num_layers_char=<nmchar>]
  [--hidden_size_chars=<hsc>]  [--hidden_size_LSTM=<hsn>]
  [--add_char_emb=<add>] [--num_heads=<nmh>]
  [--dropout=<dp>]
  [--learning_rate=<lr>] [--batch_size=<bs>]
  [--n_epochs=<ne>] [--optimizer=<op>] [--checkpoints=<ckpt>]
  [--buffer_size=<bfs>]
  [--deepness_finish=<dpsf>] [--activation_finish=<actf>]
  [--intern_size=<intf>] [--weighted_loss=<wgth>]
[--add_encoder=<addenc>]

Options:
  -h --help
  --version
  --sentence                The start sentence to predict from
  --k                       The number of tokens which must be predicted
  --log_dir                 Logs directory, where the checkpoints will be stored
  --gpu_train=<gpu>         Boolean, True is using a gpu. (Requires tensorflow-gpu) [default: True]
  --mode=<md>               The tensorflow mode, in ['train', 'train_eval', 'eval'] [default: train_eval]
  --embedding_size=<ems>    The word embedding dimension [default: 300]
  --char_emb_size=<em>      The chosen character embedding size [default: 100]
  --num_LSTM_layers=<nmlstm> The number of LSTM units [default:2]
  --num_layers_char=<nmchar> The number of LSTM units for char embedding [default:2]
  --hidden_size_chars=<hsc>  The hidden size of the character embedding Bi-LSTM. [default: 25]
  --hidden_size_LSTM=<hsn>  The hidden size of the LSTM. [default: 600]
  --add_char_emb=<add>      Boolean, True if char embedding is needed [default: True]
  --num_heads=<nmh>         The number of head for the attention encoder layer [default: 10]
  --dropout=<dp>            The dropout value. [default: 0.5]
  --learning_rate=<lr>      The learning rate for the optimizer [default: 1e-3]
  --batch_size=<bs>         The batch size for the training [default: 24]
  --n_epochs=<ne>           Number of epochs to train the network on [default: 100]
  --optimizer=<op>          The chosen tf optimizer, in ['sgd', 'adam', 'adagrad', 'rmsprop'] [default: sgd]
  --checkpoints=<ckpt>      Save checkpoints every ckpt steps [default: 5000]
  --buffer_size=<bfs>       Buffer size for shuffling  [default: 500]
  --deepness_finish=<dpsf>  The layer depth for the dense finish [default: 2]
  --activation_finish=<actf> The activation function for the dense finish [default: leaky_relu]
  --intern_size=<intf>      The internal dimension of finish dense layers [default: 3000]
  --weighted_loss=<wgth>    Boolean, indicates if loss must be weighted or not [default: True]
  --add_encoder=<addenc>     Boolean, indicates if encodre must be kept [default: True]
"""

import os
import logging

import tensorflow as tf
from docopt import docopt
from functools import partial
from numpy import genfromtxt
from glob import glob

from utils import generate_dataset, create_full_vocab
from reader import input_fn_gen
from model import model_fn

logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))

def str_to_bool(s):
    if s == 'True':
        return True
    elif s == 'False':
        return False
    else:
        raise ValueError

if __name__ == "__main__":
    # Get the arguments
    args = docopt(__doc__, version="0.3")
    # print(args)
    # Transform it into a params dic
    params = {}
    for k in args.keys():
        k2 = k.replace("<", "").replace(">", "").replace("-", "")
        try:  # Convert strings to int or floats when required
            params[k2] = int(args[k])
        except:
            try:
                params[k2] = float(args[k])
            except:
                try:
                    params[k2] = str_to_bool(args[k])
                except:
                    params[k2] = args[k]
    print(params)
    if params['num_LSTM_layers'] is None or params['num_layers_char'] is None:
        params['num_LSTM_layers'] = 2
        params['num_layers_char'] = 2
    # Checking if logdir already has the emb files:
    params['embedding_path'] = os.path.join(params['log_dir'], 'embedding')

    # Path to vocabs for indexing
    params["word_emb_vocab"] = os.path.join(
        params["embedding_path"], "vocab.txt"
    )
    params["char_vocab"] = os.path.join(
        params["embedding_path"], "chars.txt"
    )

    # Create dataset files if not already done:
    params['data_path'] = os.path.join(params['log_dir'], 'dataset_lm')

    # Get nb_chars, nb_labels, & nb_words for params (used in model):
    with open(params["word_emb_vocab"], "r", encoding="utf-8") as f:
        lines = f.readlines()
        params["vocab_size"] = len(lines)
        params["max_len_sent"] = max(map(len, lines))
    with open(params["char_vocab"], "r", encoding="utf-8") as f:
        params["nb_chars"] = len(f.readlines())

    params['frequencies'] = genfromtxt(os.path.join(
                                  params['embedding_path'],
                                  'freq.txt'),
                                delimiter="\t")
    params['activation_finish'] = tf.nn.leaky_relu
    # Create input functions
    input_fn = partial(input_fn_gen, mode='train', params=params)
    input_eval = partial(input_fn_gen, mode='eval', params=params)

    # Session config for tensorflow
    config = tf.ConfigProto(
        allow_soft_placement=True, log_device_placement=True
    )
    config.gpu_options.allow_growth = True

    config = (
        tf.estimator.RunConfig(save_checkpoints_steps=params["checkpoints"])
        .replace(save_summary_steps=1)
        .replace(session_config=config)
    )

    # Generate estimator object
    estimator = tf.estimator.Estimator(
        model_fn=model_fn,
        model_dir=params["log_dir"],
        params=params,
        config=config,
    )

    # Decoding indices:
    with open(params['word_emb_vocab'], "r", encoding='utf-8') as f:
        lines = f.readlines()

    dic = {i: lines[i] for i in range(len(lines))}
    n = len(dic)
    dic[n] = 'pad'
    dic[n+1] = 'pad'

    def predict_k(start_sent, k):
        acc_sent = start_sent
        for i in range(k):
            input_predict = partial(input_fn_gen, mode='predict', sent=acc_sent)
            for _, predictions in enumerate(estimator.predict(input_predict)):
                acc_sent = acc_sent + ' ' + dic[predictions['label']].rstrip('\n')
        return acc_sent

    print(predict_k(params['sentence'], params['k']))