from __future__ import unicode_literals
__author__ = "mozman <mozman@gmx.at>"

from contextlib import contextmanager

from ..legacy import graphics as legacy
from ..lldxf.tags import DXFTag, Tags
from ..lldxf.classifiedtags import ClassifiedTags
from ..lldxf.attributes import DXFAttr, DXFAttributes, DefSubclass
from ..lldxf import const
from ..legacy.facemixins import PolyfaceMixin, PolymeshMixin
from ..tools.rgb import int2rgb, rgb2int
from ..tools import float2transparency, transparency2float

none_subclass = DefSubclass(None, {
    'handle': DXFAttr(5),
    'owner': DXFAttr(330),  # Soft-pointer ID/handle to owner BLOCK_RECORD object
})

entity_subclass = DefSubclass('AcDbEntity', {
    'paperspace': DXFAttr(67, default=0),  # 0 .. modelspace, 1 .. paperspace
    'layer': DXFAttr(8, default='0'),  # layername as string
    'linetype': DXFAttr(6, default='BYLAYER'),  # linetype as string, special names BYLAYER/BYBLOCK
    'lineweight': DXFAttr(370),  # lineweight enum value. Stored and moved around as a 16-bit integer
    'ltscale': DXFAttr(48, default=1.0),  # linetype scale
    'invisible': DXFAttr(60, default=0),  # invisible .. 1, visible .. 0
    'color': DXFAttr(62, default=256),  # dxf color index, 0 .. BYBLOCK, 256 .. BYLAYER
    'true_color': DXFAttr(420, dxfversion='AC1018'),  # true color as 0x00RRGGBB 24-bit value
    'color_name': DXFAttr(430, dxfversion='AC1018'),  # color name as string
    'transparency': DXFAttr(440, dxfversion='AC1018'),  # transparency value 0x020000TT 0 = fully transparent / 255 = opaque
    'shadow_mode': DXFAttr(284, dxfversion='AC1021'),  # shadow_mode
    # 0 = Casts and receives shadows
    # 1 = Casts shadows
    # 2 = Receives shadows
    # 3 = Ignores shadows

})

# noinspection PyUnresolvedReferences
class ModernGraphicEntityExtension(object):
    @property
    def rgb(self):
        return int2rgb(self.get_dxf_attrib('true_color'))

    @rgb.setter  # line.rgb = (12, 34, 56)
    def rgb(self, rgb):
        self.set_dxf_attrib('true_color', rgb2int(rgb))

    @property
    def transparency(self):
        return transparency2float(self.get_dxf_attrib('transparency'))

    @transparency.setter  # line.transparency = 0.50
    def transparency(self, transparency):
        # 0.0 = opaque & 1.0 if 100% transparent
        self.set_dxf_attrib('transparency', float2transparency(transparency))

class ModernGraphicEntity(legacy.GraphicEntity, ModernGraphicEntityExtension):
    """ Default graphic entity wrapper, allows access to following dxf attributes:
     - handle
     - owner handle
     - layer
     - linetype
     - color
     - paperspace
     - ltscale
     - invisible
     - true_color (as int), requires DXF Version AC1018 (AutoCAD R2004)
     - color_name, requires DXF Version AC1018 (AutoCAD R2004)
     - transparency, requires DXF Version AC1018 (AutoCAD R2004)
     - shadow_mode, requires DXF Version AC1021 (AutoCAD R2007)

     Wrapper for all unsupported graphic entities.
    """
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass)

_LINETEMPLATE = """  0
LINE
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbLine
 10
0.0
 20
0.0
 30
0.0
 11
1.0
 21
1.0
 31
1.0
"""

line_subclass = DefSubclass('AcDbLine', {
    'start': DXFAttr(10, xtype='Point2D/3D'),
    'end': DXFAttr(11, xtype='Point2D/3D'),
    'thickness': DXFAttr(39, default=0),
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
})

class Line(legacy.Line, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_LINETEMPLATE)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, line_subclass)

_POINT_TPL = """  0
POINT
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbPoint
 10
0.0
 20
0.0
 30
0.0
"""
point_subclass = DefSubclass('AcDbPoint', {
    'location': DXFAttr(10, 'Point2D/3D'),
    'thickness': DXFAttr(39, None),
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
})

class Point(legacy.Point, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_POINT_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, point_subclass)

_CIRCLE_TPL = """  0
CIRCLE
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbCircle
 10
0.0
 20
0.0
 30
0.0
 40
1.0
"""
circle_subclass = DefSubclass('AcDbCircle', {
    'center': DXFAttr(10, xtype='Point2D/3D'),
    'radius': DXFAttr(40),
    'thickness': DXFAttr(39, default=0.0),
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
})

class Circle(legacy.Circle, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_CIRCLE_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, circle_subclass)

_ARC_TPL = """  0
ARC
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbCircle
 10
0.0
 20
0.0
 30
0.0
 40
1.0
100
AcDbArc
 50
0
 51
360
"""

arc_subclass = DefSubclass('AcDbArc', {
    'start_angle': DXFAttr(50),
    'end_angle': DXFAttr(51),
})

class Arc(legacy.Arc, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_ARC_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, circle_subclass, arc_subclass)

_TRACE_TPL = """  0
TRACE
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbTrace
 10
0.0
 20
0.0
 30
0.0
 11
0.0
 21
0.0
 31
0.0
 12
0.0
 22
0.0
 32
0.0
 13
0.0
 23
0.0
 33
0.0
"""
trace_subclass = DefSubclass('AcDbTrace', {
    'vtx0': DXFAttr(10, xtype='Point2D/3D'),
    'vtx1': DXFAttr(11, xtype='Point2D/3D'),
    'vtx2': DXFAttr(12, xtype='Point2D/3D'),
    'vtx3': DXFAttr(13, xtype='Point2D/3D'),
    'thickness': DXFAttr(39, default=0.0),
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
})

class Trace(legacy.Trace, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_TRACE_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, trace_subclass)

class Solid(Trace):
    TEMPLATE = ClassifiedTags.from_text(_TRACE_TPL.replace('TRACE', 'SOLID'))

_3DFACE_TPL = """  0
3DFACE
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbFace
 10
0.0
 20
0.0
 30
0.0
 11
0.0
 21
0.0
 31
0.0
 12
0.0
 22
0.0
 32
0.0
 13
0.0
 23
0.0
 33
0.0
"""
face_subclass = DefSubclass('AcDbFace', {
    'vtx0': DXFAttr(10, xtype='Point3D'),
    'vtx1': DXFAttr(11, xtype='Point3D'),
    'vtx2': DXFAttr(12, xtype='Point3D'),
    'vtx3': DXFAttr(13, xtype='Point3D'),
    'invisible_edge': DXFAttr(70, default=0),
})

class Face(legacy.Face, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_3DFACE_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, face_subclass)

_TEXT_TPL = """  0
TEXT
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbText
 10
0.0
 20
0.0
 30
0.0
 40
1.0
  1
TEXTCONTENT
 50
0.0
 51
0.0
  7
STANDARD
 41
1.0
 71
0
 72
0
 11
0.0
 21
0.0
 31
0.0
100
AcDbText
 73
0
"""
text_subclass = (
    DefSubclass('AcDbText', {
        'insert': DXFAttr(10, xtype='Point2D/3D'),
        'height': DXFAttr(40),
        'text': DXFAttr(1),
        'rotation': DXFAttr(50, default=0.0),  # in degrees (circle = 360deg)
        'oblique': DXFAttr(51, default=0.0),  # in degrees, vertical = 0deg
        'style': DXFAttr(7, default='STANDARD'),  # text style
        'width': DXFAttr(41, default=1.0),  # width FACTOR!
        'text_generation_flag': DXFAttr(71, default=0),  # 2 = backward (mirr-x), 4 = upside down (mirr-y)
        'halign': DXFAttr(72, default=0),  # horizontal justification
        'align_point': DXFAttr(11, xtype='Point2D/3D'),
        'thickness': DXFAttr(39, default=0.0),
        'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
    }),
    DefSubclass('AcDbText', {'valign': DXFAttr(73, default=0)}))

class Text(legacy.Text, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_TEXT_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, *text_subclass)

_POLYLINE_TPL = """  0
POLYLINE
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDb2dPolyline
 66
1
 70
0
 10
0.0
 20
0.0
 30
0.0
"""

polyline_subclass = DefSubclass('AcDb2dPolyline', {
    'elevation': DXFAttr(10, xtype='Point3D'),
    'flags': DXFAttr(70, default=0),
    'default_start_width': DXFAttr(40, default=0.0),
    'default_end_width': DXFAttr(41, default=0.0),
    'm_count': DXFAttr(71, default=0),
    'n_count': DXFAttr(72, default=0),
    'm_smooth_density': DXFAttr(73, default=0),
    'n_smooth_density': DXFAttr(74, default=0),
    'smooth_type': DXFAttr(75, default=0),
    'thickness': DXFAttr(39, default=0.0),
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
})

class Polyline(legacy.Polyline, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_POLYLINE_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, polyline_subclass)

    def post_new_hook(self):
        super(Polyline, self).post_new_hook()
        self.update_subclass_specifier()

    def update_subclass_specifier(self):
        # For dxf attribute access not the name of the subclass is important, but
        # the order of the subclasses 1st, 2nd, 3rd and so on.
        # The 3rd subclass is the AcDb3dPolyline or AcDb2dPolyline subclass
        subclass = self.tags.subclasses[2]
        subclass[0] = DXFTag(100, self.get_mode())

    def cast(self):
        mode = self.get_mode()
        if mode == 'AcDbPolyFaceMesh':
            return Polyface.convert(self)
        elif mode == 'AcDbPolygonMesh':
            return Polymesh.convert(self)
        else:
            return self

class Polyface(Polyline, PolyfaceMixin):
    """ PolyFace structure:
    POLYLINE
      AcDbEntity
      AcDbPolyFaceMesh
    VERTEX - Vertex
      AcDbEntity
      AcDbVertex
      AcDbPolyFaceMeshVertex
    VERTEX - Face
      AcDbEntity
      AcDbFaceRecord
    SEQEND
    """
    @staticmethod
    def convert(polyline):
        return Polyface(polyline.tags, polyline.drawing)

class Polymesh(Polyline, PolymeshMixin):
    """ PolyMesh structure:
    POLYLINE
      AcDbEntity
      AcDbPolygonMesh
    VERTEX
      AcDbEntity
      AcDbVertex
      AcDbPolygonMeshVertex
    """
    @staticmethod
    def convert(polyline):
        return Polymesh(polyline.tags, polyline.drawing)

_VERTEX_TPL = """ 0
VERTEX
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbVertex
100
AcDb2dVertex
 10
0.0
 20
0.0
 30
0.0
 40
0.0
 41
0.0
 42
0.0
 70
0
"""
vertex_subclass = (
    DefSubclass('AcDbVertex', {}),  # subclasses[2]
    DefSubclass('AcDb2dVertex', {  # subclasses[3]
        'location': DXFAttr(10, xtype='Point2D/3D'),
        'start_width': DXFAttr(40, default=0.0),
        'end_width': DXFAttr(41, default=0.0),
        'bulge': DXFAttr(42, default=0),
        'flags': DXFAttr(70),
        'tangent': DXFAttr(50),
        'vtx0': DXFAttr(71),
        'vtx1': DXFAttr(72),
        'vtx2': DXFAttr(73),
        'vtx3': DXFAttr(74),
    })
)

EMPTY_VERTEX_SUBCLASS = Tags()

class Vertex(legacy.Vertex, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_VERTEX_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, *vertex_subclass)

    def post_new_hook(self):
        self.update_subclass_specifier()

    def update_subclass_specifier(self):
        def set_subclass(subclassname):
            subclass = self.tags.subclasses[3]
            subclass[0] = DXFTag(100, subclassname)

        if self.is_3d_polyline_vertex:  # flags & const.VTX_3D_POLYLINE_VERTEX
            set_subclass('AcDb3dPolylineVertex')
        elif self.is_face_record:  # (flags & Vertex.FACE_FLAGS) == const.VTX_3D_POLYFACE_MESH_VERTEX:
            set_subclass('AcDbFaceRecord')
            self.tags.subclasses[2] = EMPTY_VERTEX_SUBCLASS  # clear subclass AcDbVertex
        elif self.is_poly_face_mesh_vertex:  # flags & Vertex.FACE_FLAGS == Vertex.FACE_FLAGS:
            set_subclass('AcDbPolyFaceMeshVertex')
        elif self.is_polygon_mesh_vertex:  # flags & const.VTX_3D_POLYGON_MESH_VERTEX:
            set_subclass('AcDbPolygonMeshVertex')
        else:
            set_subclass('AcDb2dVertex')

    @staticmethod
    def fix_tags(tags):
        """ If subclass[2] is not 'AcDbVertex', insert empty subclass
        """
        if tags.subclasses[2][0].value != 'AcDbVertex':
            tags.subclasses.insert(2, EMPTY_VERTEX_SUBCLASS)

class SeqEnd(legacy.SeqEnd):
    TEMPLATE = ClassifiedTags.from_text("  0\nSEQEND\n  5\n0\n330\n 0\n100\nAcDbEntity\n")
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass)

_LWPOLYLINE_TPL = """  0
LWPOLYLINE
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbPolyline
 90
0
 70
0
"""

lwpolyline_subclass = DefSubclass('AcDbPolyline', {
    'elevation': DXFAttr(38, default=0.0),
    'thickness': DXFAttr(39, default=0.0),
    'flags': DXFAttr(70, default=0),
    'const_width': DXFAttr(43, default=0.0),
    'count': DXFAttr(90),
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
})

LWPOINTCODES = (10, 20, 40, 41, 42)

class LWPolyline(ModernGraphicEntity):
    TEMPLATE = ClassifiedTags.from_text(_LWPOLYLINE_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, lwpolyline_subclass)

    @property
    def AcDbPolyline(self):
        return self.tags.subclasses[2]

    @property
    def closed(self):
        return bool(self.dxf.flags & const.LWPOLYLINE_CLOSED)

    @closed.setter
    def closed(self, status):
        flagsnow = self.dxf.flags
        if status:
            self.dxf.flags = flagsnow | const.LWPOLYLINE_CLOSED
        else:
            self.dxf.flags = flagsnow & (~const.LWPOLYLINE_CLOSED)

    def __len__(self):
        return self.dxf.count

    def __iter__(self):
        """ Yielding tuples of (x, y, start_width, end_width, bulge), start_width, end_width and bulge is 0 if not
        present.
        """
        def get_vertex():
            point.append(attribs.get(40, 0))
            point.append(attribs.get(41, 0))
            point.append(attribs.get(42, 0))
            return tuple(point)

        point = None
        attribs = {}
        for tag in self.AcDbPolyline:
            if tag.code in LWPOINTCODES:
                if tag.code == 10:
                    if point is not None:
                        yield get_vertex()
                    point = list(tag.value)
                    attribs = {}
                else:
                    attribs[tag.code] = tag.value
        if point is not None:
            yield get_vertex()  # last point

    def get_rstrip_points(self):
        last0 = 4
        for point in self:
            while point[last0] == 0 and last0 > 1:
                last0 -= 1
            yield tuple(point[:last0+1])

    def append_points(self, points):
        """ Append new *points* to polyline, *points* is a list of (x, y, [start_width, [end_width, [bulge]]])
        tuples. Set start_width, end_width to 0 to ignore (x, y, 0, 0, bulge).
        """
        tags = self.AcDbPolyline

        def append_point(point):

            def add_tag_if_not_zero(code, value):
                if value != 0.0:
                    tags.append(DXFTag(code, value))
            tags.append(DXFTag(10, (point[0], point[1])))  # x, y values
            try:
                add_tag_if_not_zero(40, point[2])  # start width, default=0
                add_tag_if_not_zero(41, point[3])  # end width, default=0
                add_tag_if_not_zero(42, point[4])  # bulge, default=0
            except IndexError:
                pass

        for point in points:
            append_point(point)

        self._update_count()

    def _update_count(self):
        self.dxf.count = len(self.AcDbPolyline.find_all(10))

    get_points = __iter__

    def set_points(self, points):
        """ Remove all points and append new *points*, *points* is a list of (x, y, [start_width, [end_width, [bulge]]])
        tuples. Set start_width, end_width to 0 to ignore (x, y, 0, 0, bulge).
        """
        self.discard_points()
        self.append_points(points)

    @contextmanager
    def points(self):
        points = list(self)
        yield points
        self.set_points(points)

    @contextmanager
    def rstrip_points(self):
        points = list(self.get_rstrip_points())
        yield points
        self.set_points(points)

    def discard_points(self):
        self.AcDbPolyline.remove_tags(codes=LWPOINTCODES)
        self.dxf.count = 0

    def __getitem__(self, index):
        """ Returns polyline point at *index* as (x, y, start_width, end_width, bulge) tuple, start_width, end_width and
        bulge is 0 if not present.
        """
        if index < 0:
            index += self.dxf.count
        for x, point in enumerate(self):
            if x == index:
                return point
        raise IndexError(index)

_BLOCK_TPL = """  0
BLOCK
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbBlockBegin
  2
BLOCKNAME
  3
BLOCKNAME
 70
0
 10
0.0
 20
0.0
 30
0.0
  1

"""
block_subclass = (
    DefSubclass('AcDbEntity', {'layer': DXFAttr(8, default='0')}),
    DefSubclass('AcDbBlockBegin', {
        'name': DXFAttr(2),
        'name2': DXFAttr(3),
        'description': DXFAttr(4),
        'flags': DXFAttr(70),
        'base_point': DXFAttr(10, xtype='Point2D/3D'),
        'xref_path': DXFAttr(1, default=""),
    })
)

class Block(legacy.GraphicEntity, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_BLOCK_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, *block_subclass)

_ENDBLOCK_TPL = """  0
ENDBLK
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbBlockEnd
"""
endblock_subclass = (
    DefSubclass('AcDbEntity', {'layer': DXFAttr(8, default='0')}),
    DefSubclass('AcDbBlockEnd', {}),
)

class EndBlk(ModernGraphicEntity):
    TEMPLATE = ClassifiedTags.from_text(_ENDBLOCK_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, *endblock_subclass)

_INSERT_TPL = """  0
INSERT
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbBlockReference
  2
BLOCKNAME
 10
0.0
 20
0.0
 30
0.0
 41
1.0
 42
1.0
 43
1.0
 50
0.0
"""

insert_subclass = DefSubclass('AcDbBlockReference', {
    'attribs_follow': DXFAttr(66, default=0),
    'name': DXFAttr(2),
    'insert': DXFAttr(10, xtype='Point2D/3D'),
    'xscale': DXFAttr(41, default=1.0),
    'yscale': DXFAttr(42, default=1.0),
    'zscale': DXFAttr(43, default=1.0),
    'rotation': DXFAttr(50, default=0.0),
    'column_count': DXFAttr(70, default=1),
    'row_count': DXFAttr(71, default=1),
    'column_spacing': DXFAttr(44, default=0.0),
    'row_spacing': DXFAttr(45, default=0.0),
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
})

class Insert(legacy.Insert, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_INSERT_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, insert_subclass)

_ATTDEF_TPL = """  0
ATTDEF
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbText
 10
0.0
 20
0.0
 30
0.0
 40
1.0
  1
DEFAULTTEXT
 50
0.0
 51
0.0
 41
1.0
  7
STANDARD
 71
0
 72
0
 11
0.0
 21
0.0
 31
0.0
100
AcDbAttributeDefinition
  3
PROMPTTEXT
  2
TAG
 70
0
 73
0
 74
0
"""

attdef_subclass = (
    DefSubclass('AcDbText', {
        'insert': DXFAttr(10, xtype='Point2D/3D'),
        'thickness': DXFAttr(39, default=0.0),
        'height': DXFAttr(40),
        'text': DXFAttr(1),
        'rotation': DXFAttr(50, default=0.0),
        'width': DXFAttr(41, default=1.0),
        'oblique': DXFAttr(51, default=0.0),
        'style': DXFAttr(7, default='STANDARD'),
        'text_generation_flag': DXFAttr(71, default=0),
        'halign': DXFAttr(72, default=0),
        'align_point': DXFAttr(11, xtype='Point2D/3D'),
        'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
    }),
    DefSubclass('AcDbAttributeDefinition', {
        'prompt': DXFAttr(3),
        'tag': DXFAttr(2),
        'flags': DXFAttr(70),
        'field_length': DXFAttr(73, default=0),
        'valign': DXFAttr(74, default=0),
    })
)

class Attdef(legacy.Attdef, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_ATTDEF_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, *attdef_subclass)

_ATTRIB_TPL = """  0
ATTRIB
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbText
 10
0.0
 20
0.0
 30
0.0
 40
1.0
  1
DEFAULTTEXT
 50
0.0
 51
0.0
 41
1.0
  7
STANDARD
 72
0
 11
0.0
 21
0.0
 31
0.0
100
AcDbAttribute
  2
TAG
 70
0
 71
0
 73
0
 74
0
"""
attrib_subclass = (
    DefSubclass('AcDbText', {
        'insert': DXFAttr(10, xtype='Point2D/3D'),
        'thickness': DXFAttr(39, default=0.0),
        'height': DXFAttr(40),
        'text': DXFAttr(1),
        'rotation': DXFAttr(50, default=0.0),  # error in DXF description, because placed in 'AcDbAttribute'
        'width': DXFAttr(41, default=1.0),  # error in DXF description, because placed in 'AcDbAttribute'
        'oblique': DXFAttr(51, default=0.0),  # error in DXF description, because placed in 'AcDbAttribute'
        'style': DXFAttr(7, default='STANDARD'),  # error in DXF description, because placed in 'AcDbAttribute'
        'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),  # error in DXF description, because placed in 'AcDbAttribute'
        'halign': DXFAttr(72, default=0),
        'align_point': DXFAttr(11, xtype='Point2D/3D'),
    }),
    DefSubclass('AcDbAttribute', {
        'tag': DXFAttr(2),
        'flags': DXFAttr(70),
        'field_length': DXFAttr(73, default=0),
        'text_generation_flag': DXFAttr(71, default=0),
        'valign': DXFAttr(74, default=0),
    })
)

class Attrib(legacy.Attrib, ModernGraphicEntityExtension):
    TEMPLATE = ClassifiedTags.from_text(_ATTRIB_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, *attrib_subclass)
_ELLIPSE_TPL = """  0
ELLIPSE
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbEllipse
 10
0.0
 20
0.0
 30
0.0
 11
1.0
 21
0.0
 31
0.0
 40
1.0
 41
0.0
 42
6.283185307179586
"""

ellipse_subclass = DefSubclass('AcDbEllipse', {
    'center': DXFAttr(10, xtype='Point2D/3D'),
    'major_axis': DXFAttr(11, xtype='Point2D/3D'),  # relative to the center
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
    'ratio': DXFAttr(40),
    'start_param': DXFAttr(41),  # this value is 0.0 for a full ellipse
    'end_param': DXFAttr(42),  # this value is 2*pi for a full ellipse
})

class Ellipse(ModernGraphicEntity):
    TEMPLATE = ClassifiedTags.from_text(_ELLIPSE_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, ellipse_subclass)
_RAY_TPL = """ 0
RAY
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbRay
 10
0.0
 20
0.0
 30
0.0
 11
1.0
 21
0.0
 31
0.0
"""
ray_subclass = DefSubclass('AcDbRay', {
    'start': DXFAttr(10, xtype='Point3D'),
    'unit_vector': DXFAttr(11, xtype='Point3D'),
})

class Ray(legacy.GraphicEntity):
    TEMPLATE = ClassifiedTags.from_text(_RAY_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, ray_subclass)

class XLine(Ray):
    TEMPLATE = ClassifiedTags.from_text(_RAY_TPL.replace('RAY', 'XLINE'))

_SHAPE_TPL = """  0
SHAPE
  5
0
330
0
100
AcDbEntity
  8
0
100
AcDbShape
 10
0.0
 20
0.0
 30
0.0
 40
1.0
  2
NAME
 50
0.0
 41
1.0
 51
0.0
"""

shape_subclass = DefSubclass('AcDbShape', {
    'thickness': DXFAttr(39, default=0.0),
    'insert': DXFAttr(10, xtype='Point2D/3D'),
    'size': DXFAttr(40),
    'name': DXFAttr(2),
    'rotation': DXFAttr(50, default=0.0),
    'xscale': DXFAttr(41, default=1.0),
    'oblique': DXFAttr(51, default=0.0),
    'extrusion': DXFAttr(210, xtype='Point3D', default=(0.0, 0.0, 1.0)),
})

# SHAPE is not tested with real world DXF drawings!
class Shape(ModernGraphicEntity):
    TEMPLATE = ClassifiedTags.from_text(_SHAPE_TPL)
    DXFATTRIBS = DXFAttributes(none_subclass, entity_subclass, shape_subclass)


from pytest_mock import MockFixture
from agents_playground.core.time_utilities import MS_PER_SEC

from agents_playground.core.waiter import Waiter

class TestWaiter:
  def test_waiting(self, mocker: MockFixture) -> None:
    patched_sleep = mocker.patch('agents_playground.core.waiter.sleep')
    waiter = Waiter()
    waiter.wait(5)
    patched_sleep.assert_called_once_with(5)

  def test_waiting_until_deadline(self, mocker: MockFixture) -> None:
    RIGHT_NOW = 15000
    DEADLINE = 33000
    TIME_TO_WAIT = (DEADLINE - RIGHT_NOW)/MS_PER_SEC
    mocker.patch('agents_playground.core.time_utilities.TimeUtilities.now', return_value=RIGHT_NOW)
    waiter = Waiter()
    waiter.wait = mocker.Mock()
    waiter.wait_until_deadline(DEADLINE)
    waiter.wait.assert_called_once_with(TIME_TO_WAIT)

  def test_not_waiting_for_the_past(self, mocker: MockFixture) -> None:
    RIGHT_NOW = 15000
    DEADLINE = 13000 # In the past
    TIME_TO_WAIT = (DEADLINE - RIGHT_NOW)/MS_PER_SEC
    mocker.patch('agents_playground.core.time_utilities.TimeUtilities.now', return_value=RIGHT_NOW)
    waiter = Waiter()
    waiter.wait = mocker.Mock()
    waiter.wait_until_deadline(DEADLINE)
    waiter.wait.assert_not_called()


import time
import base64

from django.core.files.base import ContentFile
from django.template import loader
from rest_framework import viewsets
from rest_framework.response import Response
from django.core.cache import cache
from django.db import transaction
from django.http import HttpResponse, JsonResponse
from rest_framework.decorators import api_view
from rest_framework.views import APIView

from .serialzer import *
from rest_framework.authentication import TokenAuthentication
from rest_framework.permissions import IsAuthenticated
from rest_framework.authtoken.models import Token

# Create your views here.

# ------------redis----------
# stop:  /etc/init.d/redis-server stop
# start:  /etc/init.d/redis-server start

# ------------User------------
class UserView(viewsets.ModelViewSet):
    authentication_classes = [TokenAuthentication]
    permission_classes = [IsAuthenticated]

    queryset = User.objects.all()
    serializer_class = UserSerializer

# --------------vehicles-------------
class VehiclesView(viewsets.ModelViewSet):
    queryset = Vehicle.objects.all()
    serializer_class = VehicleSerializer

    def create(self, request, *args, **kwargs):
        cache.clear()
        return super().create(request, *args, **kwargs)

    def list(self, request, *args, **kwargs):
        if cache.get('vehicles'):
            serializer = cache.get('vehicles')
            return Response(serializer)
        else:
            queryset = self.filter_queryset(self.get_queryset())

            page = self.paginate_queryset(queryset)
            if page is not None:
                serializer = self.get_serializer(page, many=True)
                return self.get_paginated_response(serializer.data)

            serializer = self.get_serializer(queryset, many=True)
            cache.set('vehicles', serializer.data)
            return Response(serializer.data)

    def retrieve(self, request, *args, **kwargs):
        vehicle_id = kwargs.get('pk')
        if cache.get('vehicles/' + vehicle_id):
            serializer = cache.get('vehicles/' + vehicle_id)
            return Response(serializer)
        else:
            instance = self.get_object()
            serializer = self.get_serializer(instance)
            cache.set('vehicles/' + vehicle_id, serializer.data, timeout=60)
            return Response(serializer.data)

    def update(self, request, *args, **kwargs):
        cache.clear()
        return super().update(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        cache.clear()
        return super().destroy(request, *args, **kwargs)

# ------------Cars----------------
class CarsView(viewsets.ModelViewSet):
    queryset = Car.objects.all()
    serializer_class = CarSerializer

    def create(self, request, *args, **kwargs):
        cache.clear()
        return super().create(request, *args, **kwargs)

    def list(self, request, *args, **kwargs):
        if cache.get('cars'):
            data = cache.get('cars')
            return Response(data)
        else:
            queryset = self.filter_queryset(self.get_queryset())

            page = self.paginate_queryset(queryset)
            if page is not None:
                serializer = self.get_serializer(page, many=True)
                return self.get_paginated_response(serializer.data)

            serializer = self.get_serializer(queryset, many=True)
            cache.set('cars', serializer.data)
            return Response(serializer.data)

    def retrieve(self, request, *args, **kwargs):
        car_id = kwargs.get('pk')
        if cache.get('cars/' + car_id):
            return Response(cache.get('cars/' + car_id))
        else:
            instance = self.get_object()
            serializer = self.get_serializer(instance)
            cache.set('cars/' + car_id, serializer.data)
            return Response(serializer.data)

    def update(self, request, *args, **kwargs):
        cache.clear()
        return super().update(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        cache.clear()
        return super().destroy(request, *args, **kwargs)

# ----------------Trucks---------------
class TruckView(viewsets.ModelViewSet):
    queryset = Truck.objects.all()
    serializer_class = TruckSerializer

    def create(self, request, *args, **kwargs):
        cache.clear()
        return super().create(request, *args, **kwargs)

    def list(self, request, *args, **kwargs):
        if cache.get('trucks'):
            data = cache.get('trucks')
            return Response(data)
        else:
            queryset = self.filter_queryset(self.get_queryset())

            page = self.paginate_queryset(queryset)
            if page is not None:
                serializer = self.get_serializer(page, many=True)
                return self.get_paginated_response(serializer.data)

            serializer = self.get_serializer(queryset, many=True)
            cache.set('trucks', serializer.data)
            return Response(serializer.data)

    def retrieve(self, request, *args, **kwargs):
        truck_id = kwargs.get('pk')
        if cache.get('trucks/' + truck_id):
            return Response(cache.get('trucks/' + truck_id))
        else:
            instance = self.get_object()
            serializer = self.get_serializer(instance)
            cache.set('trucks/' + truck_id, serializer.data)
            return Response(serializer.data)

    def update(self, request, *args, **kwargs):
        cache.clear()
        return super().update(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        cache.clear()
        return super().destroy(request, *args, **kwargs)

# --------------Bike--------------
class BikeView(viewsets.ModelViewSet):
    queryset = Bike.objects.all()
    serializer_class = BikeSerializer

    def create(self, request, *args, **kwargs):
        cache.clear()
        return super().create(request, *args, **kwargs)

    def list(self, request, *args, **kwargs):
        if cache.get('bikes'):
            data = cache.get('bikes')
            return Response(data)
        else:
            queryset = self.filter_queryset(self.get_queryset())

            page = self.paginate_queryset(queryset)
            if page is not None:
                serializer = self.get_serializer(page, many=True)
                return self.get_paginated_response(serializer.data)

            serializer = self.get_serializer(queryset, many=True)
            cache.set('bikes', serializer.data)
            return Response(serializer.data)

    def retrieve(self, request, *args, **kwargs):
        bike_id = kwargs.get('pk')
        if cache.get('bikes/' + bike_id):
            return Response(cache.get('bikes/' + bike_id))
        else:
            instance = self.get_object()
            serializer = self.get_serializer(instance)
            cache.set('bikes/' + bike_id, serializer.data)
            return Response(serializer.data)

    def update(self, request, *args, **kwargs):
        cache.clear()
        return super().update(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        cache.clear()
        return super().destroy(request, *args, **kwargs)

# ------------Shop----------------
class ShopView(viewsets.ModelViewSet):
    queryset = Shop.objects.all()
    serializer_class = ShopSerializer

    def create(self, request, *args, **kwargs):
        cache.clear()
        return super().create(request, *args, **kwargs)

    def list(self, request, *args, **kwargs):
        if cache.get('shops'):
            data = cache.get('shops')
            return Response(data)
        else:
            queryset = self.filter_queryset(self.get_queryset())

            page = self.paginate_queryset(queryset)
            if page is not None:
                serializer = self.get_serializer(page, many=True)
                return self.get_paginated_response(serializer.data)

            serializer = self.get_serializer(queryset, many=True)
            cache.set('shops', serializer.data)
            return Response(serializer.data)

    def retrieve(self, request, *args, **kwargs):
        shop_id = kwargs.get('pk')
        if cache.get('shops/' + shop_id):
            return Response(cache.get('shops/' + shop_id))
        else:
            instance = self.get_object()
            serializer = self.get_serializer(instance)
            cache.set('shops/' + shop_id, serializer.data)
            return Response(serializer.data)

    def update(self, request, *args, **kwargs):
        cache.clear()
        return super().update(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        cache.clear()
        return super().destroy(request, *args, **kwargs)

# -----------------Transaction API------------------------

@transaction.atomic
@api_view(['POST', 'GET'])
def transaction_api_post(request):
    if request.method == 'GET':
        query_id = request.GET.get('id', None)
        if not query_id:
            return JsonResponse({'error': 'Missing transaction ID.'}, status=400)
        data = Vehicle.objects.get(pk=query_id)
        if not data:
            return JsonResponse({'error': 'Objects does`t exists.'}, status=400)
        time.sleep(10)
        data.lp_number = 5001
        data.save()
        return JsonResponse({'data': 'Object exists'})

    if request.method == 'POST':
        data = request.data

        vehicle = data.get('vehicle', None)
        vehicle = Vehicle(lp_number=vehicle.get('lp_number', None),
                          wheel_count=vehicle.get('wheel_count', None),
                          manufacture=vehicle.get('manufacture', None),
                          model_name=vehicle.get('model_name', None),
                          file=vehicle.get('file', None),
                          )
        vehicle.save()

        # serializer = VehicleSerializer(data=data.get('vehicle', None))
        # if serializer.is_valid():
        #     serializer.save()
        # else:
        #     raise serializers.ValidationError()

        bike = data.get('bike', None)
        bike = Bike(lp_number=bike.get('lp_number', None),
                    wheel_count=bike.get('wheel_count', None),
                    manufacture=bike.get('manufacture', None),
                    model_name=bike.get('model_name', None),
                    milage=bike.get('milage', None),
                    file=bike.get('file', None),
                    )
        bike.save()

        # serializer = BikeSerializer(data=data.get('bike', None))
        # if serializer.is_valid():
        #     serializer.save()
        # else:
        #     raise serializers.ValidationError()

        truck = data.get('truck', None)
        truck = Truck(lp_number=truck.get('lp_number', None),
                      wheel_count=truck.get('wheel_count', None),
                      manufacture=truck.get('manufacture', None),
                      model_name=truck.get('model_name', None),
                      file=truck.get('file', None),
                      max_goods_weight=truck.get('max_goods_weight', None)
                      )
        truck.save()

        # serializer = TruckSerializer(data=data.get('truck', None))
        # if serializer.is_valid():
        #     serializer.save()
        # else:
        #     raise serializers.ValidationError()

        shop = data.get('shop', None)
        shop = Shop(name=shop.get('name', None))
        shop.save()

        # serializer = ShopSerializer(data=data.get('shop', None))
        # if serializer.is_valid():
        #     serializer.save()
        # else:
        #     raise serializers.ValidationError()

        car = data.get('car', None)
        car = Car(lp_number=car.get('lp_number', None),
                  wheel_count=car.get('wheel_count', None),
                  manufacture=car.get('manufacture', None),
                  model_name=car.get('model_name', None),
                  is_air_conditioned=car.get('is_air_conditioned', None),
                  has_roof_top=car.get('has_roof_top', None),
                  file=car.get('file', None),
                  )
        car.save()

        # serializer = CarSerializer(data=data.get('car', None))
        # if serializer.is_valid():
        #     serializer.save()
        # else:
        #     raise serializers.ValidationError()

        return HttpResponse('Posted Successfully')

# ---------------middleware---------------
class CustomMiddleware(APIView):
    def get(self, request):
        data = request.data
        return Response(data, 200)

# ------------csrf------------------
@api_view(['GET', 'POST'])
def csrf_fun(request):
    if request.method == 'GET':
        html = loader.get_template('csrf.html')
        return HttpResponse(html.render({}, request))
    if request.method == 'POST':
        data = request.data
        return Response(data, 200)

@api_view(['GET', 'POST'])
def csrf_fun_ajax(request):
    if request.method == 'GET':
        html = loader.get_template('csrf_ajax.html')
        return HttpResponse(html.render({}, request))
    if request.method == "POST":
        data = request.data
        print(data)
        return Response(data.values(), 200)

# -------------------file upload--------------

@api_view(['GET', 'POST'])
def file_html(request):
    if request.method == 'GET':
        html = loader.get_template('file upload.html')
        return HttpResponse(html.render({}, request))
    if request.method == 'POST':
        data = request.data
        name = data.get('name')
        uploaded_file = request.FILES['document']
        file_content = uploaded_file.read()

        # Encode the content as base64
        encoded_content = base64.b64encode(file_content).decode('utf-8')

        student = Student(name=name)
        student.file.save(str(name) + '.pdf', ContentFile(file_content))
        student.save()
        return Response({'message': 'File uploaded successfully.'}, 200)


from django.conf.urls import url
from basic_app import views

# Template tagging 这里app_name是template的专有标签，用于在html pages中获取html的相对路径
# 需要一个名字来标注路径
app_name = 'basic_app'

urlpatterns = [
    url(r'^relative/$',views.relative,name='relative'),
    url(r'^other/$',views.other,name='other')
]


import copy
import os
import joblib
import time

import numpy as np
import pandas as pd

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from torch.optim import lr_scheduler
from torch.utils.data import DataLoader, Dataset
from tqdm import tqdm
from transformers import AdamW, AutoConfig, AutoModel, AutoTokenizer, DataCollatorWithPadding

from common import set_seed
from dataset import ClassificationDataset
from model import ClassificationModel, MeanPooling

# Suppress warnings
import warnings
warnings.filterwarnings("ignore")
# For descriptive error messages
os.environ['CUDA_LAUNCH_BLOCKING'] = "1"

train_df = pd.read_csv("data/train.csv")
test_df = pd.read_csv("data/test.csv")

CONFIG = {
    "seed": 42,
    "epochs": 5,
    "model_name": "microsoft/deberta-v3-base",
    "n_accumulate": 2,
    "train_batch_size": 8,
    "valid_batch_size": 16,
    "max_length": 512,
    "learning_rate": 3e-5,
    "min_lr": 1e-6,
    "T_max": 500,
    "weight_decay": 0.01,
    "num_classes": 3,
    "device": torch.device("cuda:0" if torch.cuda.is_available() else "cpu"),
  }

CONFIG["tokenizer"] = AutoTokenizer.from_pretrained(CONFIG['model_name'])

TRAIN_DIR = "/home/mil/k-tanaka/essay_feedback/essay_feedback/data/train"
TEST_DIR = "/home/mil/k-tanaka/essay_feedback/essay_feedback/data/test"

set_seed(CONFIG['seed'])

encoder = LabelEncoder()
train_df['discourse_effectiveness_label'] = encoder.fit_transform(train_df['discourse_effectiveness'])

with open("le.pkl", "wb") as fp:
    joblib.dump(encoder, fp)

def train_one_epoch(model, optimizer, scheduler, dataloader, device):
    model.train()

    total = 0
    running_loss = 0.0
    correct = 0
    lr = []
    bar = tqdm(dataloader, total=len(dataloader))
    steps = len(dataloader)
    for step, data in enumerate(bar):
        ids = data["input_ids"].to(device, dtype=torch.long)
        mask = data["attention_mask"].to(device, dtype=torch.long)
        targets = data["target"].to(device, dtype=torch.long)
        batch_size = ids.size(0)
        outputs = model(ids, mask)
        loss = criterion(outputs, targets)
        loss = loss / CONFIG['n_accumulate']
        loss.backward()
        if (step + 1) % CONFIG['n_accumulate'] == 0 or step == steps:
            optimizer.step()
            optimizer.zero_grad()
            if scheduler:
                scheduler.step()

        running_loss += loss.item() * batch_size
        total += batch_size
        _, predictions = outputs.max(1)
        correct += (predictions == targets).float().sum().item()
        epoch_loss = running_loss / total
        acc = correct / total
        bar.set_postfix(Loss=epoch_loss, Accuracy=acc*100, LR=optimizer.param_groups[0]['lr'])
        lr.append(optimizer.param_groups[0]['lr'])
    return epoch_loss, acc, lr

@torch.no_grad()
def evaluate(model, dataloader, device):
    model.eval()

    total = 0
    running_loss = 0.0
    correct = 0

    for data in dataloader:
        ids = data["input_ids"].to(device, dtype=torch.long)
        mask = data["attention_mask"].to(device, dtype=torch.long)
        targets = data["target"].to(device, dtype=torch.long)

        batch_size = ids.size(0)

        outputs = model(ids, mask)

        loss = criterion(outputs, targets)

        running_loss += loss.item() * batch_size
        total += batch_size

        _, predictions = outputs.max(1)
        correct += (predictions == targets).float().sum().item()

    epoch_loss = running_loss / total
    acc = correct / total

    print("Validation Loss: {:.4f} Accuracy: {:.2f}%".format(epoch_loss, acc * 100))
    return epoch_loss, acc

def start_training(model, optimizer, scheduler, device, num_epochs):
    start = time.time()
    best_epoch_loss = np.inf
    history = {"Train Loss": [], "Valid Loss": [], "Train Acc": [], "Valid Acc": [], "LR": []}

    for epoch in range(1, num_epochs + 1):
        print("Epoch: ", epoch)
        train_epoch_loss, train_epoch_acc, epoch_lr = train_one_epoch(
            model, optimizer, scheduler, dataloader=train_loader, device=CONFIG["device"]
        )

        val_epoch_loss, valid_epoch_acc = evaluate(
            model, valid_loader, device=CONFIG["device"]
        )

        history["Train Loss"].append(train_epoch_loss)
        history["Valid Loss"].append(val_epoch_loss)
        history["Train Acc"].append(train_epoch_acc)
        history["Valid Acc"].append(valid_epoch_acc)
        history["LR"].extend(epoch_lr)

        # deep copy the model
        if val_epoch_loss <= best_epoch_loss:
            print(
                f"Validation Loss Improved ({best_epoch_loss} ---> {val_epoch_loss})"
            )
            best_epoch_loss = val_epoch_loss
            best_epoch_acc = valid_epoch_acc
            best_model_wts = copy.deepcopy(model.state_dict())
            PATH = f"best_feedback.bin"
            torch.save(model.state_dict(), PATH)
            # Save a model file from the current directory
            print(f"Model Saved")

        print()

    end = time.time()
    time_elapsed = end - start
    print(
        "Training complete in {:.0f}h {:.0f}m {:.0f}s".format(
            time_elapsed // 3600,
            (time_elapsed % 3600) // 60,
            (time_elapsed % 3600) % 60,
        )
    )

    print(
        "Best Loss: {:.4f} Best Accuracy: {:.2f}".format(
            best_epoch_loss, best_epoch_acc * 100
        )
    )

    # load best model weights
    model.load_state_dict(best_model_wts)

    return model, history

df_train, df_valid = train_test_split(train_df, test_size=0.2, random_state=42, stratify = train_df.discourse_effectiveness_label)
train_dataset = ClassificationDataset(df_train, tokenizer=CONFIG["tokenizer"], max_length=CONFIG["max_length"], data_path=TRAIN_DIR)
valid_dataset = ClassificationDataset(df_valid, tokenizer=CONFIG["tokenizer"], max_length=CONFIG["max_length"], data_path=TRAIN_DIR)

collate_fn = DataCollatorWithPadding(tokenizer=CONFIG['tokenizer'])
train_loader = DataLoader(
    train_dataset,
    batch_size=CONFIG["train_batch_size"],
    collate_fn=collate_fn,
    num_workers=2,
    shuffle=True,
    pin_memory=True,
)

valid_loader = DataLoader(
    valid_dataset,
    batch_size=CONFIG["valid_batch_size"],
    collate_fn=collate_fn,
    num_workers=2,
    shuffle=False,
    pin_memory=True,
)

model = ClassificationModel(CONFIG['model_name'],CONFIG['num_classes'])
model.to(CONFIG['device'])

optimizer = AdamW(model.parameters(), lr=CONFIG['learning_rate'])
scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CONFIG['T_max'], eta_min=CONFIG['min_lr'])

w_adequate = 1-len(train_df[train_df['discourse_effectiveness'] == 'adequate'])/len(train_df)
w_effective = 1-len(train_df[train_df['discourse_effectiveness'] == 'effective'])/len(train_df)
w_ineffective = 1-len(train_df[train_df['discourse_effectiveness'] == 'ineffective'])/len(train_df)
class_weights = torch.tensor(
    [w_adequate, w_effective, w_ineffective]
).cuda()

criterion = nn.CrossEntropyLoss(weight=class_weights)

model, history = start_training(model, optimizer, scheduler, device=CONFIG['device'], num_epochs=CONFIG['epochs'])


import argparse
import os

from kernelupdater import SelectedKernel, CommandRunner, KernelUpdater, KERNEL_ROOT_DIR, ConfigToCopyChooser

def main():
    parser = argparse.ArgumentParser(description="Gentoo: builds latest merged kernel, installs it, and does all"
                                                 " necessary post processes")
    parser.add_argument("-f", "--force", help="Force rebuild, reinstall and post processes", action="store_true")
    args = parser.parse_args()

    selected_kernel = SelectedKernel()

    if os.path.exists(KERNEL_ROOT_DIR + "/linux/.config") and not args.force:
        print("A config file exists already in " + KERNEL_ROOT_DIR + "/linux - skipping")
        exit(1)

    command_runner = CommandRunner()
    config_to_copy_chooser = ConfigToCopyChooser(selected_kernel=selected_kernel)

    chosen_config = config_to_copy_chooser.choose_config_file()

    kernel_updater = KernelUpdater(config_file=chosen_config, a_selected_kernel=selected_kernel,
                                   a_command_runner=command_runner)
    kernel_updater.update_kernel()


from collections import deque

"""
# Definition for a Node.
class Node:
    def __init__(self, val = 0, neighbors = []):
        self.val = val
        self.neighbors = neighbors
"""
class Solution:
    def cloneGraph(self, node: 'Node') -> 'Node':
        '''
        1. use dfs or bfs to iterate through all nodes
        2. store original nodes in dictionary
        3. make sure no original nodes are stored twice (keep a visited list)
        4. store a cloned node as value to go with each original node
        5. the cloned node will only be given a val
        6. to set the cloned node's neighbors,
        '''

        clones = {}
        visited = []

        q = deque()
        if not node:
            return None
        q.append(node)

        #create dictionary of originals with clones as values
        while q:
            curr = q.popleft()
            visited.append(curr)
            for neighbor in curr.neighbors:
                if neighbor not in visited:
                    q.append(neighbor)
            clones[curr] = Node(curr.val)

        #connect clones to their neighbors
        for original, clone in clones.items():
            for neighbor in original.neighbors:
                cloned_neighbor = clones[neighbor]
                clone.neighbors.append(cloned_neighbor)

        return clones[node]


import MeCab

def tokenize(text):
    wakati = MeCab.Tagger("-O wakati")
    wakati.parse("")
    return wakati.parse(text).strip().split(" ")

def extract_keyphrase_candidates(text):
    tagger = MeCab.Tagger()
    tagger.parse("")

    node = tagger.parseToNode(text)

    keyphrase_candidates = []
    phrase = []
    phrase_noun = []
    is_adj_candidate = False
    is_multinoun_candidate = False

    while node:
        # adjectives + nouns
        if node.feature.startswith('形容詞'):
            is_adj_candidate = True
            phrase.append(node.surface)
        if node.feature.startswith("名詞") and is_adj_candidate:
            phrase.append(node.surface)
        elif len(phrase) >= 2:
            keyphrase_candidates.append(phrase)

            is_adj_candidate = False
            phrase = []

        # multiple nouns
        if node.feature.startswith("名詞"):
            phrase_noun.append(node.surface)
            is_multinoun_candidate = True
        elif len(phrase_noun) >= 2:
            keyphrase_candidates.append(phrase_noun)

            is_multinoun_candidate = False
            phrase_noun = []
        else:
            is_multinoun_candidate = False
            phrase_noun = []

        node = node.next

    return keyphrase_candidates


import PySimpleGUI as sg                        # Part 1 - The import

class Tela:
    def __init__(self):
        #layout:
        layout = [
             [sg.Text("Diretório de Origem: "), sg.Input()],
             [sg.Text("Diretório de Destino: "), sg.Input()],
             [sg.Button('Ok')]
        ]
        #janela:
        window = sg.Window('Copy Diretory', layout)
        #extrair dados
        self.event, self.values = window.Read()

    def Start(self):
        return self.values


#!/usr/bin/env python
import requests
import os
import re
import json
import xmltodict
from bs4 import BeautifulSoup
import urllib.request
from config import TUMBLR_BLOGS

def scrap_tumblr(blog, num, start):
    url = f"https://{blog}.tumblr.com/api/read?type=photo&num={num}&start={start}"
    response = requests.get(url)
    xml_cleaned = re.sub("[^\x20-\x7f]+", "", response.content.decode("utf-8"))
    data = xmltodict.parse(xml_cleaned)
    with open(f"media/{blog}.json", "w") as json_file:
        json.dump(data, json_file, indent=4, sort_keys=True)
    posts = data["tumblr"]["posts"]["post"]

    def get_url_from_post(post):
        try:
            soupe = BeautifulSoup(post["regular-body"], "html.parser")
            return soupe.find("img")["src"]
        except:
            return None
        # return soupe

    for post in posts:
        try:
            # if post has photoset, walk into photoset for each photo
            photoset = post["photoset"]["photo"]
            for photo in photoset:
                url = get_url_from_post(photo)
                if url:
                    filename = url.split("/")[-1]
                    download_image("media", filename, url)
        except:
            # select the largest resolution
            # usually in the first element
            url = get_url_from_post(post)
            # print(url)
            if url:
                filename = url.split("/")[-1]
                download_image("media", filename, url)

def download_image(path, name, url):
    file_path = os.path.join(path, name)
    print(f"downloading {file_path}")
    if not os.path.isfile(file_path):
        try:
            urllib.request.urlretrieve(url, file_path)
        except Exception as e:
            print("Failed: ", e)

for blog in TUMBLR_BLOGS:
    print(f"Scrapping {blog}")
    scrap_tumblr(blog, 40, 3)

# download_image(
#     "media",
#     "jojo.jpeg",
#     "https://64.media.tumblr.com/55de0d667fa0a26de027d0b69ac8e5e7/4eef33b906bed264-68/s640x960/3f7e17c5bcae6570faa6b439bb3b7a28747ec3b4.jpg",
# )


#!/usr/local/bin/python3.7
# -*- coding: utf-8 -*-
from typing import List

class Solution:
    def combinationSum3(self, k: int, n: int) -> List[List[int]]:
        """k 限制了树的高度，n 限制了树的宽度。"""
        def backtrack(start, track):
            #结束条件:到大树的底部
            if sum(track) == n and len(track) == k: return res.append(track[:])
            if sum(track) > n or len(track) > k: return

            for i in range(start, 10):
                # 做选择
                track.append(i)
                # 进入下一行决策
                backtrack(i + 1, track)
                # 撤销选择
                track.pop()

        res = []
        if n <= 0 or k <= 0: return []
        backtrack(1, [])
        return res

    def combinationSum3_1(self, k: int, n: int) -> List[List[int]]:
        """k 限制了树的高度，n 限制了树的宽度。"""
        def backtrack(start, track):
            #结束条件:到大树的底部
            if sum(track) == n and len(track) == k: return res.append(track[:])
            #剪枝 (n - start + 1) < k
            if sum(track) > n or len(track) > k or len(track) + (n - start + 1) < k: return

            # 考虑选择当前位置
            track.append(start)
            backtrack(start + 1, track)
            track.pop()
            # 考虑不选择当前位置
            backtrack(start + 1, track)

        res = []
        if n <= 0 or k <= 0: return []
        backtrack(1, [])
        return res

def main():
    param = 3
    param2 = 7
    solution = Solution()
    ret = solution.combinationSum3(param, param2)
    print(ret)
    ret = solution.combinationSum3_1(param, param2)
    print(ret)

'''216. 组合总和 III

找出所有相加之和为 n 的 k 个数的组合。组合中只允许含有 1 - 9 的正整数，并且每种组合中不存在重复的数字。

说明：

所有数字都是正整数。
解集不能包含重复的组合。
示例 1:

输入: k = 3, n = 7
输出: [[1,2,4]]
示例 2:

输入: k = 3, n = 9
输出: [[1,2,6], [1,3,5], [2,3,4]]

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/combination-sum-iii
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
'''
if __name__ == '__main__':
    main()


import sys
from collections import deque

inf = float('inf')

n, x, y = map(int, sys.stdin.readline().split())
g = [[] for _ in range(n)]
for i in range(n-1):
    g[i].append(i+1)
    g[i+1].append(i)
g[x-1].append(y-1)
g[y-1].append(x-1)

def bfs(i):
    dist = [inf] * n
    dist[i] = 0
    q = deque()
    q.append((i, 0))
    visited = set()

    while q:
        x, d = q.popleft()
        dist[x] = min(dist[x], d)
        visited.add(x)
        for y in g[x]:
            if y in visited:
                continue
            q.append((y, d+1))
    return dist

def main():
    res = [0] * (n + 1)
    for i in range(n):
        dist = bfs(i)
        for d in dist:
            res[d] += 1
    for c in res[1:-1]:
        print(c // 2)

if __name__ == "__main__":
    main()


#!/usr/bin/python
import sys

import gflags

from simulations_code import simulate
import market

FLAGS = gflags.FLAGS

def main():
    params_df = simulate.params_df

    markets = market.Markets(3600, 0)
    for index, row in params_df.iterrows():
        hour = row['hour']
        markets.reset(seconds=3600 * hour)
        simulate.simulate(markets, row)

if __name__ == "__main__":
    try:
        argv = FLAGS(sys.argv)
    except gflags.FlagsError as e:
        print("%s\nUsage: %s ARGS\n%s" % (e, sys.argv[0], FLAGS))
        sys.exit(1)

    main()


import matplotlib.pyplot as plt

GC_cont = []
GC1 = []
GC2 = []
GC3 = []

# read columns of file
file_name = '3a_gc123.txt'
file = open(file_name)
for line in file:
	if (line[0] != '#'):
		data = line.split()
		if (len(data) > 1):
			GC_cont.append(float(data[1]))
			GC1.append(float(data[2]))
			GC2.append(float(data[3]))
			GC3.append(float(data[4]))

file.close()

#plot
x_min = 20
x_max = 80
fig = plt.figure(figsize=(11, 5))
plt.rc('grid',ls="--")

sub = fig.add_subplot(1,2,1)
y_min = 0
y_max = 100

sub.plot([0, 100], [0, 100], ls='--', c='0.5')
sub.plot(GC_cont,GC1,'ob',label='1')
sub.plot(GC_cont,GC2,'og',label='2')
sub.set_xlabel('GC of genome [%]')
sub.set_xlim(x_min,x_max)
sub.xaxis.set_ticks(range(x_min,x_max+1,10))
sub.set_ylabel('GC of codon position [%]')
sub.set_ylim(y_min,y_max)
sub.yaxis.set_ticks(range(y_min,y_max+1,10))
sub.grid()
sub.legend(loc="upper left")

sub = fig.add_subplot(1,2,2)
sub.plot([0, 100], [0, 100], ls='--', c='0.5')
sub.plot(GC_cont,GC3,'or',label='3')
sub.set_xlabel('GC of genome [%]')
sub.set_xlim(x_min,x_max)
sub.xaxis.set_ticks(range(x_min,x_max+1,10))
sub.set_ylabel('GC of codon position [%]')
sub.set_ylim(y_min,y_max)
sub.yaxis.set_ticks(range(y_min,y_max+1,10))
sub.grid()
sub.legend(loc="upper left")

plt.show()
# save plot to file
fig.savefig("3a_gc123.pdf", bbox_inches='tight')


from __future__ import print_function
import networkx as nx
import numpy as np
import collections

def read_graph(FLAGS, edgeFile):
    print('loading graph...')

    if FLAGS.weighted:
        G = nx.read_edgelist(edgeFile, nodetype=int, data=(('weight', float),), create_using=nx.DiGraph())
    else:
        G = nx.read_edgelist(edgeFile, nodetype=int, create_using=nx.DiGraph())
        for edge in G.edges():
            G[edge[0]][edge[1]]['weight'] = 1

    if not FLAGS.directed:
        G = G.to_undirected()

    return G

def read_edgelist(inputFileName):
    f = open(inputFileName, 'r')
    lines = f.readlines()
    f.close()

    edgelist = []
    for line in lines:
        l = line.strip('\n\r').split(' ')
        edge = (int(l[0]), int(l[1]))
        edgelist.append(edge)
    return edgelist

def read_feature(inputFileName):
    f = open(inputFileName, 'r')
    lines = f.readlines()
    f.close()

    features = []
    for line in lines[1:]:
        l = line.strip('\n\r').split(' ')
        features.append(l)
    features = np.array(features, dtype=np.float32)

    return features

def write_embedding(embedding_result, outputFileName):
    f = open(outputFileName, 'w')
    N, dims = embedding_result.shape

    for i in range(N):
        s = ''
        for j in range(dims):
            if j == 0:
                s = str(i) + ',' + str(embedding_result[i, j])
            else:
                s = s + ',' + str(embedding_result[i, j])
        f.writelines(s + '\n')
    f.close()


from django.contrib import admin
from django.urls import path
from commath import views

urlpatterns = [
    path('admin/', admin.site.urls),
    path('',views.index),
    path('Decimal32/',views.Decimal32),
    path('Decimal64/',views.Decimal64),
    path('solve/',views.datasolve),
]


import os
import sys
import numpy as np
import pandas as pd
import warnings
from scipy import stats
from matplotlib import pyplot as plt
import seaborn as sns
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import time
from sklearn import linear_model

warnings.filterwarnings("ignore")
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
n_iter_cat = 2000
n_iter_lgb = 10000
n_fold = 16

def load_data(file_path, file_name):
    X = pd.read_hdf(file_path+file_name+"_x.hdf", 'data')
    Y = pd.read_hdf(file_path+file_name+"_y.hdf", 'data')
    features = X.columns.values
    X, Y = np.array(X.values), np.array(Y.values).reshape(-1, )
    print(X.shape, Y.shape)
    return X, Y, features

def load_submission(submission_path, submission_data):
    submission_x = pd.read_hdf(submission_data)
    submission = pd.read_csv(submission_path, index_col='seg_id')
    return submission, np.array(submission_x.values)

def train_lasso(x_train, y_train, submission_x, features, a):
    predict_submission = np.zeros(len(submission_x))

    # reg = linear_model.Lasso(alpha=0.1)
    # reg.fit(x_train, y_train)
    # predict_submission = reg.predict(submission_x)

    # KFold
    kfold = KFold(n_splits=n_fold, shuffle=True)
    mae = 0
    for fold_, (train_id, valid_id) in enumerate(kfold.split(x_train, y_train)):
        print(f"fold {fold_}")
        train_x, train_y = x_train[train_id], y_train[train_id]
        valid_x, valid_y = x_train[valid_id], y_train[valid_id]
        reg = linear_model.Lasso(alpha=a)
        reg.fit(train_x, train_y)
        reg.coef_

        predict_submission += reg.predict(submission_x) / kfold.n_splits
        predict_valid = reg.predict(valid_x)
        mae += mean_absolute_error(predict_valid, valid_y) / kfold.n_splits

    return predict_submission, mae

if __name__ == '__main__':
    file_dir = 'train/'
    output_dir = 'output_add/'
    train_data, train_label, features = load_data(file_dir, "train")

    # submission
    submission_df, submission_x = load_submission(file_dir + "sample_submission.csv", file_dir + "submission.hdf")

    # train and predict
    for a in [1e-10, 1e-9, 1e-8, 1e-7, 1e-6]:
        lasso_start = time.time()
        submission_ttf, lasso_mae = train_lasso(train_data, train_label, submission_x, features, a)
        lasso_end = time.time()

        # save submission
        timestamp = str(time.time()).split(".")[0]
        output_txt = output_dir + timestamp + '_result.txt'
        submission_df['time_to_failure'] = submission_ttf
        submission_df.to_csv(output_dir + timestamp + 'a=' + str(a) + '_submission.csv', index=True)

        # save result
        with open(output_txt, 'w+') as output:
            print(f"LASSO mae: {lasso_mae}, use {lasso_end - lasso_start}s", file=output)
        print(f"LASSO mae,alpha=, {a}, :, {lasso_mae}, use {lasso_end - lasso_start}s")


# coding: utf-8
# sehr einfache Erzeugung des Markups für vollständige Seiten
# jeweils 3 Abschnitte:
# - begin
# - content
# - end
# bei der Liste wird der content-Abschnitt wiederholt
# beim Formular nicht

import codecs
import os.path
import string

from .Erstellen import Mitarbeiter
from mako.template import Template
from mako.lookup import TemplateLookup

#--------------------------------------------------------
class View_cl(object):
#---------------------------------------------------------

    #------------------------------------------------------
    def __init__(self, path_spl):
        #------------------------------------------------------
        self.path_s = os.path.join(path_spl, "template")
        self.lookup_o = TemplateLookup(directories=self.path_s)

    #------------------------------------------------------
    def create_p(self, template_spl, data_opl):

        template_o = self.lookup_o.get_template(template_spl)
        markup_s = template_o.render(data_o = data_opl)

        return markup_s

    def createMitarbeiter_E(self, data_opl):
        markup_s = ''
        markup_s += self.readFile_p('list0.tpl')

        markupV_s = self.readFile_p('list1.tpl')
        lineT_o = string.Template(markupV_s)
        # mehrfach nutzen, um die einzelnen Zeilen der Tabelle zu erzeugen
        for id_s in data_opl:
            data_a = data_opl[str(id_s)]
            markup_s += lineT_o.safe_substitute (name=data_a[0] # HIER müssen Sie eine Ergänzung vornehmen - erl
            ,   vorname=data_a[1]
            ,   akagra=data_a[2]
            ,   taetigkeit=data_a[3]
            )
        markup_s += self.readFile_p('list2.tpl')

        return self.create_p('liste.tpl', data_opl) #markup_s

    def createList_px(self, data_opl): #createMitarbeiter_e
    #------------------------------------------------------
        # hier müsste noch eine Fehlerbehandlung ergänzt werden !
        markup_s = ''
        markup_s += self.readFile_p('list0.tpl')

        markupV_s = self.readFile_p('list1.tpl')
        lineT_o = string.Template(markupV_s)
        # mehrfach nutzen, um die einzelnen Zeilen der Tabelle zu erzeugen
        for id_s in data_opl:
            data_a = data_opl[str(id_s)]
            markup_s += lineT_o.safe_substitute (name_s=data_a[0] # HIER müssen Sie eine Ergänzung vornehmen - erl
            ,   vorname_s=data_a[1]
            ,   akadem_s=data_a[2]
            ,   taetigkeit_s=data_a[3]
            )
        markup_s += self.readFile_p('list2.tpl')

        return self.create_p('liste.tpl', data_opl) #markup_s
    #------------------------------------------------------
    def createForm_px(self, id_spl, data_opl):
    #------------------------------------------------------
    # hier müsste noch eine Fehlerbehandlung ergänzt werden !
        markup_s = ''
        markup_s += self.readFile_p('form0.tpl')
        markupV_s = self.readFile_p('form1.tpl')
        lineT_o = string.Template(markupV_s)
        markup_s += lineT_o.safe_substitute (name1_s=data_opl[0] # HIER müssen Sie eine Ergänzung vornehmen - erl
        ,   vorname1_s=data_opl[1]
        ,   matrnr1_s=data_opl[2]
        ,   sem1_s=data_opl[3]
        ,   name2_s=data_opl[4]
        ,   vorname2_s=data_opl[5]
        ,   matrnr2_s=data_opl[6]
        ,   sem2_s=data_opl[7]
        ,   id_s=id_spl
        )

        markup_s += self.readFile_p('form2.tpl')
        return markup_s

    #------------------------------------------------------
    def readFile_p(self, fileName_spl):
        #------------------------------------------------------
        content_s = ''
        with codecs.open(os.path.join('content', fileName_spl), 'r', 'utf-8') as fp_o:
            content_s = fp_o.read()
        return content_s
# EOF


from typing import List
class Solution:
    def verifyPreorder(self, preorder: List[int]) -> bool:
        index = 0

        def validate(left, right) -> bool:
            nonlocal index
            if index >= len(preorder):
                return True
            val = preorder[index]

            if left < val < right:
                index += 1
                return validate(left, val) or validate(val, right)
            else:
                return False

        return validate(float('-inf'), float('inf'))

leetcode/255. Verify Preorder Sequence in Binary Search Tree.py


import os
import glob
import xarray as xr

from kerchunk.hdf import SingleHdf5ToZarr
from kerchunk.combine import auto_dask, MultiZarrToZarr

from dask.distributed import Client, LocalCluster, progress

def safe_auto_dask(
    urls,
    single_driver,
    single_kwargs,
    mzz_kwargs,
    n_batches,
    remote_protocol=None,
    remote_options=None,
    filename=None,
    output_options=None,
):
    """Batched tree combine using dask.

    If you wish to run on a distributed cluster (recommended), create
    a client before calling this function.

    Parameters
    ----------
    urls: list[str]
        input dataset URLs
    single_driver: class
        class with ``translate()`` method
    single_kwargs: to pass to single-input driver
    mzz_kwargs: passed to ``MultiZarrToZarr`` for each batch
    n_batches: int
        Number of MZZ instances in the first combine stage. Maybe set equal
        to the number of dask workers, or a multple thereof.
    remote_protocol: str | None
    remote_options: dict
        To fsspec for opening the remote files
    filename: str | None
        Ouput filename, if writing
    output_options
        If ``filename`` is not None, open it with these options

    Returns
    -------
    reference set
    """
    import dask

    # Use logging for thread-safe error reporting
    # import logging
    # logpath = 'kerchunk-dask.log'
    # logger = logging.getLogger('log')
    # logger.setLevel(logging.INFO)
    # ch = logging.FileHandler(logpath, 'w')
    # ch.setFormatter(logging.Formatter('%(message)s'))
    # logger.addHandler(ch)

    # make delayed functions
    def safe_single(x):
        try:
            single_driver(x, **single_kwargs).translate()
        except OSError as e:
            # logger.info(f"File: <{x}>, Error: <{e}>")
            print(f"File: <{x}>, Error: <{e}>")
            return None

    single_task = dask.delayed(safe_single)
    post = mzz_kwargs.pop("postprocess", None)
    inline = mzz_kwargs.pop("inline_threshold", None)
    # TODO: if single files produce list of reference sets (e.g., grib2)

    def safe_batch(u_all, x_all):
        # First, remove empty results
        good = [i for i in range(len(x_all)) if x_all[i] is not None]
        u = [u_all[i] for i in good]
        x = [x_all[i] for i in good]
        return MultiZarrToZarr(
            u,
            indicts=x,
            remote_protocol=remote_protocol,
            remote_options=remote_options,
            **mzz_kwargs,
        ).translate()

    batch_task = dask.delayed(safe_batch)

    # sort out kwargs
    dims = mzz_kwargs.get("concat_dims", [])
    dims += [k for k in mzz_kwargs.get("coo_map", []) if k not in dims]
    kwargs = {"concat_dims": dims}
    if post:
        kwargs["postprocess"] = post
    if inline:
        kwargs["inline_threshold"] = inline
    for field in ["remote_protocol", "remote_options", "coo_dtypes", "identical_dims"]:
        if field in mzz_kwargs:
            kwargs[field] = mzz_kwargs[field]
    final_task = dask.delayed(
        lambda x: MultiZarrToZarr(
            x, remote_options=remote_options, remote_protocol=remote_protocol, **kwargs
        ).translate(filename, output_options)
    )

    # make delayed calls
    tasks = [single_task(u) for u in urls]
    tasks_per_batch = -(-len(tasks) // n_batches)
    tasks2 = []
    for batch in range(n_batches):
        ibatch = slice(batch * tasks_per_batch, (batch + 1) * tasks_per_batch)
        in_tasks = tasks[ibatch]
        u = urls[ibatch]
        if in_tasks:
            # skip if on last iteration and no remaining tasks
            tasks2.append(batch_task(u, in_tasks))
    return dask.compute(final_task(tasks2))[0]

if __name__ == '__main__':

    import time
    print(f"Starting at {time.strftime('%c', time.localtime())}")
    start_time = time.time()
    # Launch Dask cluster for parallelization
    if not "cluster" in locals():
        print("Launching dask cluster")
        cluster = LocalCluster()
        client = Client(cluster)
    else:
        print("Using existing dask cluster")

    # Read the file list (or generate it if it doesn't exist)
    cachefile = "goesfiles-all"
    if os.path.exists(cachefile):
        print("Using cached list of goes files.")
        with open(cachefile, 'r') as f:
            all_paths = sorted(f.read().splitlines())
    else:
        print("Generating new GOES file list")
        base_path = '/css/geostationary/BackStage/GOES-17-ABI-L1B-FULLD/2022/'
        # base_path = '/css/geostationary/BackStage/GOES-17-ABI-L1B-FULLD/2022/001'
        all_paths = sorted(glob.glob(os.path.join(base_path, '**/*.nc'), recursive=True))
        with open(cachefile, 'w') as f:
            f.write("\n".join(all_paths))

    # # Try 20 days' worth of files
    all_paths = [f for f in all_paths if "2022/00" in f or "2022/01" in f]
    print(f"Processing {len(all_paths)} files")

    # Load first dataset to get common dimensions
    d0 = xr.open_dataset(all_paths[0])
    common_dims = list(d0.dims.keys())

    print("Beginning processing...")
    result = safe_auto_dask(
        urls=all_paths,
        single_driver=SingleHdf5ToZarr,
        single_kwargs={},
        mzz_kwargs={
            "coo_map": {"t": "cf:t"},
            "concat_dims": ["t"],
            "identical_dims": common_dims
        },
        n_batches=sum(client.ncores().values()),
        filename="daskresult.json"
    )

    end_time = time.time()
    elapsed = end_time - start_time
    nfiles = len(all_paths)
    rate = float(nfiles) / elapsed
    print(f"Processed {nfiles} files in {elapsed:.2f} seconds ({rate:.2f} files/sec)")
    print(f"Done at {time.strftime('%c', time.localtime())}")


from src.core.db import DbService
from src.core.models import SalesRecord
import logging

log = logging.getLogger('root')

class Loader:
    COLUMNS = {'order_id', 'order_date', 'ship_date', 'ship_mode',
               'customer_id', 'customer_name', 'segment', 'country',
               'city', 'state', 'postal_code', 'region', 'product_id',
               'category', 'sub_category', 'product_name', 'sales'}

    MESSAGES = {
        'db_connection': 'Starting connection to Database',
        'saving': 'Bulk saving data',
        'complete': 'Save transaction is completed',
        'error': "Data couldn't be loaded. An error occurred during the transaction: {}",
        'not_connected': "Couln't connect to database. Records not being saved."
    }

    def __init__(self, data):
        self.data = data

    def load_data(self):
        self.build_log(log.info, 'db_connection')
        db = DbService()

        if db.is_active:
            try:
                self.build_log(log.info, 'saving')
                db.bulk_save(records=self.data.to_dict(orient='records'),
                             cls=SalesRecord)
                self.build_log(log.info, 'complete')
            except Exception as e:
                self.build_log(log.error, 'error', e)
        else:
            self.build_log(log.critical, 'not_connected')
            raise ConnectionError

    def build_log(self, logger, message_key, *args):
        template = self.MESSAGES[message_key]
        message = template.format(*args)
        logger(message, extra={'feature': 'LOAD'})


import json
import re
import os, sys
import os.path
from pathlib import Path
import tokenizer
from bs4 import BeautifulSoup
import zipfile
import time
import math
from nltk.stem.snowball import SnowballStemmer
from lxml.html.clean import Cleaner
from lxml import etree

count = 0
inverted_index = {}
invalid_pages = []
json_error_pages = []
error_list = []
total_word = {}
doc_id = {}
token_count = {}
tf_dict ={}

def checkEnglish(str):
    try:
        str.encode('ascii')
    except UnicodeEncodeError:
        return False
    else:
        print("All Good")
        return True

def sanitize(dirty_html):
    cleaner = Cleaner(page_structure=True,
                  meta=True,
                  embedded=True,
                  links=True,
                  style=True,
                  processing_instructions=True,
                  inline_style=True,
                  scripts=True,
                  javascript=True,
                  comments=True,
                  frames=True,
                  forms=True,
                  annoying_tags=True,
                  remove_unknown_tags=True,
                  safe_attrs_only=True,
                  safe_attrs=frozenset(['src','color', 'href', 'title', 'class', 'name', 'id']),
                  remove_tags=('span', 'font', 'div')
                  )

    return cleaner.clean_html(dirty_html)

def getContent(filename):
    global json_error_pages
    global error_list
    global count
    global inverted_index
    f = open(filename, "r")
    try:
        data = json.load(f)
    except:
        json_error_pages.append(filename)
        return
    url = data['url']
    content = data['content']
    """
    if content == "":
        print('Empty Doc')
        return
    if len(content) < 5:
        print('Empty Doc')
        return
    """
    soup = BeautifulSoup(content, 'html.parser')
    for script in soup(["script", "style"]):
        script.extract()
    text_string = soup.get_text(strip = True)
    text_tokens = tokenizer.tokenize(text_string)
    total_word[url] = len(text_tokens)
    token_count[url] ={}
    for token in text_tokens:

        if token in token_count[url].keys():
            token_count[url][token] += 1
        else:
            token_count[url][token]  = 1
        if token in inverted_index.keys() and url not in inverted_index[token]:
            inverted_index[token][0] += 1
            inverted_index[token].append(url)
        else:
            lst = [1, url]
            inverted_index[token] = lst

    #self.maxWordFile.flush()
    doc_id[url] = count
    count += 1
    print(count)
    if count % 10000 == 0:
        writeToDisk(inverted_index, count)
        inverted_index = {}

def getContent_nonUni(filename):
    global count
    #print(url)
    count += 1
    """
    f = tokenize.open(filename)
    if not isinstance(text, bytes):
    text = text.encode('utf-8')
    f = text + (pad * chr(pad)).encode("utf-8")
    #data1 = f.read()
    #data1.encode('ascii', 'ignore')
    data1 = json.loads(f.read())
    print(data1['url'])
    count += 1
    print(count)
    """

def buildIndex(directory):
    global invalid_pages
    for filename in os.listdir(directory):
        f = os.path.join(directory, filename)
        # checking if it is a file

        if os.path.isfile(f):
            try:
                getContent(f)
            except UnicodeDecodeError:
                #print("We are trying to decode now")
                #getContent_nonUni(f)
                invalid_pages.append(f)
                #continue

        elif os.path.isdir(f):
            buildIndex(f)

# Convert all queries from the input into a list
def convertQueryToList(somestring):
    global queries_list
    queries_list = list()
    for queries in somestring.split(" "):
            queries_list.append(queries.strip())

def search(query, someList, output_dir):
    global count
    postings_list = []
    p = SnowballStemmer("english")
    # Remove duplicates
    someList = list(set(someList))
    for element in someList:
        element_l = element.lower()
        element_l = p.stem(element_l)
        flag = False

        for filename in os.listdir(output_dir):
            f = os.path.join(output_dir, filename)
            with open(f) as index_file:
                data_c = (index_file.read())
            temp_index = json.loads(data_c)
            if element_l in temp_index.keys():
                posting = temp_index[element_l]
                #print(f'Keyword {element} was stemmed into {element_l} found in these pages:{posting}')
                posting.pop(0)
                for posting in list(set(posting)):
                    postings_list.append(posting)
                print(type(postings_list))
                print(len(postings_list))
                break
        '''
        if not flag:
            print(f'Search Query {element} not found')

    if len(postings_list) > 1:
        t = len(postings_list) # number of files contains the token
        df = t/count
        idf = math.log(count/(df+1))
        for token in someList:
            token = token.lower()
            print(token)
            tfidf = {}
            #doc_id = 0
            for url in postings_list:
                #tc = token_count[url][token]#the number of token found in that url
                #tf = tc / total_word[url] #num of token found / #total word count
                #tfidf[url] = (doc_id[url], tf * idf)
            #tf_dict[token] = dict(sorted(tfidf.items(), key=lambda item: item[1][1],reverse=True))
        result = 0
        all_q = {}
        q_count = 0

        for q in someList:
            if q.lower() != "and":
                q_count +=1

        for q in someList:
            for k,v in someList[q].items():
                 if k not in all_q:
                     all_q[k] = [v[0],v[1], 1]
                 else:
                     all_q[k][2] +=1
                     if all_q[k][1]> v[1]:
                         all_q[k][1] = v[1]

        for k,v in dict(sorted(all_q.items(),key=lambda item: item[1][1],reverse=True)).items():
            if v[2] == q_count:
                result +=1
                print(f"RANKED RESULTS")
                print(f"url: {k}, docID: {v[0]}")

        if(result == 0):
            print("No match found")
            return
        '''

    repeats = "~~~~~~~~~" * 20
    if len(someList) > 1:
        intersection = set.intersection(*postings_list)
        #print(postings_list[0].intersection(*postings_list))
        print(f'{repeats}')
        if len(intersection) > 0:
            print(f"RANKED RESULTS")
            print(f'Search query {query} found in {len(intersection)} pages: {intersection}')
        else:
            print(f'Search Query: {query} NOT FOUND')
        print(f'{repeats}')
    elif len(someList) == 1 and len(postings_list) > 1:
        print(f'{repeats}')
        print(f"RANKED RESULTS")
        print(f'Search query {query} found in {len(postings_list)} pages:')
        print(*postings_list, sep = "\n")
        print(f'{repeats}')
    else:
        print(f'NO RESULTS FOUND')

def writeToDisk(index, count = 100, filenames = 'output_indexes/output'):
    filename = filenames + str(count) + ".json"
    with open (filename, "w") as outfile:
        json_object = json.dumps(index, indent=4, sort_keys=True)
        outfile.write(json_object)

def mergeIndexes(output_dir):
    global inverted_index
    for filename in os.listdir(output_dir):
        f = os.path.join(output_dir, filename)
        with open(f) as index_file:
            data_c = (index_file.read())
            temp_index = json.loads(data_c)

        for posting in temp_index.keys():
            if posting in inverted_index.keys():
                temp_list  = temp_index[posting]
                temp_list.pop(0)
                for page in temp_list:
                    if page not in inverted_index[posting]:
                        inverted_index[posting].append(page)
                        inverted_index[posting][0] += 1
            else:
                inverted_index[posting] = temp_index[posting]

def splitIndex(index, chunks=50):
    "Return list of split indexes"
    return_list = [dict() for idx in range(chunks)]
    idx = 0
    for k,v in index.items():
        return_list[idx][k] = v
        if idx < chunks-1:  # indexes start at 0
            idx += 1
        else:
            idx = 0
    return return_list

if __name__ == '__main__':
    directory = sys.argv[1]
    query_directory = sys.argv[2]
    output_directory = sys.argv[3]
    tf_directory = sys.argv[4]
    total_word_file = tf_directory + '/total_word.json'
    doc_id_file = tf_directory + '/doc_id.json'
    token_count_file = tf_directory + '/token_count.json'
    """
    start = time.time()
    output_check = Path(output_directory)
    query_check = Path(query_directory)
    tf_check = Path(tf_directory)
    if not output_check.exists():
        os.mkdir(output_directory)
    if not query_check.exists():
        os.mkdir(query_directory)
    if not tf_check.exists():
        os.mkdir(tf_directory)
    output_index = 'inverted_index_final.json'
    buildIndex(directory)

    # Merge all Indexes
    mergeIndexes(output_directory)

    # Write final tf index to disk:
    with open (total_word_file, "w") as outfile:
        json_object = json.dumps(total_word, indent=4, sort_keys=True)
        outfile.write(json_object)

    with open (doc_id_file, "w") as outfile:
        json_object = json.dumps(doc_id, indent=4, sort_keys=True)
        outfile.write(json_object)

    with open (token_count_file, "w") as outfile:
        json_object = json.dumps(token_count, indent=4, sort_keys=True)
        outfile.write(json_object)

    split_indexes = splitIndex(inverted_index, 100)

    dict_count = 1

    query_subdirectory = query_directory + '/query_dict'

    for index in split_indexes:
        writeToDisk(index, dict_count, query_subdirectory)
        dict_count+=1

    # Time measurement
    end = time.time()
    mins = (end - start)/60
    print(f'Start Time: {start}')
    print(f'End Time: {end}')
    print(f'Time taken: {mins}')
    """

    #Querying the DB
    flag = True
    while(flag):
        queries_input = input('Enter your query here: ')
        convertQueryToList(queries_input)
        # Load the tf indexes

        with open(total_word_file) as index_file:
            data_c = (index_file.read())
        total_word = json.loads(data_c)

        with open(doc_id_file) as index_file:
            data_c = (index_file.read())
        doc_id = json.loads(data_c)

        with open(token_count_file) as index_file:
            data_c = (index_file.read())
        token_count = json.loads(data_c)

        print(f'Queries are: {queries_list}')
        start2 = time.time()
        for filename in os.listdir(query_directory):
            f = os.path.join(query_directory, filename)
            with open(f) as index_file:
                data_c = (index_file.read())
            temp_index = json.loads(data_c)
            count += len(temp_index)
        print(f'~~~~~~~~~Global Count = {count}~~~~~~~~~~~~')
        search(queries_input, queries_list, query_directory)
        # Time measurement
        end2 = time.time()
        mins = ((end2 - start2)/60) * 1000
        print(f'Time taken for query: {mins} ms')
        response = input("Would you like to search again. Type any key for yes or n/N for no\n")
        if response.lower() == "n":
            flag = False
            print(f'Thank you for using this search engine. Goodbye!')


from asyncpg import UniqueViolationError
from fastapi import APIRouter, Depends, HTTPException, Path, Response
from sqlalchemy.exc import IntegrityError
from starlette import status

from internal.config.logger import logger
from internal.service.blog_service import ApplicationService
from internal.usecase.utils.exception import NoContentError, DuplicateError
from internal.dto.blog import (
    BlogDetailRead,
    BlogResponse,
    BlogReadAllResponse,
    BlogRequest,
)

router = APIRouter(
    prefix='/blogs',
    tags=['Blog'],
)

@router.post(
    path='',
    response_model=BlogResponse,
    status_code=status.HTTP_201_CREATED
)
async def blog_create(
        dto: BlogRequest,
        application_service: ApplicationService = Depends()
) -> BlogResponse:
    try:
        return await application_service.create(dto)
    except (UniqueViolationError, IntegrityError) as error:
        logger.info(f"Email or phone already exists.: {error}")
        raise HTTPException(409, detail="Email or phone already exists.")
    except Exception as error:
        raise HTTPException(400, detail=str(error))

@router.get(
    path='',
    response_model=BlogReadAllResponse,
    status_code=status.HTTP_200_OK,
)
async def blog_list(
        application_service: ApplicationService = Depends()
) -> BlogReadAllResponse:

    try:
        application = BlogReadAllResponse(
            applications=[ap async for ap in application_service.list()]
        )
    except NoContentError as e:
        logger.info(f"NoContentError: {e}")
        raise HTTPException(status_code=404, detail=str(e))
    except Exception as e:
        logger.exception(e)
        raise HTTPException(status_code=500, detail="Internal Server Error")
    return application

@router.get(
    path='/{application_id}',
    response_model=BlogDetailRead,
    status_code=status.HTTP_200_OK,
)
async def blog_retrieve(
        application_id: str,
        application_service: ApplicationService = Depends()
) -> BlogDetailRead:

    try:
        application = await application_service.retrieve(application_id)
    except NoContentError as e:
        logger.info(f"NonContentError: {e}")
        raise HTTPException(status_code=404, detail=str(e))
    except Exception as e:
        logger.exception(e)
        raise HTTPException(status_code=500, detail="Internal Server Error")

    return application

@router.put(
    path='/{application_id}',
    response_model=BlogResponse,
    status_code=status.HTTP_202_ACCEPTED,
    responses={
        204: {
            "description": "User is updated, but data fetch is failed.",
        }
    }
)
async def blog_put(
        dto: BlogRequest,
        application_id: str = Path(title="The ID of the application"),
        application_service: ApplicationService = Depends()
) -> BlogResponse:

    try:
        content = await application_service.put(application_id, dto)
    except DuplicateError as e:
        logger.info(f"DuplicateError: {e}")
        raise HTTPException(status_code=409, detail=f"User or email is already exists: {dto}")
    except NoContentError as e:
        if "404" in str(e):
            logger.info(f"NoContentError: User not found: id={id}")
            raise HTTPException(status_code=404, detail=f"User not found: id={id}")
        else:
            logger.warning(f"NoContentError: User({dto}) is updated, but data fetch is failed.")
            return Response(status_code=204)
    except ValueError as e:
        logger.info(f"ValueError: {e}")
        raise HTTPException(status_code=400, detail=str(e))
    except Exception as e:
        logger.exception(e)
        raise HTTPException(status_code=500, detail="Internal Server Error")

    return content

@router.delete(
    path='/{application_id}',
    status_code=status.HTTP_204_NO_CONTENT,
)
async def blog_delete(
        application_id: str,
        application_service: ApplicationService = Depends()
) -> None:

    try:
        content = await application_service.delete(application_id)
    except NoContentError:
        logger.info(f"NoContentError: User not found: id={id}")
        raise HTTPException(status_code=404, detail=f"User not found: id={id}")
    except Exception as e:
        logger.exception(e)
        raise HTTPException(status_code=500, detail="Internal Server Error")
    return content


from calendar import month
from urllib import request
from django.shortcuts import render
from django.http import HttpResponse , HttpResponseNotFound, HttpResponseRedirect

monthly_challenge = {
    'january':'This is my first views',
    'feburary':'This is februray Month and we should try to Improve our skills',
    'march': 'learn Django every 20 mins in a day',
    'april':'Eat Healthy',
    'may':'Eat Healthy2',
    'june': 'learn Django every 20 mins in a day',
    'july':'This is februray Month and we should try to Improve our skills',
    'august': 'learn Django every 20 mins in a day',
    'september': 'learn Django every 20 mins in a day',
    'november':'This is my second views',
    'december':'This is my third views',
}

# Create your views here.

def monthly_challenges_by_number(request,month):
    try:
        monthly_challenges_text = list(monthly_challenge.keys())
        forward_month = monthly_challenges_text[month]

        return HttpResponseRedirect('/challenges/'+ forward_month)
    except:
        return HttpResponseNotFound('This is not supported..!')

def monthly_challenges(request,month):
    try:
        monthly_challenges_text = monthly_challenge[month]
        return HttpResponse(monthly_challenges_text)
    except:
        return HttpResponseNotFound("This is not Supported")


import logging
import time
from contextlib import contextmanager
from typing import Callable, Generator
from urllib import parse

import requests

from .types import AccessToken, DeviceFlowVerificationCodes

CLIENT_ID = 'da5e9528b63f1bd10fd8'

@contextmanager
def github_device_flow(write_oauth_token: Callable[[AccessToken], None]) -> Generator[
    DeviceFlowVerificationCodes, None, None
]:
    verification_codes: DeviceFlowVerificationCodes = _get_verification_codes()
    yield verification_codes
    access_token: AccessToken = _wait_for_authorization(verification_codes.device_code, verification_codes.expires_in,
                                                        verification_codes.interval)
    write_oauth_token(access_token)

def _get_verification_codes() -> DeviceFlowVerificationCodes:
    res = requests.post('https://github.com/login/device/code', {'client_id': CLIENT_ID})
    res.raise_for_status()
    r = dict(parse.parse_qsl(res.text))
    return DeviceFlowVerificationCodes(**r)

def _wait_for_authorization(device_code: str, expires_in: int, sleep_interval: int) -> AccessToken:
    time.sleep(sleep_interval)
    expires_in -= sleep_interval

    while expires_in > 0:
        try:
            return _get_access_token(device_code)
        except Exception as e:
            logging.debug(e)
            time.sleep(sleep_interval)
            expires_in -= sleep_interval
    else:
        raise Exception('Verification code expired')

def _get_access_token(device_code: str) -> AccessToken:
    res = requests.post('https://github.com/login/oauth/access_token', {
        'client_id': CLIENT_ID,
        'device_code': device_code,
        'grant_type': 'urn:ietf:params:oauth:grant-type:device_code'
    })
    res.raise_for_status()
    r = dict(parse.parse_qsl(res.text))

    if 'error' in r:
        raise Exception(r['error_description'], r['error'])

    return AccessToken(**r)


import math
import numpy as np
from scipy import signal
from matplotlib import pyplot as plt

# 3.1 Batch mode training using least squares
# - Supervised learning of network weights
def gen_data(rbfNodes):
    stepSize = 0.1
    interval = [0, 2 * math.pi]
    trainStart = 0
    testStart = 0.05
    # The points at which the functions should be evaluated
    trainX = np.arange(interval[0] + trainStart, interval[1], stepSize)
    testX = np.arange(interval[0] + testStart, interval[1], stepSize)

    # noise_train = np.random.normal(0, 0.1, len(trainX))
    # noise_test = np.random.normal(0, 0.1, len(testX))

    noise_train = 0
    noise_test = 0

    # sin(2x)
    trainSinY = np.sin(2 * trainX) + noise_train
    testSinY = np.sin(2 * testX) + noise_test
    # print(testSinY)

    # square(2x)
    trainSqY = np.ones(len(trainSinY))
    testSqY = np.ones(len(testSinY))
    trainSqY[np.where(trainSinY >= 0)] = 1
    trainSqY[np.where(trainSinY < 0)] = -1
    testSqY[np.where(testSinY >= 0)] = 1
    testSqY[np.where(testSinY < 0)] = -1

    # RBF nodes
    rbfSigma = 1
    intervalOffset = math.pi / math.exp(math.pi)  # Just having fun with math. Don't know of this offset is necessary whatsoever.
    rbfMu = np.linspace(interval[0] + intervalOffset, interval[1] - intervalOffset, rbfNodes)
    return rbfMu,rbfSigma,trainX,testX,trainSinY,testSinY,trainSqY,testSqY

def calc_phi(X, rbfMu, rbfSigma):
    Phi = np.zeros((X.shape[0], len(rbfMu)))
    for i in range(X.shape[0]):
        for j in range(len(rbfMu)):
            Phi[i][j] = gaussianTransfer(X[i], rbfMu[j], rbfSigma)
        # Phi[i][j] = 1/(1+ np.exp(Phi[i][j]))
    # print(Phi[i][j])
    return Phi

def predict_square(testX,Phi,w):
    predicted = np.ones(len(testX))
    predicted[np.where(np.dot(Phi, w) >=0)] = 1
    predicted[np.where(np.dot(Phi, w) <0)] = -1
    return predicted

def predict_sin(Phi,w):
    return np.dot(Phi, w)

# Sequential learning of RBF
def online_delta_rule(epochs, train_ip, train_op, eta,phi):
    MSE = []
    W = np.random.rand(1,phi.shape[1]) * 0.5

    for iter in range(epochs):
        for ind,input in enumerate(phi):
            input = np.reshape(input,(1, len(input)))
            error = train_op[ind] - np.dot(input, W.T)
            delta_w = eta * np.dot(input.T, error)
            W += delta_w.T
        # print(W)
        predicted = np.dot(phi,W.T)

        mse = np.mean(np.abs((predicted - train_op)))

        if iter>1:
            MSE.append(mse)

    plt.plot(range(2, epochs), MSE, 'g-')
    plt.xlabel('Epochs')
    plt.ylabel('Residual error')
    plt.title('Plot of Residual error Vs epochs')
    plt.show()
    return MSE, W

def gaussianTransfer(x, mu, sigma):
    phi_i = np.exp(np.divide(-np.power((x-mu), 2), 2*np.power(sigma, 2)))
    return phi_i

for i in range(10,11):
    rbfMu, rbfSigma, trainX, testX, trainSinY, testSinY, trainSqY, testSqY = gen_data(i)
    Phi_train = calc_phi(trainX, rbfMu, rbfSigma)
    Phi_train = np.concatenate((Phi_train, np.ones((Phi_train.shape[0], 1))), axis=1)
    Phi_test = calc_phi(testX, rbfMu, rbfSigma)
    Phi_test = np.concatenate((Phi_test, np.ones((Phi_test.shape[0], 1))), axis=1)
    MSE, W = online_delta_rule(30000,trainX,trainSinY,0.0001, Phi_train)
    prediction = np.dot(Phi_train,W.T)
    test_pred = np.dot(Phi_test,W.T)
    plt.plot(trainX, trainSinY, 'k--', label='actual function')
    plt.plot(trainX,prediction,'r.',label='train prediction')
    plt.plot(testX,test_pred,'bo', label='test prediction')
    plt.legend()
    plt.title('For 6 RBF nodes')
    plt.show()
    print(MSE[-1])