# -*- coding: utf-8 -*-

import os
import tempfile
import resources
import relativism
from qgis.gui import *
from qgis.core import *
from PyQt4.QtGui import *
from PyQt4.QtCore import *

try:
    import mapnik2 as mapnik
except ImportError:
    import mapnik

# repair compatibility with mapnik2 development series
if hasattr(mapnik,'Box2d'):
    mapnik.Envelope = mapnik.Box2d

MAPNIK_VERSION = None

if hasattr(mapnik,'mapnik_version'):
    MAPNIK_VERSION = mapnik.mapnik_version()

import sync

# Use pdb for debugging
#import pdb
# These lines allow you to set a breakpoint in the app
#pyqtRemoveInputHook()
#pdb.set_trace()

#TODO - support for Composer
# http://trac.osgeo.org/qgis/changeset/13361

try:
    from pygments import highlight
    from pygments.lexers import XmlLexer
    from pygments.formatters import HtmlFormatter
    HIGHLIGHTING = True
except:
    HIGHLIGHTING = False

from imageexport import ImageExport
from text_editor import TextEditor

NAME = 'Quantumnik'

class Quantumnik(QObject):
    def __init__(self, iface):
        QObject.__init__(self)
        self.iface = iface
        self.canvas = iface.mapCanvas()

        # Fake canvas to use in tab to overlay the quantumnik layer
        self.qCanvas = None
        self.qCanvasPan = None
        self.qCanvasZoomIn = None
        self.qCanvasZoomOut = None
        self.tabWidget = None

        self.mapnik_map = None
        self.using_mapnik = False
        self.from_mapfile = False
        self.loaded_mapfile = None
        self.been_warned = False
        self.last_image_path = None
        self.dock_window = None
        self.keyAction = None
        self.keyAction2 = None
        self.keyAction3 = None

    def initGui(self):
        self.action = QAction(QIcon(":/mapnikglobe.png"), QString("Create Mapnik Canvas"),
                              self.iface.mainWindow())
        self.action.setWhatsThis("Create Mapnik Canvas")
        self.action.setStatusTip("%s: render with Mapnik" % NAME)
        QObject.connect(self.action, SIGNAL("triggered()"), self.toggle)

        self.action4 = QAction(QString("View live xml"), self.iface.mainWindow())
        QObject.connect(self.action4, SIGNAL("triggered()"), self.view_xml)

        self.action3 = QAction(QString("Export Mapnik xml"),
                               self.iface.mainWindow())
        QObject.connect(self.action3, SIGNAL("triggered()"), self.save_xml)

        self.action5 = QAction(QString("Load Mapnik xml"),
                               self.iface.mainWindow())
        QObject.connect(self.action5, SIGNAL("triggered()"), self.load_xml)

        self.action6 = QAction(QString("Load Cascadenik mml"),
                               self.iface.mainWindow())
        QObject.connect(self.action6, SIGNAL("triggered()"), self.load_mml)

        self.action7 = QAction(QString("Export Map Graphics"), self.iface.mainWindow())
        QObject.connect(self.action7, SIGNAL("triggered()"),
                        self.export_image_gui)

        self.helpaction = QAction(QIcon(":/mapnikhelp.png"),"About",
                                  self.iface.mainWindow())
        self.helpaction.setWhatsThis("%s Help" % NAME)
        QObject.connect(self.helpaction, SIGNAL("triggered()"), self.helprun)

        self.iface.addToolBarIcon(self.action)

        self.iface.addPluginToMenu("&%s" % NAME, self.action)
        self.iface.addPluginToMenu("&%s" % NAME, self.helpaction)
        self.iface.addPluginToMenu("&%s" % NAME, self.action3)
        self.iface.addPluginToMenu("&%s" % NAME, self.action4)
        self.iface.addPluginToMenu("&%s" % NAME, self.action5)
        self.iface.addPluginToMenu("&%s" % NAME, self.action6)
        self.iface.addPluginToMenu("&%s" % NAME, self.action7)

        # > QGIS 1.2
        if hasattr(self.iface,'registerMainWindowAction'):
            self.keyAction2 = QAction(QString("Switch to QGIS"), self.iface.mainWindow())
            self.iface.registerMainWindowAction(self.keyAction2, "Ctrl+[")
            self.iface.addPluginToMenu("&%s" % NAME, self.keyAction2)
            QObject.connect(self.keyAction2, SIGNAL("triggered()"),self.switch_tab_qgis)

            self.keyAction3 = QAction(QString("Switch to Mapnik"), self.iface.mainWindow())
            self.iface.registerMainWindowAction(self.keyAction3, "Ctrl+]")
            self.iface.addPluginToMenu("&%s" % NAME, self.keyAction3)
            QObject.connect(self.keyAction3, SIGNAL("triggered()"),self.switch_tab_mapnik)

    def unload(self):
        self.iface.removePluginMenu("&%s" % NAME,self.action)
        self.iface.removePluginMenu("&%s" % NAME,self.helpaction)
        self.iface.removePluginMenu("&%s" % NAME,self.action3)
        self.iface.removePluginMenu("&%s" % NAME,self.action4)
        self.iface.removePluginMenu("&%s" % NAME,self.action5)
        self.iface.removePluginMenu("&%s" % NAME,self.action6)
        self.iface.removePluginMenu("&%s" % NAME,self.action7)
        self.iface.removeToolBarIcon(self.action)
        if self.keyAction:
            self.iface.unregisterMainWindowAction(self.keyAction)
        if self.keyAction2:
            self.iface.unregisterMainWindowAction(self.keyAction2)
        if self.keyAction3:
            self.iface.unregisterMainWindowAction(self.keyAction3)

    def export_image_gui(self):
        flags = Qt.WindowTitleHint | Qt.WindowSystemMenuHint | Qt.WindowStaysOnTopHint
        export = ImageExport(self,flags)
        export.show()

    def view_xml(self,m=None):
        if not self.dock_window:
            self.dock_window = TextEditor(self)
            self.iface.mainWindow().addDockWidget( Qt.BottomDockWidgetArea,
                                                   self.dock_window )
            if not self.using_mapnik:
                # http://trac.osgeo.org/qgis/changeset/12955 - render starting signal
                QObject.connect(self.canvas, SIGNAL("renderComplete(QPainter *)"),
                                self.checkLayers)

        self.dock_window.show()
        if self.loaded_mapfile:
            # if we have loaded a map xml or mml
            # so lets just display the active file
            xml = open(self.loaded_mapfile,'rb').read()
        else:
            if not m:
                # regenerate from qgis objects
                e_c = sync.EasyCanvas(self,self.canvas)
                m = e_c.to_mapnik()
            if hasattr(mapnik,'save_map_to_string'):
                xml = mapnik.save_map_to_string(m)
            else:
                (handle, mapfile) = tempfile.mkstemp('.xml', 'quantumnik-map-')
                os.close(handle)
                mapnik.save_map(m,str(mapfile))
                xml = open(mapfile,'rb').read()
        e = self.canvas.extent()
        bbox = '%s %s %s %s' % (e.xMinimum(),e.yMinimum(),
                                e.xMaximum(),e.yMaximum())
        cmd = '\n<!-- nik2img.py mapnik.xml out.png -d %s %s -e %s -->\n' % (self.canvas.width(), self.canvas.height(), bbox)
        try:
            if self.mapnik_map:
                cmd += '<!-- <MinScaleDenominator>%s</MinScaleDenominator> -->\n' % (self.mapnik_map.scale_denominator())
        except:
            pass

        code = xml + cmd
        if HIGHLIGHTING:
            highlighted = highlight(code, XmlLexer(), HtmlFormatter(linenos=False, nowrap=False, full=True))
            self.dock_window.textEdit.setHtml(highlighted)
        else:
            self.dock_window.textEdit.setText(xml + cmd)

    def helprun(self):
        infoString = QString("Written by Dane Springmeyer\nhttps://github.com/springmeyer/quantumnik")
        QMessageBox.information(self.iface.mainWindow(),"About %s" % NAME,infoString)

    def toggle(self):
        if self.using_mapnik:
            self.stop_rendering()
        else:
            self.start_rendering()

    def proj_warning(self):
        self.been_warned = True
        ren = self.canvas.mapRenderer()
        if not ren.hasCrsTransformEnabled() and self.canvas.layerCount() > 1:
            if hasattr(self.canvas.layer(0),'crs'):
                if not self.canvas.layer(0).crs().toProj4() == ren.destinationCrs().toProj4():
                    QMessageBox.information(self.iface.mainWindow(),"Warning","The projection of the map and the first layer do not match. Mapnik may not render the layer(s) correctly.\n\nYou likely need to either enable 'On-the-fly' CRS transformation or set the Map projection in your Project Properties to the projection of your layer(s).")
            else:
                if not self.canvas.layer(0).srs().toProj4() == ren.destinationSrs().toProj4():
                    QMessageBox.information(self.iface.mainWindow(),"Warning","The projection of the map and the first layer do not match. Mapnik may not render the layer(s) correctly.\n\nYou likely need to either enable 'On-the-fly' CRS transformation or set the Map projection in your Project Properties to the projection of your layer(s).")

    def save_xml(self):
        # need to expose as an option!
        relative_paths = True

        mapfile = QFileDialog.getSaveFileName(None, "Save file dialog",
                                              'mapnik.xml', "Mapfile (*.xml)")
        if mapfile:
            e_c = sync.EasyCanvas(self,self.canvas)
            mapfile_ = str(mapfile)
            base_path = os.path.dirname(mapfile_)
            e_c.base_path = base_path
            m = e_c.to_mapnik()
            mapnik.save_map(m,mapfile_)

            if relative_paths:
                relativism.fix_paths(mapfile_,base_path)

    def make_bundle(self): pass
        # todo: accept directory name
        # move mapfile and all file based datasources
        # into that folder and stash some docs inside
        # provide option to zip and upload to url on the fly

    def set_canvas_from_mapnik(self):
        # set up keyboard shortcut
        # > QGIS 1.2
        if hasattr(self.iface,'registerMainWindowAction'):
            if not self.keyAction:
                # TODO - hotkey does not work on linux....
                self.keyAction = QAction(QString("Refresh " + NAME), self.iface.mainWindow())
                self.iface.registerMainWindowAction(self.keyAction, "Ctrl+r")
                self.iface.addPluginToMenu("&%s" % NAME, self.keyAction)
                QObject.connect(self.keyAction, SIGNAL("triggered()"),self.toggle)

        self.mapnik_map.zoom_all()
        e = self.mapnik_map.envelope()
        crs = QgsCoordinateReferenceSystem()
        srs = self.mapnik_map.srs
        if srs == '+init=epsg:900913':
            # until we can look it up in srs.db...
            merc = "+init=EPSG:900913"
            crs.createFromProj4(QString(merc))
        elif 'init' in srs:
            # TODO - quick hack, needs regex and fallbacks
            epsg = srs.split(':')[1]
            crs.createFromEpsg(int(epsg))
        else:
            if srs == '+proj=latlong +datum=WGS84':
                # expand the Mapnik srs a bit
                # http://trac.mapnik.org/ticket/333
                srs = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'
            crs.createFromProj4(QString(srs))
        if hasattr(self.canvas.mapRenderer(),'setDestinationCrs'):
            self.canvas.mapRenderer().setDestinationCrs(crs)
        else:
            self.canvas.mapRenderer().setDestinationSrs(crs)
        if not crs.isValid():
            QMessageBox.information(self.iface.mainWindow(),
                                    "Warning","Projection not understood")
            return
        QObject.connect(self.canvas, SIGNAL("renderComplete(QPainter *)"),
                        self.render_dynamic)
        self.canvas.setExtent(QgsRectangle(e.minx,e.miny,e.maxx,e.maxy))
        self.canvas.refresh()

    def set_mapnik_to_canvas(self):
        QObject.connect(self.canvas, SIGNAL("renderComplete(QPainter *)"),
                        self.render_dynamic)
        self.canvas.refresh()

    def refresh_loaded_mapfile(self):
        if self.mapfile_format == 'Cascadenik mml':
            self.load_mml(refresh=True)
        else:
            self.load_xml(refresh=True)

    def load_mml(self,refresh=False):
        self.from_mapfile = True
        self.mapfile_format = 'Cascadenik mml'
        if self.loaded_mapfile and refresh:
            mapfile = self.loaded_mapfile
        else:
            mapfile = QFileDialog.getOpenFileName(None, "Open file dialog",
                                                  '', "Cascadenik MML (*.mml)")
        if mapfile:
            self.mapnik_map = mapnik.Map(1,1)
            import cascadenik
            if hasattr(cascadenik,'VERSION'):
                major = int(cascadenik.VERSION.split('.')[0])
                if major < 1:
                    from cascadenik import compile
                    compiled = '%s_compiled.xml' % os.path.splitext(str(mapfile))[0]
                    open(compiled, 'w').write(compile(str(mapfile)))
                    mapnik.load_map(self.mapnik_map, compiled)
                elif major == 1:
                    output_dir = os.path.dirname(str(mapfile))
                    cascadenik.load_map(self.mapnik_map,str(mapfile),output_dir,verbose=False)
                elif major > 1:
                    raise NotImplementedError('This nik2img version does not yet support Cascadenik > 1.x, please upgrade nik2img to the latest release')
            else:
                from cascadenik import compile
                compiled = '%s_compiled.xml' % os.path.splitext(str(mapfile))[0]
                #if os.path.exits(compiled):
                    #pass
                open(compiled, 'w').write(compile(str(mapfile)))
                mapnik.load_map(self.mapnik_map, compiled)

            if self.loaded_mapfile and refresh:
                self.set_mapnik_to_canvas()
            else:
                self.set_canvas_from_mapnik()
            self.loaded_mapfile = str(mapfile)

    def load_xml(self,refresh=False):
        # TODO - consider putting into its own layer:
        # https://trac.osgeo.org/qgis/ticket/2392#comment:4
        self.from_mapfile = True
        self.mapfile_format = 'xml mapfile'
        if self.loaded_mapfile and refresh:
            mapfile = self.loaded_mapfile
        else:
            mapfile = QFileDialog.getOpenFileName(None, "Open file dialog",
                                                  '', "XML Mapfile (*.xml)")
        if mapfile:
            self.mapnik_map = mapnik.Map(1,1)
            mapnik.load_map(self.mapnik_map,str(mapfile))
            if self.loaded_mapfile and refresh:
                self.set_mapnik_to_canvas()
            else:
                self.set_canvas_from_mapnik()
            self.loaded_mapfile = str(mapfile)

    def finishStopRender(self):
        self.iface.mapCanvas().setMinimumSize(QSize(0, 0))

    def stop_rendering(self):
        # Disconnect all the signals as we disable the tool
        QObject.disconnect(self.qCanvas, SIGNAL("renderComplete(QPainter *)"),
                           self.render_dynamic)
        QObject.disconnect(self.qCanvas,
                           SIGNAL("xyCoordinates(const QgsPoint&)"),
                           self.updateCoordsDisplay)
        QObject.disconnect(self.canvas, SIGNAL("renderComplete(QPainter *)"),
                           self.checkLayers)
        QObject.disconnect(self.canvas, SIGNAL("extentsChanged()"),
                           self.checkExtentsChanged)
        QObject.disconnect(self.canvas, SIGNAL("mapToolSet(QgsMapTool *)"),
                           self.mapToolSet)
        self.using_mapnik = False
        # If the current tab is quantumnik then we need to update the extent
        # of the main map when exiting to make sure they are in sync
        if self.tabWidget.currentIndex() == 1:
            self.mapnikMapCoordChange()
        # Need to restore the main map instead of the mapnik tab
        tabWidgetSize = self.tabWidget.size()
        mapCanvasExtent = self.iface.mapCanvas().extent()
        self.iface.mapCanvas().setMinimumSize(tabWidgetSize)
        self.iface.mainWindow().setCentralWidget(self.iface.mapCanvas())
        self.iface.mapCanvas().show()
        # Set the canvas extent to the same place it was before getting
        # rid of the tabs
        self.iface.mapCanvas().setExtent(mapCanvasExtent)
        self.canvas.refresh()

        # null out some vars
        self.qCanvasPan = None
        self.qCanvasZoomIn = None
        self.qCanvasZoomOut = None
        # We have to let the main app swizzle the screen and then
        # hammer it back to the size we want
        QTimer.singleShot(1, self.finishStopRender)

    def create_mapnik_map(self):
        if not self.been_warned:
            self.proj_warning()
        self.easyCanvas = sync.EasyCanvas(self,self.canvas)
        self.mapnik_map = self.easyCanvas.to_mapnik()
        if self.dock_window:
            self.view_xml(self.mapnik_map)

    @property
    def background(self):
        return sync.css_color(self.canvas.backgroundBrush().color())

    # Provide a hack to try and find the map coordinate status bar element
    # to take over while the mapnik canvas is in play.
    def findMapCoordsStatus(self):
        coordStatusWidget = None
        sb = self.iface.mainWindow().statusBar()
        for x in sb.children():
            # Check if we have a line edit
            if isinstance(x, QLineEdit):
                # Now check if the text does not contain a ':'
                if not ':' in x.text():
                    # we have our coord status widget
                    coordStatusWidget = x
        return coordStatusWidget

    def finishStartRendering(self):
        self.tabWidget.setMinimumSize(QSize(0, 0))
        self.canvas.refresh()

    def start_rendering(self):
        if self.from_mapfile and not self.canvas.layerCount():
            self.refresh_loaded_mapfile()
        else:
            self.from_mapfile = False
            # http://trac.osgeo.org/qgis/changeset/12926
            # TODO - if not dirty we don't need to create a new map from scratch...
            self.create_mapnik_map()
            # Need to create a tab widget to toss into the main window
            # to hold both the main canvas as well as the mapnik rendering
            mapCanvasSize = self.canvas.size()
            mapCanvasExtent = self.iface.mapCanvas().extent()
            newWidget = QTabWidget(self.iface.mainWindow())
            sizePolicy = QSizePolicy(QSizePolicy.Expanding,
                                     QSizePolicy.Preferred)
            sizePolicy.setHorizontalStretch(0)
            sizePolicy.setVerticalStretch(0)
            sizePolicy.setHeightForWidth(newWidget.sizePolicy().hasHeightForWidth())
            newWidget.setSizePolicy(sizePolicy)
            newWidget.setSizeIncrement(QSize(0, 0))
            newWidget.setBaseSize(mapCanvasSize)
            newWidget.resize(mapCanvasSize)
            # Very important: Set the min size of the tabs to the size of the
            # original canvas.  We will then let the main app take control
            # and then use a one shot timer to set the min size back down.  It
            # is a hack, but allows us to keep the canvas and tab size correct.
            newWidget.setMinimumSize(mapCanvasSize)

            # This is the new blank canvas that we will use the qpainter
            # from to draw the mapnik image over.
            self.qCanvas = QgsMapCanvas(self.iface.mainWindow())
            self.qCanvas.setCanvasColor(QColor(255,255,255))
            self.qCanvas.enableAntiAliasing(True)
            self.qCanvas.useImageToRender(False)
            self.qCanvas.show()

            # A set of map tools for the mapnik canvas
            self.qCanvasPan = QgsMapToolPan(self.qCanvas)
            self.qCanvasZoomIn = QgsMapToolZoom(self.qCanvas,False)
            self.qCanvasZoomOut = QgsMapToolZoom(self.qCanvas,True)
            self.mapToolSet(self.canvas.mapTool())

            # Add the canvas items to the tabs
            newWidget.addTab(self.canvas, "Main Map")
            newWidget.addTab(self.qCanvas, "Mapnik Rendered Map")
            self.tabWidget = newWidget
            # Add the tabs as the central widget
            self.iface.mainWindow().setCentralWidget(newWidget)
            # Need to set the extent of both canvases as we have just resized
            # things
            self.canvas.setExtent(mapCanvasExtent)
            self.qCanvas.setExtent(mapCanvasExtent)
            # Hook up to the tabs changing so we can make sure to update the
            # rendering in a lazy way... i.e. a pan in the main canvas will
            # not cause a redraw in the mapnik tab until the mapnik tab
            # is selected.
            self.connect(self.tabWidget,SIGNAL("currentChanged(int)"),
                         self.tabChanged)
            # Grab the maptool change signal so the mapnik canvas tool
            # can stay in sync.
            # TODO: We need to get the in/out property for the zoom tool
            # exposed to the python bindings.  As it stands now, we can
            # not tell what direction the tool is going when we get this
            # signal and it is a zoom tool.
            QObject.connect(self.canvas, SIGNAL("mapToolSet(QgsMapTool *)"),
                            self.mapToolSet)
            # Catch any mouse movements over the mapnik canvas and
            # sneek in and update the cord display
            ## This is a hack to find the status element to populate with xy
            self.mapCoords = self.findMapCoordsStatus()
            QObject.connect(self.qCanvas,
                            SIGNAL("xyCoordinates(const QgsPoint&)"),
                            self.updateCoordsDisplay)
            # Get the renderComplete signal for the qCanvas to allow us to
            # render the mapnik image over it.
            QObject.connect(self.qCanvas, SIGNAL("renderComplete(QPainter *)"),
                            self.render_dynamic)
            # Get the renderComplete signal for the main canvas so we can tell
            # if there have been any layer changes and if we need to re-draw
            # the mapnik image.  This is mainly for when the mapnik tab is
            # active but layer changes are happening.
            QObject.connect(self.canvas, SIGNAL("renderComplete(QPainter *)"),
                            self.checkLayers)
            QObject.connect(self.canvas, SIGNAL("extentsChanged()"),
                            self.checkExtentsChanged)
            self.using_mapnik=True
            # We use a single shot timer to let the main app resize the main
            # window with us holding a minsize we want, then we reset the
            # allowable min size after the main app has its turn.  Hack, but
            # allows for the window to be rezised with a new main widget.
            QTimer.singleShot(1, self.finishStartRendering)

    def updateCoordsDisplay(self, p):
        if self.mapCoords:
            capturePyString = "%.5f,%.5f" % (p.x(),p.y())
            capture_string = QString(capturePyString)
            self.mapCoords.setText(capture_string)

    def mapToolSet(self, tool):
        # something changed here in recent QGIS versions causing:
        # exceptions when closing QGIS because these objects are None
        if tool:
            if isinstance(tool,QgsMapToolPan):
                self.qCanvas.setMapTool(self.qCanvasPan)
            elif isinstance(tool,QgsMapToolZoom):
                # Yet another hack to find out if the tool we are using is a
                # zoom in or out
                if tool.action().text() == QString("Zoom In"):
                    self.qCanvas.setMapTool(self.qCanvasZoomIn)
                else:
                    self.qCanvas.setMapTool(self.qCanvasZoomOut)
            else:
                self.qCanvas.setMapTool(self.qCanvasPan)

    def switch_tab_qgis(self):
        if self.tabWidget:
            self.tabWidget.setCurrentIndex(0)

    def switch_tab_mapnik(self):
        if self.tabWidget:
            self.tabWidget.setCurrentIndex(1)

    def tabChanged(self, index):
        if index == 0:
            self.mapnikMapCoordChange()
        else:
            self.mainMapCoordChange()

    def mainMapCoordChange(self):
        # print "coordChange"
        self.mapnik_map = self.easyCanvas.to_mapnik(self.mapnik_map)
        self.qCanvas.setExtent(self.iface.mapCanvas().extent())
        self.qCanvas.refresh()

    def mapnikMapCoordChange(self):
        # print "coordChange"
        self.canvas.setExtent(self.qCanvas.extent())
        self.canvas.refresh()

    # Here we are checking to see if we got a new extent on the main
    # canvas even though we are in the mapnik tab... in that case we have
    # done something like zoom to full extent etc.
    def checkExtentsChanged(self):
        if self.tabWidget:
            if self.tabWidget.currentIndex() == 1:
                self.mainMapCoordChange()

    # Here we are checking to see if we got a render complete on the main
    # canvas even though we are in the mapnik tab... in that case we have
    # a new layer etc.
    def checkLayers(self, painter=None):
        if self.tabWidget:
            if self.tabWidget.currentIndex() == 1:
                # There was a change in the main canvas while we are viewing
                # the mapnik canvas (i.e. layer added/removed etc) so we
                # need to refresh the mapnik map
                self.mapnik_map = self.easyCanvas.to_mapnik(self.mapnik_map)
                self.qCanvas.refresh()
                # We also make sure the main map canvas gets put back to the
                # current extent of the qCanvas incase the main map got changed
                # as a side effect since updates to it are lazy loaded on tab
                # change.
                self.canvas.setExtent(self.qCanvas.extent())
                # We make sure to update the XML viewer if
                # if is open
                if self.dock_window:
                    self.view_xml(self.mapnik_map)
        if self.dock_window:
            self.view_xml()

    def render_dynamic(self, painter):
        if self.mapnik_map:
            w = painter.device().width()
            h = painter.device().height()
            # using canvas dims leads to shift in QGIS < 1.3...
            #w = self.canvas.width()
            #h = self.canvas.height()
            try:
                self.mapnik_map.resize(w,h)
            except:
                self.mapnik_map.width = w
                self.mapnik_map.height = h
            if self.qCanvas:
                can = self.qCanvas
            else:
                can = self.canvas
            try:
                e = can.extent()
            except:
                can = self.canvas
                e = can.extent()
            bbox = mapnik.Envelope(e.xMinimum(),e.yMinimum(),
                                   e.xMaximum(),e.yMaximum())
            self.mapnik_map.zoom_to_box(bbox)
            im = mapnik.Image(w,h)
            mapnik.render(self.mapnik_map,im)
            if os.name == 'nt':
                qim = QImage()
                qim.loadFromData(QByteArray(im.tostring('png')))
                painter.drawImage(0,0,qim)
            else:
                qim = QImage(im.tostring(),w,h,QImage.Format_ARGB32)
                painter.drawImage(0,0,qim.rgbSwapped())
            can.refresh()


import geopandas as gpd
import shapely

from cartagen4py.utils.geometry.line import extend_line_with_point
from cartagen4py.utils.network import *

def collapse_roundabouts(roads, roundabouts, crossroads=None, maximum_diameter=None):
    """
    Collapse detected roundabouts below the provided diameter to a point and return the new network.
    Parameters
    ----------
    network : geopandas GeoDataFrame of LineStrings.
        The road network where roundabouts will be collapsed.
    roundabouts : geopandas GeoDataFrame of Polygons.
        The polygons representing the faces of the network detected as roundabouts.
    maximum_diameter : float, optional.
        The diameter, in meter, below which roundabouts are collapsed.
        Default value is set to None.
    """

    # Retrieve crs for output
    crs = roads.crs

    # Convert geodataframe to list of dicts
    roundabouts = roundabouts.to_dict('records')
    roads = roads.to_dict('records')
    if crossroads is not None:
        crossroads = crossroads.to_dict('records')

    # Create a list of all the roads geometry of the network
    network = []
    for n in roads:
        network.append(n['geometry'])

    # Calculate the spatial index on roads
    tree = shapely.STRtree(network)

    # This list will store the indexes of interior roads to throw away
    nokeep = []

    # Looping through roundabouts
    for roundabout in roundabouts:
        # Retrieve geometry and centroid of the roundabout
        polygon = roundabout['geometry']
        centroid = polygon.centroid
        linear_ring = polygon.exterior
        # Calculate the diameter of the roundabout, which is the closest distance between its centroid and border
        diameter = shapely.distance(centroid, linear_ring)

        # Check if diameter is below the given parameter before proceeding
        if maximum_diameter is not None:
            if diameter >= maximum_diameter:
                continue

        connected_crossroads = []
        if crossroads is not None:
            for crossroad in crossroads:
                if crossroad['roundabout'] == roundabout['cid']:
                    connected_crossroads.append(crossroad)

        if len(connected_crossroads) > 0:
            cgeom = []
            for c in connected_crossroads:
                cgeom.append(c['geometry'])
            face = shapely.ops.unary_union([polygon, *cgeom])
        else:
            face = polygon

        # Selecting intersecting roads index
        intersecting = tree.query(face)

        full_ring = face.exterior

        # Future storage of internal (the roundabout roads themselves)
        # and external (the entrances/exits of the roundabout)
        internals = []
        externals = []

        # Looping through intersecting roads index
        for i in intersecting:
            # Get the road geometry
            r = network[i]
            if shapely.intersects(full_ring, r):
                # If the road is contained within the linear ring of the roundabout, it is interior
                if shapely.contains(full_ring, r):
                    internals.append(i)
                # If the road is only touching the linear ring of the roundabout but is not contained within its polygon, it is exterior
                elif shapely.touches(full_ring, r):
                    if shapely.contains(linear_ring, r):
                        # Add the road for removal
                        if i not in nokeep:
                            nokeep.append(i)
                    else:
                        externals.append(i)

        # Loop through internal roads
        for i in internals:
            # Add the internal road for removal
            if i not in nokeep:
                nokeep.append(i)

        # Future storage for multiple external roads entering the roundabout at the same point
        noextend = []
        # Loop through external roads index
        for external in externals:
            # Retrieve external road geometry
            e = roads[external]['geometry']
            # Retrieve coordinates of start and end of the road line
            start, end = e.coords[0], e.coords[-1]
            # Creating points for the start and end
            first = shapely.Point(start)
            last = shapely.Point(end)

            # If the first point is closer to the centroid of the roundabout than the last
            if shapely.distance(first, centroid) < shapely.distance(last, centroid):
                # If the point as not already been extended to the centroid
                if start not in noextend:
                    # Extend the line from the start to the centroid
                    roads[external]['geometry'] = extend_line_with_point(e, centroid, position='start')
                    # Add the point if an other road passes through it to avoid overlapping
                    noextend.append(start)
            # Else, do the same for the last point
            else:
                if end not in noextend:
                    roads[external]['geometry'] = extend_line_with_point(e, centroid, position='end')
                    noextend.append(end)

    # Storing the collapsed network into a new list
    collapsed = []
    for i, n in enumerate(roads):
        # Check if the index is to be kept
        if i not in nokeep:
            collapsed.append(n)

    return gpd.GeoDataFrame(collapsed, crs=crs)


import operator
import spacy

from flask import Flask
from flask import request, jsonify

app = Flask(__name__)

class TextClassificationModel(object):
    def __init__(self, path_to_model):
        # Load the best trained Text Classification model
        self.nlp_model = spacy.load(path_to_model)

    def predict(self, sentence):
        # Transform the input sentence to SpaCy's Doc with predicted categories
        doc = self.nlp_model(sentence)
        return doc.cats

nlp_model = TextClassificationModel('./training/model-best/')

@app.route('/predict', methods=['GET'])
def predict_category_for_input_sentence():
    sentence = request.args.get('sentence')
    cats = nlp_model.predict(sentence=sentence)

    best_category = max(cats.items(), key=operator.itemgetter(1))[0]
    best_score = [value for key, value in cats.items() if key == best_category][0]

    return jsonify({'code': 200, 'category': best_category, 'score': best_score})

if __name__ == '__main__':
    app.run(debug=True, host='127.0.0.1', port='5000')


from typing import List
from random import randint
import pygame
from map.map import Map

from main_object import Object
from main_object.object import TYPE_BLOCK
from tank.tank import TankBuilder
from .controller import Controller, TYPE_BOT
from tank import Tank
from blocks import WATER_BLOCK

class TankBot(Controller):

  def __init__(self):
    self.amount_bots = 5
    super().__init__(TYPE_BOT)
    self.step = randint(50, 100)
    self.players: List[Tank] = []
    self.tanks: List[Tank] = []

  def update(self, map: Map):
      if self.amount_bots - len(self.tanks) > 0:
        for _ in range(0, self.amount_bots - len(self.tanks)):
          new_bot = TankBuilder.random_generate()
          map.add_object(new_bot)
          self.tanks.append(new_bot)

      if self.step == 0:
        self.step = randint(50, 100)
      self.step -= 1

      for player in self.players:
        if not player.status():
          self.players.remove(player)

      for tank in self.tanks:
        if not tank.status():
          self.tanks.remove(tank)

      for player in self.players:
        x = player.sprite.centerx
        y = player.sprite.centery

        for tank in self.tanks:
          if tank != player and tank.status():
              x_ = tank.sprite.centerx
              y_ = tank.sprite.centery

              l_x = x
              if x_ < x:
                l_x = x_
              l_y = y
              if y_ < y:
                l_y = y_
              layout_x = pygame.Rect(l_x, y_ - 10, abs(x - x_), 20)
              layout_y = pygame.Rect(x_ - 10, l_y, 20, abs(y_ - y))

              #pygame.draw.rect(window, 'red', layout_x)
              #pygame.draw.rect(window, 'red', layout_y)

              flag_x = True
              flag_y = True
              for object in map.objects:
                if object.type is TYPE_BLOCK and object.type_block is not WATER_BLOCK:
                  if layout_x.colliderect(object.sprite):
                    flag_x = False
                  if layout_y.colliderect(object.sprite):
                    flag_y = False

              if x_ - 10 <= x and x_ + 10 >= x and flag_y:
                if y < y_:
                  tank.direct = 0
                else:
                  tank.direct = 2
                tank.shoot()
              elif y_ - 10 <= y and y_ + 10 >= y and flag_x:
                if x < x_:
                  tank.direct = 3
                else:
                  tank.direct = 1
                tank.shoot()
              else:
                if self.step != 0:
                  tank.update_direct(tank.direct)
                else:
                  tank.update_direct(randint(0, 4))

  def set(self, tank: Tank):
      self.players.append(tank)

  def set_objects(self, tanks: List[Tank]):
      self.tanks = tanks


from __future__ import annotations

from asyncio import Task, ensure_future, sleep # pylint: disable=unused-import; typing
import json
from collections import defaultdict
from datetime import datetime, timedelta
from logging import getLogger
import typing
from typing import Callable, DefaultDict, Dict, Iterator, List, Mapping, Optional, cast
from urllib.parse import urlsplit

from . import error
from .core import RewriteFunc
from .jsonredis import RedisSortedSet
from .micro import Collection, Object, User
from .util import expect_type, randstr

if typing.TYPE_CHECKING:
    from .micro import Application

StatisticFunc = Callable[[datetime], float]

class Analytics:
    """Analytics unit.

    .. attribute:: definitions

       Statistic definitions by topic.

    .. attribute:: statistics

       Statistics over time by topic.

    .. attribute:: referrals

       Referrals from other sites.

    .. attribute:: app

       Context application.
    """

    def __init__(self, *, definitions: Mapping[str, StatisticFunc] = {},
                 app: 'Application') -> None:
        self.definitions = {
            'users': self._count_users,
            'users-actual': self._count_users_actual,
            'users-active': self._count_users_active,
            **definitions
        } # type: Dict[str, StatisticFunc]
        self.statistics = {topic: Statistic(topic, app=app) for topic in self.definitions}
        self.referrals = Referrals(app=app)
        self.app = app

    def collect_statistics(self, *, _t: datetime = None) -> None:
        """Collect statistics for all topics."""
        t = _t or self.app.now()
        for topic, f in self.definitions.items():
            # pylint: disable=protected-access; Statistic is a friend
            self.statistics[topic]._add(Point(t, f(t))) # type: ignore

    def start_collect_statistics(self) -> 'Task[None]':
        """Start the collect statistics job."""
        async def _run() -> None:
            t = self.app.now().replace(hour=0, minute=15, second=0)
            while True:
                t += timedelta(days=1)
                await sleep((t - self.app.now()).total_seconds())
                self.collect_statistics(_t=t)
                getLogger(__name__).info('Collected statistics for %s', format(t, '%d %b %Y'))
        return cast('Task[None]', ensure_future(_run()))

    def _count_users(self, t: datetime) -> float:
        # pylint: disable=unused-argument; part of API
        return len(self.app.users)

    def _count_users_actual(self, t: datetime) -> float:
        # pylint: disable=unused-argument; part of API
        return sum(1 for user in self._actual_users())

    def _count_users_active(self, t: datetime) -> float:
        return sum(1 for user in self._actual_users()
                   if t - user.authenticate_time <= timedelta(days=30)) # type: ignore

    def _actual_users(self) -> Iterator[User]:
        return (user for user in self.app.users
                if user.authenticate_time - user.create_time >= timedelta(days=1))

class Statistic:
    """See :ref:`Statistic`.

    .. attribute:: topic

       Topic the statistic is about.

    .. attribute:: app

       Context application.
    """

    def __init__(self, topic: str, *, app: 'Application') -> None:
        self.topic = topic
        self.app = app
        self._key = 'analytics.statistics.{}'.format(self.topic)

    def get(self, *, user: Optional[User]) -> List['Point']:
        """See :http:get:`/api/analytics/statistics/(topic)`."""
        if not user in self.app.settings.staff: # type: ignore
            raise error.PermissionError()
        return [Point.parse(json.loads(p.decode())) for p # type: ignore
                in self.app.r.r.zrange(self._key, 0, -1)] # type: ignore

    def json(self, *, user: User = None) -> Dict[str, object]:
        """See :meth:`micro.JSONifiable.json`."""
        return {'items': [p.json() for p in self.get(user=user)]}

    def _add(self, p):
        self.app.r.r.zadd(self._key, {json.dumps(p.json()): p.t.timestamp()})

class Point:
    """See :ref:`Point`."""

    def __init__(self, t: datetime, v: float) -> None:
        self.t = t
        self.v = v

    @staticmethod
    def parse(data: Dict[str, object]) -> 'Point':
        """See :meth:`micro.JSONifiableWithParse.parse`."""
        v = data.get('v')
        if not isinstance(v, (float, int)):
            raise TypeError()
        return Point(datetime.fromisoformat(expect_type(str)(data.get('t'))), float(v))

    def json(self) -> Dict[str, object]:
        """See :meth:`micro.JSONifiable.json`."""
        return {'t': self.t.isoformat(), 'v': self.v}

    def __eq__(self, other: object) -> bool:
        return isinstance(other, Point) and self.t == other.t and self.v == other.v

class Referral(Object):
    """See :ref:`Referral`."""

    def __init__(self, *, id: str, app: 'Application', url: str, time: str) -> None:
        super().__init__(id=id, app=app)
        self.url = url
        self.time = datetime.fromisoformat(time)

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> Dict[str, object]:
        return {
            **super().json(restricted=restricted, include=include, rewrite=rewrite),
            'url': self.url,
            'time': self.time.isoformat()
        }

class Referrals(Collection[Referral]):
    """See :ref:`Referrals`."""

    def __init__(self, *, app: 'Application') -> None:
        super().__init__(RedisSortedSet('analytics.referrals', app.r.r),
                         check=lambda key: self.app.check_user_is_staff(), app=app) # type: ignore

    def add(self, url: str, *, user: Optional[User]) -> Referral:
        """See :http:post:`/api/analytics/referrals`."""
        # pylint: disable=unused-argument; part of API
        if not urlsplit(url).scheme:
            raise error.ValueError('Relative url {}'.format(url))
        referral = Referral(id=randstr(), app=self.app, url=url, time=self.app.now().isoformat())
        self.app.r.oset(referral.id, referral)
        self.app.r.r.zadd(self.ids.key, {referral.id.encode(): -referral.time.timestamp()})
        return referral

    def summarize(self, period: tuple[datetime, datetime] = None) -> list[tuple[str, int]]:
        """See :http:get:`/api/analytics/referrals/summary?period`."""
        self.app.check_user_is_staff()

        if period is None:
            end = self.app.now()
            period = (end - timedelta(days=7), end)
        # Since the scores of the entries are timestamps * -1, we must also multiply the given
        # timestamps by -1 and invert the order so it is still a valid interval
        period = (-period[1].timestamp(), -period[0].timestamp())
        keys = [key.decode() for key in self.app.r.r.zrangebyscore(self.ids.key, *period)]

        result: DefaultDict[str, int] = defaultdict(lambda: 0)
        referrals = self.app.r.omget(keys, default=AssertionError, expect=expect_type(Referral))
        for referral in referrals:
            result[referral.url] += 1

        return sorted(result.items(), key=lambda x: (-x[1], x[0])) # type: ignore[misc]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: Susanne Reinhardt & Rafal Kowalewski

Calculates nearest neighbor distances within datasets and between both
imaging rounds. Results are saved as csv in a subfolder called called
'AdditionalOutputs'.
For the crossNND distance between imaging rounds, the coordinates of the respective
partner in the other dataset are saved as additional columns in the _resi..hdf5
files as well as their distance. This file is then called _resi.._info.hdf5

"""
import numpy as _np
import numpy as np
import pandas as pd
import sys
import os
from sklearn.neighbors import NearestNeighbors as NN

from Functions import tools

def nn_analysis(
    x1, x2,
    y1, y2,
    z1, z2,
    nn_count,
    same_channel,
):
    '''
    Searches the nearest neighbor of dataset 2 coordinates in dataset 1.

    Parameters
    ----------
    x1 : pandas series
    x2 : pandas series
    y1 : pandas series
    y2 : pandas series
    z1 : pandas series
    z2 : pandas series
        Coordinates of two datasets.
    nn_count : int
        Input value for sklearn's NearestNeighbors function. Sets up until
        which order higher nearest neighbors should be calculated,
        f.ex. 1st, 2nd or until 3rd NND
    same_channel : bool
        if True, both datasets are equal and the NND distribution is calculated
        within one dataset.
        if False, both datasets differ and the nearest neighbor of each
        coordinate in dataset 2 is searched in dataset 1.

    Returns
    -------
    nnd : list
        Nearest Neighbor Distance. Optionally higher NNDs in further columns.
    nn_indices : list
        Index of nearest neighbors in dataset 1 to dataset 2 coordinates.
    nn_partner : np array
        Coordinates of nearest neighbors in dataset 1 to dataset 2 coordinates.

    '''
    if z1 is not None: # 3D
        input1 = _np.stack((x1, y1, z1)).T
        input2 = _np.stack((x2, y2, z2)).T
    else: # 2D
        input1 = _np.stack((x1, y1)).T
        input2 = _np.stack((x2, y2)).T
    if same_channel:
        model = NN(n_neighbors=nn_count+1)
    else:
        model = NN(n_neighbors=nn_count)
    model.fit(input1)
    nnd, indices = model.kneighbors(input2)

    if same_channel:
        nnd = nnd[:, 1:] # ignore the zero distance

        # Get indices for first NND partners (if same+channel = True the first
        # column corresponds to the distance measured to itself)
        nn_indices = [sublist[1] for sublist in indices]

    else:
        # Get indices for first NND partners
        nn_indices = [sublist[0] for sublist in indices]

    nn_partner = np.array([input1[nn_indices[i]] for i in range(len(input2))])

    return nnd, nn_indices, nn_partner

def nearest_neighbor(channel1_name, file_resi1, channel2_name, file_resi2, px_size):
    '''
    Initiates nearest neighbor calculations betweend two datasets than can be
    equal or different.
    Saves calculated distances in .csv format. In case of two different
    datasets it adds columns to the resi hdf5 files with the coordinates
    of the nearest neighbor in the other file and the distance to it.

    Parameters
    ----------
    channel1_name : string
        Name of channel 1.
    file_resi1 : string
        Path and filename of the hdf5 file containing the channel 1 RESI locs.
    channel2_name : string
        Name of channel 2.
    file_resi2 : string
        Path and filename of the hdf5 file containing the channel 2 RESI locs.
    px_size : ing
        Pixel size in nm..

    '''

    # NND within channel or crossNND betwen channels?
    # used for avoiding zero distances (to self)
    same_channel = file_resi1 == file_resi2

    """load Resi files"""

    df_resi1 = pd.read_hdf(file_resi1, key = 'locs')
    x1 = df_resi1['x']*px_size  # Input is in px
    y1 = df_resi1['y']*px_size
    try:
        z1 = df_resi1['z'] # Input is in nm
        flag_3D_1 = True
    except:
        z1 = None
        flag_3D_1 = False

    if not same_channel: # load second channel
        df_resi2 = pd.read_hdf(file_resi2, key = 'locs')
        x2 = df_resi2['x']*px_size  # Input is in px
        y2 = df_resi2['y']*px_size
        try:
            z2 = df_resi2['z'] # Input is in nm
            flag_3D_2 = True
        except:
            z2 = None
            flag_3D_2 = False

        if not flag_3D_1 == flag_3D_2:
            sys.exit("Both datasets have to be either 2D or 3D.")
    else: # same channel
        x2 = x1
        y2 = y1
        z2 = z1

    flag_3D = flag_3D_1

    if same_channel:
        nn_count = 1 # only first nearest neighbor
        # The first NN will be calculated to itself, so we will later discard
        # the first column and take only the second column corresponding to the
        # the conventioanl first neareast neighbor.
    else:
        nn_count = 1 # only first cross nearest neighbor

    nnd, nn_indices, nn_partner = nn_analysis(
        x1, x2,
        y1, y2,
        z1, z2,
        nn_count,
        same_channel,
    )

    # Create subfolder for nearest neighbor csv files
    try:
        os.mkdir(os.path.split(file_resi1)[0] + '/AdditionalOutputs')
    except OSError:
        pass
    path =  os.path.split(file_resi1)[0] + "/AdditionalOutputs/"
    fname = path + os.path.split(file_resi1)[1]

    # save nn distances as csv
    if same_channel:
        np.savetxt('%s_nn_%s.csv' %(fname[:-5], channel1_name), nnd)
    else:
        np.savetxt('%s_nn_%s_to_%s.csv' %(fname[:-5], channel1_name, channel2_name), nnd)

    # Extend the content of file_resi_2 with columns containing the coordinates
    # to its nearest neighbors in file_resi1 as well as their distance and
    # orientation in the xy plane to each other

    if not same_channel:
        df_resi2['crossNND_ID'] = nn_indices
        df_resi2['crossNND_x'] = nn_partner[:,0]/px_size # Back from nm to pix
        df_resi2['crossNND_y'] = nn_partner[:,1]/px_size
        if flag_3D:
            df_resi2['crossNND_z'] = nn_partner[:,2]/px_size # save this in pix
        df_resi2['crossNND'] = nnd
        if flag_3D:
            df_resi2['crossNND_dxy'] = np.sqrt((df_resi2['x']-df_resi2['crossNND_x'])**2+
                                                 (df_resi2['y']-df_resi2['crossNND_y'])**2)*px_size
            df_resi2['crossNND_dz'] = (df_resi2['z']-df_resi2['crossNND_z']*px_size)
        df_resi2['orientation'] = tools.angle(df_resi2['x'], df_resi2['y'], df_resi2['crossNND_x'], df_resi2['crossNND_y'])

        path = os.path.split(file_resi2)[0] + "/"
        filename_old = os.path.split(file_resi2)[1]
        filename_new = '%s_info.hdf5' % (filename_old[:-5])
        tools.picasso_hdf5(df_resi2, filename_new, filename_old, path)


from pyspark.sql import SparkSession
from pyspark import SparkContext
from pyspark.sql import SQLContext
from pyspark.sql.types import BooleanType, StructField,StructType, IntegerType, DateType, LongType, DoubleType, StringType

spark = SparkSession.builder.appName("Query1_DFAPI").getOrCreate()

fileSchema = spark.read.options(header="true", inferSchema = "true").csv("/home/mikeg/Documents/databases/TPC-H/TPC-H_Tools_v3.0.0/dbgen/tmpqueries/schema/lineitem.csv").schema
lineitem = spark.read.format("csv").options(inferSchema = "true" , header = "false").schema(fileSchema).option("delimiter", "|").csv("hdfs://localhost:9000/user/diplomma/data/data/lineitem.tbl")

partSchema = spark.read.options(header="true", inferSchema = "true").csv("/home/mikeg/Documents/databases/TPC-H/TPC-H_Tools_v3.0.0/dbgen/tmpqueries/schema/part.csv").schema
part = spark.read.format("csv").options(inferSchema = "true" , header = "false").schema(partSchema).option("delimiter", "|").csv("hdfs://localhost:9000/user/diplomma/data/data/part.tbl")

lineitem.registerTempTable("lineitem")
part.registerTempTable("part")

sqlString= "select \
        sum(l_extendedprice) / 7.0 as avg_yearly \
from \
        lineitem, \
        part \
where \
        p_partkey = l_partkey \
        and p_brand = 'Brand#23' \
        and p_container = 'MED BOX' \
        and l_quantity < ( \
                select \
                        0.2 * avg(l_quantity) \
                from \
                        lineitem \
                where \
                        l_partkey = p_partkey \
        );"

res = spark.sql(sqlString)
res.show()


import requests
from telebot import *
from telebot.types import ReplyKeyboardMarkup, KeyboardButton
import webbrowser
import json
import random
TOKEN= '6165585308:AAFqLhFGa10yIc-4S0ktm_E3_oxqO8lBtto'

bot = TeleBot(TOKEN)

keyboard = ReplyKeyboardMarkup(resize_keyboard=True)
keyboard.add(KeyboardButton('👍Годные тайтлы'))
keyboard.add(KeyboardButton('🤷♂Рандомное аниме'))

keyboard.add(KeyboardButton('Об авторе и боте'))

#-=-=-=-=-Получение списка anime-=-=-=-=-=-=-=-=
url = "https://api.jikan.moe/v4/anime"

response = requests.get(url)

data = response.json()
anime_titles = [item['title'] for item in data['data']]

# for i in anime_titles:
#     print(i)

#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-==-=-=

@bot.message_handler(regexp='автор')
def about_author(message):
    bot.send_message(message.chat.id, 'Автор Наум')

@bot.message_handler(regexp='🤷♂Рандомное Аниме')
def godnie_taitls(message):
    bot.send_message(message.chat.id, f'Нашел для тебя: {anime_titles[random.randint(1,20)]}', reply_markup=keyboard)

@bot.message_handler(regexp='👍Годные тайтлы')
def godnie_taitl(message):
    bot.send_sticker(message.chat.id, 'CAACAgIAAxkBAAEJOoRkfzAXYWnuBMaa7SOhGnKLtchlIQACriIAAvThCUilx87YPT22ey8E')
    a = '\n'.join(str(x) for x in anime_titles)
    bot.send_message(message.chat.id, a)

@bot.message_handler(commands=['help', 'start'])
def send_welcome(message):
    bot.send_message(message.chat.id, "Я бот, который советует тебе аниме, чтобы тебе не было скучно", reply_markup=keyboard)
    bot.send_sticker(message.chat.id, 'CAACAgUAAxkBAAEJNUlkfJxisSAeZ5X3M1am1HJlQXwdrgACogMAAh4QYVWLQatEtVFxri8E')

@bot.message_handler(regexp='открой сайт')
def opener(message):
    webbrowser.open('https://animego.org/')

bot.infinity_polling()


import streamlit as st
import time,datetime,pytz,os,json
import pandas as pd
import altair as alt
from PIL import Image

from timeplus import *

st.set_page_config(layout="wide")
col_img, col_txt, col_link = st.columns([1,8,5])
with col_img:
    image = Image.open("detailed-analysis@2x.png")
    st.image(image, width=100)
with col_txt:
    st.title("Real-time Insights for Github")
with col_link:
    st.markdown("[Source Code](https://github.com/timeplus-io/streamlit_apps/blob/main/pages/203_%F0%9F%92%BB_github_dashboard.py)", unsafe_allow_html=True)

env = Environment().address(st.secrets["TIMEPLUS_HOST"]).apikey(st.secrets["TIMEPLUS_API_KEY"]).workspace(st.secrets["TIMEPLUS_TENANT"])

# a wrapper to run batch SQL, return headers and rows
def batchQuery(bathSQL):
    q=Query(env=env).sql(query=bathSQL).create()
    header=q.metadata()["result"]["header"]
    rows=[]
    for event in q.result():
        if event.event != "metrics" and event.event != "query":
            for row in json.loads(event.data):
                rows.append(row)
    q.cancel()
    q.delete()
    return header,rows

def show_table_for_query(sql,table_name,row_cnt):
    st.code(sql, language="sql")
    query = Query(env=env).sql(query=sql).create()
    col = [h["name"] for h in query.metadata()["result"]["header"]]
    def update_table(row,name):
        data = {}
        for i, f in enumerate(col):
            data[f] = row[i]
            #hack show first column as more friendly datetime diff
            if i==0 and isinstance(row[i], str):
                data[f]=datetime.datetime.strptime(row[i], '%Y-%m-%dT%H:%M:%S').replace(tzinfo=pytz.UTC)
                # Get current UTC datetime with timezone awareness
                current_datetime = datetime.datetime.now(pytz.UTC)
                minutes=divmod((current_datetime-data[f]).total_seconds(),60)
                data[f]=f"{row[i]} ({int(minutes[0])} min {int(minutes[1])} sec ago)"

        df = pd.DataFrame([data], columns=col)
        if name not in st.session_state:
            st.session_state[name] = st.table(df)
        else:
            st.session_state[name].add_rows(df)
    # iterate query result
    limit = row_cnt
    count = 0
    for event in query.result():
        if event.event != "metrics" and event.event != "query":
            for row in json.loads(event.data):
                update_table(row,table_name)
                count += 1
                if count >= limit:
                    break
            # break the outer loop too
            if count >= limit:
                break
    query.cancel()
    query.delete()

col1, col2, col3 = st.columns([3,3,1])

with col1:
    st.header('New events every minute')
    sql="""SELECT window_end AS time,count() AS count from tumble(table(github_events),1m)
WHERE _tp_time > date_sub(now(), 2h) GROUP BY window_end"""
    st.code(sql, language="sql")
    result=batchQuery(sql)
    col = [h["name"] for h in result[0]]
    df = pd.DataFrame(result[1], columns=col)
    c = alt.Chart(df).mark_line(point=alt.OverlayMarkDef()).encode(x='time:T',y='count:Q',tooltip=['count'],color=alt.value('#D53C97'))
    st.altair_chart(c, use_container_width=True)

    st.header('Recent events')
    show_table_for_query('SELECT created_at,actor,type,repo FROM github_events','live_events',3)

with col2:
    #st.header('New repos')
    #show_table_for_query("""SELECT created_at,actor,repo,json_extract_string(payload,'master_branch') AS branch \nFROM github_events WHERE type='CreateEvent'""",'new_repo',3)

    st.header('Default branch for new repos')
    sql="""SELECT payload:master_branch AS branch,count(*) AS cnt
FROM table(github_events) WHERE _tp_time>date_sub(now(),1h) AND type='CreateEvent'
GROUP BY branch ORDER BY cnt DESC LIMIT 3"""
    st.code(sql, language="sql")
    result=batchQuery(sql)
    col = [h["name"] for h in result[0]]
    df = pd.DataFrame(result[1], columns=col)
    base = alt.Chart(df).encode(theta=alt.Theta('cnt:Q',stack=True),color=alt.Color('branch:N',legend=None),tooltip=['branch','cnt'])
    pie=base.mark_arc(outerRadius=80)
    text = base.mark_text(radius=110, size=16).encode(text='branch:N')
    st.altair_chart(pie+text, use_container_width=True)

    st.header('Hot repos')
    sql="""SELECT max(created_at) AS followed_at, repo, count(distinct actor) AS new_followers
FROM table(github_events) WHERE _tp_time>date_sub(now(),10m) AND type ='WatchEvent'
GROUP BY repo ORDER BY new_followers DESC LIMIT 3
"""
    show_table_for_query(sql,'star_table',3)

# show a changing single value for total events
with col3:
    st.header('Event count')
    st.code("SELECT count(*) FROM github_events EMIT periodic 1s", language="sql")
    with st.empty():
        #show the initial events first
        sql="select count(*) from table(github_events)"
        cnt=batchQuery(sql)[1][0][0]
        st.metric(label="Github events", value="{:,}".format(cnt))
        st.session_state.last_cnt=cnt

        #create a streaming query to update counts
        sql=f"select {cnt}+count(*) as events from github_events emit periodic 1s"
        query = Query(env=env).sql(query=sql).create()
        def update_row(row):
            delta=row[0]-st.session_state.last_cnt
            if (delta>0):
                st.metric(label="Github events", value="{:,}".format(row[0]), delta=row[0]-st.session_state.last_cnt, delta_color='inverse')
                st.session_state.last_cnt=row[0]
        # iterate query result
        limit = 200
        count = 0
        for event in query.result():
            if event.event != "metrics" and event.event != "query":
                for row in json.loads(event.data):
                    update_row(row)
                    count += 1
                    if count >= limit:
                        break
                # break the outer loop too
                if count >= limit:
                    break
        query.cancel()
        query.delete()


#!/usr/bin/env python3
from typing import Iterable, List
from flask import Flask, jsonify, request, Response
from janome.analyzer import Analyzer
from janome.tokenizer import Token, Tokenizer
from janomeutils import ChunkFilter, token_to_morae

app = Flask(__name__)
tokenizer = Tokenizer(mmap=True)
analyzer = Analyzer(tokenizer=tokenizer, token_filters=[ChunkFilter()])

@app.route('/api/analyze', methods=['POST'])
def analyze() -> Response:
    sentence: str = request.get_data(as_text=True)
    chunks: Iterable[List[Token]] = analyzer.analyze(sentence)
    return jsonify(
        [[{'surface': token.surface, 'morae': list(token_to_morae(token)), 'pos': token.part_of_speech}
          for token in chunk] for chunk in chunks]
    )

@app.route('/', methods=['GET'])
def homepage() -> Response:
    return app.send_static_file('index.html')

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)


import pandas as pd
import time

def smoothing_3(df):
    df_copy = df.copy()
    max_index = df.shape[0] - 1
    for i in range(max_index-1):
        mean = df.iloc[i:i+3]["quantity"].mean()
        df_copy["quantity"].iloc[i+1] = mean
    df_copy["quantity"].iloc[0] = df.iloc[0:2]["quantity"].mean()
    df_copy["quantity"].iloc[-1] = df.iloc[-2:]["quantity"].mean()
    return df_copy

def smoothing_5(df):
    df_copy = df.copy()
    max_index = df.shape[0] - 1
    for i in range(0, max_index-3):
        mean = df.iloc[i:i+5]["quantity"].mean()
        df_copy["quantity"].iloc[i+2] = mean
    df_copy["quantity"].iloc[0] = df.iloc[0:2]["quantity"].mean()
    df_copy["quantity"].iloc[1] = df.iloc[0:3]["quantity"].mean()
    df_copy["quantity"].iloc[-1] = df.iloc[-2:]["quantity"].mean()
    df_copy["quantity"].iloc[-2] = df.iloc[-3:]["quantity"].mean()
    return df_copy

def smoothing_5_new(df):
    df_copy = df.copy()
    max_index = df.shape[0] - 1
    for i in range(0, max_index-3):
        mean = df.iloc[i:i+5]["quantity"].mean()
        df_copy["quantity"].iloc[i+2] = mean
    return df_copy.iloc[2:-2]

def smoothing_7_new(df):
    start = time.time()
    df_copy = df.copy()
    max_index = df.shape[0] - 1
    for i in range(0, max_index-5):
        mean = df.iloc[i:i+7]["quantity"].mean()
        df_copy["quantity"].iloc[i+3] = mean
    end = time.time()
    print("time for smoothing", (end - start), " seconds")
    return df_copy.iloc[3:-3]

if __name__ == "__main__":
    import matplotlib.pyplot as plt
    data = pd.read_csv(
        "~/PycharmProjects/seasonality_hypothesis/data_generated/aggregated_complete_outliers_removed_seas.csv",
        names=["dt_week", "quantity"])

    df_1 = smoothing(data)
    df_2 = smoothing_5(data)
    data["dt_week"] = data["dt_week"].apply(lambda x: pd.to_datetime(x, format="%Y-%m-%d"))
    data = data.set_index("dt_week")
    df_1["dt_week"] = df_1["dt_week"].apply(lambda x: pd.to_datetime(x, format="%Y-%m-%d"))
    df_1 = df_1.set_index("dt_week")
    df_2["dt_week"] = df_2["dt_week"].apply(lambda x: pd.to_datetime(x, format="%Y-%m-%d"))
    df_2 = df_2.set_index("dt_week")
    #plt.plot(df_1["quantity"], marker=".", label=' 3 point average')
    plt.plot(df_2["quantity"], marker=".", label=' 5 point average')
    #plt.plot(data["quantity"], marker=".", label='overall data aggregated')
    plt.legend(loc="upper left")
    plt.xlabel("dt_weeks")
    plt.ylabel("aggregated quantities")
    plt.title("5 point average")
    plt.show()


import pandas as pd
import json

input_excel_file = "G:\Django\input_excel_file.xlsx"
df = pd.read_excel(input_excel_file)

df = df.fillna("NULL")

grouped_data = df.groupby(['Customer ID', 'age'])

output_data = []

for (customer_id, age), group in grouped_data:
    customer_entry = {
        "Customer ID": str(customer_id),
        "age": str(age),
        "panelList": []
    }

    panels_dict = {}

    for index, row in group.iterrows():
        panel_name = row["panel_name"]
        panel_code = row["panel_code"]
        parameter_entry = {
            "parameterName": row["Parameter Name"],
            "unit": row["Units"],
            "parameterCode": str(row["Parameter Code"]),
            "value": str(row["Result"]),
            "lowerRange": str(row["Low Range"]),
            "upperRange": str(row["High Range"]),
            "displayRange": f"{row['Low Range']}-{row['High Range']}"
        }

        if panel_name not in panels_dict:
            panels_dict[panel_name] = {
                "panel_name": panel_name,
                "panel_code": panel_code,
                "parameters": []
            }

        panels_dict[panel_name]["parameters"].append(parameter_entry)

    customer_entry["panelList"] = list(panels_dict.values())
    output_data.append(customer_entry)

output_json = json.dumps(output_data, indent=4)

with open("final_output.json", "w") as json_file:
    json_file.write(output_json)

print("JSON transformation complete.")


# -*- coding: utf-8 -*-
"""
Thanks to the author Ruhan Sa, who is the TA of IR project 3 in Fall 2015
"""
print("In Python")
import json
import sys
import urllib.request
from collections import defaultdict

count = 1
langs = {'text_de','text_ru','text_en'}
core = sys.argv[2]
model = {'DFR', 'TFIDF', 'BM25'}
countRows = 0
outf = open(sys.argv[1] + '.txt', 'w+')
count = 1;
sortingList = []
QueryList = []
QueryDict = defaultdict(list)
writingDictionary = {}
with open('queries.txt', encoding="utf-8") as f:
	for line in f:
		query = line.strip('s\n').replace(':', '')[4:]
		query = urllib.parse.quote(query)
		countRows = 1
		for lang in langs:
			inurl = ('http://ec2-35-163-27-180.us-west-2.compute.amazonaws.com:8983/solr/' + core +
					'/select?df=' + lang + '&fl=score,id&indent=on&q=' + query + '&rows=20&wt=json')
			qid = count
			IRModel = core
			data = urllib.request.urlopen(inurl).read()
			docs = json.loads(data.decode('utf-8'))['response']['docs']
			rank = 1
			for doc in docs:
				countRows += 1
				if qid == 10 or qid == 20:
					score = float((str(doc['score']) + str(countRows) + '5'))
					sortingList.append(score)
					writingDictionary.update({str('Q' + str(count) + str(score)): str(doc['id'])})
				else:
					score = float((str(doc['score']) + str(countRows)))
					sortingList.append(score)
					writingDictionary.update({str('Q' + str(count) + str(score)): str(doc['id'])})
		sortingList.sort(reverse=True)
		QueryDict.setdefault('Q' + str(count), sortingList)
		QueryList.append('Q' + str(count))
		sortingList = []
		count += 1
countRows = 1
count = 1
for query in QueryList:
	for key in QueryDict.get(query):
		print('Q' + str(count) + str(key))
		if countRows > 20:
			break
		else:
			if count > 9:
				outf.write('0' + str(count) + ' ' + 'Q' + str(count) + ' ' + writingDictionary.get('Q' + str(count) + str(key)) + ' ' + str(countRows) + ' ' + str(key)[:-1] + ' ' + core + '\n')
			else:
				outf.write('00' + str(count) + ' ' + 'Q' + str(count) + ' ' + writingDictionary.get('Q' + str(count) + str(key)) + ' ' + str(countRows) + ' ' + str(key)[:-1] + ' ' + core + '\n')
			countRows += 1
	count += 1
	countRows = 1
outf.close()


import os
import unittest
from decimal import Decimal
from datetime import datetime as dt
from datetime import timedelta
from functools import reduce

from singer import metadata
import tap_tester.connections as connections
import tap_tester.menagerie   as menagerie
import tap_tester.runner      as runner

from base import TestAdrollBase
from test_client import TestClient

class TestAdrollReportsFieldSelection(TestAdrollBase):
    """
    Test that with all fields selected for the 'ad_reports' stream  we replicate data as expected.
    Test that with only automatic fields selected for the 'ad_reports' stream  we replicate data as expected.
    """

    def name(self):
        return "tap_tester_adroll_reports_fields"

    def testable_streams(self):
        return self.expected_incremental_streams()

    @classmethod
    def setUpClass(cls):
        print("\n\nTEST SETUP\n")
        cls.client = TestClient()

    @classmethod
    def tearDownClass(cls):
        print("\n\nTEST TEARDOWN\n\n")

    def format_expected_data(self, records_by_stream):
        """Type the data to match expected tap output"""
        for stream, stream_records in records_by_stream.items():
            print("Altering records for {} ".format(stream))
            for record in stream_records:
                for key, value in record.items():
                    if type(value) == float:
                        record[key] = Decimal(str(value))

    def test_run(self):
        """
        Verify that for each stream you can get data when no fields are selected
        and only the automatic fields are replicated.
        """

        print("\n\nRUNNING {}\n\n".format(self.name()))

        # Initialize start_date state to capture ad_reports records
        self.START_DATE = self.timedelta_formatted(self.REPORTS_START_DATE, -1)
        self.END_DATE = self.REPORTS_END_DATE
        print("INCREMENTAL STREAMS RELY ON A STATIC DATA SET. SO WE TEST WITH:\n" +
              "  START DATE 1 | {}\n".format(self.START_DATE) +
              "  END DATE 2 | {}".format(self.END_DATE))

        # ensure data exists for sync streams and set expectations
        expected_records_all = {x: [] for x in self.expected_streams()} # all fields selected
        expected_records_auto = {x: [] for x in self.expected_streams()} # no fields selected
        for stream in self.testable_streams():
            start_date = self.parse_date(self.START_DATE)
            end_date = self.parse_date(self.END_DATE)
            existing_objects = self.client.get_all(stream, start_date, end_date)

            assert existing_objects, "Test data is not properly set for {}, test will fail.".format(stream)
            print("Data exists for stream: {}".format(stream))
            for obj in existing_objects:
                expected_records_all[stream].append(obj)
                expected_records_auto[stream].append(
                    {field: obj.get(field)
                     for field in self.expected_automatic_fields().get(stream)}
                )

        # format expected data to match expected output of tap
        self.format_expected_data(expected_records_all)

        # Instantiate connection with default start/end dates
        conn_id = connections.ensure_connection(self, original_properties=False)

        # run in check mode
        check_job_name = runner.run_check_mode(self, conn_id)

        # verify check exit codes
        exit_status = menagerie.get_exit_status(conn_id, check_job_name)
        menagerie.verify_check_exit_status(self, exit_status, check_job_name)

        found_catalogs = menagerie.get_catalogs(conn_id)
        self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id))

        found_catalog_names = set(map(lambda c: c['tap_stream_id'], found_catalogs))
        diff = self.expected_check_streams().symmetric_difference( found_catalog_names )
        self.assertEqual(len(diff), 0, msg="discovered schemas do not match: {}".format(diff))
        print("discovered schemas are OK")

        ##########################################################################
        ### ALL FIELDS SYNC
        ##########################################################################

        # Select all available fields from all streams
        exclude_streams = self.expected_streams().difference(self.testable_streams())
        self.select_all_streams_and_fields(
            conn_id=conn_id, catalogs=found_catalogs, select_all_fields=True, exclude_streams=exclude_streams
        )

        catalogs = menagerie.get_catalogs(conn_id)

        # Ensure our selection worked
        for cat in found_catalogs:
            catalog_entry = menagerie.get_annotated_schema(conn_id, cat['stream_id'])
            # Verify only testable streams are selected
            selected = catalog_entry.get('annotated-schema').get('selected')
            print("Validating selection on {}: {}".format(cat['stream_name'], selected))
            if not cat['stream_name'] in self.testable_streams():
                # None expected for {'inclusion':'available'} happens when menagerie "deselects" stream
                self.assertTrue(selected is None, msg="Stream is selected, but shouldn't be.")
                continue
            self.assertTrue(selected, msg="Stream not selected.")

            # Verify all fields within each selected stream are selected
            for field, field_props in catalog_entry.get('annotated-schema').get('properties').items():
                field_selected = field_props.get('selected')
                print("\tValidating selection on {}.{}: {}".format(cat['stream_name'], field, field_selected))
                self.assertTrue(field_selected, msg="Field not selected.")

        #clear state
        menagerie.set_state(conn_id, {})

        # run sync with all fields selected
        sync_job_name_all = runner.run_sync_mode(self, conn_id)

        # Verify tap exit codes
        exit_status_all = menagerie.get_exit_status(conn_id, sync_job_name_all)
        menagerie.verify_sync_exit_status(self, exit_status_all, sync_job_name_all)

        # read target output
        record_count_by_stream_all = runner.examine_target_output_file(self, conn_id,
                                                                       self.expected_streams(),
                                                                       self.expected_primary_keys())
        replicated_row_count_all =  reduce(lambda accum,c : accum + c, record_count_by_stream_all.values())
        synced_records_all = runner.get_records_from_target_output()

        # Verify target has records for all synced streams
        for stream, count in record_count_by_stream_all.items():
            assert stream in self.expected_streams()
            if stream in self.testable_streams():
                self.assertGreater(count, 0, msg="failed to replicate any data for: {}".format(stream))
        print("total replicated row count: {}".format(replicated_row_count_all))

        ##########################################################################
        ### AUTOMATIC FIELDS SYNC
        ##########################################################################

        # Instantiate connection with default start/end dates
        conn_id = connections.ensure_connection(self, original_properties=False)

        # run in check mode
        check_job_name = runner.run_check_mode(self, conn_id)

        # verify check exit codes
        exit_status = menagerie.get_exit_status(conn_id, check_job_name)
        menagerie.verify_check_exit_status(self, exit_status, check_job_name)

        found_catalogs = menagerie.get_catalogs(conn_id)
        self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id))

        found_catalog_names = set(map(lambda c: c['tap_stream_id'], found_catalogs))
        diff = self.expected_check_streams().symmetric_difference( found_catalog_names )
        self.assertEqual(len(diff), 0, msg="discovered schemas do not match: {}".format(diff))
        print("discovered schemas are OK")

        # Select no available fields (only automatic) for  all testable streams
        self.select_all_streams_and_fields(
            conn_id=conn_id, catalogs=found_catalogs, select_all_fields=False, exclude_streams=exclude_streams
        )

        catalogs = menagerie.get_catalogs(conn_id)

        # Ensure our selection worked
        for cat in found_catalogs:
            catalog_entry = menagerie.get_annotated_schema(conn_id, cat['stream_id'])
            # Verify all streams are selected
            selected = catalog_entry.get('annotated-schema').get('selected')
            print("Validating selection on {}: {}".format(cat['stream_name'], selected))
            if not cat['stream_name'] in self.testable_streams():
                self.assertTrue(selected is None, msg="Stream is selected, but shouldn't be.")
                continue
            self.assertTrue(selected, msg="Stream not selected.")

            # Verify only automatic fields are selected
            for field, field_props in catalog_entry.get('annotated-schema').get('properties').items():
                field_selected = field_props.get('selected')
                print("\tValidating selection on {}.{}: {}".format(cat['stream_name'], field, field_selected))

                if field in self.expected_automatic_fields().get(cat['stream_name']):
                    # NOTE: AUTOMATIC FIELDS IGNORE THE SELECTED md {'selected': None}
                    print("NOTE: selection for {} is ignored by the Transformer ".format(field) +
                          " so long as 'inlcusion' = 'automatic'")
                else:
                    self.assertFalse(field_selected, msg="Field is selected but not automatic.")

        # run sync with no fields selected (only automatic)
        sync_job_name_auto = runner.run_sync_mode(self, conn_id)

        # Verify tap exit codes
        exit_status_auto = menagerie.get_exit_status(conn_id, sync_job_name_auto)
        menagerie.verify_sync_exit_status(self, exit_status_auto, sync_job_name_auto)

        # read target output
        record_count_by_stream_auto = runner.examine_target_output_file(self, conn_id,
                                                                       self.expected_streams(),
                                                                       self.expected_primary_keys())
        replicated_row_count_auto =  reduce(lambda accum,c : accum + c, record_count_by_stream_auto.values())
        synced_records_auto = runner.get_records_from_target_output()

        # Verify target has records for all synced streams
        for stream, count in record_count_by_stream_auto.items():
            assert stream in self.expected_streams()
            if stream in self.testable_streams():
                self.assertGreater(count, 0, msg="failed to replicate any data for: {}".format(stream))
        print("total replicated row count: {}".format(replicated_row_count_auto))

        # Test by Stream
        for stream in self.testable_streams():
            with self.subTest(stream=stream):
                ##########################################################################
                ### TESTING ALL FIELDS
                ##########################################################################

                data = synced_records_all.get(stream)
                record_messages_keys = [set(row['data'].keys()) for row in data['messages']]
                expected_keys = expected_records_all.get(stream)[0].keys()

                # Verify schema covers all fields
                schema_keys = set(self.expected_schema_keys(stream))
                self.assertEqual(
                    set(), set(expected_keys).difference(schema_keys),
                    msg="\nFields missing from schema: {}\n".format(set(expected_keys).difference(schema_keys))
                )

                # not a test, just logging the fields that are included in the schema but not in the expectations
                if schema_keys.difference(set(expected_keys)):
                    print("WARNING Fields missing from expectations: {}".format(schema_keys.difference(set(expected_keys))))

                # Verify that all fields are sent to the target
                for actual_keys in record_messages_keys:
                    self.assertEqual(
                        actual_keys.symmetric_difference(schema_keys), set(),
                        msg="Expected all fields, as defined by schemas/{}.json".format(stream)
                    )

                actual_records = [row['data'] for row in data['messages']]
                expected_records = expected_records_all.get(stream)

                # Verify the number of records match expectations
                self.assertEqual(len(expected_records),
                                 len(actual_records),
                                 msg="Number of actual records do match expectations. " +\
                                 "Check expectations, check for duplicate records in Target.")

                # verify there are no dup records in the target
                already_tracked = []
                for actual_record in actual_records:
                    if actual_record in already_tracked:
                        continue
                    already_tracked.append(actual_record)
                self.assertEqual(len(already_tracked), len(actual_records), msg="DUPLICATES PRESENT")

                # verify by values, that we replicated the expected records
                for actual_record in actual_records:
                    self.assertTrue(actual_record in expected_records,
                                    msg="Actual record missing from expectations\n" +
                                    "Actual Record: {}".format(actual_record))
                for expected_record in expected_records:
                    self.assertTrue(expected_record in actual_records,
                                    msg="Expected record missing from target." +
                                    "Expected Record: {}".format(expected_record))

                ##########################################################################
                ### TESTING AUTOMATIC FIELDS
                ##########################################################################

                data = synced_records_auto.get(stream)
                record_messages_keys = [set(row['data'].keys()) for row in data['messages']]
                expected_keys = self.expected_automatic_fields().get(stream)

                # Verify that only the automatic fields are sent to the target
                for actual_keys in record_messages_keys:
                    self.assertEqual(
                        actual_keys.symmetric_difference(expected_keys), set(),
                        msg="Expected automatic fields and nothing else.")

                actual_records = [row['data'] for row in data['messages']]
                expected_records = expected_records_auto.get(stream)

                #Verify the number of records match expectations
                self.assertEqual(len(expected_records),
                                 len(actual_records),
                                 msg="Number of actual records do match expectations. " +\
                                 "We probably have duplicate records.")

                # verify there are no dup records in the target
                already_tracked = []
                for actual_record in actual_records:
                    if actual_record in already_tracked:
                        continue
                    already_tracked.append(actual_record)
                self.assertEqual(len(already_tracked), len(actual_records), msg="DUPLICATES PRESENT")

                # verify by values, that we replicated the expected records
                for actual_record in actual_records:
                    self.assertTrue(actual_record in expected_records,
                                    msg="Actual record missing from expectations\n" +
                                    "Actual Record: {}".format(actual_record))
                for expected_record in expected_records:
                    self.assertTrue(expected_record in actual_records,
                                    msg="Expected record missing from target." +
                                    "Expected Record: {}".format(expected_record))

if __name__ == '__main__':
    unittest.main()


from typing import Collection

from opentelemetry.instrumentation.aiopg import wrappers
from opentelemetry.instrumentation.aiopg.package import _instruments
from opentelemetry.instrumentation.aiopg.version import __version__
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor

class AiopgInstrumentor(BaseInstrumentor):
    _CONNECTION_ATTRIBUTES = {
        "database": "info.dbname",
        "port": "info.port",
        "host": "info.host",
        "user": "info.user",
    }

    _DATABASE_SYSTEM = "postgresql"

    def instrumentation_dependencies(self) -> Collection[str]:
        return _instruments

    def _instrument(self, **kwargs):
        """Integrate with PostgreSQL aiopg library.
        aiopg: https://github.com/aio-libs/aiopg
        """

        tracer_provider = kwargs.get("tracer_provider")

        wrappers.wrap_connect(
            __name__,
            self._DATABASE_SYSTEM,
            self._CONNECTION_ATTRIBUTES,
            version=__version__,
            tracer_provider=tracer_provider,
        )

        wrappers.wrap_create_pool(
            __name__,
            self._DATABASE_SYSTEM,
            self._CONNECTION_ATTRIBUTES,
            version=__version__,
            tracer_provider=tracer_provider,
        )

    # pylint:disable=no-self-use
    def _uninstrument(self, **kwargs):
        """ "Disable aiopg instrumentation"""
        wrappers.unwrap_connect()
        wrappers.unwrap_create_pool()

    # pylint:disable=no-self-use
    def instrument_connection(self, connection, tracer_provider=None):
        """Enable instrumentation in a aiopg connection.

        Args:
            connection: The connection to instrument.
            tracer_provider: The optional tracer provider to use. If omitted
                the current globally configured one is used.

        Returns:
            An instrumented connection.
        """
        return wrappers.instrument_connection(
            __name__,
            connection,
            self._DATABASE_SYSTEM,
            self._CONNECTION_ATTRIBUTES,
            version=__version__,
            tracer_provider=tracer_provider,
        )

    def uninstrument_connection(self, connection):
        """Disable instrumentation in a aiopg connection.

        Args:
            connection: The connection to uninstrument.

        Returns:
            An uninstrumented connection.
        """
        return wrappers.uninstrument_connection(connection)


from django import forms
from blogapp.models import ContactModel
from django.core.mail import send_mail

class ContactForm(forms.ModelForm):
    class Meta:
        model = ContactModel
        fields = ('full_name','email', 'message')

    def send_email(self, message):
        send_mail(
            subject='İletişim Formundan Yeni Mesaj Var!',
            message=message,
            from_email=None,
            recipient_list=['backendwithdjango@gmail.com'],
            fail_silently=False
        )


import click
import matplotlib.pyplot as plt
from matplotlib import rc
import numpy as np
import pandas as pd

rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})
# rc('text', usetex=True)

fsl = fs = 9

@click.command()
@click.argument('csv-path', type=click.File())
@click.option('--outputpath', '-o', type=click.Path(), default="diff-selectivity.png")
@click.option('--plotnum', '-n', type=int, default=1)
def d_selectivity(csv_path, outputpath="diff-selectivity.png", plotnum=0):
    fig = plt.figure(figsize=(7., 7.))

    data = pd.read_csv(csv_path)
    data["log_CO2_H2O"] = np.log10(data["CO2 / H2O"])
    data["log_CO2_N2"] = np.log10(data["CO2 / N2"])
    uio66 = data[data.mof.str.startswith("UIO-66")]
    uio67 = data[data.mof.str.startswith("UIO-67")]
    moflabels = list(data.mof.str.replace("UIO-6[67] ", ""))

    ax = fig.subplots(ncols=1)
    sc1 = ax.scatter(uio66.log_CO2_H2O, uio66.log_CO2_N2, zorder=2, s=10, marker=">", label="UIO-66", c="sienna")
    sc2 = ax.scatter(uio67.log_CO2_H2O, uio67.log_CO2_N2, zorder=2, s=20, marker= r'$\bowtie$', label="UIO-67", c="mediumseagreen")
    ax.grid(which='major', axis='both', linestyle='-', color='0.9', zorder=10)
    ax.set_xlabel('log$_{10}$ CO$_2$/H$_2$O diffusion selectivity', fontsize=fsl)
    ax.set_ylabel('log$_{10}$ CO$_2$/N$_2$ diffusion selectivity', fontsize=fsl)

    if plotnum == 0:
        ax.set_xlim(-0.8, 3.3)
        ax.set_ylim(-1.6, .1)
        ax.set_yticks([-1., 0.])
        ax.set_xticks([0., 1., 2., 3.])

        ax.axline([0,0], slope=-1, ls="--", color=".8", zorder=1)
        ax.annotate("0", (-.1,.1), size=5, xytext=(5,-2), textcoords='offset points',
            horizontalalignment="left", verticalalignment='top', fontsize=fsl, c=".5")
        ax.axline([1,0], slope=-1, ls="--", color=".8", zorder=1)
        ax.annotate("+1", (0.9,.1), size=5, xytext=(5,-2), textcoords='offset points',
            horizontalalignment="left", verticalalignment='top', fontsize=fsl, c=".5")
        ax.axline([2,0], slope=-1, ls="--", color=".8", zorder=1)
        ax.annotate("+2", (1.9,.1), size=5, xytext=(5,-2), textcoords='offset points',
            horizontalalignment="left", verticalalignment='top', fontsize=fsl, c=".5")

        for i, mof in enumerate(data.mof):
            x, y = (data.log_CO2_H2O[i], data.log_CO2_N2[i])
            if mof in ["UIO-67"]:
                ax.annotate(moflabels[i], (x, y), size=5, textcoords='offset points', xytext=(x-4.,y - 0), horizontalalignment='right', verticalalignment='center', fontsize=fs)
            elif mof in ["UIO-66", "UIO-66 2x OH", "UIO-66 OH", "UIO-66 4x F", "UIO-67 2x N$_3$", "UIO-67 2x alkane-OC$_3$"]:
                ax.annotate(moflabels[i], (x, y), size=5, textcoords='offset points', xytext=(x+4,y + 0), horizontalalignment='left', verticalalignment='center', fontsize=fsl)

    elif plotnum == 1:
        ax.set_xlim(1.1, 3.3)
        ax.set_ylim(-0.6, .1)
        ax.set_yticks([-0.5, 0.])
        ax.set_xticks([1.5, 2., 2.5, 3.])

        ax.axline([1,0], slope=-1, ls="--", color=".8", zorder=1)
        ax.annotate("+1", (1.1,-0.1), size=5, xytext=(5,-2), textcoords='offset points',
            horizontalalignment="left", verticalalignment='top', fontsize=fsl, c=".5")
        ax.axline([2,0], slope=-1, ls="--", color=".8", zorder=1)
        ax.annotate("+2", (1.9,.1), size=5, xytext=(5,-2), textcoords='offset points',
            horizontalalignment="left", verticalalignment='top', fontsize=fsl, c=".5")

        for i, mof in enumerate(data.mof):
            x, y = (data.log_CO2_H2O[i], data.log_CO2_N2[i])
            if (1.1 < x < 3.3) and (-0.6 < y < 0.1):
                if mof in ["UIO-67", "UIO-67 2x alkane-OC$_4$", "UIO-67 branched-HNC$_5$"]:
                    ax.annotate(moflabels[i] + " - %3.1f" % (float(data.CO2[i]) * 1e4), (x, y), size=5, xytext=(x-0.03,y - 0.), horizontalalignment='right', verticalalignment='center', fontsize=fsl)
                else:
                    ax.annotate(moflabels[i] + " - %3.1f" % (float(data.CO2[i]) * 1e4), (x, y), size=5, xytext=(x+0.03,y - 0.), horizontalalignment='left', verticalalignment='center', fontsize=fsl)

    ax.legend()
    # fig.subplots_adjust(wspace=0.05, hspace=0.05)
    fig.savefig(outputpath, dpi=300, bbox_inches='tight') # , transparent=True
    plt.close(fig)

if __name__ == '__main__':
    d_selectivity()