import unittest
from server.logic.database.edge_database import EdgeDatabase
from pymongo import MongoClient
from server.config import DATABASE_CONNECTION
from server.config import DATABASE_SCHEME_EDGE_RATING

class DatabaseTestCase(unittest.TestCase):
    def setUp(self):
        """
        Set up database class and instruct to use test database.
        """
        self.database = EdgeDatabase()
        self.database.__database_name__ = "servertest"

    def tearDown(self):
        """
        Wipe test database.
        """
        client = MongoClient(DATABASE_CONNECTION['host'], DATABASE_CONNECTION['port'])
        db = client[self.database.__database_name__]
        db.authenticate(DATABASE_CONNECTION['user'], DATABASE_CONNECTION['password'])
        collection = db[DATABASE_CONNECTION['collection_rating']]
        collection.drop()
        client.close()

    def test_save_new_and_add_rating(self):
        """
        Tests all functionality of save_new() and add_rating().
        """
        edge_from = 4
        edge_to = 5
        rating = 2

        # test save_new on clean database
        self.database.save_new(edge_from, edge_to)
        self.assertTrue(self.check_edge_in_database(edge_from, edge_to, 0, 0))

        # test add_rating on 1 item database
        self.database.add_rating(edge_from, edge_to, rating)
        self.assertTrue(self.check_edge_in_database(edge_from, edge_to, rating, 1))

        # test save_new doesn't overwrite
        self.database.save_new(edge_from, edge_to)
        self.assertTrue(self.check_edge_in_database(edge_from, edge_to, rating, 1))

        edge_from_v2 = 8
        edge_to_v2 = 6

        # test save_new on non-empty database
        self.database.save_new(edge_from_v2, edge_to_v2)
        self.assertTrue(self.check_edge_in_database(edge_from, edge_to, rating, 1))
        self.assertTrue(self.check_edge_in_database(edge_from_v2, edge_to_v2, 0, 0))

        edge_from_v3 = 6
        edge_to_v3 = 6
        rating_v3 = 45

        # test add_rating on multiple item database
        self.database.add_rating(edge_from_v3, edge_to_v3, rating_v3)
        self.assertTrue(self.check_edge_in_database(edge_from, edge_to, rating, 1))
        self.assertTrue(self.check_edge_in_database(edge_from_v2, edge_to_v2, 0, 0))
        self.assertTrue(self.check_edge_in_database(edge_from_v3, edge_to_v3, rating_v3, 1))

    def test_load_methods(self):
        """
        Tests all functionality for get_average_rating(), get_amount_voted(), get_edge().
        Assumes the save methods are working correctly.
        """
        edge_from_v1 = 4
        edge_to_v1 = 8
        rating_v1 = 6
        self.database.save_new(edge_from_v1, edge_to_v1)

        # test for zero edge
        self.assertEqual(0.0, self.database.get_average_rating(edge_from_v1, edge_to_v1))
        self.assertEqual(0, self.database.get_amount_voted(edge_from_v1, edge_to_v1))
        self.assertTupleEqual((0, 0, 0), self.database.get_edge(edge_from_v1, edge_to_v1))

        # test for non existing edge
        self.assertEqual(0.0, self.database.get_average_rating(987, 1564))
        self.assertEqual(0, self.database.get_amount_voted(987, 1564))
        self.assertIsNone(self.database.get_edge(987, 1564))

        self.database.add_rating(edge_from_v1, edge_to_v1, rating_v1)

        # test for 1 added rating
        self.assertEqual(rating_v1, self.database.get_average_rating(edge_from_v1, edge_to_v1))
        self.assertEqual(1, self.database.get_amount_voted(edge_from_v1, edge_to_v1))
        self.assertTupleEqual((rating_v1, 1, rating_v1), self.database.get_edge(edge_from_v1, edge_to_v1))

    def test_bulk_methods(self):
        """
        Tests all functionality for save_new_list(), add_rating_list(), get_all_edges().
        """
        edge_from_v1 = 6
        edge_to_v1 = 7
        rating_v1 = 9

        edge_from_v2 = 9
        edge_to_v2 = 4
        rating_v2 = 5

        list_v1 = []
        list_v1.append((edge_from_v1, edge_to_v1, 0, 0))
        list_v1.append((edge_from_v2, edge_to_v2, 0, 0))

        self.database.save_new_list(list_v1)

        # check both edges in database
        self.assertTrue(self.check_edge_in_database(edge_from_v1, edge_to_v1, 0, 0))
        self.assertTrue(self.check_edge_in_database(edge_from_v2, edge_to_v2, 0, 0))

        list_v2 = []
        list_v2.append((edge_from_v1, edge_to_v1, rating_v1))
        list_v2.append((edge_from_v2, edge_to_v2, rating_v2))

        self.database.add_rating_list(list_v2)

        # check both edges in database
        self.assertTrue(self.check_edge_in_database(edge_from_v1, edge_to_v1, rating_v1, 1))
        self.assertTrue(self.check_edge_in_database(edge_from_v2, edge_to_v2, rating_v2, 1))

        all_edges = self.database.get_all_edges()

        # check tuples returned
        self.assertTupleEqual((rating_v1, 1, rating_v1), all_edges[edge_from_v1, edge_to_v1])
        self.assertTupleEqual((rating_v2, 1, rating_v2), all_edges[edge_from_v2, edge_to_v2])

    def check_edge_in_database(self, edge_from, edge_to, total_rating, amount_voted):
        """
        Utility method for direct checking if an edge is in the database.
        :return True if an edge with the given parameters is in the database, False otherwise
        """
        client = MongoClient(DATABASE_CONNECTION['host'], DATABASE_CONNECTION['port'])
        db = client[self.database.__database_name__]
        db.authenticate(DATABASE_CONNECTION['user'], DATABASE_CONNECTION['password'])
        collection = db[DATABASE_CONNECTION['collection_rating']]

        query = {DATABASE_SCHEME_EDGE_RATING['edge_from']: edge_from,
                 DATABASE_SCHEME_EDGE_RATING['edge_to']: edge_to,
                 DATABASE_SCHEME_EDGE_RATING['total_rating']: total_rating,
                 DATABASE_SCHEME_EDGE_RATING['amount_voted']: amount_voted}

        result = collection.find_one(query)
        client.close()

        if result is None:
            # nothing is found
            return False
        else:
            return True


import streamlit as st
import altair as alt
import pandas as pd

life = pd.read_csv("https://raw.githubusercontent.com/rezpe/datos_viz/master/lifecountries.csv")

chart_life = alt.Chart(life).mark_circle().encode(
    x='Country GDP',
    y='Life Expectancy',
    size='size',
    color='Continent',
    tooltip='country'
).interactive()

st.title("Esperanza de vida según el pais")

st.markdown("Hemos registrado la espearnza de vida y la renta per capita en varios paises, y hemos hecho un gráfico")

st.write(chart_life)

18. Tableau & Advanced Visualizatios/hello.py


#!/usr/bin/env python
# coding: utf-8

from flask import request, jsonify, Blueprint
import numpy as np

from lib.function import get_data, cos_sim
from lib.nlp_function import text_to_vector

api = Blueprint("api", __name__)

#取得文章をベクトルに変換
data_path = "data/text.txt"
text_list = get_data(data_path)
text_vector_list = text_to_vector(text_list)

@api.route("/get_texts", methods=["GET"])
def get_texts():
    #類似度の閾値
    thred_value = 0.8

    text = request.args.get('value', '')

    #文章をベクトル化
    text_vector = text_to_vector([text])[0]

    cos_sim_list = []
    for i, t_v in enumerate(text_vector_list):
        #類似度取得
        sim_value = cos_sim(t_v, text_vector)
        #類似度が閾値以上の文章と類似度をリストに格納
        if sim_value >= thred_value:
            cos_sim_list.append({"text":text_list[i], "value":str(sim_value)})

    #類似度が降順に並び替え
    cos_sim_list = sorted(cos_sim_list, key=lambda x: x["value"], reverse=True)

    return jsonify(cos_sim_list)


"""fireroad URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.contrib.admin.views.decorators import staff_member_required
from django.views.generic.base import RedirectView
from django.contrib import admin
from common.views import dev_login
from django.conf import settings

# admin.autodiscover()
# admin.site.login = staff_member_required(admin.site.login, login_url=settings.LOGIN_URL)

urlpatterns = [
    url(r'courses/', include('catalog.urls')),
    url(r'courseupdater/', include('courseupdater.urls')),
    url(r'recommend/', include('recommend.urls')),
    url(r'admin/', admin.site.urls),
    url(r'sync/', include('sync.urls')),
    url(r'analytics/', include('analytics.urls')),
    url(r'requirements/', include('requirements.urls')),
    url(r'', include('common.urls')),
]

# Redirect to the appropriate login page if one is specified in the settings module
if settings.LOGIN_URL:
    if settings.LOGIN_URL.strip("/") != 'dev_login':
        urlpatterns.insert(0, url(r'^admin/login/$', RedirectView.as_view(url=settings.LOGIN_URL,
                                                                          permanent=True,
                                                                          query_string=True)))
        urlpatterns.insert(0, url(r'^dev_login/$', RedirectView.as_view(url=settings.LOGIN_URL,
                                                                          permanent=True,
                                                                          query_string=True)))
    if settings.LOGIN_URL.strip("/") != 'login':
        urlpatterns.insert(0, url(r'^login/$', RedirectView.as_view(url=settings.LOGIN_URL,
                                                                    permanent=True,
                                                                    query_string=True)))


from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING, List, Optional, Union
from glob import glob

from opentrons.config import IS_ROBOT, IS_LINUX
from opentrons.drivers.rpi_drivers import types, interfaces, usb, usb_simulator
from opentrons.hardware_control.emulation.module_server.helpers import (
    listen_module_connection,
)
from opentrons.hardware_control.modules.module_calibration import (
    ModuleCalibrationOffset,
    load_module_calibration_offset,
    save_module_calibration_offset,
)
from opentrons.hardware_control.modules.types import ModuleType
from opentrons.types import Point
from .types import AionotifyEvent, BoardRevision, OT3Mount
from . import modules

if TYPE_CHECKING:
    from .api import API
    from .ot3api import OT3API

log = logging.getLogger(__name__)

MODULE_PORT_REGEX = re.compile("|".join(modules.MODULE_TYPE_BY_NAME.keys()), re.I)

class AttachedModulesControl:
    """
    A class to handle monitoring module attachment, capturing the physical
    USB port information and finally building a module object.
    """

    def __init__(
        self,
        api: Union["API", "OT3API"],
        usb: interfaces.USBDriverInterface,
    ) -> None:
        self._available_modules: List[modules.AbstractModule] = []
        self._api = api
        self._usb = usb

    @classmethod
    async def build(
        cls,
        api_instance: Union["API", "OT3API"],
        board_revision: BoardRevision,
    ) -> AttachedModulesControl:
        usb_instance = (
            usb.USBBus(board_revision)
            if not api_instance.is_simulator and IS_ROBOT
            else usb_simulator.USBBusSimulator()
        )
        mc_instance = cls(api=api_instance, usb=usb_instance)

        if not api_instance.is_simulator:
            # Do an initial scan of modules.
            await mc_instance.register_modules(mc_instance.scan())
            if not IS_ROBOT:
                # Start task that registers emulated modules.
                api_instance.loop.create_task(
                    listen_module_connection(mc_instance.register_modules)
                )

        return mc_instance

    @property
    def available_modules(self) -> List[modules.AbstractModule]:
        return self._available_modules

    async def build_module(
        self,
        port: str,
        usb_port: types.USBPort,
        type: modules.ModuleType,
        sim_model: Optional[str] = None,
    ) -> modules.AbstractModule:
        return await modules.build(
            port=port,
            usb_port=usb_port,
            type=type,
            simulating=self._api.is_simulator,
            hw_control_loop=self._api.loop,
            execution_manager=self._api._execution_manager,
            sim_model=sim_model,
        )

    async def unregister_modules(
        self, mods_at_ports: List[modules.ModuleAtPort]
    ) -> None:
        """
        De-register Modules.

        Remove any modules that are no longer found by aionotify.
        """
        removed_modules = []
        for mod in mods_at_ports:
            for attached_mod in self.available_modules:
                if attached_mod.port == mod.port:
                    removed_modules.append(attached_mod)
        for removed_mod in removed_modules:
            try:
                self._available_modules.remove(removed_mod)
            except ValueError:
                log.exception(
                    f"Removed Module {removed_mod} not found in attached modules"
                )
        for removed_mod in removed_modules:
            log.info(
                f"Module {removed_mod.name()} detached from port {removed_mod.port}"
            )
            await removed_mod.cleanup()
        self._available_modules = sorted(
            self._available_modules, key=modules.AbstractModule.sort_key
        )

    async def register_modules(
        self,
        new_mods_at_ports: Optional[List[modules.ModuleAtPort]] = None,
        removed_mods_at_ports: Optional[List[modules.ModuleAtPort]] = None,
    ) -> None:
        """
        Register Modules.

        Upon system recognition of a module being plugged in, we should
        register that module and de-register any modules that are
        no longer found on the system.
        """
        if new_mods_at_ports is None:
            new_mods_at_ports = []
        if removed_mods_at_ports is None:
            removed_mods_at_ports = []

        # destroy removed mods
        await self.unregister_modules(removed_mods_at_ports)
        self._new_mods_at_ports = new_mods_at_ports
        unsorted_mods_at_port = self._usb.match_virtual_ports(new_mods_at_ports)

        # build new mods
        for mod in unsorted_mods_at_port:
            new_instance = await self.build_module(
                port=mod.port,
                usb_port=mod.usb_port,
                type=modules.MODULE_TYPE_BY_NAME[mod.name],
            )
            self._available_modules.append(new_instance)
            log.info(
                f"Module {mod.name} discovered and attached"
                f" at port {mod.port}, new_instance: {new_instance}"
            )
        self._available_modules = sorted(
            self._available_modules, key=modules.AbstractModule.sort_key
        )

    def scan(self) -> List[modules.ModuleAtPort]:
        """Scan for connected modules and return list of
        tuples of serial ports and device names
        """
        if IS_ROBOT and IS_LINUX:
            devices = glob("/dev/ot_module*")
        else:
            devices = []

        discovered_modules = []

        for port in devices:
            symlink_port = port.split("dev/")[1]
            module_at_port = self.get_module_at_port(symlink_port)
            if module_at_port:
                discovered_modules.append(module_at_port)

        log.debug("Discovered modules: {}".format(discovered_modules))
        return discovered_modules

    @staticmethod
    def get_module_at_port(port: str) -> Optional[modules.ModuleAtPort]:
        """Given a port, returns either a ModuleAtPort
        if it is a recognized module, or None if not recognized.
        """
        match = MODULE_PORT_REGEX.search(port)
        if match:
            name = match.group().lower()
            if name not in modules.MODULE_TYPE_BY_NAME:
                log.warning(f"Unexpected module connected: {name} on {port}")
                return None
            return modules.ModuleAtPort(port=f"/dev/{port}", name=name)
        return None

    async def handle_module_appearance(self, event: AionotifyEvent) -> None:
        """Only called upon availability of aionotify. Check that
        the file system has changed and either remove or add modules
        depending on the result.

        Args:
            event: The event passed from aionotify.

        Returns:
            None
        """
        maybe_module_at_port = self.get_module_at_port(event.name)
        new_modules = None
        removed_modules = None
        if maybe_module_at_port is not None:
            if hasattr(event.flags, "DELETE"):
                removed_modules = [maybe_module_at_port]
                log.info(f"Module Removed: {maybe_module_at_port}")
            elif hasattr(event.flags, "CREATE"):
                new_modules = [maybe_module_at_port]
                log.info(f"Module Added: {maybe_module_at_port}")
            try:
                await self.register_modules(
                    removed_mods_at_ports=removed_modules,
                    new_mods_at_ports=new_modules,
                )
            except Exception:
                log.exception("Exception in Module registration")

    def get_module_by_module_id(
        self, module_id: str
    ) -> Optional[modules.AbstractModule]:
        """Returns the module with the matching serial id."""
        found_module: Optional[modules.AbstractModule] = None
        for module in self.available_modules:
            if module.device_info["serial"] == module_id:
                found_module = module
                break
        return found_module

    def load_module_offset(
        self, module_type: ModuleType, module_id: str
    ) -> Optional[ModuleCalibrationOffset]:
        log.info(f"Loading module offset for {module_type} {module_id}")
        return load_module_calibration_offset(module_type, module_id)

    def save_module_offset(
        self,
        module: ModuleType,
        module_id: str,
        mount: OT3Mount,
        slot: str,
        offset: Point,
        instrument_id: Optional[str] = None,
    ) -> Optional[ModuleCalibrationOffset]:
        log.info(f"Saving module {module} {module_id} offset: {offset} for slot {slot}")
        save_module_calibration_offset(
            offset, mount, slot, module, module_id, instrument_id
        )
        return load_module_calibration_offset(module, module_id)


import lib.img_drawLabel as draw
import lib.img_segment as seg
import lib.web_catchElementInfo as catch
import lib.web_crawl as crawl
import lib.file_utils as file

import pandas as pd
import os
import time
from func_timeout import func_set_timeout, FunctionTimedOut

is_crawl_link = False
is_read_existed_links = not is_crawl_link
is_catch_element = True
is_segment = False
is_draw_label = False

data_position = 'E:\Mulong\Datasets\dataset_webpage\page10000'
img_root = os.path.join(data_position, 'org')
label_root = os.path.join(data_position, 'label')
drawn_root = os.path.join(data_position, 'drawn')
segment_root = os.path.join(data_position, 'segment')
driver_path = 'D:\webdriver'

if is_crawl_link:
    # set the web crawler
    initial_url = "https://world.taobao.com/"
    link_num = 1
    # start crawling
    links = crawl.crawl(initial_url, link_num, 5)
    crawl.save_links(links, os.path.join(data_position, 'links.csv'))
    # read links
    csv = pd.read_csv(os.path.join(data_position, 'links.csv'))
    links = csv.link

if is_read_existed_links:

    # read links
    csv = pd.read_csv(os.path.join(data_position, 'link_10000.csv'))
    links = csv.link

print("*** %d Links Fetched ***\n" % len(links))

start_pos = 14061
end_pos = 15000
for index in range(start_pos, len(links)):
    start_time = time.clock()
    # set path
    img_org_path = os.path.join(img_root, str(index) + '.png')
    img_drawn_path = os.path.join(drawn_root, str(index) + '.png')
    img_segment_path = os.path.join(segment_root, str(index))
    label_path = os.path.join(label_root, str(index) + '.csv')

    # catch label and screenshot img and segment them into smaller size
    img, label = None, None
    catch_success = False
    if is_catch_element and '.com' in links.iloc[index]:
        # set the format of libel
        libel_format = pd.read_csv(os.path.join(data_position, 'format.csv'), index_col=0)
        url = 'http://' + links.iloc[index] if 'http://' not in links.iloc[index] else links.iloc[index]
        try:
            img, label = catch.catch(url, label_path, img_org_path, libel_format, driver_path)
        except FunctionTimedOut:
            print('Catch Time Out')
            continue

    # segment the lengthy images
    if is_segment and img is not None:
        seg.segment_img(img, 600, img_segment_path, 0)

    # read and draw label on segment img
    if is_draw_label and img is not None and label is not None:
        draw.label(label, img, img_drawn_path)

    end_time = time.clock()
    print("*** %d Time taken:%ds ***\n" % (index, int(end_time - start_time)))

    if index > end_pos:
        break


from plone.app.portlets.portlets import base
from plone.portlets.interfaces import IPortletDataProvider
from Products.CMFCore.utils import getToolByName
from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile
from zope.i18n import translate
from zope.interface import implements

from intranett.policy import IntranettMessageFactory as _
from intranett.policy.utils import get_user_profile_url

class IProjectRoomInfo(IPortletDataProvider):
    pass

class Assignment(base.Assignment):

    implements(IProjectRoomInfo)
    title = _("Project Info")

class Renderer(base.Renderer):

    render = ViewPageTemplateFile('projectroominfo.pt')
    portletTitle = _("Project Info")

    @property
    def available(self):
        return (getattr(self.context, 'getProjectRoom', None) is not None and
                'portal_factory' not in self.request.URL)

    def update(self):
        if not self.available:
            return
        context = self.context
        mtool = getToolByName(context, 'portal_membership')
        gtool = getToolByName(context, 'portal_groups')
        room = self.context.getProjectRoom()
        self.state = room.getProjectRoomState()
        self.title = room.Title()
        self.project_url = room.absolute_url()
        result = []
        for name in room.participants:
            title = name
            url = None
            if name == 'AuthenticatedUsers':
                title = translate(u'Authenticated Users (Virtual Group)',
                    domain='plone', context=self.request)
                result.append(dict(name=name, title=title, url=None))
                continue
            member = mtool.getMemberById(name)
            if member is not None:
                title = member.getProperty("fullname") or name
                url = get_user_profile_url(context, name)
            else:
                group = gtool.getGroupById(name)
                if group is not None:
                    title = group.getProperty('title') or name
            result.append(dict(name=name, title=title, url=url))
        self.participants = result

class AddForm(base.NullAddForm):

    def create(self):
        return Assignment()


#!/usr/bin/env python2

import argparse
import boto3
import datetime
import urllib
import math
import os
import logging
import gzip
import datetime
import calendar
import logging
import StringIO
import multiprocessing
import Queue
import random
import functools

logger = logging.getLogger('make_speeds')
logger.setLevel(logging.INFO)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(fmt='%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(handler)

# import make_speeds after logger has been setup
import make_speeds

valhalla_tiles = [{'level': 2, 'size': 0.25}, {'level': 1, 'size': 1.0}, {'level': 0, 'size': 4.0}]

def url_suffix(tile_level, tile_index):
  tile_set = filter(lambda x: x['level'] == tile_level, valhalla_tiles)[0]
  max_index = int(360.0/tile_set['size'] * 180.0/tile_set['size']) - 1
  num_dirs = int(math.ceil(len(str(max_index)) / 3.0))
  suffix = list(str(tile_index).zfill(3 * num_dirs))
  for i in range(0, num_dirs):
    suffix.insert(i * 3 + i, '/')
  suffix = '/' + str(tile_level) + ''.join(suffix)
  return suffix

def split(l, n):
  size = int(math.ceil(len(l)/float(n)))
  cutoff = len(l) % n
  result = []
  pos = 0
  for i in range(0, n):
    end = pos + size if cutoff == 0 or i < cutoff else pos + size - 1
    result.append(l[pos:end])
    pos = end
  return result

def interrupt_wrapper(func):
  try:
    func()
  except (KeyboardInterrupt, SystemExit):
    logger.error('Interrupted or killed')

def upload(speed_bucket, level, index, week, speed_tiles):
  logger.info('Uploading data to bucket: ' + speed_bucket)
  client = boto3.client('s3')
  prefix = week + '/'.join(url_suffix(int(level), int(index)).split('/')[:-1])
  for tile in speed_tiles:
    key = prefix + '/' + tile.split('/')[-1] + '.gz'
    logger.info('Uploading ' + tile + ' as ' + key)
    zipped = StringIO.StringIO()
    with open(tile, 'rb') as f_in, gzip.GzipFile(mode='wb',fileobj=zipped) as f_out:
      f_out.write(f_in.read())
    client.put_object(
      ACL='public-read',
      Bucket=speed_bucket,
      ContentType='application/octet-stream',
      ContentEncoding='gzip',
      Body=zipped.getvalue(),
      Key=key)

def load(histograms, sub_segments, info, lengths, loaded):
  segments = {}
  while True:
    try:
      file_name = histograms.get()
      #finished if we get the sentinel
      if not file_name:
        break
      loaded.put(file_name)
      make_speeds.addSegments(file_name, info, lengths, segments)
    except (KeyboardInterrupt, SystemExit) as e:
      raise e
    except Exception as e:
      logger.error('Failed to load segments from %s' % file_name)

  #send back the info
  sub_segments.put(segments)

def convert(level, index, week, osmlr_version, histograms, concurrency):
  url = 'http://s3.amazonaws.com/osmlr-tiles/' + osmlr_version + '/pbf' + url_suffix(level, index) + '.osmlr'
  logger.info('Fetching osmlr tile: ' + url)
  osmlr = str(level) + '_' + str(index) + '.osmlr'
  urllib.URLopener().retrieve(url, osmlr)

  logger.info('Getting segment lengths from osmlr')
  lengths = make_speeds.getLengths(osmlr)

  date = datetime.datetime.strptime(week + '/1','%Y/%W/%w')
  start = calendar.timegm(date.timetuple())
  info = {'rangeStart': start, 'rangeEnd': start + 604800, 'unitSize': 604800, 'entrySize': 3600,
    'description': 'Hourly speeds for the week starting on ' + str(date), 'level': level, 'index': index}

  logger.info('Accumulating segment speeds from histograms')
  #we share the segments dict in a process safe way
  sub_segments = multiprocessing.Queue()
  loaded = multiprocessing.Queue()
  processes = []
  #start them up and wait for them to get all the results back
  for i in xrange(concurrency):
    histograms.put(False)
  for i in xrange(concurrency):
    bound = functools.partial(load, histograms, sub_segments, info, lengths, loaded)
    processes.append(multiprocessing.Process(target=interrupt_wrapper, args=(bound,)))
    processes[-1].start()

  #then harvest all the sub_segments
  segments = {}
  for i in xrange(concurrency):
    #get out the segments and merge them all together
    sub = sub_segments.get()
    while len(sub):
      seg_id = sub.iterkeys().next()
      hours = sub.pop(seg_id)
      #since each histogram is a specific hour no two dicts will have the same hour
      segments.setdefault(seg_id, {}).update(hours)
    logger.info('Merged result segments from sub process %d' % i)

  #then make some tiles
  logger.info('Creating speed tiles')
  prefix = url_suffix(int(level), int(index)).split('/')[-1]
  return make_speeds.createSpeedTiles(lengths, prefix + '.spd', 10000, prefix + '.nex', True, segments, info), loaded, osmlr

def fetch(histogram_bucket, keys, results):
  session = boto3.session.Session()
  resource = session.resource('s3')
  for key in keys:
    try:
      dest = key.replace('/', '_')
      resource.Object(histogram_bucket, key).download_file(dest)
      if os.path.isfile(dest):
        logger.info('Downloaded s3://' + histogram_bucket + '/' + key)
        results.put(dest)
    except (KeyboardInterrupt, SystemExit) as e:
      raise e
    except Exception as e:
      pass

def download(histogram_bucket, tile_level, tile_index, week, concurrency):
  #we need to get all the histograms for all the hours of this week
  date = datetime.datetime.strptime(week + '/1','%Y/%W/%w')
  logger.info('Downloading files for week ' + '{d.year}/{d.month}/{d.day} tile {l}/{t}.fb'.format(d=date, l=tile_level, t=tile_index))
  keys = []
  for hour in xrange(0, 24 * 7):
    key = '{d.year}/{d.month}/{d.day}/{d.hour}/{l}/{t}.fb'.format(d=(date + datetime.timedelta(hours=hour)), l=tile_level, t=tile_index)
    keys.append(key)
  random.shuffle(keys)
  keys = split(keys, concurrency)

  processes = []
  downloaded = multiprocessing.Queue()
  for i in xrange(concurrency):
    bound = functools.partial(fetch, histogram_bucket, keys[i], downloaded)
    processes.append(multiprocessing.Process(target=interrupt_wrapper, args=(bound,)))
    processes[-1].start()
  for p in processes:
    if p.is_alive():
      p.join()
  return downloaded

def add_tiles(tiles, tile_level, tile_index, max_level):
  #keep this one
  tiles.append((tile_level, tile_index))
  #add the subtiles
  if tile_level < max_level:
    level = filter(lambda x: x['level'] == tile_level, valhalla_tiles)[0]
    next_level = filter(lambda x: x['level'] == tile_level + 1, valhalla_tiles)[0]
    per_row = int(360 / level['size'])
    row = tile_index / per_row
    col = tile_index - row * per_row
    scale = int(level['size'] / next_level['size'])
    row *= scale
    col *= scale
    per_row = int(360 / next_level['size'])
    #add the children recursively
    for y in range(row, row + scale):
      for x in range(col, col + scale):
        add_tiles(tiles, next_level['level'],  y*per_row + x, max_level)

if __name__ == '__main__':
  #build args
  parser = argparse.ArgumentParser()
  parser.add_argument('--environment', type=str, help='The environment prod or dev to use when computing bucket names and batch job queues and definitions', required=True)
  parser.add_argument('--tile-level', type=int, help='The tile level used to get the input data from the histogram bucket', required=True)
  parser.add_argument('--tile-index', type=int, help='The tile index used to get the input data from the histogram bucket', required=True)
  parser.add_argument('--week', type=str, help='The week used to get the input data from the histogram bucket', required=True)
  parser.add_argument('--concurrency', type=int, help='The week used to get the input data from the histogram bucket', default=1)
  parser.add_argument('--max-tile-level', type=int, help='The max tile level to generate speed tiles for. Must be at least 0 and can go up to 2', default=1)
  parser.add_argument('--osmlr-version', type=str, help='The version of osmlr to fetch when creating speed tiles', required=True)
  args = parser.parse_args()

  #generate the list of the parent tile and all its subtiles, level 0 and 1 only right now
  tiles = []
  add_tiles(tiles, args.tile_level, args.tile_index, args.max_tile_level)

  #need this to submit jobs
  batch_client = boto3.client('batch')
  histogram_bucket = 'datastore-output-' + args.environment
  speed_bucket = 'speedtiles-' + args.environment
  job_queue = 'referencetiles-' + args.environment
  job_def = 'referencetiles-' + args.environment

  try:
    #for each tile
    for tile in tiles:
      logger.info('Histogram input bucket: ' + histogram_bucket)
      logger.info('Speedtile output bucket: ' + speed_bucket)
      logger.info('Tile level: ' + str(tile[0]))
      logger.info('Tile index: ' + str(tile[1]))
      logger.info('Week: ' + args.week)
      logger.info('OSMLR: ' + args.osmlr_version)
      #go get the histogram data
      histograms = download(histogram_bucket, tile[0], tile[1], args.week, args.concurrency)
      if not histograms.empty():
        #make the speed tile
        speed_tiles, histograms, osmlr = convert(tile[0], tile[1], args.week, args.osmlr_version, histograms, args.concurrency)
        #move the speed tile to its destination
        upload(speed_bucket, tile[0], tile[1], args.week, speed_tiles)
        #create the corresponding referencetile job
        job_name = '_'.join([args.week.replace('/', '-'), str(tile[0]), str(tile[1])])
        job = {'environment': args.environment, 'tile_level': str(tile[0]), 'tile_index': str(tile[1]), 'week': args.week}
        logger.info('Submitting reference tile job ' + job_name)
        logger.info('Job parameters ' + str(job))
        submitted = batch_client.submit_job(
          jobName = job_name,
          jobQueue = job_queue,
          jobDefinition = job_def,
          parameters = job,
          containerOverrides={
            'memory': 8192,
            'vcpus': 2,
            'command': ['/scripts/ref-tile-work.py', '--environment', 'Ref::environment', '--end-week', 'Ref::week', '--weeks', '52', '--tile-level', 'Ref::tile_level', '--tile-index', 'Ref::tile_index']
          }
        )
        logger.info('Job %s was submitted and got id %s' % (job_name, submitted['jobId']))
        #clean up the files
        histograms.put(False)
        while True:
          f = histograms.get()
          if not f:
            break
          os.remove(f)
        for f in speed_tiles + [osmlr]:
          os.remove(f)
      else:
        logger.info('No histogram data')

    logger.info('Run complete')
  except (KeyboardInterrupt, SystemExit):
    logger.error('Interrupted or killed')


import argparse
import numpy as np
import os

parser = argparse.ArgumentParser()
parser.add_argument("--min_sigma_mag", type=float, default=-5)
parser.add_argument("--max_sigma_mag", type=float, default=2)
parser.add_argument("--n_sigma", type=int, default=60)
parser.add_argument("--p", type=int, default=2)
parser.add_argument("--q", type=int, default=2)
parser.add_argument("--kappa", type=float, default=0.9)

args = parser.parse_args()

sigmas = np.logspace(args.min_sigma_mag, args.max_sigma_mag, args.n_sigma)

template = """

#!/bin/bash

#PBS -l walltime=48:00:00
#PBS -l select=1:ncpus=1:mem=5gb
#PBS -J 1-{}

module load lang/cuda
module load lang/python/anaconda/pytorch

cd /home/jr19127/loss-surfaces-of-gans/scripts

# PBS ARRAY INDEX is 1-up, but bash arrays are 0-up, so leading pad with -1

sigmas=({})

# Execute code
python rmt_sigmaz_exp.py --sigma ${{sigmas[$(( ($PBS_ARRAY_INDEX-1)  ))]}} --out {}/results_$(( ($PBS_ARRAY_INDEX-1))) --p {} --q {} --kappa {:.1f}
"""
outdir = f"/work/jr19127/gan-loss-surfaces/rmt_results/vary_sigma/p{args.p}q{args.q}"
os.makedirs(outdir, exist_ok=True)

sigma_str = " ".join(["{:.7f}".format(s) for s in sigmas])
script = template.format(len(sigmas), sigma_str, outdir,  args.p, args.q, args.kappa)

with open(f"run_rmt_vary_sigmaz_p{args.p}q{args.q}.sh", "w") as fout:
    fout.write(script)


"""Interactive plots for statistics over trajectories.

"""
import warnings

import panel as pn

from ad_sensitivity_analysis.interactive import create_widgets
from ad_sensitivity_analysis.plot import stats_per_traj_static

warnings.simplefilter(action="ignore", category=RuntimeWarning)

def plot_kde_histogram_interactive(ds):
    """
    Use plot_kde_histogram() interactively in a jupyter notebook.

    Parameters
    ----------
    ds : xarray.Dataset
        Dataset created via create_rank_traj_dataset().

    Returns
    -------
    panel.layout that can be used in a jupyter notebook.
    """
    out_param = create_widgets.create_out_params_select(ds)
    width_height_font_row = create_widgets.image_font_size_row(pixel=False)

    title_widget = create_widgets.create_title_widget()
    save_field_button_latex_row = create_widgets.create_save_latex_row()

    in_params = create_widgets.create_in_params_selector(ds)
    line_slider = create_widgets.create_line_slider()
    bw_slider = pn.widgets.FloatSlider(
        name="Change the bandwidth for the kde calculation",
        start=0.05,
        end=1.5,
        step=0.05,
        value=1.0,
    )
    flow_phase_ignore_row = create_widgets.create_flow_phase_ignore_row()
    common_toggle = pn.widgets.Toggle(
        name="Common norm",
        button_type="success",
    )

    plot_pane = pn.panel(
        pn.bind(
            stats_per_traj_static.plot_kde_histogram,
            ds=ds,
            in_params=in_params,
            out_param=out_param,
            linewidth=line_slider,
            bw_adjust=bw_slider,
            flow=flow_phase_ignore_row[0],
            phase=flow_phase_ignore_row[1],
            ignore_zero_gradients=flow_phase_ignore_row[2],
            title=title_widget,
            filename=save_field_button_latex_row[0],
            width=width_height_font_row[0],
            height=width_height_font_row[1],
            common_norm=common_toggle,
            font_scale=width_height_font_row[2],
            save=save_field_button_latex_row[1],
            latex=save_field_button_latex_row[2],
        ),
    ).servable()

    return pn.Column(
        width_height_font_row,
        save_field_button_latex_row,
        flow_phase_ignore_row,
        pn.Row(
            in_params,
            pn.Column(
                out_param,
                common_toggle,
            ),
        ),
        pn.Row(
            line_slider,
            bw_slider,
            title_widget,
        ),
        plot_pane,
    )


import os
import re
import sys
from setuptools import setup, find_packages

with open("README.md", "r") as f:
    long_description = f.read()

package = 'contentstack_management'

def get_version(package):
    """
    Return package version as listed in `__version__` in `init.py`.
    """
    init_py = open(os.path.join(package, '__init__.py')).read()
    return re.search("^__version__ = ['\"]([^'\"]+)['\"]",
                     init_py, re.MULTILINE).group(1)

def get_author_name(package):
    """
    Return package author as listed in `__author__` in `init.py`.
    """
    init_py = open(os.path.join(package, '__init__.py')).read()
    return re.search("^__author__ = ['\"]([^'\"]+)['\"]",
                     init_py, re.MULTILINE).group(1)

def get_author_email(package):
    """
    Return package email as listed in `__email__` in `init.py`.
    """
    init_py = open(os.path.join(package, '__init__.py')).read()
    return re.search("^__email__ = ['\"]([^'\"]+)['\"]",
                     init_py, re.MULTILINE).group(1)

setup(
    name="contentstack-management",
    version=get_version(package),
    packages=find_packages(exclude=['tests']),
    py_modules=['_api_client', 'contentstack','common','_errors','_constant'],
    description="Contentstack API Client Library for Python",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/contentstack/contentstack-management-python",
    author=get_author_name(package),
    author_email=get_author_email(package),
    license="MIT",
    classifiers=[
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python :: 3.10",
        "Operating System :: OS Independent",
    ],
    install_requires=["bson >= 0.5.9", "requests >= 2.5.4", "requests-toolbelt >= 0.3.1"],
    extras_require={
        "dev": ["pytest>=7.0", "twine>=4.0.2", "dotenv>=0.0.5"],
    },
    python_requires=">=3.9",
)


from atom import Potential, RotatePoints, EnergyLevel, ElectronStates

class Atom:
	def __init__(self):
		self.xU   = Potential()
		self.e    = EnergyLevel()
		self.N    = ElectronStates()
		self.__RP = RotatePoints()

	def setV(self, V):
		self.xU.setV(V)
		self.e.setV(V)
		self.N.setV(V)
		self.__RP.setV(V)

	def setT(self, T):
		self.xU.setT(T)
		self.e.setT(T)
		self.N.setT(T)
		self.__RP.setT(T)

	def setZ(self, Z):
		self.xU.setZ(Z)
		self.e.setZ(Z)
		self.N.setZ(Z)
		self.__RP.setZ(Z)

	def setVTZ(self, V, T, Z):
		self.xU.setVTZ(V, T, Z)
		self.e.setVTZ(V, T, Z)
		self.N.setVTZ(V, T, Z)
		self.__RP.setVTZ(V, T, Z)

	def setTolerance(self, eps):
		self.xU.setTolerance(eps)
		self.e.setTolerance(eps)
		self.N.setTolerance(eps)
		self.__RP.setTolerance(eps)


import os
import pysam
import argparse

############################################
############################################
## PARSE ARGUMENTS
############################################
############################################

Description = 'Remove singleton reads from paired-end BAM file i.e if read1 is present in BAM file without read 2 and vice versa.'
Epilog = """Example usage: bampe_rm_orphan.py <BAM_INPUT_FILE> <BAM_OUTPUT_FILE>"""

argParser = argparse.ArgumentParser(description=Description, epilog=Epilog)

## REQUIRED PARAMETERS
argParser.add_argument('BAM_INPUT_FILE', help="Input BAM file sorted by name.")
argParser.add_argument('BAM_OUTPUT_FILE', help="Output BAM file sorted by name.")

## OPTIONAL PARAMETERS
argParser.add_argument('-fr', '--only_fr_pairs', dest="ONLY_FR_PAIRS", help="Only keeps pairs that are in FR orientation on same chromosome.",action='store_true')
args = argParser.parse_args()

############################################
############################################
## HELPER FUNCTIONS
############################################
############################################

def makedir(path):

    if not len(path) == 0:
        try:
            #!# AVI: changed because of race conditions if directory exists, original code:  os.makedirs(path)
            os.makedirs(path, exist_ok=True)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise

############################################
############################################
## MAIN FUNCTION
############################################
############################################

def bampe_rm_orphan(BAMIn,BAMOut,onlyFRPairs=False):

    ## SETUP DIRECTORY/FILE STRUCTURE
    OutDir = os.path.dirname(BAMOut)
    makedir(OutDir)

    ## COUNT VARIABLES
    totalReads = 0; totalOutputPairs = 0; totalSingletons = 0; totalImproperPairs = 0

    ## ITERATE THROUGH BAM FILE
    EOF = 0
    SAMFin = pysam.AlignmentFile(BAMIn,"rb")  #!# AVI: changed to new API from pysam.Samfile
    SAMFout = pysam.AlignmentFile(BAMOut, "wb",header=SAMFin.header)   #!# AVI: changed to new API from pysam.Samfile
    currRead = next(SAMFin)     #!# AVI: adapted for the use of the iterator, original code: currRead = SAMFin.next()

    for read in SAMFin.fetch(until_eof=True): #!# AVI: added .fetch() to explicitly use new API
        totalReads += 1
        if currRead.qname == read.qname:
            pair1 = currRead; pair2 = read

            ## FILTER FOR READS ON SAME CHROMOSOME IN FR ORIENTATION
            if onlyFRPairs:
                if pair1.tid == pair2.tid:

                    ## READ1 FORWARD AND READ2 REVERSE STRAND
                    if not pair1.is_reverse and pair2.is_reverse:
                        if pair1.reference_start <= pair2.reference_start:
                            totalOutputPairs += 1
                            SAMFout.write(pair1)
                            SAMFout.write(pair2)
                        else:
                            totalImproperPairs += 1

                    ## READ1 REVERSE AND READ2 FORWARD STRAND
                    elif pair1.is_reverse and not pair2.is_reverse:
                        if pair2.reference_start <= pair1.reference_start:
                            totalOutputPairs += 1
                            SAMFout.write(pair1)
                            SAMFout.write(pair2)
                        else:
                            totalImproperPairs += 1

                    else:
                        totalImproperPairs += 1
                else:
                    totalImproperPairs += 1
            else:
                totalOutputPairs += 1
                SAMFout.write(pair1)
                SAMFout.write(pair2)

            ## RESET COUNTER
            try:
                totalReads += 1
                currRead = next(SAMFin)   #!# AVI: adapted for the use of the iterator, original code: currRead = SAMFin.next()
            except:
                StopIteration
                EOF = 1

        ## READS WHERE ONLY ONE OF A PAIR IS IN FILE
        else:
            totalSingletons += 1
            pair1 = currRead
            currRead = read

    if not EOF:
        totalReads += 1
        totalSingletons += 1
        pair1 = currRead

    ## CLOSE ALL FILE HANDLES
    SAMFin.close()
    SAMFout.close()

    LogFile = os.path.join(OutDir,'%s_bampe_rm_orphan.log' % (os.path.basename(BAMOut[:-4])))
    SamLogFile = open(LogFile,'w')
    SamLogFile.write('\n##############################\n')
    SamLogFile.write('FILES/DIRECTORIES')
    SamLogFile.write('\n##############################\n\n')
    SamLogFile.write('Input File: ' + BAMIn + '\n')
    SamLogFile.write('Output File: ' + BAMOut + '\n')
    SamLogFile.write('\n##############################\n')
    SamLogFile.write('OVERALL COUNTS')
    SamLogFile.write('\n##############################\n\n')
    SamLogFile.write('Total Input Reads = ' + str(totalReads) + '\n')
    SamLogFile.write('Total Output Pairs = ' + str(totalOutputPairs) + '\n')
    SamLogFile.write('Total Singletons Excluded = ' + str(totalSingletons) + '\n')
    SamLogFile.write('Total Improper Pairs Excluded = ' + str(totalImproperPairs) + '\n')
    SamLogFile.write('\n##############################\n')
    SamLogFile.close()

############################################
############################################
## RUN FUNCTION
############################################
############################################

bampe_rm_orphan(BAMIn=args.BAM_INPUT_FILE,BAMOut=args.BAM_OUTPUT_FILE,onlyFRPairs=args.ONLY_FR_PAIRS)

############################################
############################################
############################################
############################################


from Pessoa import *
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import smtplib
from selenium import webdriver
import time
class Professor(Pessoa):

    def __init__(self,nome,rg,cpf,phone,email,nasc,idade ,sexo ,salario, materia,tempo,carga,exp,desc,plus,novo,atual):
        super().__init__(nome,rg,cpf,phone,email,nasc,idade ,sexo)
        self.materia = materia
        self.salario = salario
        self.tempo = tempo  # quantas horas de trabalho
        self.carga = carga  # horas semanais
        self.exp = exp  # tempo de experiência
        self.desc = desc  # desconto
        self.plus = plus  # aumento
        self.novo = self.salario * plus / 100 + self.salario  # novo salário
        self.atual = self.novo * (-desc) / 100 + self.novo  # salário final

    def dados(self):
        super(Professor, self).dados()
        print(f'Matéria {self.materia}')
        print(f'Salário base R$ {self.salario:.2f}')
        print(f'Horas de trabalho diárias {self.tempo:.0f} hrs')
        print(f'Horas semanais {self.carga:.0f} hrs')
        print(f'Tempo de trabalho {self.exp:.0f} ano(s)')
        print(f'Desconto no salário {self.desc} %')
        print(f'Acréscimo no {self.plus} %')
        if self.tempo > 10:
            bon = self.tempo * 110 / 100
            print(f'Bonificação de 10% no salário este mês irá receber R$ {bon:.2f}')
        else:
            print(f'Não houve nenhuma bonificação seu salário ficou em R$ {self.salario:.2f}')
        if self.carga >= 45 and self.exp >= 5:
            plus = self.salario * 145 / 100
            print(f'Seu salário irá receber um aumento fixo de R$ {plus:.2f}')
        else:
            print(f'O salário ficou em R$ {self.atual}')
            print(f'Infelizmente não foi cumprido os pré requisitos. Não desista ')

    def write(self,jan, nome):
        try:
            a = open(jan,'at')
        except:
            print(f'Erro no cadastro')
        else:
            a.write(f'{nome}')

    def __str__(self):
        super(Professor, self).__str__()
        return f'{self.nome} ,{self.rg},{self.cpf},{self.phone},{self.email}, ' \
               f'{self.nasc} , {str(self.idade)},  {self.sexo} , {self.materia} , ' \
               f'{self.salario} , {self.tempo} , {self.carga} , {self.exp} , {self.desc} , {self.plus} ,' \
               f'{self.novo} ,  {self.atual}'

    def send_mail(self,assunto, de, para):
        body = f"Professor {self.nome}\n" \
               f"{self.nome} portador do RG {self.rg} e CPF {self.cpf}, nascido no ano {self.nasc} com idade no " \
               f"ano corrente {self.idade} anos, de sexo {self.sexo} leciona matéria {self.materia} seu salário base é de R$" \
               f"{self.salario} seu salário stual com descontos ou acréscimos é de R$ {self.atual}"
        msg = MIMEMultipart()
        msg['Subject'] =  "" + assunto
        msg['From'] = "" + de
        msg['To'] = "" + para
        password = ""
        msg.attach(MIMEText(body,'plain'))

        server = smtplib.SMTP('smtp.gmail.com', port=587)
        server.starttls()
        server.login(msg['From'],password)
        server.sendmail(msg['From'],msg['To'],msg.as_string())
        server.quit()

    def wpp(self):

        texto = f'Professor {self.nome} da {self.materia} seu salário é de R$ {self.atual}. Consulte o email para ' \
                f'demais informações '
        link = f"https://web.whatsapp.com/send?phone={self.phone}&text={texto}"
        browser = webdriver.Firefox()
        browser.maximize_window()
        browser.get('https://web.whatsapp.com')
        time.sleep(20)
        browser.get(link)
        time.sleep(40)


from string import ascii_letters
from dataclasses import dataclass
from typing import Sequence, Union

from tucker_riemopt import backend as back
from tucker_riemopt import Tucker

@dataclass()
class TuckerMatrix(Tucker):
    """Tucker representation for operators.

    :param core: The core tensor of the decomposition.
    :param factors: The list of factors of the decomposition.
    :param n: Dimensionality of source space (see ``from_dense`` method).
    :param m: Dimensionality of target space (see ``from_dense`` method).
    """
    n: Union[Sequence[int], None] = None
    m: Union[Sequence[int], None] = None

    @classmethod
    def from_dense(cls, dense_tensor: back.type(), n: Union[Sequence[int], None] = None, m: Union[Sequence[int], None] = None,
                   eps=1e-14):
        """Convert dense tensor into `TuckerMatrix` representation. The resul is tensor that represents linear operator
        from R^N to R^M.

        :param dense_tensor: Dense tensor that will be factorized.
        :param n: Sequence of ints (n_1, ..., n_a) such that N = n_1 * ... * n_a.
        :param m: Sequence of ints (m_1, ..., m_b) such that M = m_1 * ... * m_b.
        :param eps: Precision of approximation.
        :return: `TuckerMatrix` representation of the provided dense tensor.
        """
        if n is None or m is None:
            raise ValueError("n and m parameter must be specialized for matrices")
        dense_tensor = cls._hosvd(dense_tensor, eps=eps)
        return cls(dense_tensor.core, dense_tensor.factors, n, m)

    def __matmul__(self, other: Union[back.type(), Tucker, "TuckerMatrix"]):
        """Perform matrix multiplication of tensors. If other's ndim > matrix ndim, then perform batch matmul over last
        ndim modes.

        :param other: Can be dense tensor, Tucker or TuckerMatrix. If dense tensor, then treated as vector and matvec
        operation is performed. The result is also a dense tensor. If `Tucker`, then treated as vector and matvec
        operation is performed. The result is also a Tucker. Else matmul operation is performed and another matrix is
        returned.
        :return: Dense tensor, Tucker or TuckerMatrix, depends on type of `other`.
        """
        if type(other) == back.type():
            core_letters = ascii_letters[:self.ndim]
            factors_letters = []
            operand_letters = []
            result_letters = []
            reshaped_factors = []
            for i in range(self.ndim):
                reshaped_factors.append(back.reshape(self.factors[i], (self.n[i], self.m[i], -1), order="F"))
                factors_letters.append(ascii_letters[self.ndim + 2 * i: self.ndim + 2 * (i + 1)] + core_letters[i])
                operand_letters.append(factors_letters[-1][0])
                result_letters.append(factors_letters[-1][1])
            batch_letters = []
            for i in range(len(other.shape) - self.ndim):
                batch_letters.append(ascii_letters[2 * (self.ndim + 1) + i + 1])
            operand_letters = batch_letters + operand_letters
            result_letters =  batch_letters + result_letters
            return back.einsum(f"{core_letters},{','.join(factors_letters)},{''.join(operand_letters)}->{''.join(result_letters)}",
                               self.core, *reshaped_factors, other)


import base64
import os
import logging
from krita import *
from krita_stable_diffusion.interface.interfaces.vertical_interface import VerticalInterface
from krita_stable_diffusion.settings import APPLICATION_ID
from krita_stable_diffusion.settings import DEFAULT_MODEL

class Base:
    """
    Extend this class to create a new tab for use in an interface.
    :param name: The name of the tab
    :param settings_key: The key of the settings to be used
    :param widget: The widget to add to the layout
    :param layout: The layout to add the widget to
    :param default_setting_values: The default values of the settings object
    """
    inserting_image = False
    krita_instance = None
    display_name = ""
    name = ""
    settings_key = ""
    widget = None
    layout = None
    default_setting_values = {}
    log_widget = None
    image_insert_timer = None
    progressbar_update_timer = None

    @property
    def color_mode(self):
        return self.config.value("color_mode", "RGBA")

    @property
    def resolution(self):
        return float(self.config.value("resolution", 300.0))

    @property
    def width(self):
        return int(self.config.value("width", 512))

    @property
    def height(self):
        return int(self.config.value("height", 512))

    @property
    def active_document(self):
        return self.krita.activeDocument()

    @property
    def active_node(self):
        return self.active_document.activeNode()

    @property
    def selection(self):
        return self.active_document.selection()

    @property
    def x(self):
        return 0 if self.selection is None else self.selection.x()

    @property
    def y(self):
        return 0 if self.selection is None else self.selection.y()

    @property
    def krita(self):
        if not self.krita_instance:
            self.krita_instance = Krita.instance()
        return self.krita_instance

    def string_to_binary(self, st):
        return ''.join(format(ord(i), '08b') for i in str(st))

    def build_prompt(self, data, image_type=None, style=None):
        if image_type and style:
            data["prompt"] = " ".join([
                f"{style} {image_type}",
                data["prompt"]
            ])
        return data

    def get_active_layer_binary(self, pos_x, pos_y):
        """
        Saves image to path
        :param path:
        :param is_mask:
        :return:
        """
        logging.info(f"saving layer...")
        item = self.active_document.activeNode()
        pixels = item.pixelData(pos_x, pos_y, self.width, self.height)
        image_data = QImage(
            pixels,
            self.width,
            self.height,
            QImage.Format_RGB32
        )
        byte_array = QByteArray()
        buffer = QBuffer(byte_array)
        buffer.open(QIODevice.WriteOnly)
        image_data.save(buffer, "PNG", -1)
        return bytes(byte_array)

    def get_mask(self, pos_x, pos_y):
        """
        Get the mask from the top mask layer
        :return: mask as bytes
        """
        mask = None
        node = self.active_document.activeNode()
        childNodes = node.childNodes()
        for childNode in childNodes:
            if childNode.type() == "transparencymask":
                mask = childNode
                break
        if mask is None:
            return None
        pixels = mask.pixelData(pos_x, pos_y, self.width, self.height)
        image_data = QImage(
            pixels,
            self.width,
            self.height,
            QImage.Format_Indexed8
        )

        image_data.invertPixels()

        byte_array = QByteArray()
        buffer = QBuffer(byte_array)
        buffer.open(QIODevice.WriteOnly)
        image_data.save(buffer, "PNG", -1)
        return bytes(byte_array)

    def handle_outpaint(self, data):
        """
        Get the pixels from the active layer based on the location of
        outpaint layer. Also create a mask based on the overlap.
        :param data:
        :return:
        """

        childNodes = self.active_document.topLevelNodes()
        for childNode in childNodes:
            name = childNode.name()
            # get layer named outpaint
            if name == "outpaint":
                node = childNode
                pos_x = node.position().x()
                pos_y = node.position().y()

                # select the pixels in the active layer
                item = self.active_document.activeNode()
                pixels = item.pixelData(pos_x, pos_y, self.width, self.height)

                # # create a QByteArray containing 512 * 512 * 4 bytes
                # # (4 bytes per pixel) - white pixels
                mask_pixels = QByteArray(b'\xff' * (self.width * self.height * 4))

                image_data = QImage(
                    pixels,
                    self.width,
                    self.height,
                    QImage.Format_ARGB32
                )
                byte_array = QByteArray()
                buffer = QBuffer(byte_array)
                buffer.open(QIODevice.WriteOnly)
                image_data.save(buffer, "PNG", -1)

                mask_image_data = QImage(
                    mask_pixels,
                    self.width,
                    self.height,
                    QImage.Format_RGB32
                )

                # iterate over the pixels in image_data
                # and set the mask_pixels pixels to black if the pixel is not transparent
                for x in range(self.width):
                    for y in range(self.height):
                        pixel = image_data.pixel(x, y)
                        if pixel != 0:
                            mask_image_data.setPixel(x, y, 0)

                mask_byte_array = QByteArray()
                mask_buffer = QBuffer(mask_byte_array)
                mask_buffer.open(QIODevice.WriteOnly)
                mask_image_data.save(mask_buffer, "PNG", -1)

                data["pixels"] = base64.b64encode(bytes(byte_array)).decode("utf-8")
                data["mask"] = base64.b64encode(bytes(mask_byte_array)).decode("utf-8")
                data["pos_x"] = pos_x
                data["pos_y"] = pos_y
        return data

    def handle_inpaint(self, data):
        """
        Get the pixels from the active layer and mask from its transparency mask
        :param data:
        :return:
        """
        pos_x = 0
        pos_y = 0
        pixels = base64.b64encode(
            self.get_active_layer_binary(pos_x, pos_y)
        ).decode("utf-8")

        # get mask from top mask layer
        mask = self.get_mask(pos_x, pos_y)
        if mask:
            data["mask"] = base64.b64encode(mask).decode("utf-8")
        else:
            data["mask"] = mask
        data["pixels"] = pixels
        return data

    def handle_img2img(self, data):
        """
        Get the pixels from the active layer.
        :param data:
        :return:
        """
        pos_x = 0
        pos_y = 0
        pixels = base64.b64encode(
            self.get_active_layer_binary(pos_x, pos_y)
        ).decode("utf-8")
        data["pixels"] = pixels
        return data

    def handle_button_press(self, request_type, **kwargs):
        """
        Callback for the generate button.
        :param _element: passed by the button but not used
        :return: None, sends request to stable diffusion
        """
        data = {}
        for k, v in self.default_setting_values.items():
            v = self.config.value(k, v)
            data[k] = v
        self.update_progressbar(request_type, 0, 0)

        # display status label
        Application.status_label.setText(
            "Loading (first load will take a while)"
        )

        # add config options to request data
        data = self.prep_config_options(request_type, data)

        # build the prompt based on request data
        data = self.build_prompt(
            data,
            kwargs.get("image_type", None),
            kwargs.get("style", None)
        )

        data["pos_x"] = 0
        data["pos_y"] = 0
        data["model_base_path"] = self.config.value("model_base_path", "")

        # handle request types that require image data to be sent
        if request_type == "outpaint":
            data = self.handle_outpaint(data)
        elif request_type in ["inpaint"]:
            data = self.handle_inpaint(data)
        elif request_type == "img2img":
            data = self.handle_img2img(data)

        # if there is no self.active_document, create one
        if not self.active_document:
            document = self.krita.createDocument(
                self.width,
                self.height,
                "StableDiffusion",
                self.color_mode,
                "U8",
                "",
                self.resolution
            )
            self.krita.activeWindow().addView(document)

            # activate the document
            self.krita.activeDocument().setActiveNode(
                self.krita.activeDocument().rootNode()
            )

        # send request to the server
        self.send(data, request_type)

    def update_progressbar(self, reqtype, max, val):
        try:
            progress_bar = None
            if reqtype == "txt2img":
                progress_bar = Application.txt2img_progress_bar
            elif reqtype == "img2img":
                progress_bar = Application.img2img_progress_bar
            elif reqtype == "inpaint":
                progress_bar = Application.inpaint_progress_bar
            elif reqtype == "outpaint":
                progress_bar = Application.outpaint_progress_bar
            if progress_bar:
                if progress_bar.element.maximum() != max:
                    progress_bar.setRange(0, max)
                if max != 0:
                    progress_bar.setvalue(val, max)
        except Exception as e:
            pass

    def update_image_insert(self):
        if len(Application.image_queue) > 0 and not self.inserting_image:
            self.inserting_image = True
            image = Application.image_queue.pop(0)
            self.insert_image(image)

    def progressbar_timed_update(self):
        self.progressbar_update_timer = QTimer()
        self.progressbar_update_timer.timeout.connect(lambda: self.update_progressbar(
            Application.cur_reqtype,
            Application.step_total,
            Application.cur_step
        ))
        self.progressbar_update_timer.start(100)

    def image_insert_timed_update(self):
        self.image_insert_timer = QTimer()
        self.image_insert_timer.timeout.connect(self.update_image_insert)
        self.image_insert_timer.start(100)

    def byte_array(self, image):
        """
        Convert QImage to QByteArray
        :param image:
        :return: QByteArray
        """
        bits = image.bits()
        bits.setsize(image.byteCount())
        return QByteArray(bits.asstring())

    def create_layer(self, name, visible=True, type="paintLayer"):
        """
        Creates a new layer in the active document
        :param name:
        :param type:
        :return: a reference to the new layer
        """
        document = self.active_document.createNode(name, type)
        self.root_node.addChildNode(document, None)
        return document

    def add_image_from_bytes(
            self,
            reqype,
            name,
            image_bytes,
            pos_x=0,
            pos_y=0,
    ):
        """
        Adds image from bytes
        :param name:
        :param image_bytes:
        :return:
        """
        image = QImage()
        image.loadFromData(image_bytes)

        if reqype == "outpaint":
            # get active layer
            layer = self.active_document.activeNode()
        else:
            layer = self.create_layer(name)
        layer.setPixelData(self.byte_array(image), pos_x, pos_y, self.width, self.height)

        # the layer is not visible until we refresh the projection
        self.active_document.refreshProjection()

        return layer

    @property
    def root_node(self):
        doc = self.krita.activeDocument()
        if not doc:
            raise Exception("NO ACTIVE DOCUMENT")
        try:
            root = doc.rootNode()
        except RuntimeError:
            print("runtime error")
            # check if doc has been deleted
            if not doc:
                print("NO ACTIVE DOCUMENT")
            # rootNode has been deleted, fix it
            doc.setRootNode(doc.createNode("root", "paintLayer"))
            root = doc.rootNode()
        return root

    def insert_image(self, image_response):
        """
        Insert an image into the document
        :param image: The image to insert
        :return: None
        """
        image = image_response["image"]
        reqtype = image_response["reqtype"]
        pos_x = image_response["pos_x"]
        pos_y = image_response["pos_y"]
        # convert base64 image to bytes+
        image_bytes = base64.b64decode(image)
        total_layers = len(self.root_node.childNodes()) + 1
        self.add_image_from_bytes(
            reqtype,
            f"Layer {total_layers}",
            image_bytes,
            pos_x,
            pos_y
        )
        self.inserting_image = False

    def send(self, options, request_type):
        """
        Send request to stable diffusion
        :param options: The options to send
        :param request_type: The type of request to send
        :return: None
        """
        Application.stablediffusion.client.message = {
            "action": request_type,
            "options": options,
        }

    message_log = []

    def log_message(self, message, level="info"):
        """
        Log a message to the log widget
        :param message: The message to log
        :param level: The level of the message
        :return: None
        """
        self.message_log.append(f"{level.upper()} {message}")
        if self.log_widget:
            self.log_widget.widget.setPlaceholderText(
                "\n".join(self.message_log)
            )

    def tab(self):
        return self.widget, self.display_name

    def prep_config_options(self, request_type, data):
        # add config options
        do_nsfw_filter = self.config.value("do_nsfw_filter", True)
        do_watermark = self.config.value("do_watermark", True)
        data["do_nsfw_filter"] = False if do_nsfw_filter == 0 else True
        data["do_watermark"] = False if do_watermark == 0 else True

        available_models = None
        model_version = self.config.value(f"{request_type}_model_version", "v1")
        data["version"] = model_version
        if model_version == "v1":
            available_models = Application.available_models_v1
        elif model_version == "v2":
            available_models = Application.available_models_v2
        elif model_version == "v1 (community)":
            available_models = Application.available_models_custom_v1
        else:
            available_models = Application.available_models_custom_v2

        data["model_path"] = self.config.value("model_path", "")

        if available_models:
            model_name = self.config.value(f"{request_type}_model", 0)
            for index in range(len(available_models)):
                available_name = available_models[index]["name"]
                available_path = available_models[index]["path"]
                if available_name == model_name:
                    data["model_path"] = available_path
                    data[f"{request_type}_model"] = os.path.join(
                        available_path
                    )
                    break
        else:
            data["model"] = DEFAULT_MODEL

        self.config.setValue("log", Application.stablediffusion.log)
        return data

    def initialize_settings(self):
        """
        Create tab specific settings
        :return:
        """
        self.settings_key = f"ksd_{self.name}_settings"
        Application.__setattr__(self.settings_key, QSettings(
            QSettings.IniFormat,
            QSettings.UserScope,
            "krita",
            APPLICATION_ID
        ))
        self.config = Application.__getattribute__(self.settings_key)

    def reset_default_values(self):
        # set default values
        for k, v in self.default_setting_values.items():
            self.config.setValue(k, v)

    def initialize_interfaces(self, interfaces):
        if len(interfaces) == 0:
            return
        self.layout = VerticalInterface(interfaces=interfaces)
        self.widget = QWidget()
        self.widget.setLayout(self.layout)

    def __init__(self, interfaces):
        Application.__setattr__("update_progressbar", self.update_progressbar)
        Application.__setattr__("connected_to_sd", False)
        #Application.__setattr__("log_message", self.send)
        Application.__setattr__("app", self)
        # get steps from config
        self.initialize_settings()
        Application.__setattr__("cur_reqtype", None)
        Application.__setattr__("step_total", 0)
        Application.__setattr__("cur_step", 0)
        Application.__setattr__("image_queue", [])
        self.initialize_interfaces(interfaces)
        self.progressbar_timed_update()
        self.image_insert_timed_update()


from datetime import date

from fastapi import Depends, HTTPException, status
from fastapi_pagination import Params

from code.exceptions import NewsNotFound, UnsupportedSymbol
from code.services import CompanyNewsListService, NewsDetailsService, NewsListService

async def get_news(params: Params = Depends()):
    return await NewsListService(params).do()

async def get_news_by_symbol(
    symbol: str, params: Params = Depends(),
    date_from: date = None, date_to: date = None,
):
    service = CompanyNewsListService(symbol, params, date_from, date_to)

    try:
        return await service.do()
    except UnsupportedSymbol:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)

async def get_news_by_id(news_id: int):
    service = NewsDetailsService(news_id)

    try:
        return await service.do()
    except NewsNotFound:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)


from airflow import DAG
from datetime import datetime, timedelta
from airflow.operators.dummy import DummyOperator
from Delta_Ops.Delta_Ops import credits_organizer,movies_organizer

from airflow.operators.python import PythonOperator

start_date = datetime(2022, 11, 5)
current_ts = datetime.now().strftime('%Y%m%d_%H%M%S')
default_args = {
    'owner': 'train',
    'start_date': start_date,
    'retries': 1,
    'retry_delay': timedelta(seconds=5)
}

with DAG('orchestra_dag', default_args=default_args, schedule_interval='@daily', catchup=False) as dag:
    start = DummyOperator(task_id="start")
    push_credits_silver = PythonOperator(task_id='pushing_credits_silver', python_callable=credits_organizer)
    push_movies_silver = PythonOperator(task_id='pushing_movies_silver', python_callable=movies_organizer)
    stop = DummyOperator(task_id="stop")

    start >> push_credits_silver >> push_movies_silver >> stop


import time
from contextlib import contextmanager

import pytest

from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import TSV
from helpers.client import CommandRequest

cluster = ClickHouseCluster(__file__)

node1 = cluster.add_instance('node1', config_dir='configs', with_zookeeper=True)
node2 = cluster.add_instance('node2', config_dir='configs', with_zookeeper=True)
nodes = [node1, node2]

@pytest.fixture(scope="module")
def started_cluster():
    try:
        cluster.start()

        for node in nodes:
            node.query('''
CREATE TABLE simple (date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192);
'''.format(replica=node.name))
            node.query("INSERT INTO simple VALUES (0, 0)")

            node.query('''
CREATE TABLE simple2 (date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple2', '{replica}', date, id, 8192);
'''.format(replica=node.name))

        yield cluster

    finally:
        pass
        cluster.shutdown()

def test_deduplication_window_in_seconds(started_cluster):
    node = node1

    node.query("INSERT INTO simple2 VALUES (0, 0)")
    time.sleep(1)
    node.query("INSERT INTO simple2 VALUES (0, 0)") # deduplication works here
    node.query("INSERT INTO simple2 VALUES (0, 1)")
    assert TSV(node.query("SELECT count() FROM simple2")) == TSV("2\n")

    # wait clean thread
    time.sleep(2)

    assert TSV.toMat(node.query("SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple2/blocks'"))[0][0] == "1"
    node.query("INSERT INTO simple2 VALUES (0, 0)") # deduplication doesn't works here, the first hash node was deleted
    assert TSV.toMat(node.query("SELECT count() FROM simple2"))[0][0] == "3"

def check_timeout_exception(e):
    s = str(e)
    #print s
    assert s.find('timed out!') >= 0 or s.find('Return code: -9') >= 0

# Currently this test just reproduce incorrect behavior that sould be fixed
def test_deduplication_works_in_case_of_intensive_inserts(started_cluster):
    inserters = []
    fetchers = []

    for node in nodes:
        host = node.ip_address

        inserters.append(CommandRequest(['/bin/bash'], timeout=10, stdin="""
set -e
for i in `seq 1000`; do
    clickhouse-client --host {} -q "INSERT INTO simple VALUES (0, 0)"
done
""".format(host)))

        fetchers.append(CommandRequest(['/bin/bash'], timeout=10, stdin="""
set -e
for i in `seq 1000`; do
    res=`clickhouse-client --host {} -q "SELECT count() FROM simple"`
    if [[ $res -ne 1 ]]; then
        echo "Selected $res elements! Host: {}" 1>&2
        exit -1
    fi;
done
""".format(host, node.name)))

    # There were not errors during INSERTs
    for inserter in inserters:
        try:
            inserter.get_answer()
        except Exception as e:
            check_timeout_exception(e)

    # There were not errors during SELECTs
    for fetcher in fetchers:
        try:
            fetcher.get_answer()
        except Exception as e:
            pass
            # Uncomment when problem will be fixed
            # check_timeout_exception(e)


import random
import time
import cv2
import numpy as np
from ultralytics import YOLO
def checkimage(image):
    my_file = open("coco.txt", "r")
    # reading the file
    data = my_file.read()
    # replacing end splitting the text | when newline ('\n') is seen.
    class_list = data.split("\n")
    my_file.close()

    # print(class_list)

    # Generate random colors for class list
    detection_colors = []
    for i in range(len(class_list)):
        r = random.randint(0, 255)
        g = random.randint(0, 255)
        b = random.randint(0, 255)
        detection_colors.append((b, g, r))

    model = YOLO("best.pt", "v8")

    frame_wid = 640
    frame_hyt = 480

    cap = image
    #cap = cv2.VideoCapture("test-video.m4v")
        # Capture frame-by-frame
    # ret, frame = cap.read()
    # if not ret:
    #     print("Can't receive frame (stream end?). Exiting ...")
    detect_params = model.predict(source=[image], conf=0.55, save=False)

    # Convert tensor array to numpy
    DP = detect_params[0].numpy()

    return True

        # Terminate run when "Q" pressed

    # When everything done, release the capture


#!/usr/bin/env python
# -*- coding:utf-8 -*-
import torch
from torch import nn

class Reshape(nn.Module):
    def forward(self, x):
        return x.view(-1, 1, 28, 28)

net = nn.Sequential(
    Reshape(),
    # 输入通道为1，输出通道为6的卷积层   (28+4-5+1)*(28+4-5+1) = 28*28
    nn.Conv2d(1, 6, kernel_size=5, padding=2), nn.Sigmoid(),
    # 平均池化层2x2，移动不重叠   28/2 = 14
    nn.AvgPool2d(kernel_size=2, stride=2),
    # 输入通道为6，输出通道为16的卷积层  (14-5+1)*(14-5+1) = 10*10
    nn.Conv2d(6, 16, kernel_size=5), nn.Sigmoid(),
    # 平均池化层2x2，移动不重叠   10/2 = 5
    nn.AvgPool2d(kernel_size=2, stride=2),
    # 将四维卷积层变为一维向量
    nn.Flatten(),
    # 16个输出再乘以5*5的池化层转为向量为16*5*5 = 400
    nn.Linear(16 * 5 * 5, 120), nn.Sigmoid(),
    nn.Linear(120, 84), nn.Sigmoid(),
    # 输出0到9
    nn.Linear(84, 10)
)

X = torch.rand(size=(1, 1, 28, 28), dtype=torch.float32)
for layer in net:
    X = layer(X)
    print(f'{layer.__class__.__name__}output shape\t->\t{X.shape}')


from tkinter import *
from tkinter import ttk
from tkinter import tix
from tkinter import messagebox
import banco_2
import os
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import letter, A4
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.platypus import SimpleDocTemplate,Image
import webbrowser
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
import matplotlib.pyplot as plt
from matplotlib.figure import Figure
from tkcalendar import Calendar, DateEntry

# defs

def popular2():
    tv2.delete(*tv2.get_children())
    vquery = "SELECT * FROM tb_nomes ORDER BY ID"
    linhas=banco_2.dql(vquery)
    for i in linhas:
        tv2.insert("","end",values=i)

def buscar():
    tv2.delete(*tv2.get_children())
    vquery="SELECT * FROM tb_nomes WHERE nome LIKE '%"+nome_entry.get()+"%'"
    linhas=banco_2.dql(vquery)
    for i in linhas:
        tv2.insert("","end",values=i)

def printTudo():
    webbrowser.open("ListaGeral.pdf")

def imprimirTudo():
    valBarra(10000)
    try:
        vquery = "SELECT * FROM tb_nomes"
        linhas=banco_2.dql(vquery)
        cnv=canvas.Canvas("ListaGeral.pdf")
        cnv.setFont("Helvetica-Bold",24)
        cnv.drawString(200,790,'Cadastro de Clientes')

        cnv.setFont("Helvetica-Bold",12)
        cnv.drawString(40,740,'ID: ')
        cnv.drawString(70,740,'Nome: ')
        cnv.drawString(200,740,'Telefone: ')
        cnv.drawString(300,740,'UF: ')
        cnv.drawString(340,740,'Cidade: ')
        cnv.drawString(430,740,'Assinante: ')
        cnv.drawString(510,740,'Data: ')

        cnv.setFont("Helvetica",12)
        x = 40
        y = 700
        for i in linhas:
            id = i[0]
            nome = i[1]
            fone = i[2]
            estado= i[3]
            cidade = i[4]
            ass= i[5]
            data= i[6]
            cnv.drawString(x, y, str(id))
            cnv.drawString(x + 30, y, nome)
            cnv.drawString(x + 160, y, fone)
            cnv.drawString(x + 260, y, str(estado))
            cnv.drawString(x + 300, y, str(cidade))
            cnv.drawString(x + 420, y, str(ass))
            cnv.drawString(x + 470, y, str(data))
            y -= 20
        cnv.rect(20,720,550,3,fill=True,stroke=False)
        cnv.showPage()
        cnv.save()
        printTudo()
    except:
        messagebox.showerror(title="ERRO",message="Erro ao Gerar PDF")
        return
    var_barra.set(0)

def printCliente():
    webbrowser.open("cliente.pdf")

def geraRelatório():
    c=canvas.Canvas("cliente.pdf")
    codigo = codigo_entry.get()
    nome = nome_entry.get()
    telefone = telefone_entry.get()
    cidade = cidade_entry.get()
    estado = estado_entry.get()
    ass = text1.get()
    data = entry_data.get()
    c.setFont("Helvetica-Bold",24)
    c.drawString(200,790,'Ficha do Cliente')

    c.setFont("Helvetica-Bold",18)
    c.drawString(50,700,'ID: ')
    c.drawString(50,670,'Nome: ')
    c.drawString(50,640,'Telefone: ')
    c.drawString(50,610,'Cidade: ')
    c.drawString(50,580,'UF: ')
    c.drawString(50,540,'Assinante: ')
    c.drawString(50,500,'Data: ')

    c.setFont("Helvetica",18)
    c.drawString(150,700,codigo)
    c.drawString(150,670,nome)
    c.drawString(150,640,telefone)
    c.drawString(150,610,cidade)
    c.drawString(150,580,estado)
    c.drawString(150,540,ass)
    c.drawString(150,500,data)

    c.rect(20,470,550,5,fill=True,stroke=False)

    c.showPage()
    c.save()
    printCliente()

def valBarra(m):
    cont=0
    etapas=m/100
    while cont<etapas:
        cont+=10
        i=0
        while i<1000000:
            i+=1
        var_barra.set(cont)
        app.update()

def limpar():
    codigo_entry.delete(0,END)
    nome_entry.delete(0,END)
    telefone_entry.delete(0,END)
    cidade_entry.delete(0,END)
    estado_entry.set(" ")
    text1.set("Não")
    entry_data.delete(0,END)

def novo():
    if nome_entry.get()=="" or telefone_entry.get()=="" or estado_entry.get()==" " or cidade_entry.get()=="" or entry_data.get()=="":
        messagebox.showinfo(title="ERRO",message="Adicione os valores")
        return
    try:
        vquery = "INSERT INTO tb_nomes (nome,fone,Estado,Cidade,assinante,data) VALUES ('"+nome_entry.get()+"','"+telefone_entry.get()+"','"+estado_entry.get()+"','"+cidade_entry.get()+"','"+text1.get()+"','"+entry_data.get()+"')"
        banco_2.dml(vquery)
    except:
        messagebox.showerror(title="ERRO",message="Erro ao Inserir")
        return
    popular2()
    limpar()
    nome_entry.focus()

def duploClick(event):
    limpar()
    tv2.selection()
    for i in tv2.selection():
        col1,col2,col3,col4,col5,col6,col7=tv2.item(i,'values')
        codigo_entry.insert(END,col1)
        nome_entry.insert(END,col2)
        telefone_entry.insert(END,col3)
        estado_entry.set(col4)
        cidade_entry.insert(END,col5)
        text1.set(col6)
        entry_data.insert(END,col7)

def apagar():
    codigo = codigo_entry.get()
    vid=-1
    itemSelection=tv2.selection()[0]
    valores=tv2.item(itemSelection,"values")
    vid=valores[0]
    try:
        vquery = "DELETE FROM tb_nomes WHERE id="+vid
        banco_2.dml(vquery)
        messagebox.showinfo(title="SUCESSO",message="Valores deletados!")
    except:
        messagebox.showwarning(title="ERRO",message="Erro ao deletar")
        return
    tv2.delete(itemSelection)

def alterar():
    codigo = codigo_entry.get()
    nome = nome_entry.get()
    telefone = telefone_entry.get()
    estado = estado_entry.get()
    cidade=cidade_entry.get()
    ass = text1.get()
    data = entry_data.get()
    vid=-1
    itemSelection=tv2.selection()[0]
    valores=tv2.item(itemSelection,"values")
    vid=valores[0]
    try:
        vquery = "UPDATE tb_nomes SET nome = '"+nome+"', fone = '"+telefone+"',Estado = '"+estado+"', Cidade = '"+cidade+"', assinante = '"+ass+"', data = '"+data+"' WHERE id="+vid
        banco_2.dml(vquery)
        messagebox.showinfo(title="SUCESSO",message="Valores alterados!")
    except:
        messagebox.showerror(title="ERRO",message="Erro ao alterar")
        return
    popular2()
    limpar()

def Menus():
    menubar = Menu(app,tearoff=0)
    app.config(menu=menubar)
    filemenu=Menu(menubar,tearoff=0)
    filemenu2=Menu(menubar,tearoff=0)

    def quit(): app.destroy()

    menubar.add_cascade(label= "Opções",menu=filemenu)
    menubar.add_cascade(label= "Sobre",menu=filemenu2)

    filemenu.add_command(label="Sair",command=quit)
    filemenu.add_command(label="Imprimir Dados",command=imprimirTudo)
    filemenu.add_command(label="Ficha do Cliente",command=geraRelatório)
    filemenu2.add_command(label="Info.")

def dashboard():
    #dashboard

    lb_dash = Label(frame_3,text="Relatório Geral",fg ='black',font=('verdana',12,'bold'))
    lb_dash.place(relx=0,rely=0)

    # clientes

    cont = "SELECT COUNT(nome) FROM tb_nomes"
    valor1=banco_2.dql(cont)

    frame_cl = Frame(frame_3,bd=4,background='white')
    frame_cl.place(relx=0,rely=0.08,relwidth=0.3,relheight=0.2)
    lb_cl = Label(frame_cl,text="Número de Clientes",fg ='black',font=('verdana',8,'bold'),background='white')
    lb_cl.place(relx=0.1,rely=0)
    frame_qcl = Frame(frame_cl,background='blue')
    frame_qcl.place(relx=0,rely=0,relwidth=0.05,relheight=0.25)
    lb_cl_valor = Label(frame_cl,text=str(valor1[0][0])+" Clientes",fg ='blue',font=('verdana',10,'bold'),background='white')
    lb_cl_valor.place(relx=0.25,rely=0.5)

    # assinantes

    cont_ass = "SELECT COUNT(nome) FROM tb_nomes WHERE assinante = 'Sim'"
    valor2=banco_2.dql(cont_ass)

    frame_ass = Frame(frame_3,bd=4,background='white')
    frame_ass.place(relx=0.31,rely=0.08,relwidth=0.3,relheight=0.2)
    lb_ass = Label(frame_ass,text="Número Assinantes",fg ='black',font=('verdana',8,'bold'),background='white')
    lb_ass.place(relx=0.1,rely=0)
    frame_qass = Frame(frame_ass,background='blue')
    frame_qass.place(relx=0,rely=0,relwidth=0.05,relheight=0.25)
    lb_ass_valor = Label(frame_ass,text=str(valor2[0][0])+" Assinantes",fg ='blue',font=('verdana',10,'bold'),background='white')
    lb_ass_valor.place(relx=0.25,rely=0.5)

    # clientes/ UF

    uf = "SELECT COUNT(nome),Estado FROM tb_nomes GROUP BY Estado"
    valor3=banco_2.dql(uf)

    frame_uf = Frame(frame_3,bd=4,background='white')
    frame_uf.place(relx=0,rely=0.30,relwidth=1,relheight=0.6)
    lb_uf = Label(frame_uf,text="Clientes / UF",fg ='black',font=('verdana',8,'bold'),background='white')
    lb_uf.place(relx=0.05,rely=0)
    frame_quf = Frame(frame_uf,background='blue')
    frame_quf.place(relx=0,rely=0,relwidth=0.02,relheight=0.13)

    list_est = []
    list_nome = []
    for i in valor3:
        list_nome.append(i[0])
        list_est.append(i[1])

    figura = plt.Figure(figsize=(10, 2.5),dpi=60)
    ax=figura.add_subplot(111)

    ax.bar(list_est,list_nome)
    totals=[]
    c=0
    for i in ax.patches:
        ax.text(i.get_x()+.4,i.get_height()+.05,str(list_nome[c]),fontsize=12,fontstyle='italic',verticalalignment='baseline',color='black')
        c+=1

    ax.patch.set_facecolor('#FFFFFF')
    ax.spines['bottom'].set_color('#CCCCCC')
    ax.spines['bottom'].set_linewidth(1)
    ax.spines['right'].set_linewidth(0)
    ax.spines['top'].set_linewidth(0)
    ax.spines['left'].set_color('#CCCCCC')
    ax.spines['left'].set_linewidth(1)

    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['left'].set_visible(False)
    ax.tick_params(bottom=False,left=False)
    ax.set_axisbelow(True)
    ax.yaxis.grid(True,color='#EEEEEE')
    ax.xaxis.grid(False)

    canva = FigureCanvasTkAgg(figura,frame_uf)
    canva.get_tk_widget().place(relx=0,rely=0.20)

def calendario():
    def print_cal():
        dataIni = calendario1.get_date()
        calendario1.destroy()
        entry_data.delete(0,END)
        entry_data.insert(END,dataIni)
        calData.destroy()

    calendario1 = Calendar(frame_1,fg="gray75",bg="blue",font=("Times",'9','bold'),locale='pt_br')
    calendario1.place(relx=0.4,rely=0.1)
    calData = Button(frame_1,text="Inserir Data",command=print_cal)
    calData.place(relx=0.55,rely=0.85,height=25,width=100)

# INTERFACE INICIAL

pastaApp = os.path.dirname(__file__)

app = tix.Tk()
app.title("Sistema de Cadastros")
app.geometry("700x500")
app.resizable(True,True)
app.maxsize(width=900,height=700)
app.minsize(width=500,height=400)

# ABAS

nb=ttk.Notebook(app)
nb.place(relx=0,rely=0,relwidth=1,relheight=1)

tb1=Frame(nb,background="#dde")
tb2=Frame(nb,background="#dde")

nb.add(tb1,text="Cadastro")
nb.add(tb2,text="Relatório")

# Frames Cadastro
frame_1 = Frame(tb1,bd=4,highlightbackground='black',highlightthickness=3,bg='#99ABB5')
frame_1.place(relx=0.02,rely=0.02,relwidth=0.96,relheight=0.46)

frame_2 = Frame(tb1,bd=4,highlightbackground='black',highlightthickness=3)
frame_2.place(relx=0.02,rely=0.5,relwidth=0.96,relheight=0.46)

frame_3 = Frame(tb2,bd=4,highlightbackground='black',highlightthickness=3)
frame_3.place(relx=0.02,rely=0.02,relwidth=0.96,relheight=0.8)

Menus()

# Progress bar
var_barra=DoubleVar()
var_barra.set(0)

pb=ttk.Progressbar(tb1,variable=var_barra,maximum=100)
pb.place(x=0,y=0,relwidth=1,height=5)

# Botões
canvas_bt = Canvas(frame_1,bd=0,bg='#1e3743',highlightbackground='gray',highlightthickness=5)
canvas_bt.place(relx=0.19,rely=0.08,relwidth=0.22,relheight=0.19)

bt_limpar = Button(frame_1,text="Limpar", bd=2,bg='#107db2',activebackground='#108ecb',activeforeground='white',fg='white',font=('verdana',8,'bold'),command=limpar)
bt_limpar.place(relx=0.2,rely=0.1,relwidth=0.1,relheight=0.15)
bt_buscar = Button(frame_1,text="Buscar", bd=2,bg='#107db2',activebackground='#108ecb',activeforeground='white',fg='white',font=('verdana',8,'bold'),command=buscar)
bt_buscar.place(relx=0.3,rely=0.1,relwidth=0.1,relheight=0.15)

canvas_bt2 = Canvas(frame_1,bd=0,bg='#1e3743',highlightbackground='gray',highlightthickness=5)
canvas_bt2.place(relx=0.59,rely=0.08,relwidth=0.32,relheight=0.19)

bt_novo = Button(frame_1,text="Novo", bd=2,bg='#107db2',activebackground='#108ecb',activeforeground='white',fg='white',font=('verdana',8,'bold'),command=novo)
bt_novo.place(relx=0.6,rely=0.1,relwidth=0.1,relheight=0.15)
bt_alterar = Button(frame_1,text="Alterar", bd=2,bg='#107db2',activebackground='#108ecb',activeforeground='white',fg='white',font=('verdana',8,'bold'),command=alterar)
bt_alterar.place(relx=0.7,rely=0.1,relwidth=0.1,relheight=0.15)
bt_apagar = Button(frame_1,text="Apagar", bd=2,bg='#107db2',activebackground='#108ecb',activeforeground='white',fg='white',font=('verdana',8,'bold'),command=apagar)
bt_apagar.place(relx=0.8,rely=0.1,relwidth=0.1,relheight=0.15)

#Calendario

bt_calendario = Button(frame_1,text="Data",fg="blue",font=('verdana',8,'bold'),command=calendario)
bt_calendario.place(relx=0.05,rely=0.87,relwidth=0.08,relheight=0.1)
entry_data = Entry(frame_1,width=10)
entry_data.place(relx=0.14,rely=0.87,relwidth=0.1,relheight=0.1)

#balloons

balao_limpar = tix.Balloon(frame_1)
balao_limpar.bind_widget(bt_limpar,balloonmsg="Limpar todos as caixas de texto")
balao_buscar = tix.Balloon(frame_1)
balao_buscar.bind_widget(bt_buscar,balloonmsg="Filtra a tabela com os valores da entrada 'Nome'")
balao_novo = tix.Balloon(frame_1)
balao_novo.bind_widget(bt_novo,balloonmsg="Adiciona os valores digitados ao banco de dados")
balao_alterar = tix.Balloon(frame_1)
balao_alterar .bind_widget(bt_alterar,balloonmsg="Edita os valores de cadastro do cliente selecionado")
balao_apagar = tix.Balloon(frame_1)
balao_apagar .bind_widget(bt_apagar,balloonmsg="Deleta o cliente selecionado do banco de dados")

# Labels e Entrys

dashboard()

# codigo
lb_codigo = Label(frame_1,text="Código",fg='darkblue',bg='#99ABB5')
lb_codigo.place(relx=0.05,rely=0.05)

codigo_entry = Entry(frame_1)
codigo_entry.place(relx=0.05,rely=0.15,relwidth=0.08)

# nome

lb_nome = Label(frame_1,text="Nome",fg='darkblue',bg='#99ABB5')
lb_nome.place(relx=0.05,rely=0.35)

nome_entry = Entry(frame_1)
nome_entry.place(relx=0.05,rely=0.45,relwidth=0.5)

# telefone

lb_telefone = Label(frame_1,text="Telefone",fg='darkblue',bg='#99ABB5')
lb_telefone.place(relx=0.05,rely=0.6)

telefone_entry = Entry(frame_1)
telefone_entry.place(relx=0.05,rely=0.7,relwidth=0.2)

# cidade

lb_cidade = Label(frame_1,text="Cidade",fg='darkblue',bg='#99ABB5')
lb_cidade.place(relx=0.45,rely=0.60)

cidade_entry = Entry(frame_1)
cidade_entry.place(relx=0.45,rely=0.7,relwidth=0.3)

# estado

listEstados=[" ","AC","AL","AM","AP","BA","CE","DF","ES","GO","MA","MG","MS","MT","PA","PB","PE","PI","PR","RJ","RN","RO","RR","RS","SC","SE","SP","TO"]

lb_estado = Label(frame_1,text="Estado",fg='darkblue',bg='#99ABB5')
lb_estado.place(relx=0.3,rely=0.60)

estado_entry=ttk.Combobox(frame_1,values=listEstados,state="readonly")
estado_entry.set(" ")
estado_entry.place(relx=0.3,rely=0.7,relwidth=0.1)

# assinante

vass=IntVar()

lb_ass = Label(frame_1,text="Assinante?",fg='darkblue',bg='#99ABB5')
lb_ass.place(relx=0.64,rely=0.35)

text1 = StringVar()

text1.set('Não')

cb_assin=Checkbutton(frame_1, textvariable = text1, variable = text1,
                          offvalue = 'Não',
                          onvalue = 'Sim',bg='#99ABB5')
cb_assin.place(relx=0.64,rely=0.45)

# treeview Cadastro
tv2=ttk.Treeview(frame_2,columns=('id','nome','fone','estado','cidade','assinante','data'),show='headings',height=10)
tv2.column('id',minwidth=0,width=30)
tv2.column('nome',minwidth=0,width=100)
tv2.column('fone',minwidth=0,width=115)
tv2.column('estado',minwidth=0,width=50)
tv2.column('cidade',minwidth=0,width=80)
tv2.column('assinante',minwidth=0,width=20)
tv2.column('data',minwidth=0,width=40)
tv2.heading('id',text='ID')
tv2.heading('nome',text='Nome')
tv2.heading('fone',text='Telefone')
tv2.heading('estado',text='Estado')
tv2.heading('cidade',text='Cidade')
tv2.heading('assinante',text='Ass.')
tv2.heading('data',text='Data')
tv2.place(relx=0.01,rely=0.01,relwidth=0.95,relheight=0.98)
scroll = Scrollbar(frame_2,orient='vertical')
tv2.configure(yscroll=scroll.set)
scroll.place(relx=0.96,rely=0.01,relwidth=0.04,relheight=0.98)
popular2()
tv2.bind("<Double-1>",duploClick)

# FIM
app.mainloop()


import abc
import datetime
import os
import shutil
import time

from grocsvs import log

class StepChunk(object):
    __metaclass__ = abc.ABCMeta

    @staticmethod
    def get_steps(options):
        """ """
        raise Exception("this abstract staticmethod needs to be instantiated by subclasses")

    @abc.abstractmethod
    def __init__(self, options, **kwdargs):
        """
        must take as input the options instance and a dict of arguments
        """
        pass

    @abc.abstractmethod
    def outpaths(self, final):
        """
        return a dictionary of names -> output paths; final indicates if the
        paths should be for the temporary, working version (final=False) or the
        completed final version (final=True)
        """
        return

    @abc.abstractmethod
    def run(self):
        """
        actually runs the pipeline step on the arguments, given the options,
        defined in __init__
        """
        pass

    def __str__(self):
        raise Exception("not implemented")

    @property
    def results_dir(self):
        return self._results_dir(self.options)#return os.path.join(self.options.results_dir, self.__class__.__name__)

    @classmethod
    def _results_dir(cls, options):
        # private method to allow classmethod-level access to results_dir
        return os.path.join(options.results_dir, cls.__name__)

    @property
    def working_dir(self):
        return self._working_dir(self.options)#os.path.join(self.options.working_dir, self.__class__.__name__)

    @classmethod
    def _working_dir(cls, options):
        return os.path.join(options.working_dir, cls.__name__)

    @property
    def log_path(self):
        return os.path.join(self.options.log_dir, self.__class__.__name__, str(self))

    def start_logging(self):
        self.logger = log.Logger(self.log_path)
        self.logger.log("-- starting logging {} --".format(str(self)))
        self._start_time = time.time()

    def stop_logging(self):
        elapsed = time.time() - self._start_time
        self.logger.log("-> finished running step; time elapsed: {}".format(datetime.timedelta(seconds=elapsed)))
        self.logger.log("-- stopping logging --")

    @classmethod
    def clean_all_steps(cls, options):
        import logging
        results_dir = cls._results_dir(options)
        if os.path.exists(results_dir):
            logging.info("Removing {}".format(results_dir))
            shutil.rmtree(results_dir)

        working_dir = cls._working_dir(options)
        if os.path.exists(working_dir):
            logging.info("Removing {}".format(working_dir))
            shutil.rmtree(working_dir)

        log_dir = os.path.join(options.log_dir, cls.__name__)
        if os.path.exists(log_dir):
            logging.info("Removing {}".format(log_dir))
            shutil.rmtree(log_dir)

    def needs_to_run(self):
        """ checks if any of the output files are missing """
        paths = self.outpaths(final=True)
        for name, path in paths.items():
            if not os.path.exists(path):
                return True

        return False

    def finalize(self):
        temp_paths = self.outpaths(final=False)
        final_paths = self.outpaths(final=True)

        for key, working_path in temp_paths.items():
            if working_path != final_paths[key]:
                if not os.path.exists(working_path):
                    raise Exception("{} step failed to produce output: {}".format(
                        self.__class__.__name__, working_path))
                shutil.move(working_path, final_paths[key])
                time.sleep(0.1)

        assert not self.needs_to_run()