#!/usr/bin/env python
# Collecting video thumbnails URLs
# Written by Yu-Jie Lin in 2016
# Public domain, or via UNLICENSE if not applicable.

from __future__ import print_function

import argparse
import json
import logging
import os
import re
import sys
import urllib2
from datetime import datetime as dt
from os import path

import httplib2
from PIL import Image, ImageDraw, ImageEnhance, ImageFont

from apiclient.discovery import build
from oauth2client.client import OAuth2WebServerFlow
from oauth2client.file import Storage as BaseStorage
from oauth2client.tools import argparser, run_flow

__project__ = 'READYT1k'
__description__ = 'Collecting video thumbnails URLs'

API_STORAGE = '%s.dat' % __project__
DEFAULT_JSON = '%s.json' % __project__
MAX_RESULTS = 50

W, H = 1920, 1080
FONT_PATH = path.expanduser('~/.fonts/Envy Code R/Envy Code R.ttf')
FONTS = {
  16: ImageFont.truetype(FONT_PATH, 16),
  32: ImageFont.truetype(FONT_PATH, 32),
  128: ImageFont.truetype(FONT_PATH, 128),
}
FILE_BANNER = '../banner/banner.png'
CREDIT = 'Music: For Mimi by Twin Musicom <http://www.twinmusicom.org/> (CC BY 4.0)'

LOGGING_FORMAT = (
  '[%(asctime)s]'
  '[%(levelname)8s]'
  '[%(name)s:%(funcName)s:%(lineno)3d] '
  '%(message)s'
)

class Storage(BaseStorage):
  """Inherit the API Storage to suppress CredentialsFileSymbolicLinkError
  """

  def __init__(self, filename):

    super(Storage, self).__init__(filename)
    self._filename_link_warned = False

  def _validate_file(self):

    if os.path.islink(self._filename) and not self._filename_link_warned:
      print('File: %s is a symbolic link.' % self._filename)
      self._filename_link_warned = True

def auth(filename):

  # JSON from Google Developer Console / Credentials
  with open('READYT1k.secret.json') as f:
    secret = json.load(f)
  FLOW = OAuth2WebServerFlow(
    secret['installed']['client_id'],
    secret['installed']['client_secret'],
    'https://www.googleapis.com/auth/youtube',
    auth_uri='https://accounts.google.com/o/oauth2/auth',
    token_uri='https://accounts.google.com/o/oauth2/token',
  )

  storage = Storage(filename)
  credentials = storage.get()
  if credentials is None or credentials.invalid:
    credentials = run_flow(FLOW, storage, argparser.parse_args([]))

  http = httplib2.Http()
  return build("youtube", "v3", http=credentials.authorize(http))

def get_upload_playlistId(yt):

  req = yt.channels().list(part='contentDetails', mine=True)
  resp = req.execute()

  return resp['items'][0]['contentDetails']['relatedPlaylists']['uploads']

def process_video(yt, video):

  v_id = video['snippet']['resourceId']['videoId']
  logging.info('processing: %s', v_id)

  # obtain tags and duration and privacy setting
  part = 'id,contentDetails,snippet,status'
  req = yt.videos().list(part=part, id=v_id)
  resp = req.execute()

  v = resp['items'][0]
  s = v['snippet']
  t = s['thumbnails']
  t_opts = ('maxres', 'standard', 'high', 'medium', 'default')
  t_url = list(t[k]['url'] for k in t_opts if k in t)[0]

  logging.info('extracting: %s', s['title'])

  return {
    'id': v_id,
    'snippet': {
      'publishedAt': s['publishedAt'],
      'title': s['title'],
      'thumbnail': t_url,
      'tags': s.get('tags', []),
    },
    'contentDetails': {
      'duration': v['contentDetails']['duration'],
    },
    'status': {
      'privacyStatus': v['status']['privacyStatus'],
    },
  }

def update(yt, data):

  playlistId = data.get('playlistId', None)
  if not playlistId:
    logging.debug('playlistId not in data')
    playlistId = get_upload_playlistId(yt)
    data['playlistId'] = playlistId
  logging.debug('playlistId: %s', playlistId)

  token = None
  # from oldest to newest
  videos = data.get('items', [])
  ids = [v['id'] for v in videos]

  new_videos = []
  req = yt.playlistItems().list(
    part='snippet',
    playlistId=playlistId,
    maxResults=MAX_RESULTS,
    pageToken=token,
  )
  while req:
    resp = req.execute()
    items = resp['items']
    logging.debug('returned videos: %d', len(items))

    for video in items:
      if video['snippet']['resourceId']['videoId'] in ids:
        req = None
        break
      new_videos.append(process_video(yt, video))
    if req:
      req = yt.playlistItems().list_next(req, resp)

  new_videos = sorted(new_videos, key=lambda v: v['snippet']['publishedAt'])
  logging.info('new videos: %d', len(new_videos))

  videos += new_videos
  data['items'] = videos

def filter_videos(videos):

  g = (v for v in videos if v['status']['privacyStatus'] != 'private')
  g = (v for v in g if v['contentDetails']['duration'] != 'PT0S')

  return list(g)

def get_thumb_filename(p, v):

  return path.join(p, '%s.jpg' % v['id'])

def resize_image(im):

  w, h = im.size
  if w == W and h == H:
    return im

  scale = max(1.0 * W / w, 1.0 * H / h)
  new_size = int(scale * w), int(scale * h)
  im = im.resize(new_size, Image.NEAREST)

  x, y = abs(W - new_size[0]) // 2, abs(H - new_size[1]) // 2
  box = (x, y, x + W, y + H)
  im = im.crop(box)

  return im

def render_round_rect(d, r, x1, y1, x2, y2, fill):

  R = 2 * r
  d.rectangle((x1, y1 + r, x2, y2 - r), fill=fill)
  d.rectangle((x1 + r, y1, x2 - r, y2), fill=fill)
  d.ellipse((x1, y1, x1 + R, y1 + R), fill=fill)  # top-left
  d.ellipse((x2 - R, y1, x2, y1 + R), fill=fill)  # top-right
  d.ellipse((x1, y2 - R, x1 + R, y2), fill=fill)  # bottom-left
  d.ellipse((x2 - R, y2 - R, x2, y2), fill=fill)  # bottom-right

def format_time(t):

  d = t / 86400
  t -= d * 86400
  h = t / 3600
  t -= h * 3600
  m = t / 60
  t -= m * 60
  s = t

  text = '%d day%s %2d hour%s %2d minute%s %2d second%s' % (
    d, ' ' if d == 1 else 's',
    h, ' ' if h == 1 else 's',
    m, ' ' if m == 1 else 's',
    s, ' ' if s == 1 else 's',
  )

  return text

def render_frame(im, i, n, v, acc_time, tags):

  im = im.convert('RGBA')
  enc = ImageEnhance.Brightness(im)
  im = enc.enhance(0.5 + 0.5 * (i / n))

  txt = Image.new('RGBA', im.size, (255, 255, 255, 0))
  d = ImageDraw.Draw(txt)

  # render tags

  for j in range(len(tags)):

    font = FONTS[32]
    tag = tags[j][0]
    size = d.textsize(tag, font=font)
    x = 20
    y = 20 + j * 48

    fill = (255, 255, 255, 192)
    r = 8
    render_round_rect(d, r, x - r, y, x + size[0] + r, y + 40, fill)

    fill = (16, 16, 16, 228)
    d.text((x, y), tag, font=font, fill=fill)

  # render text #

  font = FONTS[128]
  size = d.textsize(str(i), font=font)
  x = (W - size[0]) // 2
  y = (H - size[1]) // 2

  fill = (128, 128, 128, 160)
  r = 32
  render_round_rect(d, r, x - r, y, x + size[0] + r, y + size[1] + r, fill)

  fill = (255, 255, 255, 228)
  d.text((x, y), str(i), font=font, fill=fill)

  # render text date

  font = FONTS[32]
  pubdate = v['snippet']['publishedAt'][:10]
  size = d.textsize(pubdate, font=font)
  x = (W - size[0]) // 2
  y = (H - size[1]) // 2 + 128

  fill = (128, 128, 128, 160)
  r = 8
  render_round_rect(d, r, x - r, y, x + size[0] + r, y + size[1] + r, fill)

  fill = (255, 255, 255, 228)
  d.text((x, y), pubdate, font=font, fill=fill)

  # render acc_time

  font = FONTS[32]
  fmt_atime = format_time(acc_time)
  size = d.textsize(fmt_atime, font=font)
  x = (W - size[0]) // 2
  y = (H - size[1]) // 2 + 128 + 64

  fill = (255, 255, 255, 160)
  r = 8
  render_round_rect(d, r, x - r, y, x + size[0] + r, y + 40, fill)

  fill = (32, 32, 32, 228)
  d.text((x, y), fmt_atime, font=font, fill=fill)

  # render progress bar

  bar = Image.new('RGBA', im.size, (255, 255, 255, 0))
  d = ImageDraw.Draw(bar)
  gout = 20
  w_barout = W - 2 * gout
  h_barout = 32
  gin = 4
  w_barin = w_barout - 2 * gin

  fill = (128, 128, 128, 128)
  d.rectangle((gout, H - h_barout - gout, W - gout, H - gout), fill=fill)

  fill = (192, 192, 192, 228)
  w = 1.0 * w_barin * i // n
  p = (gout + gin, H - h_barout - gout + gin, gout + gin + w, H - gout - gin)
  d.rectangle(p, fill=fill)

  # merge into image

  im = Image.alpha_composite(im, txt)
  im = Image.alpha_composite(im, bar)

  return im

def add_tags(tags, new_tags):

  for t in new_tags:
    tags[t] = tags.get(t, 0) + 1

def top_tags(tags, N):

  tags = zip(tags.keys(), tags.values())
  tags = sorted(tags, key=lambda t: t[1], reverse=True)
  return tags[:N]

def main():

  logger = logging.getLogger()

  p = argparse.ArgumentParser(description=__description__)
  p.add_argument('-d', '--debug', action='store_true')
  p.add_argument('-s', '--storage', default=API_STORAGE,
                 help='the credential file (default: %(default)s)')
  p.add_argument('-j', '--json', default=DEFAULT_JSON,
                 help='cached videos in JSON (default: %(default)s)')
  p.add_argument('--auth', action='store_true',
                 help='authorize the script and exit')
  p.add_argument('-u', '--update', action='store_true',
                 help='update the list')
  p.add_argument('-c', '--count', action='store_true',
                 help='count the filtered list')
  p.add_argument('-D', '--download', type=str, metavar='DIR',
                 help='download thumbnails of videos from the filtered list')
  p.add_argument('-g', '--generate', type=str, metavar='DIR',
                 help='download thumbnails of videos from the filtered list')
  args = p.parse_args()

  logging.basicConfig(format=LOGGING_FORMAT)
  if args.debug:
    logger.setLevel(logging.DEBUG)
  else:
    logger.setLevel(logging.INFO)

  if args.auth or args.update:
    yt = auth(args.storage)
    if args.auth:
      return

  data = {}
  if path.exists(args.json):
    with open(args.json) as f:
      data = json.load(f)
    logging.debug('%s loaded' % args.json)

  if args.update:
    update(yt, data)
    with open(args.json, 'w') as f:
      json.dump(data, f)
  videos = data.get('items', [])
  logging.info('total videos: %d', len(videos))

  if args.count or args.download or args.generate:
    videos = filter_videos(videos)
    logging.info('filtered videos: %d', len(videos))

    if args.download:
      if not path.exists(args.download):
        os.makedirs(args.download)

      for v in videos:
        filename = get_thumb_filename(args.download, v)
        if path.exists(filename):
          continue
        thumb = v['snippet']['thumbnail']
        logging.info('downloading %s' % thumb)
        uf = None
        try:
          uf = urllib2.urlopen(thumb)
          with open(filename, 'wb') as f:
            f.write(uf.read())
          logging.info('saved: %s' % filename)
        except Exception as e:
          raise e
        finally:
          if uf:
            uf.close()

    if args.generate:
      if not args.download:
        logging.error('--download is required')
        sys.exit(1)
      if not path.exists(args.generate):
        os.makedirs(args.generate)

      t_open = resize_image(Image.open('READYT1k.png')).convert('RGBA')

      # rending opening frames

      filename = path.join(args.generate, 'open%04d.png' % 1)
      logging.debug('saving frame: %s' % filename)
      t_open.save(filename)

      dots = 5
      for i in range(dots):
        t = t_open.copy()
        d = ImageDraw.Draw(t)
        x = W // 2
        y = H - H // 4
        r = 40
        D = 180
        fill = (192, 192, 192, 64)
        for j in range(dots - i):
          p = (x - r + (j - 2) * D, y - r, x + r + (j - 2) * D, y + r)
          d.ellipse(p, fill=fill)

        filename = path.join(args.generate, 'open%04d.png' % (i + 2))
        logging.debug('saving frame: %s' % filename)
        t.save(filename)

      # render main frames

      # n = len(videos)
      n = 1000
      acc_time = 0
      tags = {}
      RE_T = re.compile('PT((?P<m>\d+)M)?((?P<s>\d+)S)?')
      for i in range(n):
        if i < len(videos):
          v = videos[i]
        else:
          v = {
            'contentDetails': {
              'duration': 'PT1M3S',
            },
            'snippet': {
              'publishedAt': '2016-02-03',
              'tags': [
                'test',
              ],
            },
          }
        m = RE_T.match(v['contentDetails']['duration'])
        acc_time += int(m.group('m') or 0) * 60 + int(m.group('s') or 0)
        add_tags(tags, v['snippet']['tags'])

        if 'id' in v:
          t = Image.open(get_thumb_filename(args.download, v))
        else:
          t = Image.new('RGBA', (W, H), (255, 255, 255, 0))
        t = resize_image(t)
        t = render_frame(t, i + 1, n, v, acc_time, top_tags(tags, 20))

        filename = path.join(args.generate, 'main%04d.png' % (i + 1))
        logging.debug('saving frame: %s' % filename)
        t.save(filename)

      drange = (
        videos[0]['snippet']['publishedAt'][:10],
        videos[n - 1]['snippet']['publishedAt'][:10],
      )
      dfmt = '%Y-%m-%d'
      ddiff = dt.strptime(drange[1], dfmt) - dt.strptime(drange[0], dfmt)
      fmt = 'published date range: %s to %s (%d days)'
      logging.info(fmt % (drange[0], drange[1], ddiff.days))
      logging.info('accumulated runtime: %s' % format_time(acc_time))
      total_tags = sum(tags.values())
      avg_tags = total_tags / n
      logging.info('total tags: %d (avg. %d/video)' % (total_tags, avg_tags))
      for i, tag in enumerate(top_tags(tags, 5)):
        logging.info('#%d tag: %s (%d)' % (i + 1, tag[0], tag[1]))

      # render ending frames

      t_banner = resize_image(Image.open(FILE_BANNER)).convert('RGBA')
      last_t = t

      # add audio credit
      d = ImageDraw.Draw(t_banner)
      size = d.textsize(CREDIT, font=FONTS[32])
      x, y = (W - size[0] - 20, H - size[1] - 20)
      fill = (128, 128, 128, 160)
      r = 8
      render_round_rect(d, r, x - r / 2, y, x + size[0], y + size[1] + r / 4, fill)
      fill = (255, 255, 255, 228)
      d.text((x, y), CREDIT, font=FONTS[32], fill=fill)

      end_frames = 100 - 1
      for i in range(end_frames + 1):
        t = Image.blend(last_t, t_banner, 1.0 * i / end_frames)

        filename = path.join(args.generate, 'end%04d.png' % (i + 1))
        logging.debug('saving frame: %s' % filename)
        t.save(filename)

if __name__ == '__main__':
  main()


"""Tests for google3.third_party.java_src.tsunami.plugin_server.py.common.net.requests_http_client."""

from unittest import mock

from absl.testing import absltest
from absl.testing import parameterized
import requests

from tsunami.plugin_server.py.common.net.http.host_resolver_http_adapter import HostResolverHttpAdapter
from tsunami.plugin_server.py.common.net.http.http_header_fields import HttpHeaderFields
from tsunami.plugin_server.py.common.net.http.http_method import HttpMethod

class HostResolverHttpAdapterTest(parameterized.TestCase):

  @classmethod
  def setUpClass(cls):
    super().setUpClass()
    cls.custom_adapter = HostResolverHttpAdapter(5, 10)

  def setUp(self):
    super().setUp()
    self.addCleanup(mock.patch.stopall)
    # Mock of requests's HTTPAdapter
    response = requests.Response()
    response.status_code = 200
    mock.patch.object(
        requests.adapters.HTTPAdapter,
        'send',
        return_value=response,
    ).start()
    # Mock hostname lookup
    self.mock_getaddrinfo = mock.patch('socket.getaddrinfo').start()

  @parameterized.named_parameters(
      ('with_hostname', 'vuln-app.com'),
      ('with_ipv4', '199.21.82.88'),
      (
          'with_ipv6',
          '[2001:0db8:85a3:0000:0000:8a2e:0370:7334]',
      ),
  )
  def test_send_dispatches_with_host_header(self, host):
    url = 'http://{}:8080/send'.format(host)
    request = self._prepare_request(url)

    self.custom_adapter.send(request)

    requests.adapters.HTTPAdapter.send.assert_called_with(request)
    self.assertEqual(
        request.headers.get(HttpHeaderFields.HOST.value), '{}:8080'.format(host)
    )

  def test_send_without_target_ip_dispatches_default_hostname(self):
    url = 'http://vuln-app.com:8080/send'
    request = self._prepare_request(url)

    self.custom_adapter.send(request)

    requests.adapters.HTTPAdapter.send.assert_called_with(request)
    self.assertEqual(request.url, url)

  def test_send_when_hostname_resolves_to_ip_uses_default_hostname(self):
    url = 'http://vuln-app.com:8080/send'
    ip = '199.21.82.88'
    request = self._prepare_request(url)

    self.mock_getaddrinfo.return_value = [ip]
    self.custom_adapter.send(request, ip=ip)

    requests.adapters.HTTPAdapter.send.assert_called_once_with(request)
    self.assertEqual(
        request.headers.get(HttpHeaderFields.HOST.value), 'vuln-app.com:8080'
    )
    self.assertEqual(request.url, url)

  def test_send_when_hostname_is_the_ip_uses_default_hostname(self):
    ip = '2001:0db8:85a3:0000:0000:8a2e:0370:7334'
    url = 'http://[{}]:8080/send'.format(ip)
    request = self._prepare_request(url)

    self.custom_adapter.send(request, ip=ip)

    requests.adapters.HTTPAdapter.send.assert_called_once_with(request)
    self.assertEqual(
        request.headers.get(HttpHeaderFields.HOST.value),
        '[{}]:8080'.format(ip))
    self.assertEqual(request.url, url)

  def test_send_when_hostname_is_case_insensitive(self):
    url = 'http://vuln-APP.com:8080/send'
    ip = '199.21.82.88'
    request = self._prepare_request(url)

    self.custom_adapter.send(request, ip=ip)

    requests.adapters.HTTPAdapter.send.assert_called_once_with(request)
    self.assertEqual(
        request.headers.get(HttpHeaderFields.HOST.value), 'vuln-APP.com:8080'
    )
    self.assertEqual(request.url, 'http://199.21.82.88:8080/send')

  def test_send_when_hostname_does_not_resolve_to_ipv4_uses_ipv4(self):
    url = 'http://vuln-app.com:8080/send'
    ip = '199.21.82.88'
    request = self._prepare_request(url)

    self.mock_getaddrinfo.return_value = ['1.1.1.1']
    self.custom_adapter.send(request, ip=ip)

    requests.adapters.HTTPAdapter.send.assert_called_once_with(request)
    self.assertEqual(
        request.headers.get(HttpHeaderFields.HOST.value), 'vuln-app.com:8080'
    )
    self.assertEqual(request.url, 'http://199.21.82.88:8080/send')

  def test_send_when_hostname_does_not_resolve_to_ipv6_uses_ipv6(self):
    url = 'http://vuln-app.com:8080/send'
    ip = '2001:0db8:85a3:0000:0000:8a2e:0370:7334'
    request = self._prepare_request(url)

    self.mock_getaddrinfo.return_value = ['1.1.1.1']
    self.custom_adapter.send(request, ip=ip)

    requests.adapters.HTTPAdapter.send.assert_called_once_with(request)
    self.assertEqual(
        request.headers.get(HttpHeaderFields.HOST.value), 'vuln-app.com:8080'
    )
    self.assertEqual(
        request.url,
        'http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8080/send',
    )

  def _prepare_request(self, url):
    request = requests.Request(
        method=HttpMethod.GET, url=url, data=b'HTML content'
    )
    request = request.prepare()
    request.url = url
    return request

if __name__ == '__main__':
  absltest.main()


import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
import tokens
import discord
from discord.ext import commands
from discord import app_commands
import datetime
import pytz
import requests
import asyncio

apiurl="https://developer-lostark.game.onstove.com/gamecontents/challenge-abyss-dungeons"
key=tokens.apikey

class GameContents(commands.Cog):
    def __init__(self, bot) -> None:
        self.bot = bot

    @app_commands.command(name="도비스", description="금주의 도전 어비스 던전을 확인합니다.")
    async def abyss(self, interaction: discord.Interaction) -> None:
        embed = self.abyssofweek()
        await interaction.response.send_message(embed=embed)

    def abyssofweek(self):
        url=apiurl
        headers={'accept': 'application/json', 'authorization': 'bearer ' + key,'Content-Type':'application/json'}
        try:
            response=requests.get(url,headers=headers,verify=False).json()
            dungeonlist=[]
            for dungeon in response:
                dungeonlist.append(dungeon.get("Name"))
            embed=discord.Embed(title="**:calendar_spiral: 금주의 도전 어비스 던전**", timestamp=datetime.datetime.now(pytz.timezone('UTC')), color=0x00ff00)
            embed.add_field(name=f':video_game: {str(dungeonlist[0])}',value="\n",inline="False")
            embed.add_field(name=f':video_game: {str(dungeonlist[1])}',value="\n",inline="False")
            embed.set_footer(text="Made by.ㅈㅇㅈ#0081")
            return embed
        except Exception as e:
            embed=discord.Embed(title="오류 발생",timestamp=datetime.datetime.now(pytz.timezone('UTC')),color=0x9B0000)
            embed.add_field(name=e,value="\n",inline=False)
            return embed

async def setup(bot) -> None:
    await bot.add_cog(GameContents(bot))


# This file contains basic operations on PGM files
# read, write, generating histogram, cumulative histogram
# Calculating the mean and the standard deviation
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
from PIL import Image
import tkinter as tk
import PIL.ImageTk
from collections import Counter

# this function takes a pgm file and return a 2 dimensions numpy array
def readPGM(name):
    pgmf = open(name, 'rb')
    lines = pgmf.readlines()
    i = 1
    while True:
        if lines[i].decode()[0] == "#":
            del lines[i]
        else:
            break
    (width, height) = [int(i) for i in lines[1].split()]
    (maxGreyLevel) = [int(i)
                      for i in lines[2] .split()]  # idk why it returns an array
    i = 0
    for line in lines[3:]:
        line = line.decode()
        row = []
        numbers = [int(i) for i in line.split()]
        if i == 0:
            arr = np.copy(numbers)
        else:
            arr = np.append(arr, np.array(numbers))
        i += 1

    arr.resize((height, width))
    return (height, width, maxGreyLevel[0], arr)

# write a pgm file from a 2d numpy array
def writePGM(fileName, arr, width, height, maxGreyLevel):
    pgmf = open(fileName, 'w')
    pgmf.write("P2\n")
    pgmf.write(str(width)+" ")
    pgmf.write(str(height)+"\n")
    pgmf.write(str(maxGreyLevel)+"\n")
    for line in arr:
        ch = np.array2string(line)
        ch = ch[1:len(ch)-1]
        pgmf.write(ch)
        pgmf.write("\n")

# return the mean (moyenne) and the standard deviation (écart type)
def meanSTD(arr):
    return (arr.mean(), np.std(arr))

# return a dictionary representing the histogram

def histogram(arr, maxGreyLevel):
    hist = Counter(arr.flatten())
    cumulativeHist = np.zeros(maxGreyLevel+1, dtype=int)
    # print(sorted(hist.items()))
    for k in hist:
        cumulativeHist[k] = hist[k]
    i = 1
    while i < maxGreyLevel+1:
        cumulativeHist[i] += cumulativeHist[i-1]
        i += 1
    return (hist, cumulativeHist)

def plotHistogram(hist):
    plt.clf()
    plt.bar(hist.keys(), hist.values())
    plt.show(block=False)

def plotCumulativeHist(cumulativeHist,maxGreyLevel):
    plt.clf()
    plt.bar(np.arange(maxGreyLevel+1), cumulativeHist)
    plt.show(block=False)

def plotImage(arr):
    plt.clf()
    plt.imshow(Image.fromarray(arr))
    plt.show(block=False)
# print(np.shape(arr))
# print(np.shape(arr[0]))

# a = np.array([[1,2],[3,4]])
# print(a)
# print(a.flatten())

# -------------------- window ---------------------
# window = tkinter.Tk()

# window.title("OpenCV and Tkinter")
# canvas = tkinter.Canvas(window, width=width, height=height)
# canvas.pack()
# image = plt.imshow(Image.fromarray(arr))
# image = PIL.ImageTk.PhotoImage(image=PIL.Image.fromarray(arr))
# canvas.create_image(0, 0, image=image, anchor=tkinter.NW)
# window.mainloop()
# -------------------- window ---------------------

# plt.show()


import torch
from torch import nn
from torch.nn import functional as F
from torch.nn.utils.rnn import pack_padded_sequence,pad_packed_sequence

DEVICE = torch.device('cuda')

class model(nn.Module):
  # initialize model
  def __init__(self,args):
    super(model, self).__init__()
    outtoks, embsize, hsize = args.ntoks, args.esz, args.hsz
    self.outemb = nn.Embedding(outtoks,args.esz)
    self.posemb = nn.Embedding(args.posembs,args.esz)
    self.qemb = nn.Embedding(args.rtoks,args.edgesz)
    self.vtoks = args.vtoks
    self.dlstm = nn.LSTMCell(args.esz*2 + args.edgesz, hsize)
    self.outlin = nn.Linear(hsize, outtoks)
    self.entemb_drop = nn.Dropout(args.drop)

    self.sigmoid = nn.Sigmoid()
    self.tanh = nn.Tanh()
    self.maxlen = args.max
    self.outtoks = outtoks
    self.endtok = args.endtok
    self.edgesz = args.edgesz
    self.embsize = embsize
    self.hsize = hsize

  # lstm decoder
  def decode_single(self, k, h, c):
    h, c = self.dlstm(k,(h,c))
    decoded = self.outlin(h.contiguous())
    return decoded, h, c
  # language model
  def langModel(self, outp, pos, q, h=None, c=None):

    outp, outlen = outp
    pos, poslen = pos
    q, qlen = q

    outp = self.outemb(outp)
    outp = self.entemb_drop(outp)
    pos = self.posemb(pos)
    pos = self.entemb_drop(pos)

    q = self.qemb(q)
    q = self.entemb_drop(q)

    decin = torch.cat([outp,pos,q],dim=-1)
    decin = decin.transpose(0,1)

    h  = torch.zeros(outp.size(0), self.hsize).to(DEVICE)
    c = torch.zeros(outp.size(0), self.hsize).to(DEVICE)

    outputs = []
    for k in decin:
        o, h, c = self.decode_single(k, h, c)
        outputs.append(o)
    o = torch.stack(outputs,1)
    return o

  def forward(self,outp,pos,q):

    o = self.langModel(outp,pos,q)

    return o


import io
import os
import shutil
from datetime import datetime
from functools import reduce
from typing import Optional, Union

import pandas as pd
import torch
import torch.nn as nn
from PIL import Image
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
from torch.optim import Optimizer

from sidewalk_widths_extractor.typing import _DEVICE, _PATH

def load_checkpoint(source_path: _PATH, module: Union[nn.Module, Optimizer]) -> Union[None, int]:
    """Load a nn.Module checkpoint.

    Args:
        source_path (_PATH): the path to the checkpoint file.
        module (nn.Module): nn.Module to-be-loaded from.

    Returns:
        Union[None, int]: returns the epoch number if there is one, otherwise return None.
    """
    d = torch.load(source_path)
    module.load_state_dict(d["state_dict"])
    return d["epoch"] if "epoch" in d else None

def save_checkpoint(
    target_path: _PATH,
    module: Union[nn.Module, Optimizer],
    epoch: Optional[int] = None,
    filename: str = "checkpoint",
    format_type: str = ".pth.tar",
) -> None:
    """Save a nn.Module checkpoint.

    Args:
        target_path (_PATH): the path to the target checkpoint file.
        module (nn.Module): nn.Module to-be-saved.
        epoch (Optional[int], optional): The epoch number. Defaults to None.
        filename (str, optional): checkpoint file's name. Defaults to "checkpoint".
        format_type (str, optional): file format. Defaults to ".pth.tar".
    """
    mkdir(target_path)
    cp = {"state_dict": module.state_dict()}
    if epoch:
        cp["epoch"] = epoch
    torch.save(
        cp,
        os.path.join(target_path, filename + format_type),
    )

def mkdir(target_path: _PATH) -> None:
    """
    Create a directory if it doesn't exist.

    Args:
        target_path (_PATH): The path to the directory
    """
    if not os.path.exists(target_path):
        os.makedirs(target_path)

def archive(source_path: _PATH, target_path: _PATH, archive_format: str = "zip") -> None:
    """
    Archive the given folder.

    Args:
        source_path (_PATH): The path to the source folder.
        target_path (_PATH): The path to the target folder.
        archive_format (str): The archive format
    """
    base = os.path.basename(target_path)
    name = base.split(".")[0]
    archive_from = os.path.dirname(source_path)
    archive_to = os.path.basename(source_path.strip(os.sep))
    shutil.make_archive(name, archive_format, archive_from, archive_to)
    shutil.move(f"{name}.{archive_format}", target_path)

def get_device() -> _DEVICE:
    """
    Get the available device.

    Returns:
        _DEVICE: device identifier
    """
    return (
        torch.device(f"cuda:{torch.cuda.current_device()}") if torch.cuda.is_available() else "cpu"
    )

def create_path_with_timestamp(
    root: Optional[_PATH], prefix: Optional[str] = None, postfix: Optional[str] = None
) -> str:
    """Create a path with current timestamp.

    Args:
        root (Optional[_PATH]): root path.
        prefix (Optional[str], optional): string to be added before the timestamp. Defaults to None.
        postfix (Optional[str], optional): string to be added after the timestamp. Defaults to None.

    Returns:
        str: path string
    """
    timestamp = datetime.now().strftime(r"%d-%m-%Y %H-%M-%S")
    out, name = [], None
    if root:
        out.append(root)
    if prefix:
        name = prefix
    if name:
        name += " " + timestamp
    else:
        name = timestamp
    if postfix:
        name += " " + postfix
    out.append(name)
    return os.path.join(*out)

def save_writer_scalars(log_path: _PATH, filename: str = "scalars.csv") -> None:
    event = EventAccumulator(path=log_path)
    event.Reload()
    tags = event.Tags()["scalars"]

    if len(tags) == 0:
        return

    dfs = []
    for tag in tags:
        dfs.append(
            pd.DataFrame(event.Scalars(tag))
            .rename(columns={"value": tag})
            .drop(["wall_time"], axis=1)
        )
    out = reduce(lambda left, right: pd.merge(left, right, on=["step"], how="outer"), dfs)
    out.to_csv(os.path.join(log_path, filename), index=False)

def save_writer_figures(log_path: _PATH, folder_name: str = "figures") -> None:
    event = EventAccumulator(path=log_path, size_guidance={"images": 0})
    event.Reload()
    tags = event.Tags()["images"]

    if len(tags) == 0:
        return

    figure_folder = os.path.join(log_path, folder_name)
    mkdir(figure_folder)

    for tag in tags:
        for item in event.Images(tag):
            step = item.step
            encoded = item.encoded_image_string
            img = Image.open(io.BytesIO(encoded))
            path = os.path.join(figure_folder, str(step))
            mkdir(path)
            img.save(os.path.join(path, f"{tag.replace('/','_')}.png"))


import asyncio
import pickle
import grpc
import logging

from .grpc import (
    scheduler_pb2 as scheduler_worker_messages,
    scheduler_pb2_grpc as scheduler_worker_service
)

from apscheduler.schedulers.asyncio import AsyncIOScheduler
from sqlalchemy.orm import sessionmaker

from app.core.settings import get_settings
from app.db.base import create_session_factory
from app.schemas.task import Task
from .services.scheduler import add_task_to_scheduler, start_tasks
from .services.logger import init_logger

class SchedulerWorkerService(scheduler_worker_service.SchedulerWorkerServicer):
    def __init__(
        self,
        scheduler: AsyncIOScheduler,
        sessionmaker: sessionmaker,
    ):
        self.scheduler = scheduler
        self.sessionmaker = sessionmaker

    async def add_task(
        self,
        request: scheduler_worker_messages.AddTaskRequest,
        context: grpc.aio.ServicerContext,
    ) -> scheduler_worker_messages.ResponseStub:
        task_obj: Task = pickle.loads(request.task_obj)
        async with self.sessionmaker() as session:
            logger.info(f'add_task -> {task_obj.search_phrase}')
            await add_task_to_scheduler(
                task_obj=task_obj,
                session=session,
                scheduler=self.scheduler,
                timeout=request.timeout,
            )
        return scheduler_worker_messages.ResponseStub()

    async def remove_task(
        self,
        request: scheduler_worker_messages.TaskIdRequest,
        context: grpc.aio.ServicerContext,
    ) -> scheduler_worker_messages.RemoveResponse:
        logger.info(f'remove_task -> {request.task_id}')
        self.scheduler.remove_job(str(request.task_id))

        return scheduler_worker_messages.RemoveResponse(success=True)

async def serve_worker() -> None:

    # create basics for worker
    settings = get_settings()
    scheduler = AsyncIOScheduler()
    contextsession = create_session_factory(settings.postgres_uri)

    # setup server
    server = grpc.aio.server()
    scheduler_worker_service.add_SchedulerWorkerServicer_to_server(
        server=server,
        servicer=SchedulerWorkerService(
            scheduler=scheduler,
            sessionmaker=contextsession,
        ),
    )

    server.add_insecure_port(settings.worker_socket)
    logger.info("Starting server on %s", settings.worker_socket)
    await server.start()

    logger.info("Starting tasks from db")
    await start_tasks(contextsession, scheduler, settings.parse_timeout)

    logger.info("Starting scheduler")
    scheduler.start()

    await server.wait_for_termination()

if __name__ == "__main__":
    # simple logger
    file_name = __file__.split("/")[-1]
    init_logger(file_name, logging.DEBUG)
    logger = logging.getLogger(file_name)

    # run worker
    asyncio.run(serve_worker())


from flask_pymongo import PyMongo
from flask import Flask
app=Flask(__name__)
app.config["MONGO_URI"]="mongodb://localhost:27017/Jovian"
#how to check if the database is connected
mongo=PyMongo(app)
print(mongo.db)
JOBS=mongo.db.Jobs.find() # JOBS is a cursor
jobs=[job for job in JOBS] # jobs is a list
print(jobs)


#%%
import pandas as pd
import pickle
import numpy as np

import gensim
import sklearn.metrics

#%%
#--Read in data
D2V_WOstop = gensim.models.Doc2Vec.load(r'..\data\process\D2V_WOstop')
df = pd.read_csv(r'..\data\df_cb_main_combined.csv', index_col=0, encoding='utf-8', error_bad_lines=False).dropna(subset=['Review']).drop_duplicates(['Author', 'Game'])

#Acquire feature and key word lists
D2VFeatures = D2V_WOstop.vocab.keys()

#Setup target keyword group cluster
numOfGroups = [10, 30, 100, 300, 1000]
numOfGroups = [300]

#%%
for n in numOfGroups:
    numOfGroup = n
    #--Read in corresponding keyword group file
    df_keyword = pd.read_csv(r'..\data\output\keywordGroup_hierarchy_' + str(numOfGroup) + '.csv', encoding='utf-8')

    #--Make into keyword groups
    keyWords = []
    for g in range(numOfGroup):
        key = df_keyword.query('cluster == @g + 1').keyword.tolist()
        keyWords.append(key)

    keyGroups = {
        'groupNo': np.arange(numOfGroup) + 1,
        'keyWords': keyWords,
        'keyWords_filtered': None,
        'centroid': None
    }

    df_keyGroups = pd.DataFrame(keyGroups)

    keyWords_filtered = []
    centroids = []
    for index, row in df_keyGroups.iterrows():
        #Remove key words outside the D2V space
        filtered = [w for w in row['keyWords'] if w in D2VFeatures]
        keyWords_filtered.append(filtered)

        #Compute centroid vector of each group
        try: centroid = np.mean(D2V_WOstop[filtered], axis=0)
        except: centroid = None
        centroids.append(centroid)

    df_keyGroups['keyWords_filtered'] = keyWords_filtered
    df_keyGroups['centroid'] = centroids

    #Leave only non-null groups
    df_keyGroups = df_keyGroups[pd.notnull(df_keyGroups['centroid'])]

    #%%
    #--Acquire score of each key word group
    #By the average distance of the docvec and each key word group
    #Cosine similarity here: -1 ~ 1; -1=opposite; 1=same
    for g in range(len(df_keyGroups)):
        scores = []
        try:
            gvec = df_keyGroups['centroid'][g].reshape(1, -1)
            for index, row in df.iterrows():
                dvec = D2V_WOstop.docvecs['id_' + str(index)].reshape(1, -1)
                score = sklearn.metrics.pairwise.cosine_similarity(gvec, dvec)[0, 0]

                scores.append(score)

            df['group' + str(g + 1)] = scores
        #Some groups contain to word
        except: pass

    #--Review and save
    # df.drop('Review', axis=1).to_csv(r'..\data\process\score_' + str(numOfGroup) + '_doc2vec.csv')
    pickle.dump(df, open(r'..\data\process\score_' + str(numOfGroup) + '_doc2vec.p', 'wb'))
    df.query('CoreID > 0')

#--Print top games of each keygroup
#Also the score must be the highest among all scores of the game to be chosen
if False:
    CORE_GAMES = pd.read_csv(r'..\data\core_games.csv', encoding='utf-8', header=None)[0].tolist()
    for i in np.arange(numOfGroup) + 1:
        print('Games with top scores in group' + str(i))
        maxScore = df[df['group' + str(i)] == np.max(df.filter(regex='^group\d+$'), axis=1)]
        topScoreGames = df.sort_values(by='group' + str(i))[:100]
        top_notcore = topScoreGames[topScoreGames.Core == 0]['Game']
        print(top_notcore)


#!/usr/bin/env python
import os, json
FileList = []
FileListJson = {}
FileListJson['video'] = []
with open("config/vidStreamer-sample.json") as jsonFile:
	data = json.load(jsonFile)
print(data)

print(json.dumps(FileListJson))
oldpath = path = os.getcwd()
path = data["rootFolder"]
print(path)

os.chdir(path)

import glob
extList = ['.mp4','.avi']
# FileList = glob.glob('./*.js')
for root, dirs, files in os.walk('./'):
	for file in files:
		for ext in extList:
			if file.endswith(ext):
				print(root)
				print(dirs)
				jsonObj = {}
				jsonObj['name'] = file
				jsonObj['path'] = os.path.join(root, file).replace('\\','/').replace('./', '/')
				FileListJson['video'].append(jsonObj)
				FileList.append(os.path.join(root, file).replace('\\','/'))

if len(FileList) < 1:
	print('no file found!')
else:
	for f in FileList:
		print(f)
os.chdir(oldpath)
print(json.dumps(FileListJson))
with open('./config/VidList.json','w+') as outFile:
	outFile.write(json.dumps(FileListJson))


import argparse
from tqdm import tqdm
import tensorflow as tf
from tensorflow import keras

from quality_net_utilities import *

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Trains a feature-aware normalization model.')

    parser.add_argument('--dataset_path',dest='dataset_path',
                        action='store',type=str,default=None,
                        help="Path to hdf5 dataset.")
    parser.add_argument('--input_height',dest = 'input_height',
                        action = 'store',type = int,default = 256,
                        help = 'The file extension for all images.')
    parser.add_argument('--input_width',dest = 'input_width',
                        action = 'store',type = int,default = 256,
                        help = 'The file extension for all images.')

    parser.add_argument('--log_every_n_steps',dest = 'log_every_n_steps',
                        action = 'store',type = int,default = 100,
                        help = 'How often are the loss and global step logged.')
    parser.add_argument('--save_summary_folder',dest = 'save_summary_folder',
                        action = 'store',type = str,default = 'summaries',
                        help = 'Directory where summaries are saved.')
    parser.add_argument('--save_checkpoint_folder',dest = 'save_checkpoint_folder',
                        action = 'store',type = str,default = 'checkpoints',
                        help = 'Directory where checkpoints are saved.')
    parser.add_argument('--batch_size',dest = 'batch_size',
                        action = 'store',type = int,default = 4,
                        help = 'Size of mini batch.')
    parser.add_argument('--number_of_epochs',dest = 'number_of_epochs',
                        action = 'store',type = int,default = 50,
                        help = 'Number of epochs in the training process.')
    parser.add_argument('--learning_rate',dest = 'learning_rate',
                        action = 'store',type = float,default = 0.001,
                        help = 'Learning rate for the SGD optimizer.')

    #Data augmentation
    for arg in [
        ['brightness_max_delta',8. / 255.,float],
        ['saturation_lower',0.9,float],['saturation_upper',1.1,float],
        ['hue_max_delta',0.2,float],
        ['contrast_lower',0.9,float],['contrast_upper',1.1,float],
        ['min_jpeg_quality',90,int],['max_jpeg_quality',100,int],
        ['probability',0.5,float]]:
        parser.add_argument('--{}'.format(arg[0]),dest=arg[0],
                            action='store',type=arg[2],default=arg[1])

    args = parser.parse_args()

    data_augmentation_params = {
        'brightness_delta':args.brightness_max_delta,
        'saturation_lower':args.saturation_lower,
        'saturation_upper':args.saturation_upper,
        'hue_delta':args.hue_max_delta,
        'contrast_lower':args.contrast_lower,
        'contrast_upper':args.contrast_upper,
        'min_jpeg_quality':args.min_jpeg_quality,
        'max_jpeg_quality':args.max_jpeg_quality,
        'probability':args.probability}

    print("Setting up network...")
    mobilenet_v2 = keras.applications.DenseNet121(
        input_shape=[args.input_height,args.input_width,3],
        include_top=False,classes=2,pooling='max')
    quality_net = quality_net_model(
        mobilenet_v2,args.input_height,args.input_width)

    print("Setting up data generator...")
    flipper = Flipper()
    colour_augmenter = ColourAugmentation(**data_augmentation_params)

    def augment(x):
        return flipper(colour_augmenter(x))

    data_generator = DataGenerator(args.dataset_path,augment)
    def load_generator():
        for image,label in data_generator.generate():
            yield image,label
    generator = load_generator
    output_types = (tf.float32,tf.float32)
    output_shapes = (
        tf.TensorShape((args.input_height,args.input_width,3)),
        tf.TensorShape((1)))
    tf_dataset = tf.data.Dataset.from_generator(
        generator,output_types=output_types,output_shapes=output_shapes)
    tf_dataset = tf_dataset.repeat()
    tf_dataset = tf_dataset.batch(args.batch_size)
    tf_dataset = tf_dataset.prefetch(args.batch_size*5)

    print("Setting up training...")
    auc = tf.keras.metrics.AUC()
    acc = Accuracy()
    rec = Recall()
    prec = Precision()
    loss_fn = keras.losses.BinaryCrossentropy()
    quality_net.compile(
        optimizer=keras.optimizers.Adam(learning_rate=args.learning_rate),
        loss=loss_fn, metrics=[auc,acc,rec,prec])
    steps_per_epoch = data_generator.n_images // args.batch_size

    print(steps_per_epoch)
    print("Training...")
    tensorboard_callback = keras.callbacks.TensorBoard(
        log_dir=args.save_summary_folder)
    checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
        args.save_checkpoint_folder + '-{epoch:02d}')
    lr_callback = tf.keras.callbacks.ReduceLROnPlateau(
        'loss',min_lr=1e-6)

    all_callbacks = [
        tensorboard_callback,checkpoint_callback,lr_callback]

    quality_net.fit(
        x=tf_dataset,batch_size=args.batch_size,epochs=args.number_of_epochs,
        callbacks=all_callbacks,steps_per_epoch=steps_per_epoch)


"""PRE gen hook
"""
import logging
import os
import shutil
import sys
import git

_logger = logging.getLogger()
TMP_DIR = './tmp/'

def pre_hook():
    """Flag the PRE gen HOOK execution"""
    _logger.warning("PRE Gen hook executed")

    try:

        # Default value = Volt
        theme = "{{cookiecutter.theme}}"
        repo  = 'https://github.com/app-generator/django-dashboard-volt.git'

        if 'material-dashboard' == theme:
            repo  = 'https://github.com/app-generator/django-dashboard-material.git'

        if 'soft-ui' == theme:
            repo  = 'https://github.com/app-generator/django-soft-ui-dashboard.git'

        if 'datta-able' == theme:
            repo  = 'https://github.com/app-generator/django-datta-able.git'

        if 'star-admin' == theme:
            repo  = 'https://github.com/app-generator/django-star-admin.git'

        print(' Cloning theme: <' + theme + '>, REPO: ' + repo )

        repo = git.Repo.clone_from( repo, TMP_DIR, branch='master')
        print(' ...done')

    except OSError as e:
        _logger.warning(f"Error: {e}")
        sys.exit(0)

    # Exit with success
    sys.exit(0)

if __name__ == "__main__":
    pre_hook()


# -*- coding: utf-8 -*-
from dajaxice.decorators import dajaxice_register
from dajax.core import Dajax
from family.models import Member, postimage
from django.template.defaultfilters import linebreaksbr

def show_sex_in_chinese(sex):
      if sex == 'M':
             return u'男'
      else:
             return u'女'

def get_first_thumbnail_of_member(object):
         try:
           r=object.postimage_set.all().order_by('id')[0]
           if r:
                return r.thumbnail.url
         except IndexError:
           return 'https://lishefei.s3.amazonaws.com/lis/images/lhz/placeholder.png'

def get_nth_item_image_of_member(object,n):
    r=list(object.postimage_set.all().order_by('id')[n-1:n])
    if r:
     return r[0].image.url
    return ''

@dajaxice_register
def get_member(req, pk):
    dajax = Dajax()
    p = Member.objects.get(pk=pk)
    dajax.assign('#image', 'src', get_first_thumbnail_of_member(p))
    dajax.assign('#name', 'innerHTML', p.name)
    dajax.assign('#sex', 'innerHTML', show_sex_in_chinese(p.sex))
    dajax.assign('#desc', 'innerHTML', linebreaksbr(p.desc))
    dajax.assign('#image1', 'src', get_nth_item_image_of_member(p,2))
    dajax.assign('#image2', 'src', get_nth_item_image_of_member(p,3))
    dajax.assign('#image3', 'src', get_nth_item_image_of_member(p,4))
    dajax.assign('#image4', 'src', get_nth_item_image_of_member(p,5))

    return dajax.json()


from matplotlib.axes import Axes
import matplotlib.pyplot as plt
import pandas as pd

from base import Threaded
from data import store as store
from utils import ui, logger

_logger = logger.get_logger()

class Chart(Threaded):
    figure: plt.Figure
    ax: plt.Axes
    ticker: str
    history: pd.DataFrame

    def __init__(self, ticker: str, days: int = 365, live: bool = False):
        if not store.is_ticker(ticker):
            raise ValueError(f'Ticker {ticker} is invalid')
        if days <= 10:
            raise ValueError(f'Days must be greater than 10')

        self.ticker = ticker.upper()
        self.days = days
        self.live = live
        self.history = pd.DataFrame()
        self.company: dict = {}
        self.figure, self.ax = plt.subplots(figsize=ui.CHART_SIZE)

        plt.style.use(ui.CHART_STYLE)
        plt.margins(x=0.1)
        plt.subplots_adjust(bottom=0.15)

        self.figure.canvas.manager.set_window_title(f'{self.ticker}')
        self.ax.grid(which='major', axis='both')

    @Threaded.threaded
    def fetch_history(self):
        self.task_state = 'None'

        _logger.info(f'Fetching history for {self.ticker}...')

        self.task_ticker = self.ticker
        self.history = store.get_history(self.ticker, days=self.days, live=self.live)
        self.company = store.get_company(self.ticker)

        self.ax.set_title(f'{self.company["name"]} (daily)')

        self.task_state = 'Done'

    def plot_ohlc(self) -> tuple[plt.Figure, plt.Axes]:
        if self.history.empty:
            _logger.info(f'Need history for {self.ticker}. Fetching...')
            self.fetch_history()

        self.ax.secondary_yaxis('right')

        if self.history.iloc[-1]['close'] < 30.0:
            self.ax.yaxis.set_major_formatter('{x:.2f}')
        else:
            self.ax.yaxis.set_major_formatter('{x:.0f}')

        # Setup values
        interval = 10
        width1 = 0.5
        width2 = 0.05
        color_up = 'green'
        color_dn = 'red'
        length = len(self.history)

        # Ticks
        ticks = self.history.index[::interval]
        self.ax.set_xticks(ticks)
        self.ax.tick_params(axis='x', labelrotation=45)
        labels = [self.history.iloc[i]['date'].strftime(ui.DATE_FORMAT_MDY) for i in range(0, length, interval)]
        self.ax.set_xticklabels(labels)

        # Find up and down days
        up = self.history[self.history.close >= self.history.open]
        dn = self.history[self.history.close < self.history.open]

        # Up prices
        self.ax.bar(up.index, up.high - up.close, width2, bottom=up.close, color=color_up)
        self.ax.bar(up.index, up.close - up.open, width1, bottom=up.open,  color=color_up)
        self.ax.bar(up.index, up.low - up.open,   width2, bottom=up.open,  color=color_up)

        # Down prices
        self.ax.bar(dn.index, dn.high - dn.open,  width2, bottom=dn.open,  color=color_dn)
        self.ax.bar(dn.index, dn.close - dn.open, width1, bottom=dn.open,  color=color_dn)
        self.ax.bar(dn.index, dn.low - dn.close,  width2, bottom=dn.close, color=color_dn)

        return (self.figure, self.ax)

    def plot_history(self, close: bool = False) -> plt.Figure:
        if self.history.empty:
            _logger.info(f'Need history for {self.ticker}. Fetching...')
            self.fetch_history()

        self.ax.secondary_yaxis('right')

        if self.history.iloc[-1]['close'] < 30.0:
            self.ax.yaxis.set_major_formatter('{x:.2f}')
        else:
            self.ax.yaxis.set_major_formatter('{x:.0f}')

        length = len(self.history)
        dates = [self.history.iloc[index]['date'].strftime(ui.DATE_FORMAT_MDY) for index in range(length)]

        # Grid and ticks
        self.ax.set_xticks(range(0, length+1, int(length/12)))
        self.ax.tick_params(axis='x', labelrotation=45)

        # Plot Highs & Lows using simple lines
        if close:
            self.ax.plot(dates, self.history.close, '-b', linewidth=0.8)
        else:
            self.ax.plot(dates, self.history.high, '-g', linewidth=0.5)
            self.ax.plot(dates, self.history.low, '-r', linewidth=0.5)
            self.ax.fill_between(dates, self.history.high, self.history.low, facecolor='gray', alpha=0.4)

        return self.figure

if __name__ == '__main__':
    import sys
    import logging
    from utils import logger

    logger.get_logger(logging.DEBUG)

    if len(sys.argv) > 1:
        chart = Chart(sys.argv[1], days=180)
    else:
        chart = Chart('IBM')

    # figure = chart.plot_ohlc()
    # figure = chart.plot_history(close=True)
    figure = chart.plot_chart()
    plt.figure(figure)
    plt.show()


import torch.nn as nn
from typing import Tuple, Optional, Iterable, List

def build_mlp(
    _in: int,
    hidden: Tuple[int, ...],
    out: Optional[int] = None,
    add_batchnorm: bool = False,
    activations: Iterable[nn.Module] = [nn.LeakyReLU()]
) -> List[nn.Module]:
    layers: List[nn.Module] = []
    h_prev = _in
    for h in hidden:
        layers.extend([
            nn.Linear(in_features=h_prev, out_features=h),
        ])
        layers.extend(activations)
        if add_batchnorm:
            layers.append(nn.BatchNorm1d(h))

        h_prev = h

    if out is not None:
        layers.append(nn.Linear(in_features=h_prev, out_features=out))

    return layers


import sys # remove the path of ROS
if '/opt/ros/kinetic/lib/python2.7/dist-packages' in sys.path:
    sys.path.remove('/opt/ros/kinetic/lib/python2.7/dist-packages')
import os
import cv2
import inspect
import torch
import numpy as np
import importlib
from torch import nn
from os.path import join as jn
from torch.nn import functional as F
from collections import OrderedDict
import torch.optim.lr_scheduler as lrs
from warmup_scheduler import GradualWarmupScheduler

import pytorch_lightning as pl
from .metrics import compare_psnr, compare_ssim

class MInterface(pl.LightningModule):
    def __init__(self, model_name, loss, lr, **kargs):
        super().__init__()
        self.save_hyperparameters(ignore=["callbacks"])
        self.configure_loss()
        self.model = self.load_model()
        self.teacher_model = self.load_model()

        if kargs['train']:
            # training params
            self.loss_weight = kargs['loss_weight'] # [loss_BC, loss_SC, loss_TG, loss_SG]
            self.training_stage = kargs['training_stage']
            assert self.training_stage in [1, 2]

            # load teacher model during second stage training
            if self.training_stage == 2:
                state_dict = self.load_state_dict_from_checkpoint(kargs['ckpt_path'])
                self.model.load_state_dict(state_dict)
                # load teacher model
                self.teacher_model.load_state_dict(state_dict)
                self.teacher_model.freeze()
                self.teacher_model.eval()
        else:
            # test params
            self.save_path = kargs['save_path']
            self.has_gt = kargs['has_gt']

    def forward(self, event, blur):
        return self.model(event, blur)

    def training_step(self, batch, batch_idx):
        eger_blur2sharp, eger_largeblur2sharp, eger_largeblur2blur, blur, large_blur = batch
        B, ev_C, ev_H, ev_W = eger_blur2sharp.shape
        im_C, im_H, im_W = blur.shape[-3:]
        assert im_H / ev_H == im_W / ev_W

        # assemble inputs
        egers = torch.cat((eger_blur2sharp[:,None,...],  eger_largeblur2blur[:,None,...], eger_largeblur2sharp[:,None,...]), 1).reshape((B*3,ev_C,ev_H,ev_W)) # B*3,T*4,H,W
        blurs = torch.cat((blur[:,None,...], large_blur[:,None,...], large_blur[:,None,...]), 1).reshape((B*3,im_C,im_H,im_W))

        # process by network
        L, SAN_E = self(egers, blurs)
        L = L.clamp(0,1).reshape((B,3,im_C,im_H,im_W))
        SAN_E = SAN_E.reshape((B,3,im_C,im_H,im_W))

        ## first stage training
        if self.training_stage == 1:
            # compute loss
            loss_BC, loss_SC = self.compute_loss(L, SAN_E, blur)
            loss = self.loss_weight[0] * loss_BC + self.loss_weight[1] * loss_SC

        ## second stage training
        else:
            # generate pseudo-ground-truth by teacher model
            L_teacher, _ = self.teacher_model(eger_blur2sharp, blur)
            L_teacher = L_teacher.clamp(0,1)

            # generate down-sampled results
            rand_scale = ((np.random.randint(ev_H, im_H+1) // 32) * 32) / im_H
            lr_large_blur = self.image_downsampling(large_blur, rand_scale)
            lr_L_lb2s, _ = self(eger_largeblur2sharp, lr_large_blur)
            lr_L_lb2s = lr_L_lb2s.clamp(0,1)

            # generate down-sampled pseudo-ground-truth
            lr_L_teacher = self.image_downsampling(L_teacher, rand_scale)

            # compute loss
            loss_BC, loss_SC, loss_TG, loss_SG = self.compute_loss(L, SAN_E, blur, lr_L_lb2s, L_teacher, lr_L_teacher)
            loss = self.loss_weight[0] * loss_BC + self.loss_weight[1] * loss_SC +\
                 self.loss_weight[2] * loss_TG + self.loss_weight[3] * loss_SG

        self.log('loss', loss, on_step=False, on_epoch=True, prog_bar=True)
        return loss

    def test_step(self, batch, batch_idx):
        if self.has_gt:
            eger, blur, gt, prefix = batch
        else:
            eger, blur, prefix = batch

        # process by network
        B, N = eger.shape[:2]
        for n in range(N):
            L, _ = self(eger[:,n,...], blur[:,n,...])
            L = L.clamp(0,1)

            # save results
            if self.save_path != None:
                for b in range(B):

                    # save reconstruction
                    result = (L[b,...].permute(1,2,0).squeeze().cpu().detach().numpy() * 255).astype(np.uint8)
                    save_name = prefix[b] + '_%03d.png' %(n)
                    cv2.imwrite(jn(self.result_root, save_name), result)

                    # save blurry inputs
                    blurry_img = (blur[b,n,...].permute(1,2,0).squeeze().cpu().detach().numpy() * 255).astype(np.uint8)
                    cv2.imwrite(jn(self.blur_root, save_name), blurry_img)

                    # save gt and compute metrics
                    if self.has_gt:
                        groundtruth = (gt[b,n,...].permute(1,2,0).squeeze().cpu().detach().numpy() * 255).astype(np.uint8)
                        cv2.imwrite(jn(self.gt_root, save_name), groundtruth)

                        # compute metrics
                        psnr, ssim = compare_psnr(groundtruth, result), compare_ssim(groundtruth, result)
                        self.log('psnr', psnr, on_step=True, on_epoch=True, prog_bar=True)
                        self.log('ssim', ssim, on_step=True, on_epoch=True, prog_bar=True)

    def on_test_epoch_start(self):
        # ensure directories
        if self.save_path != None:
            self.result_root = jn(self.save_path, 'Result')
            if not os.path.exists(self.result_root):
                os.makedirs(self.result_root)
            self.blur_root = jn(self.save_path, 'Blur')
            if not os.path.exists(self.blur_root):
                os.makedirs(self.blur_root)

            if self.has_gt:
                self.gt_root = jn(self.save_path, 'GroundTruth')
                if not os.path.exists(self.gt_root):
                    os.makedirs(self.gt_root)

    def compute_loss(self, L, SAN_E, blur, lr_L_lb2s=None, L_teacher=None, lr_L_teacher=None):
        # seperate results
        L_b2s, L_lb2b, L_lb2s = L[:,0,...], L[:,1,...], L[:,2,...]
        SAN_E_b2s, SAN_E_lb2b, SAN_E_lb2s = SAN_E[:,0,...], SAN_E[:,1,...], SAN_E[:,2,...]

        # compute first-stage losses
        loss_BC = self.loss_function(L_lb2b, blur)
        loss_SC = self.loss_function(SAN_E_lb2s / SAN_E_b2s, SAN_E_lb2b)

        if self.training_stage == 1:
            return loss_BC, loss_SC
        else:
            # compute second-stage losses
            loss_TG = self.loss_function(L_teacher, L_lb2s)
            loss_SG = self.loss_function(lr_L_teacher, lr_L_lb2s) # may need to be modified
            return loss_BC, loss_SC, loss_TG, loss_SG

    def image_downsampling(self, img, scale_factor, mode='bicubic'):
        img = F.interpolate(img, scale_factor=scale_factor, mode=mode).clamp(0,1)
        return img

    def load_state_dict_from_checkpoint(self, checkpoint_path):
        state_dict = torch.load(checkpoint_path)['state_dict']
        new_state_dict = OrderedDict()
        for k, v in state_dict.items():
            if k[:6] == 'model.': # find keys starting with `model.`
                name = k[6:] # remove `model.`
                new_state_dict[name] = v
        return new_state_dict

    def configure_optimizers(self):
        if hasattr(self.hparams, 'weight_decay'):
            weight_decay = self.hparams.weight_decay
        else:
            weight_decay = 0
        optimizer = torch.optim.Adam(
            self.parameters(), lr=self.hparams.lr, weight_decay=weight_decay)

        if self.hparams.lr_scheduler is None:
            return optimizer
        else:
            if self.hparams.lr_scheduler == 'step':
                scheduler = lrs.StepLR(optimizer,
                                       step_size=self.hparams.lr_decay_steps,
                                       gamma=self.hparams.lr_decay_rate)
            elif self.hparams.lr_scheduler == 'cosine':
                scheduler = lrs.CosineAnnealingLR(optimizer,
                                                  T_max=self.hparams.lr_decay_steps,
                                                  eta_min=self.hparams.lr_decay_min_lr)
            else:
                raise ValueError('Invalid lr_scheduler type!')

            if self.hparams.warmup_epoch > 0:
                scheduler_warmup = GradualWarmupScheduler(optimizer, multiplier=1, total_epoch=self.hparams.warmup_epoch, after_scheduler=scheduler)
                return [optimizer], [scheduler_warmup]
            else:
                return [optimizer], [scheduler]

    def configure_loss(self):
        loss = self.hparams.loss.lower()
        if loss == 'mse':
            self.loss_function = F.mse_loss
        elif loss == 'l1':
            self.loss_function = F.l1_loss
        else:
            raise ValueError("Invalid Loss Type!")

    def load_model(self):
        name = self.hparams.model_name
        # Change the `snake_case.py` file name to `CamelCase` class name.
        # Please always name your model file name as `snake_case.py` and
        # class name corresponding `CamelCase`.
        camel_name = ''.join([i.capitalize() for i in name.split('_')])
        try:
            Model = getattr(importlib.import_module(
                '.'+name, package=__package__), camel_name)
        except:
            raise ValueError(
                f'Invalid Module File Name or Invalid Class Name {name}.{camel_name}!')

        return self.instancialize(Model)

    def instancialize(self, Model, **other_args):
        """ Instancialize a model using the corresponding parameters
            from self.hparams dictionary. You can also input any args
            to overwrite the corresponding value in self.hparams.
        """
        class_args = inspect.getargspec(Model.__init__).args[1:]
        inkeys = self.hparams.keys()
        args1 = {}
        for arg in class_args:
            if arg in inkeys:
                args1[arg] = getattr(self.hparams, arg)
        args1.update(other_args)
        return Model(**args1)


#!/usr/bin/env python
# coding: utf-8

# In[12]:

import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

test= pd.read_csv("C:\\Users\\akshansh\\Desktop\\python programs\\test.csv")
print(test)

# In[15]:

test=sns.load_dataset("titanic")
test.head()

# In[16]:

sns.lineplot(x="age",y="fare",data=test,hue="sex")
plt.show()

# In[17]:

sns.lineplot(x="age",y="fare",data=test,hue="sex",style="sex")
plt.show()

# In[28]:

sns.lineplot(x="age",y="fare",data=test,hue="sex",marker='o')
plt.show()

# In[30]:

sns.barplot(x="age",y="fare",data=test,hue="sex")
plt.show()

# In[31]:

sns.barplot(x="age",y="fare",data=test,hue="sex",palette="rocket")
plt.show()

# In[54]:

plt.figure(figsize=(10,6))
sns.barplot(x="age",y="fare",data=test,hue="sex")
plt.show()

# In[65]:

plt.figure(figsize=(10,6))
sns.histplot(x="age",y="fare",data=test,hue="sex",bins=10)
plt.show()

# In[66]:

plt.figure(figsize=(10,6))
sns.scatterplot(x="age",y="fare",data=test,hue="sex")
plt.show()

# In[ ]:


"""
Aggregate feature subsets from multiple runs from feature generation
code in gen_mol_fea.py.

Example:
    python src/agg_runs.py --res_dir out/BL2/2020-4-24 --fea dd --prfx BL2
"""
import warnings
warnings.filterwarnings('ignore')

import os
import sys
from pathlib import Path
from time import time
import argparse
from glob import glob

import numpy as np
import pandas as pd

filepath = Path(__file__).resolve().parent

# Utils
from utils.utils import load_data

def parse_args(args):
    parser = argparse.ArgumentParser(description='Aggregate feature subsets from multiple runs.')
    parser.add_argument('--res_dir', required=True, default=None, type=str,
                        help='Global dir where multiple runs were dumped (default: None).')
    parser.add_argument('--fea', required=True, default='dd', type=str,
                        help='Feature type (default: dd).')
    parser.add_argument('--prfx', default=None, type=str,
                        help='Name prefix for the output file (default: None).')
    args = parser.parse_args(args)
    return args

def run(args):
    res_dir = Path( args['res_dir']).resolve()
    fea_name = args['fea']
    prfx = args['prfx']

    run_dirs = glob(str(res_dir/'*ids*'))
    fea_dfs = []
    for i, r in enumerate(run_dirs):
        dpath = sorted(Path(r).glob(f'*{fea_name}*'))[0]
        if not dpath.exists():
            continue
        df = load_data(dpath)

        if 'smiles' in df.columns:
            df = df.rename(columns={'smiles': 'SMILES'})
        if 'name' in df.columns:
            df = df.rename(columns={'name': 'TITLE'})

        fea_dfs.append(df)
    fea_df = pd.concat(fea_dfs, axis=0)
    del fea_dfs

    print(fea_df.shape)
    fea_df = fea_df.drop_duplicates(subset=['TITLE']).reset_index(drop=True)
    print(fea_df.shape)

    fea_df.to_parquet(f'{res_dir}/{prfx}.{fea_name}.parquet')
    print('Done.')

def main(args):
    args = parse_args(args)
    args = vars(args)
    run(args)

if __name__ == '__main__':
    main(sys.argv[1:])


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@time    : 2020/3/30 10:30
@file    : convert_pictures.py
@author  : yitong.feng
@contact: yitong.feng@majorbio.com
"""

import fitz
from skimage import io
import cairosvg
import sys

def trim(img):
    img2 = img.sum(axis=2)
    img2_r = img2.sum(axis=1)
    img2_c = img2.sum(axis=0)
    (row,col) = img2.shape
    tempr0=0
    tempr1=0
    tempc0=0
    tempc1=0
    for r in range(0,row):
        if img2_r[r]!=765*col:
            tempr0=r
            break
    for r in range(row-1,0,-1):
        if img2_r[r]!=765*col:
            tempr1=r
            break
    for c in range(0,col):
        if img2_c[c]!=765*row:
            tempc0=c
            break
    for c in range(col-1,0,-1):
        if img2_c[c]!=765*row:
            tempc1=c
            break
    # print((tempr0, tempr1, tempc0, tempc1))
    new_img=img[tempr0:tempr1+3,tempc0:tempc1+3,0:3]
    return new_img

def convert(pic, *args, **kargs):
    if pic.endswith('.svg'):
        pdf = pic.split('.svg')[0] + '.pdf'
        cairosvg.svg2pdf(url=pic, write_to=pdf)
        pic = pdf
    pdfDoc = fitz.open(pic)
    zoom_x = 2
    zoom_y = 2
    mat = fitz.Matrix(zoom_x, zoom_y)
    pix = pdfDoc[0].getPixmap(matrix = mat)
    png = pic.split('.pdf')[0] + '.png'
    if 'out' in kargs:
        png = kargs['out']
    pix.writePNG(png)
    trim_ = True
    try:
        trim_ = kargs['trim']
    except:
        pass
    if trim_:
        im = io.imread(png)
        im = trim(im)
        io.imsave(png, im)

if __name__ == '__main__':
    if len(sys.argv) == 2:
        _, pic = sys.argv
        png = pic.split('.pdf')[0] + '.png'
        convert(pic, trim=False)
        convert(pic, out=png + '_trim.png')
    elif len(sys.argv) == 3:
        _, pic, out = sys.argv
        convert(pic, out=out, trim=False)
        convert(pic, out=out + '_trim.png', trim=True)
    else:
        exit("USAGE:python $s picture out")


import math
import traceback
from typing import List

import cv2
import d3dshot
import numpy as np
from cv2 import DMatch
from numpy import ndarray

class CustomErr(Exception):
    pass

d3dshot_mgr = d3dshot.create(capture_output='numpy')

color_board_img = cv2.imread('imgs/cb.png')
draw_board_img = cv2.imread('imgs/db.png')
pen_setting_img = cv2.imread('imgs/ps.png')

class Color:
    RED = (255, 0, 0)
    GREEN = (0, 255, 0)
    BLUE = (0, 0, 255)

def to_int(data):
    if isinstance(data, (list, tuple, set, ndarray)):
        return [int(item) for item in data]
    elif isinstance(data, (str, float, int)):
        return int(data)
    else:
        raise NotImplementedError(f'{type(data)} is not supported')

def distance(p1, p2):
    return math.sqrt(
        ((p1[0] - p2[0]) ** 2) + ((p1[1] - p2[1]) ** 2)
    )

def draw_rectangle(img, pt1, pt2, color=Color.GREEN, thickness=2, text=None):
    cv2.rectangle(img, pt1, pt2, color=color, thickness=thickness)
    if text:
        cv2.putText(img, text, pt1, fontFace=3, fontScale=1, color=color)

def find_pos(sub_img, src_img, text, tolerance=0.5, method='sift', grey_mode=False, preview=1):
    if grey_mode:
        src = cv2.cvtColor(src_img.copy(), cv2.COLOR_RGB2GRAY)
        sub = cv2.cvtColor(sub_img.copy(), cv2.COLOR_RGB2GRAY)
    else:
        src = src_img.copy()
        sub = sub_img.copy()

    src_h, src_w, *_ = src.shape
    sub_h, sub_w, *_ = sub.shape

    if method == 'legacy':
        # Legacy template matching
        result = cv2.matchTemplate(sub, src, cv2.TM_CCOEFF_NORMED)
        y, x = np.unravel_index(result.argmax(), result.shape)
        cv2.rectangle(src, (x, y), (x + sub_w, y + sub_h), color=(0, 255, 0), thickness=5)
        preview = src_img
        pt1 = (x, y)
        pt2 = (x + src_w, y + src_h)
    elif method == 'sift':
        # SIFT
        sift = cv2.SIFT_create()
        src_kp, src_des = sift.detectAndCompute(src, None)
        sub_kp, sub_des = sift.detectAndCompute(sub, None)
        bf = cv2.BFMatcher()
        matches = bf.knnMatch(sub_des, src_des, k=2)
        good: List[List[DMatch]] = []
        for m, n in matches:
            if m.distance < tolerance * n.distance:
                good.append([m])
        try:
            # Calculate offset and Draw final point
            if len(good) < 2:
                raise CustomErr(f'{text} feature point not enough ({len(good)})')

            x, y = to_int(src_kp[good[0][0].trainIdx].pt)
            offset_x, offset_y = to_int(sub_kp[good[0][0].queryIdx].pt)
            factor = np.divide(
                distance(to_int(src_kp[good[0][0].trainIdx].pt), to_int(src_kp[good[1][0].trainIdx].pt)),
                distance(to_int(sub_kp[good[0][0].queryIdx].pt), to_int(sub_kp[good[1][0].queryIdx].pt))
            )
            if np.isnan(x) or np.isnan(y):
                raise CustomErr(f'{text}:Invalid raw position {x, y}')
            if np.isnan(offset_x) or np.isnan(offset_y):
                raise CustomErr(f'{text}:Invalid offset {offset_x, offset_y}')
            if np.isnan(factor):
                raise CustomErr(f'{text}:Invalid factor {factor}')

            pt1 = (x - offset_x * factor), (y - offset_y * factor)
            pt2 = np.add(pt1, [sub_w * factor, sub_h * factor])
            draw_rectangle(src_img, to_int(pt1), to_int(pt2), thickness=5, text=text)

            # Draw feature point
            if preview > 1:
                preview = cv2.drawMatchesKnn(sub_img, sub_kp, src_img, src_kp, good[:2], None, flags=2)
            else:
                preview = src_img
        except CustomErr as e:
            print(e)
            preview = src_img
            pt1 = pt2 = 0
        except Exception:
            print(traceback.format_exc())
            preview = src_img
            pt1 = pt2 = 0
    else:
        raise NotImplementedError

    # Show preview
    show(preview)
    return pt1, pt2

def show(img, width=1280, height=720, title='Image'):
    cv2.imshow(title, cv2.resize(img, (width, height)))

def screenshot():
    screen_shot = d3dshot_mgr.screenshot()
    screen_shot = cv2.cvtColor(screen_shot, cv2.COLOR_BGR2RGB)
    return screen_shot

if __name__ == '__main__':
    while True:
        screen = screenshot()
        find_pos(color_board_img, screen, text='Color', preview=2)
        # find_pos(draw_board_img, screen, tolerance=0.9, text='Draw', grey_mode=True)
        # find_pos(pen_setting_img, screen, tolerance=0.9, text='Pen', grey_mode=True)
        cv2.waitKey(1)


import datetime
from datetime import *
from models import Scale, Observation

def age_check(agent_behavior, observation, scale):
    """
    :param agent_behavior:  Metrics to be used the agent.
    :type agent_behavior: dict
    :param observation: Content and metadata of message received and on which the trust is calculated.
    :type observation: Observation
    :param scale: The trust scale used by the agent
    :type scale: Scale
    :return: An age punishment value that is equal to the scale maximum value for recent publications and falls within
    [default, max) if it exceeded the allowed lifetime.
    :rtype: float or int
    """

    now = datetime.utcnow().timestamp()
    age = observation.details['content_trust.publication_date'] + \
        agent_behavior['content_trust.max_lifetime_seconds']

    if now < age:  # within allowed lifetime
        return scale.maximum_value()
    else:  # exceeded lifetime
        if 'content_trust.age_grace_period_seconds' in agent_behavior:
            grace_value = (
                now - age) / agent_behavior['content_trust.age_grace_period_seconds']
            if grace_value < 1.0:  # within grace period
                return (1-grace_value) * scale.maximum_value()

        return scale.default_value()  # no grace period or grace period exceeded


from typing import List
import re
from collections import Counter

class Solution:
    def shortestCompletingWord(self, licensePlate: str,
                               words: List[str]) -> str:
        regex = re.compile('[^a-zA-Z]')
        license = regex.sub('', licensePlate)
        counter1 = Counter(license.lower())
        res = ''
        for word in words:
            if self.count(counter1, word):
                if res == '' or len(word) < len(res):
                    res = word
        return res

    def count(self, counter1, word):
        counter2 = Counter(word)
        counter2.subtract(counter1)
        return all([c >= 0 for c in counter2.values()])


# leet494

from collections import defaultdict
from typing import List

def init(dims, filled_value = 0):
    if not dims: return filled_value
    return [init(dims[1:]) for _ in range(dims[0])]

class Solution:
    def findTargetSumWays(self, A: List[int], target: int) -> int:
        f = {0: 1}
        for x in A:
            f, g = defaultdict(int), f
            for t, cnt in g.items():
                f[t - x] += cnt
                f[t + x] += cnt
        return f[target]

print(Solution().findTargetSumWays([1,1,1,1,1], 3))


# Restaurant Rater
# Author: Ryan Berns
# Date Created: 2017-08-14

from urllib.request import urlopen
from bs4 import BeautifulSoup
import urllib.request
import re
import requests
import pprint
import json
import csv
from twilio.rest import Client

# Information for the yelp API
accountSID = '*******************************'
authToken = '*******************************'
client = Client(accountSID, authToken)

# Phone number used to text users
myTwilioNumber = '+99999999'

# Use a browser emulator so google allows scrapes
class AppURLopener(urllib.request.FancyURLopener):
    version = "Mozilla/5.0"
appId ='**************'
appSecret ='************'
data = {'grant_type': 'client_credentials',
        'client_id': appId,
        'client_secret': appSecret}
token = requests.post('https://api.yelp.com/oauth2/token', data=data)
accessToken = token.json()['access_token']
url = 'https://api.yelp.com/v3/businesses/search'
headers = {'Authorization': 'bearer %s' % accessToken}

# Start app and ask for input information from user
print("Welcome to my restaurant rating app!\n")
userName = input("Please enter your name. (First name and first initial of last name):\t")

# App persists user information in a csv.  When a user logs in again, those same preferences are found and do not need to be entered again.
try:
    reader = csv.reader(open(r'C:\localData\userData.csv'))
    userDirectory={}
    for row in reader:
        if len(row) > 0:
            userDirectory[row[0]] = row[1:]
except FileNotFoundError:
    userDirectory={}

if userName in userDirectory:
    print("Welcome back",userName)
    myCellPhone = userDirectory[userName][0]
    googlePref = int(userDirectory[userName][1])
    yelpPref = int(userDirectory[userName][2])
else:
    phoneNumberMessage = str("Hello, "+userName+". Can you please input your mobile number?  Just numbers! (no dashes please)\t")
    myCellPhone = input(phoneNumberMessage)
    googlePref = input("On a scale of 1 - 3 (3 being the best), how much do you value google reviews?\t")

    # Handle errors if user enters invalid scaling for google
    try:
        googlePref = int(googlePref)
    except ValueError:
        googlePref = input("Please enter as a number. (1,2 or 3). Try again:\t")
    if int(googlePref) > 3:
        googlePref = input("You can only choose 1,2 or 3.  how much do you value google reviews?\t")
    elif int(googlePref) < 1:
        googlePref = input("You can only choose 1,2 or 3.  how much do you value google reviews?\t")

    # Add yelp review value
    yelpPref = input("On a scale of 1 - 3 (3 being the best), how much do you value yelp reviews?\t")

    # Handle errors if user enters invalid scaling for yelp
    try:
        yelpPref = int(yelpPref)
    except ValueError:
        yelpPref = input("Please enter as a number. (1,2 or 3). Try again:\t")
    if int(yelpPref) > 3:
        yelpPref = input("You can only choose 1,2 or 3.  how much do you value yelp reviews?\t")
    elif int(yelpPref) < 1:
        yelpPref = input("You can only choose 1,2 or 3.  how much do you value yelp reviews?\t")

    userValues = str(myCellPhone+" "+str(googlePref)+" "+str(yelpPref)+" ").split()
    userDirectory[userName] = userValues
    with open(r'C:\localData\userData.csv', 'w') as outfile:
        csv_writer = csv.writer(outfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        for k,v in userDirectory.items():
            csv_writer.writerow([k] + v)
# Extract information from Google and Yelp while ensuring we get a rating back
restart="Y"
while restart=="Y":
    restart="N"
    nm = input("What is the name of the restaurant?\t")
    ask = str("What is the location of "+nm+"?\t")
    loct = input(ask)
    print("")
    nmUp=nm.replace(" ","+")
    loctUp = loct.replace(" ","+")
### Google ###
    query = ("https://www.google.com/search?safe=off&q="+nmUp+"+"+loctUp+"+google+review")
    opener = AppURLopener()

    response = opener.open(query)
    soup = BeautifulSoup(response,'html.parser')

    # Rating
    output = soup.find('span', {'class':'_kgd'})

    if output==None:
        print("We can\'t find that restaurant.  Please try again.")
        restart="Y"
    else:
        restart="N"
    # Number of Reviews
    output2 = soup.find('span', {'style': 'color:#777'})
    # Error handling in case the app cannot find the restaurant on google reviews
    try:
        output2Tmp = ''.join(re.findall(r'\d',str(output2.get_text())))
    except AttributeError:
        print('Sorry. We can\'t find that restaurant on google. Please check to make sure you inputted the restaurant correctly.\n')

### YELP ###
    params = {'location': loct,
              'term': nm,
              'limit': '1'
             }

    resp = requests.get(url=url, params=params, headers=headers)

    outputY = resp.json()
    subsetList = outputY['businesses']
    # Error handling in case the app cannot find the restaurant on yelp
    try:
        subsetDict = dict(subsetList[0])
    except IndexError:
        print('Sorry. We can\'t find that restaurant on yelp. Please check to make sure you inputted the restaurant correctly.\n')
        restart="Y"

def yourRating(yelpPref=yelpPref,googlePref=googlePref):
    ratio = float(googlePref/yelpPref)
    googleNum = (float(output.get_text())*float(output2Tmp)*ratio)
    yelpNum = (float(subsetDict['rating'])*float(subsetDict['review_count']))
    den = ((float(output2Tmp)*ratio)+float(subsetDict['review_count']))
    yourRtng = round(((googleNum+yelpNum)/den),2)
    return yourRtng

yourRtng = yourRating()
# Print Output to screen
print("Based on your preferences, our recommended rating for", subsetDict['name'],"is",yourRtng,".\n\n")
print("########## Google ##########")
print("Rating:\t",output.get_text())
print("Total reviews:\t",output2Tmp)
print("")
print("######### Yelp ##########")
print("Rating:\t",subsetDict['rating'])
print("Total reviews:\t",subsetDict['review_count'])
print("")
print("Address:")
for i in subsetDict['location']['display_address']:
    print(i)
print(subsetDict['display_phone'])
if subsetDict['is_closed']==True:
    print("Unfortunately",subsetDict['name'],"closed and no longer exists.")

# Build Text Output for text message
restaurant_nm = subsetDict['name']
yourRtngFormatted = str(yourRtng)
googleRtng = output.get_text()
yelpRtng = str(subsetDict['rating'])
yelpCnt = str(subsetDict['review_count'])
restaurantPhone = subsetDict['display_phone']
address = ""
for i in subsetDict['location']['display_address']:
    address += str(i+"\n")
txtOutput = str(restaurant_nm+"\nRecommended Rating:\t"+yourRtngFormatted+"\nGoogle\n"+"Rating:\t"+googleRtng+"\nTotal reviews:\t"+output2Tmp+"\n\nYelp\n"+"Rating:\t"+yelpRtng+"\nTotal reviews:\t"+yelpCnt+"\n\n"+address+"\n"+restaurantPhone)

# Send Text Message
try:
    message = client.messages.create(body=txtOutput, from_=myTwilioNumber, to=myCellPhone)
except:
    print("If the app creator wasn\'t so cheap, you would\'ve been texted this information to your mobile phone as well.\nHope you enjoy your meal!")


from collections import deque

def search(index, n, graph):
    dp = [0] * (n+1)
    visited = [0] * (n+1)

    q = deque()
    q.append(index)
    visited[index] = 1
    while q:
        top = q.popleft()

        for data in graph[top]:
            if visited[data] == 0:
                visited[data] = 1
                dp[data] = dp[top] + 1
                q.append(data)

    return dp.count(max(dp))

def solution(n, edge):
    answer = 0

    graph = [[] for _ in range(n+1)]

    for data in edge:
        a, b = data[0], data[1]
        graph[a].append(b)
        graph[b].append(a)

    answer = search(1, n, graph)
    return answer


#coding=utf8
#__author__=='luocheng'

TAG = 'cn'
start = 1
end =51
switch = 33
devide = 49.0
from collections import defaultdict

run2id2fls_ndcg = defaultdict(lambda:defaultdict(lambda:0))
run2id2sls_ndcg = defaultdict(lambda:defaultdict(lambda:0))
run2id2ndcg = defaultdict(lambda:defaultdict(lambda:0))

for l in open('./'+TAG+'.flseva.pertopic.csv').readlines()[1:]:
    segs=  l.strip().split(',')
    runname = segs[0]
    id = segs[1]
    dsharp = float(segs[3])
    run2id2fls_ndcg[runname][id] = dsharp

for l in open('./'+TAG+'.slseva.pertopic.csv').readlines()[1:]:
    segs=  l.strip().split(',')
    runname = segs[0]
    id = segs[1]
    dsharp = float(segs[3])
    run2id2sls_ndcg[runname][id] = dsharp

for l in open('./'+TAG+'.ndcg.pertopic.csv').readlines():
    segs=  l.strip().split(',')
    runname = segs[0]
    id = segs[1]
    ndcg= float(segs[2])
    run2id2ndcg[runname][id] = ndcg

def evaid():
    rtr = list()
    for i in range(start,end,1):
        if i <10:
            rtr.append('000'+str(i))
        if i >10 and i <100:
            rtr.append('00'+str(i))
        if i >=100:
            rtr.append('0'+str(i))
    return rtr
run2coarsesum =defaultdict(lambda:0.0)
run2finesum =defaultdict(lambda:0.0)

fout = open(TAG+'.dr.pertopic.csv','w')
fout.write('runname,id,coarse,fine,ndcg\n')
for r in sorted(list(run2id2ndcg.keys())):
    for id in evaid():
        if int(id) < switch:
            run2coarsesum[r] += run2id2fls_ndcg[r][id]
            run2finesum[r] += run2id2sls_ndcg[r][id]
        else:
            run2coarsesum[r] += run2id2ndcg[r][id]
            run2finesum[r] += run2id2ndcg[r][id]

        fout.write(r+','+id+','+str(run2id2fls_ndcg[r][id])+','+str(run2id2sls_ndcg[r][id])+','+str(run2id2ndcg[r][id])+'\n')
fout.close()

fout = open(TAG+'.dr.perrun.csv','w')
fout.write('runname,coarse-score,fine-score\n')
for r in sorted(list(run2id2fls_ndcg.keys())):
    fout.write(r+','+str(run2coarsesum[r]/devide)+','+str(run2finesum[r]/devide)+'\n')
fout.close()


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

class RNN(nn.Module):
	name = 'rnn_unroll'
	# def __init__(self, hidden_size=128, forecast=5, relu=False, deep=False):
	def __init__(self, hidden_size=256, forecast=5, deep=True, lag=6):
		super(RNN, self).__init__()
		# self.relu = relu
		self.lag = lag # temporal dimension
		# self.lag = 5 # temporal dimension
		self.steps = 10 # spatial dimension (optional ?)
		self.hidden_size = hidden_size
		self.forecast = forecast

		hsize = hidden_size
		if deep:
			self.inp = nn.Sequential(
				nn.Linear(self.lag, hsize),
				nn.ReLU(),
				nn.Linear(hsize, hsize),
				nn.ReLU(),
				nn.Linear(hsize, hsize),
				nn.ReLU(),
				# nn.Dropout(0.5),
			)
			self.out = nn.Sequential(
				# nn.Dropout(0.5),
				nn.ReLU(),
				nn.Linear(hsize, hsize),
				nn.ReLU(),
				nn.Linear(hsize, hsize),
				nn.ReLU(),
				nn.Linear(hsize, self.lag),
			)
		else:
			self.name += '_short'
			self.inp = nn.Sequential(
				nn.Linear(self.lag, hsize),
			)
			self.out = nn.Sequential(
				nn.Linear(hsize, self.lag),
			)

		self.rnn = nn.LSTM(hidden_size, hidden_size, 1)
		self.bsize = 32

	def step(self, input, hidden=None):
		# seqlen = 1 for stepwise eval
		# in: batch x inputsize
		if input is not None:
			input = self.inp(input)
			input = input.unsqueeze(0)
		else:
			input = torch.zeros(1, self.bsize, self.hidden_size).to(self.device)
		# in: seqlen x batch x inputsize
		output, hidden = self.rnn(input, hidden)
		# out: seqlen x batch x hiddensize
		output = self.out(output.squeeze(0))
		return output, hidden

	def forward(self, inputs, hidden=None, dump=False):
		steps = len(inputs)
		# lastKnown = steps - self.forecast - 1
		outputs = []

		for ii in range(steps):
			output, hidden = self.step(inputs[ii], hidden)
			outputs.append(output)

		outputs = torch.stack(outputs, dim=0)
		# return outputs, hidden
		if dump:
			return outputs, hidden
		else:
			return outputs

	def params(self, lr=0.001):
		criterion = nn.MSELoss().cuda()
		opt = optim.SGD(self.parameters(), lr=lr)
		sch = optim.lr_scheduler.StepLR(opt, step_size=20, gamma=0.5)
		return criterion, opt, sch

	def format_batch(self, data, wrap=True, normalize=1):
		# raw   : batch x timelen x seqlen
		data /= normalize
		data = torch.transpose(torch.transpose(data, 2, 1), 1, 0)
		# fmt   : seqlen x batch x timelen
		sequence = list(torch.split(data, 1, dim=0))

		for ti in range(len(sequence)):
			sequence[ti] = sequence[ti].to(self.device).float().squeeze(0)

		Xs = list(reversed(sequence[1:]))
		Ys = list(reversed(sequence[:-1])) # predict 1 stop back
		Ys = torch.stack(Ys, dim=0)

		return Xs, Ys


import torch
import numpy as np
import random

def divide_by_if_not_none(t,weight):
    if weight is None:
        return t
    else:
        return t/weight

def setup_random_seed(seed):
    if seed==-1:
        print('No seed was specified, leaving everthing at random. Use --seed to specify a seed if you want repeatable results.')
    else:
        print('Setting the random seed to {:}'.format(seed))
        random.seed(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)

def setup_device(desired_gpu=None):
    print('Device setup:')
    print('-------------')
    if torch.cuda.is_available() and (desired_gpu is not None):
        device = torch.device('cuda:' + str(desired_gpu))
        print('Setting the default tensor type to torch.cuda.FloatTensor')
        torch.set_default_tensor_type(torch.cuda.FloatTensor)
        print('Setting the CUDA device to {}'.format(desired_gpu))
        torch.cuda.set_device(desired_gpu)
    else:
        device = 'cpu'
        print('Setting the default tensor type to torch.FloatTensor')
        torch.set_default_tensor_type(torch.FloatTensor)
        print('Device is {}'.format(device))

    return device

def print_all_parameters(model):

    print('\n Model parameters:\n')
    for pn,pv in model.named_parameters():
        print('{} = {}\n'.format(pn, pv))

def compute_number_of_parameters(model, print_parameters=False):

    nr_of_fixed_parameters = 0
    nr_of_optimized_parameters = 0
    print('\nModel parameters:\n')
    print('-----------------')
    for pn, pv in model.named_parameters():
        if print_parameters:
            print('{} = {}'.format(pn, pv))
        current_number_of_parameters = np.prod(list(pv.size()))
        print('{}: # of parameters = {}\n'.format(pn,current_number_of_parameters))
        if pv.requires_grad:
            nr_of_optimized_parameters += current_number_of_parameters
        else:
            nr_of_fixed_parameters += current_number_of_parameters

    print('Number of fixed parameters = {}'.format(nr_of_fixed_parameters))
    print('Number of optimized parameters = {}'.format(nr_of_optimized_parameters))
    overall_nr_of_parameters = nr_of_fixed_parameters + nr_of_optimized_parameters
    print('Overall number of parameters = {}\n'.format(overall_nr_of_parameters))

    nr_of_pars = dict()
    nr_of_pars['fixed'] = nr_of_fixed_parameters
    nr_of_pars['optimized'] = nr_of_optimized_parameters
    nr_of_pars['overall'] = overall_nr_of_parameters

    return nr_of_pars

def _freeze_or_unfreeze_parameters(shooting_block, parameters, requires_grad_val):
    pass

    # get all the parameters that we are optimizing over
    pars_list = list(shooting_block.named_parameters())

    # put them all in a dictionary
    pars = dict()
    for v in pars_list:
        pars[v[0]] = v[1]

    for pn in parameters:
        if requires_grad_val:
            print('Unfreezing {}'.format(pn))
        else:
            print('Freezing {}'.format(pn))
        pars[pn].requires_grad = requires_grad_val

def unfreeze_parameters(shooting_block,parameters_to_unfreeze):
    _freeze_or_unfreeze_parameters(shooting_block=shooting_block, parameters=parameters_to_unfreeze, requires_grad_val=True)

def freeze_parameters(shooting_block,parameters_to_freeze):
    _freeze_or_unfreeze_parameters(shooting_block=shooting_block, parameters=parameters_to_freeze, requires_grad_val=False)


# -*- coding: utf-8 -*-

import pandas as pd
import numpy as np
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.model_selection import train_test_split
from bert import Bert
from data_loader import full_generator
from model import get_model
import json

# prepare y
# open data
dataframe = pd.read_csv("input/occupation_seq.csv", encoding = "cp1257").fillna(0)
LPKdict=pd.read_excel("input/ISCO_relationship.xlsx").fillna(0)#, encoding = "cp1257"

LPKdict.lpk4=LPKdict.lpk4.astype(int)

# create hierarchical structure

lpkindexmap = {}

for i in range(1,5):
    # to split 4 digit code into additional 1, 2, 3 digits codes
    dataframe[f'LPK_{i}_kodas'] = dataframe['s_veik_prof'].astype(str).str[0:i].astype(int)
    # add names to professions
    dataframe = dataframe.join(LPKdict.set_index('lpk4'), on=f'LPK_{i}_kodas')
    dataframe.rename(columns={'lpk_pav': f'LPK_{i}_pav'}, inplace=True)

    # dictionary for prediction
    lpkindexmap[i] = sorted(dataframe.set_index(f'LPK_{i}_kodas').to_dict()[f'LPK_{i}_pav'].items())

dataframe=dataframe.drop(['s_veik_prof'], axis=1)

with open('lpkindexmap.json', 'w') as f:
    json.dump(lpkindexmap, f)

#make sparse matrix as we need to save memory
mlb = MultiLabelBinarizer(sparse_output=True)

y = pd.DataFrame()
for i in range(1,5):
    d = (dataframe[['id', f'LPK_{i}_kodas']]
        .drop_duplicates()
        .groupby('id')[f'LPK_{i}_kodas']
        .apply(lambda g: g.values.tolist())
        .to_dict())

    df = pd.DataFrame(d.items(), columns=['id', f'lpk{i}'])
    d = pd.DataFrame(mlb.fit_transform(df[f'lpk{i}']), columns=[f'sparselpk{i}'])
    if y.empty:
        y = df
    y = y.join(pd.DataFrame(d, index=df.index))

# prepare x

#open file
edu = pd.read_csv("input/education_seq.csv", encoding = "cp1257").fillna("")
abstract=pd.read_csv("input/descriptions.csv", encoding = "cp1257").fillna("NA")

# make id for program
edu = edu.assign(ind=(edu['kodas']).astype('category').cat.codes)
edu=pd.merge(edu, abstract, on='kodas',  how='left').fillna("NA")

# prepare sparse dataset
edu_sparse=edu.groupby('id')['ind'].apply(lambda g: g.values.tolist()).to_dict()
edu_sparse=pd.DataFrame(edu_sparse.items(), columns=['id', 'programos_kodai'])

mlb = MultiLabelBinarizer(sparse_output=True)
mlb_transform=pd.DataFrame(mlb.fit_transform(edu_sparse['programos_kodai']), columns=['sparse_programoskodai'])

x_sp=edu_sparse.join(pd.DataFrame(mlb_transform, index=edu_sparse.index))

# prepare text dataset
x_txt=edu.groupby(['id'])['trans_abstract'].apply(','.join).reset_index()
dataframe=pd.merge(x_txt, x_sp, on='id',  how='left')

# join x and y variables for final dataset and drop cases where a person
# has education but has no occupation
dataframe=pd.merge(dataframe, y, on='id',  how='left')

dataframe=dataframe.dropna()
print(dataframe.iloc[1])

#make BERT from text input

bert = Bert()

translated_abstracts_df = dataframe[['id', 'trans_abstract']]

embeddings_df = pd.DataFrame()

for embedding in bert.make_embeddings(translated_abstracts_df[['trans_abstract']]):
    embeddings_df.append(pd.DataFrame(embedding), ignore_index=True)

result = pd.concat([translated_abstracts_df, embeddings_df], axis=1, sort=False)

#final data for training
X=result.values[:,2:]
Y = dataframe.values[:,5:9]

X_train, X_test, y_train, y_test = train_test_split(X, Y, random_state=42, test_size=0.2)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, random_state=42, test_size=0.2)
print(X_train.shape)
print(y_train.shape)

batch_size = 128
number_of_batches = int(np.ceil(X_train.shape[0]/batch_size))
val_batch_size = int(np.ceil(X_val.shape[0]/number_of_batches))
number_of_batches

#training
model = get_model(X_train, y_train)

history = model.fit_generator(
    generator=full_generator(X_train, y_train[:,1], y_train[:,2], y_train[:,3], y_train[:,0], batch_size, number_of_batches, True),
    steps_per_epoch=number_of_batches,
    validation_data=full_generator(X_val, y_val[:,1], y_val[:,2], y_val[:,3], y_val[:,0], val_batch_size, number_of_batches, False),
    validation_steps=number_of_batches,
    epochs=30, verbose=1)

gen_data=full_generator(X_test, y_test[:,1], y_test[:,2], y_test[:,3], y_test[:,0],val_batch_size, number_of_batches, False, infinite=False)
res = model.evaluate_generator(gen_data)
print(res)