# -*- coding: utf8 -*-
__author__ = 'Clemens Prescher'

from PyQt4 import QtCore, QtGui
import pyqtgraph as pg

from .ModifiedPlotItem import ModifiedPlotItem

from pyqtgraph.exporters.ImageExporter import ImageExporter
from pyqtgraph.exporters.SVGExporter import SVGExporter

class SpectrumWidget(QtGui.QWidget):
    mouse_left_clicked = QtCore.pyqtSignal(float, float)
    mouse_moved = QtCore.pyqtSignal(float, float)

    def __init__(self, *args, **kwargs):
        super(SpectrumWidget, self).__init__(*args, **kwargs)
        self._layout = QtGui.QVBoxLayout()
        self._layout.setContentsMargins(0, 0, 0, 0)

        self.create_plot_items()
        self.create_data_items()

        self.setLayout(self._layout)
        self._plot_item.connect_mouse_move_event()

    def create_plot_items(self):
        self._pg_layout_widget = pg.GraphicsLayoutWidget()
        self._pg_layout = pg.GraphicsLayout()
        self._pg_layout.setContentsMargins(0, 0, 0, 0)
        self._pg_layout.layout.setVerticalSpacing(0)

        self._plot_item = ModifiedPlotItem(enableMouseInteraction=True)
        self._plot_item.showAxis('top', show=True)
        self._plot_item.showAxis('right', show=True)
        self._plot_item.getAxis('top').setStyle(showValues=False)
        self._plot_item.getAxis('right').setStyle(showValues=False)
        self._plot_item.getAxis('left').setStyle(showValues=True)
        self._plot_item.setLabel('bottom', 'v (cm<sup>-1</sup>)')
        self._plot_item.mouse_left_clicked.connect(self.mouse_left_clicked)
        self._plot_item.mouse_moved.connect(self.mouse_moved)
        self._pg_layout.addItem(self._plot_item, 0, 0)

        self._pg_layout_widget.addItem(self._pg_layout)
        self._layout.addWidget(self._pg_layout_widget)

    def create_data_items(self):
        self._data_item = pg.PlotDataItem(pen=pg.mkPen("#fff", width=1.5))
        self._plot_item.addItem(self._data_item)

    def add_item(self, pg_item):
        self._plot_item.addItem(pg_item)

    def plot_data(self, x, y):
        self._data_item.setData(x, y)

    def get_data(self):
        return self._data_item.getData()

    def set_xlabel(self, label_string):
        self._plot_item.setLabel('bottom', label_string)

    def save_graph(self, filename):
        self._pg_layout.setContentsMargins(20, 20, 20, 20)
        QtGui.QApplication.processEvents()
        if filename.endswith('.png'):
            exporter = ImageExporter(self._pg_layout)
            exporter.export(filename)
        elif filename.endswith('.svg'):
            self._invert_color()
            exporter = SVGExporter(self._pg_layout)
            exporter.export(filename)
            self._norm_color()
        self._pg_layout.setContentsMargins(0, 0, 0, 0)
        QtGui.QApplication.processEvents()

    def _invert_color(self):
        self._plot_item.getAxis('bottom').setPen('k')
        self._plot_item.getAxis('top').setPen('k')
        self._plot_item.getAxis('left').setPen('k')
        self._plot_item.getAxis('right').setPen('k')
        self._data_item.setPen(pg.mkPen("#000", width=1.5))

    def _norm_color(self):
        self._plot_item.getAxis('bottom').setPen('w')
        self._plot_item.getAxis('top').setPen('w')
        self._plot_item.getAxis('left').setPen('w')
        self._plot_item.getAxis('right').setPen('w')
        self._data_item.setPen(pg.mkPen("#FFF", width=1.5))


from rest_framework.test import APITestCase
from rest_framework.views import status
from api.application_form.models import ApplicationForm
from api.application_form.serializers import ApplicationFormSerializer

class ApplicationFormViewSetTest(APITestCase):
    def setUp(self):
        ApplicationForm.objects.create(
            first_name="Bobby",
            last_name="Bass",
            gender="M",
            nationality="USA",
            location="USA",
            email="bbass@rocketmail.com",
            phone="18002254452"
        )

    def test_post_application_form(self):
        self.client.post("/api/v1/applicationForms/", {
            "first_name": "Krikor",
            "last_name": "Ailanjian",
            "gender": "M",
            "nationality": "USA",
            "location": "USA",
            "email": "kailanjian@gmail.com",
            "phone": "6268251906"
        })

        self.assertEqual(2, len(ApplicationForm.objects.all()))

    def test_get_application_form(self):
        response = self.client.get("/api/v1/applicationForms/")

        expected = ApplicationForm.objects.all()
        serialized = ApplicationFormSerializer(expected, many=True)
        self.assertEqual(response.data['results'], serialized.data)
        self.assertEqual(response.status_code, status.HTTP_200_OK)


import datetime
import json
import copy
from Match import Match
from Round import Round
from enum import Enum
import config

class Frame(object):
    def __init__(self, j):
        self.__dict__ = json.loads(j)

class MatchState(Enum):
    WARMUP = 'warmup'
    LIVE = 'live'
    OVER = 'gameover'
    SWAP = 'intermission'

class ParserStat:
    now = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d_%Hh%M')
    match = Match(now, '')

    def __init__(self):
        self.frame = 0
        self.state = MatchState.WARMUP
        self.round_count = 0
        self.mvps = 0
        self.scores = 0

    def parse_round(self):
        player = self.frame.player
        map = self.frame.map
        side = self.frame.player['team'] if 'team' in self.frame.player else 0
        print('SIDE', side)
        t_score = map['team_t']['score']
        ct_score = map['team_ct']['score']
        nbr = map['round']
        kill = player['state']['round_kills'] if 'round_kills' in player['state'] else 0
        killhs = player['state']['round_killhs'] if 'round_killhs' in player['state'] else 0
        assist = player['state']['round_assists'] if 'round_assists' in player['state'] else 0
        death = player['state']['round_death'] if 'round_death' in player['state'] else 0
        score = player['match_stats']['score'] if 'score' in player['match_stats'] else 0
        mvp = player['match_stats']['mvps'] if 'mvps' in player['match_stats'] else 0
        self.match.add_round(copy.copy(Round(side, t_score, ct_score, nbr, kill, killhs, assist, death, score, mvp)))

    def parse_data(self, post_data):
        self.frame = Frame(post_data)
        if not ('previously' in self.frame.__dict__):
            return 0
        if 'map' in self.frame.__dict__:
            if self.state != MatchState.LIVE and self.frame.round['phase'] == MatchState.LIVE.value:
                print('LIVE')
                self.state = MatchState.LIVE
                self.match.update_map(self.frame.map['name'])
            print('[FRAME]')
            print(self.frame.__dict__)
            if self.frame.map['phase'] == MatchState.WARMUP.value:
                print("warmup")
                return 0
            elif self.state == MatchState.LIVE and self.frame.map['phase'] == MatchState.LIVE.value:
                # First round
                if self.frame.player['steamid'] == config.Steam['yupin_id']:
                    if self.frame.round['phase'] == 'over' and not ('phase' in self.frame.previously['map']):
                        print("[ROUND_OVER]")
                        self.parse_round()
            elif self.state == MatchState.LIVE and self.frame.map['phase'] == MatchState.SWAP.value and not ('phase' in self.frame.previously['map']):
                print('SWAP')
                self.parse_round()
            elif self.state != MatchState.WARMUP and self.frame.map['phase'] == MatchState.OVER.value:
                self.state = MatchState.WARMUP
                self.parse_round()
                # Export vers db
                print("[GAME_OVER]")
                print(self.match.__dict__)
                for rnd in self.match.rounds:
                    print(rnd.__dict__)
                print("END")
                print("END")


import matplotlib.pyplot as plt
from matplotlib import animation
import numpy as np
from numpy import random
import pandas as pd

fig = plt.figure()
ax1 = plt.axes(xlim=(0, 1), ylim=(-0.1,1.2))
line, = ax1.plot([], [], lw=2)
plt.xlabel('r')
plt.ylabel('z')

linecolor = "blue"
ax1.axvline(0, linewidth = 0.8)
plotlays, plotcols = [2], [linecolor,linecolor]
lines = []
ax1.set_aspect('equal','box')
for index in range(2):
    lobj = ax1.plot([],[],lw=2,color=plotcols[index])[0]
    lines.append(lobj)

def init():
    for line in lines:
        line.set_data([],[])
    return lines

x1,y1 = [],[]
x2,y2 = [],[]

imax = 1200

def integers(a, b):
    return list(range(a, b+1))

rend = []
zend = []
rtop = []
ztop = []
for i in integers(1,imax):
    rvalues = pd.read_csv('solutions/r_values_' + str(i) + '.csv')
    zvalues = pd.read_csv('solutions/z_values_' + str(i) + '.csv')

    indmax = np.argmax(zvalues['z_values'].values)
    rtop.append(rvalues['r_values'].values[indmax])
    ztop.append(zvalues['z_values'].values[indmax])
    rend.append(rvalues['r_values'].values[-1])
    zend.append(zvalues['z_values'].values[-1])

ax1.plot(rend,zend,color='red')
ax1.plot(rtop,ztop,color='green')

def animate(i):
    rvalues = pd.read_csv('solutions/r_values_' + str(i) + '.csv')
    zvalues = pd.read_csv('solutions/z_values_' + str(i) + '.csv')

    x1 = rvalues['r_values']
    y1 = zvalues['z_values']

    x2 = -1.0*x1
    y2 = y1

    xlist = [x1, x2]
    ylist = [y1, y2]

    #for index in range(0,1):
    for lnum,line in enumerate(lines):
        line.set_data(xlist[lnum], ylist[lnum]) # set data for each line separately.

    return lines

# call the animator.  blit=True means only re-draw the parts that have changed.
anim = animation.FuncAnimation(fig, animate, init_func=init,
                               frames=integers(1,imax), interval=5, blit=True)

#anim.save('traces.mp4', fps=80, dpi = 200)
plt.show()


import sys
import sortutil

randArr = sortutil.getRandomArray(10,10,100)

sortutil.printArr(randArr)

def MAX_HEAPFY(arr,index,maxL):
    left=index*2+1
    right=index*2+2
    maxIndex=index
    if(left<=maxL and arr[left]>arr[index]):
        maxIndex=left
    if(right<=maxL and arr[right]>arr[maxIndex]):
        maxIndex=right
    if(maxIndex!=index):
        sortutil.swap(arr,maxIndex,index)
        MAX_HEAPFY(arr,maxIndex,maxL)

def BUILD_MAXHEAP(arr):
    length = len(arr)
    start = (length-1)//2
    while start>=0:
        MAX_HEAPFY(arr,start,length-1)
        start=start-1

def HEAP_SORT(arr):
    BUILD_MAXHEAP(arr)
    start = len(arr)
    endNum=len(arr)-1
    while endNum>0:
        sortutil.swap(arr,0,endNum)
        endNum=endNum-1
        MAX_HEAPFY(arr,0,endNum)

HEAP_SORT(randArr)
sortutil.printArr(randArr)


import os
import pprint

from autohdl import build
from autohdl import verilog
from autohdl.hdlLogger import log_call, logging
log = logging.getLogger(__name__)

class InstanceException(Exception):
  def __init__(self, iString):
      Exception.__init__(self, iString)

@log_call
def get_instances(files):
  """
  Input: list of files;
  Output: dictionary
    module name:
      path     : file_path
      instances: set(instance1, instance2,..)
  """
  parsed = {}
  if files:
    if type(files) is not list:
      files = [files]
    for afile in files:
      if os.path.splitext(afile)[1] not in ['.v']:
        continue
      try:
        new = verilog.get_instances(afile)
        parsed.update(new)
      except IOError as e:
        log.debug(e)
        log.warning("Can't open file: "+afile)
  return parsed

@log_call
def resolve_undef(instance, in_file, _parsed = {}):
#  print 'searching instance ', instance
#  print 'undef in file: ', in_file
  if instance in _parsed:
    return {instance: _parsed[instance]}

  dep_files = build.getDepPaths(in_file)
  if dep_files:
    for i in dep_files:
      parsed = get_instances([i])
#      print parsed
#      raw_input('next')
      for i in parsed:
        if i not in _parsed:
#          print 'added'
          _parsed[i] = parsed[i]
    if instance in _parsed:
      return {instance: _parsed[instance]}

def asNetlist(inFile, config):
  netlist = os.path.splitext(inFile)[0]+'.ngc'
  if os.path.exists(netlist):
    if config.get('depNetlist'):
      config['depNetlist'].append(netlist)
    else:
      config['depNetlist'] = [netlist]
    return True

@log_call
def analyze(parsed, config = {}, _ignore = set()):
  """
  Input: dictionary
  Output: dictionary
  """
#  _ignore.update(set(build.getParam(iKey='ignore_undefined_instances', iDefault=[])))
  for module in parsed:
    val = parsed[module]
    in_file = os.path.abspath(val['path'])
    for instance in val['instances']:
      if (instance, in_file) not in _ignore and instance not in parsed:
        parsed_new = resolve_undef(instance, in_file)
        if not parsed_new:
          if not asNetlist(in_file, config):
            log.warning("Undefined instance="+instance+' in file='+in_file)
          _ignore.add((instance, in_file))
        else:
          return parsed_new


import requests
import json

url = 'https://apimobile.meituan.com/group/v4/poi/pcsearch/1'
data = {
    'uuid': '6772ac9a728846b7b5a9.1566722473.1.0.0',
    'userid': -1,
    'limit': 32,
    'offset': 0,
    'cateId': 1,
    'q': '大兴区',
    'areaId': -1
}

r = requests.post(url, data)
with open('./info.json', 'w') as f:
	json.dump(r.json(), f)
print(r.text)


class Solution:
    def is_fives_digit(self, s: str) -> bool:
        return s == 'D' or s == 'L' or s == 'V'

    def get_digit_count(self, s: str, c: str, pos: int) -> int:
        count = 0
        for i in range(pos, len(s)):
            if s[i] == c:
                count += 1
            else:
                break
        return count

    def romanToInt(self, s: str) -> int:
        val, pos = 0, 0
        digit_val = {
            'M': 1000, 'D': 500,
            'C': 100,  'L': 50,
            'X': 10,   'V': 5,   'I': 1
        }
        sub_comb = {
            'CM': 900, 'CD': 400,
            'XC': 90,  'XL': 40,
            'IX': 9,   'IV': 4
        }
        while pos < len(s):
            if self.is_fives_digit(s[pos]):
                val = val + digit_val[s[pos]]
                pos = pos + 1
            elif s[pos:pos+2] in sub_comb:
                val = val + sub_comb[s[pos:pos+2]]
                pos = pos + 2
            else:
                dig_count = self.get_digit_count(s, s[pos], pos)
                val = val + (digit_val[s[pos]] * dig_count)
                pos = pos + dig_count
        return val


# Copyright 2022 The KerasCV Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Title: Training a KerasCV model for Imagenet Classification
Author: [ianjjohnson](https://github.com/ianjjohnson)
Date created: 2022/07/25
Last modified: 2022/07/25
Description: Use KerasCV to train an image classifier using modern best practices
"""

import sys

import tensorflow as tf
from absl import flags
from tensorflow.keras import callbacks
from tensorflow.keras import losses
from tensorflow.keras import metrics
from tensorflow.keras import optimizers

import keras_cv
from keras_cv import models
from keras_cv.datasets import imagenet

"""
## Overview
KerasCV makes training state-of-the-art classification models easy by providing implementations of modern models, preprocessing techniques, and layers.
In this tutorial, we walk through training a model against the Imagenet dataset using Keras and KerasCV.
This tutorial requires you to have KerasCV installed:
```shell
pip install keras-cv
```
"""

"""
## Setup and flags

"""

flags.DEFINE_string(
    "model_name", None, "The name of the model in KerasCV.models to use."
)
flags.DEFINE_string("imagenet_path", None, "Directory from which to load Imagenet.")
flags.DEFINE_string(
    "backup_path", None, "Directory which will be used for training backups."
)
flags.DEFINE_string(
    "weights_path", None, "Directory which will be used to store weight checkpoints."
)
flags.DEFINE_string(
    "tensorboard_path", None, "Directory which will be used to store tensorboard logs."
)
flags.DEFINE_integer(
    "batch_size",
    128,
    "Batch size for training and evaluation. This will be multiplied by the number of accelerators in use.",
)
flags.DEFINE_boolean(
    "use_xla", True, "Whether or not to use XLA (jit_compile) for training."
)
flags.DEFINE_float(
    "initial_learning_rate",
    0.05,
    "Initial learning rate which will reduce on plateau. This will be multiplied by the number of accelerators in use",
)
flags.DEFINE_string(
    "model_kwargs",
    "{}",
    "Keyword argument dictionary to pass to the constructor of the model being trained",
)

FLAGS = flags.FLAGS
FLAGS(sys.argv)

if FLAGS.model_name not in models.__dict__:
    raise ValueError(f"Invalid model name: {FLAGS.model_name}")

CLASSES = 1000
IMAGE_SIZE = (224, 224)
EPOCHS = 250

"""
We start by detecting the type of accelerators we have available and picking an
appropriate distribution strategy accordingly. We scale our learning rate and
batch size based on the number of accelerators being used.
"""

# Try to detect an available TPU. If none is present, default to MirroredStrategy
try:
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
    strategy = tf.distribute.TPUStrategy(tpu)
except ValueError:
    # MirroredStrategy is best for a single machine with one or multiple GPUs
    strategy = tf.distribute.MirroredStrategy()
print("Number of accelerators: ", strategy.num_replicas_in_sync)

BATCH_SIZE = FLAGS.batch_size * strategy.num_replicas_in_sync
INITIAL_LEARNING_RATE = FLAGS.initial_learning_rate * strategy.num_replicas_in_sync

"""
## Data loading
This guide uses the
[Imagenet dataset](https://www.tensorflow.org/datasets/catalog/imagenet2012).
Note that this requires manual download and preprocessing. You can find more
information about preparing this dataset at keras_cv/datasets/imagenet/README.md
"""

train_ds = imagenet.load(
    split="train",
    tfrecord_path=FLAGS.imagenet_path,
    batch_size=BATCH_SIZE,
    img_size=IMAGE_SIZE,
)
test_ds = imagenet.load(
    split="validation",
    tfrecord_path=FLAGS.imagenet_path,
    batch_size=BATCH_SIZE,
    img_size=IMAGE_SIZE,
)

"""
Next, we augment our dataset.
We define a set of augmentation layers and then apply them to our input dataset.
"""

AUGMENT_LAYERS = [
    keras_cv.layers.RandomFlip(),
    keras_cv.layers.RandAugment(value_range=(0, 255), magnitude=0.3),
    keras_cv.layers.CutMix(),
]

@tf.function
def augment(img, label):
    inputs = {"images": img, "labels": label}
    for layer in AUGMENT_LAYERS:
        inputs = layer(inputs)
    return inputs["images"], inputs["labels"]

train_ds = train_ds.map(augment, num_parallel_calls=tf.data.AUTOTUNE).prefetch(
    tf.data.AUTOTUNE
)
test_ds = test_ds.prefetch(tf.data.AUTOTUNE)

"""
Now we can begin training our model. We begin by loading a model from KerasCV.
"""

with strategy.scope():
    model = models.__dict__[FLAGS.model_name]
    model = model(
        include_rescaling=True,
        include_top=True,
        classes=CLASSES,
        input_shape=IMAGE_SIZE + (3,),
        **eval(FLAGS.model_kwargs),
    )

"""
Next, we pick an optimizer. Here we use Adam with a constant learning rate.
Note that learning rate will decrease over time due to the ReduceLROnPlateau callback.
"""

optimizer = optimizers.SGD(learning_rate=INITIAL_LEARNING_RATE, momentum=0.9)

"""
Next, we pick a loss function. We use CategoricalCrossentropy with label smoothing.
"""

loss_fn = losses.CategoricalCrossentropy(label_smoothing=0.1)

"""
Next, we specify the metrics that we want to track. For this example, we track accuracy.
"""

with strategy.scope():
    training_metrics = [metrics.CategoricalAccuracy()]

"""
As a last piece of configuration, we configure callbacks for the method.
We use EarlyStopping, BackupAndRestore, and a model checkpointing callback.
"""

callbacks = [
    callbacks.ReduceLROnPlateau(
        monitor="val_loss", factor=0.1, patience=10, min_delta=0.001, min_lr=0.0001
    ),
    callbacks.EarlyStopping(patience=20),
    callbacks.BackupAndRestore(FLAGS.backup_path),
    callbacks.ModelCheckpoint(FLAGS.weights_path, save_weights_only=True),
    callbacks.TensorBoard(log_dir=FLAGS.tensorboard_path, write_steps_per_second=True),
]

"""
We can now compile the model and fit it to the training dataset.
"""

model.compile(
    optimizer=optimizer,
    loss=loss_fn,
    metrics=training_metrics,
    jit_compile=FLAGS.use_xla,
)

model.fit(
    train_ds,
    batch_size=BATCH_SIZE,
    epochs=EPOCHS,
    callbacks=callbacks,
    validation_data=test_ds,
)


#!/usr/bin/python3
# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import pafy
import os
import re
import json
import youtube_dl
import sys
import sqlite3

#reload(sys)
#sys.setdefaultencoding('utf-8')

mysong =[]
def getKey():
    with open('../config.json') as f:
        data = json.load(f)
    return data['key']

def getArrMysongs():
    return mysong

def strclear(s=''):
    s = re.sub('\/|\(.*\)|\[.*\]', '', s)
    if '-' in s:  #去除一些不必要的字串
        s = s[0:s.index('-')]
    if '"' in s:  #去除一些不必要的字串
        s = s[0:s.index('"')]
    return s
def printissue():
    for row in mysong:
        print("\nArtist:"+row[0],"\nSong:"+row[1])

# 根據關鍵字搜尋影片,
# @param string key, Youtube Key
# @param string q, 搜尋關鍵字
# @return string videoId, 影片ID
def search_hot(key,q):
    url = 'https://www.googleapis.com/youtube/v3/search'
    dic = {'part':'snippet','key':key,'type':'video','q':q,'order':'viewCount','maxResults':1}
    r = requests.get(url,params=dic)
    json_data = json.loads(r.text)
    videoId = json_data['items'][0]['id']['videoId']
    return videoId

def download_v2(videoID,artist,name):
    url = 'https://www.youtube.com/watch?v='+videoID
    ydl_opts = {
        'format': 'bestaudio/best',
        'outtmpl': "./music/"+artist+" - "+name+'.%(ext)s',
        'postprocessors': [{
            'key': 'FFmpegExtractAudio',
            'preferredcodec': 'mp3',
            'preferredquality': '192',
        }]
    }
    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
        ydl.download([url])
#outtmpl


# coding: utf-8

import torch
from torch import nn
from torch.nn import functional as F
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence

from torch.nn.utils import weight_norm

from nnsvs.base import BaseModel, PredictionType
from nnsvs.mdn import MDNLayer, mdn_get_most_probable_sigma_and_mu
from nnsvs.dsp import TrTimeInvFIRFilter
from nnsvs.multistream import split_streams

def WNConv1d(*args, **kwargs):
    return weight_norm(nn.Conv1d(*args, **kwargs))

class ResnetBlock(nn.Module):
    def __init__(self, dim, dilation=1):
        super().__init__()
        self.block = nn.Sequential(
            nn.LeakyReLU(0.2),
            nn.ReflectionPad1d(dilation),
            WNConv1d(dim, dim, kernel_size=3, dilation=dilation),
            nn.LeakyReLU(0.2),
            WNConv1d(dim, dim, kernel_size=1),
        )
        self.shortcut = WNConv1d(dim, dim, kernel_size=1)

    def forward(self, x):
        return self.shortcut(x) + self.block(x)

class Conv1dResnet(BaseModel):
    def __init__(self, in_dim, hidden_dim, out_dim, num_layers=4, dropout=0.0):
        super().__init__()
        model = [
            nn.ReflectionPad1d(3),
            WNConv1d(in_dim, hidden_dim, kernel_size=7, padding=0),
        ]
        for n in range(num_layers):
            model.append(ResnetBlock(hidden_dim, dilation=2**n))
        model += [
            nn.LeakyReLU(0.2),
            nn.ReflectionPad1d(3),
            WNConv1d(hidden_dim, out_dim, kernel_size=7, padding=0),
        ]

        self.model = nn.Sequential(*model)

    def forward(self, x, lengths=None):
        return self.model(x.transpose(1,2)).transpose(1,2)

@torch.no_grad()
def _shallow_ar_inference(out, stream_sizes, analysis_filts):
    from torchaudio.functional import lfilter

    out_streams = split_streams(out, stream_sizes)
    # back to conv1d friendly (B, C, T) format
    out_streams = map(lambda x: x.transpose(1, 2), out_streams)

    out_syn = []
    for sidx, os in enumerate(out_streams):
        out_stream_syn = torch.zeros_like(os)
        a = analysis_filts[sidx].get_filt_coefs()
        # apply IIR filter for each dimiesion
        for idx in range(os.shape[1]):
            # NOTE: scipy.signal.lfilter accespts b, a in order,
            # but torchaudio expect the oppsite; a, b in order
            ai = a[idx].view(-1).flip(0)
            bi = torch.zeros_like(ai)
            bi[0] = 1
            out_stream_syn[:, idx, :] = lfilter(os[:, idx, :], ai, bi, clamp=False)
        out_syn += [out_stream_syn]

    out_syn = torch.cat(out_syn, 1)
    return out_syn.transpose(1, 2)

class Conv1dResnetSAR(Conv1dResnet):
    """Conv1dResnet with shallow AR structure

    Args:
        stream_sizes (list): Stream sizes
        ar_orders (list): Filter dimensions for each stream.
    """
    def __init__(self, in_dim, hidden_dim, out_dim, num_layers=4, dropout=0.0,
            stream_sizes=[180, 3, 1, 15], ar_orders=[20, 200, 20, 20]):
        super().__init__(in_dim, hidden_dim, out_dim, num_layers, dropout)
        self.stream_sizes = stream_sizes

        self.analysis_filts = nn.ModuleList()
        for s, K in zip(stream_sizes, ar_orders):
            self.analysis_filts += [TrTimeInvFIRFilter(s, K+1)]

    def preprocess_target(self, y):
        assert sum(self.stream_sizes) == y.shape[-1]
        ys = split_streams(y, self.stream_sizes)
        for idx, yi in enumerate(ys):
            ys[idx] = self.analysis_filts[idx](yi.transpose(1,2)).transpose(1,2)
        return torch.cat(ys, -1)

    def inference(self, x, lengths=None):
        out = self.model(x.transpose(1, 2)).transpose(1, 2)
        return _shallow_ar_inference(out, self.stream_sizes, self.analysis_filts)

class FeedForwardNet(BaseModel):
    def __init__(self, in_dim, hidden_dim, out_dim, num_layers=2, dropout=0.0):
        super(FeedForwardNet, self).__init__()
        self.first_linear = nn.Linear(in_dim, hidden_dim)
        self.hidden_layers = nn.ModuleList(
            [nn.Linear(hidden_dim, hidden_dim) for _ in range(num_layers)])
        self.last_linear = nn.Linear(hidden_dim, out_dim)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, lengths=None):
        h = self.relu(self.first_linear(x))
        for hl in self.hidden_layers:
            h = self.dropout(self.relu(hl(h)))
        return self.last_linear(h)

class LSTMRNN(BaseModel):
    def __init__(self, in_dim, hidden_dim, out_dim, num_layers=1, bidirectional=True,
            dropout=0.0):
        super(LSTMRNN, self).__init__()
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers
        self.num_direction =  2 if bidirectional else 1
        self.lstm = nn.LSTM(in_dim, hidden_dim, num_layers,
            bidirectional=bidirectional, batch_first=True, dropout=dropout)
        self.hidden2out = nn.Linear(self.num_direction*self.hidden_dim, out_dim)

    def forward(self, sequence, lengths):
        sequence = pack_padded_sequence(sequence, lengths, batch_first=True)
        out, _ = self.lstm(sequence)
        out, _ = pad_packed_sequence(out, batch_first=True)
        out = self.hidden2out(out)
        return out

class LSTMRNNSAR(LSTMRNN):
    """LSTM-RNN with shallow AR structure

    Args:
        stream_sizes (list): Stream sizes
        ar_orders (list): Filter dimensions for each stream.
    """
    def __init__(self, in_dim, hidden_dim, out_dim, num_layers=1, bidirectional=True,
            dropout=0.0, stream_sizes=[180, 3, 1, 15], ar_orders=[20, 200, 20, 20]):
        super().__init__(in_dim, hidden_dim, out_dim, num_layers,
            bidirectional, dropout)

        self.stream_sizes = stream_sizes
        self.analysis_filts = nn.ModuleList()
        for s, K in zip(stream_sizes, ar_orders):
            self.analysis_filts += [TrTimeInvFIRFilter(s, K+1)]

    def preprocess_target(self, y):
        assert sum(self.stream_sizes) == y.shape[-1]
        ys = split_streams(y, self.stream_sizes)
        for idx, yi in enumerate(ys):
            ys[idx] = self.analysis_filts[idx](yi.transpose(1,2)).transpose(1,2)
        return torch.cat(ys, -1)

    def inference(self, x, lengths=None):
        out = self.forward(x, lengths)
        return _shallow_ar_inference(out, self.stream_sizes, self.analysis_filts)

class RMDN(BaseModel):
    def __init__(self, in_dim, hidden_dim, out_dim, num_layers=1, bidirectional=True,
            dropout=0.0, num_gaussians=8, dim_wise=False):
        super(RMDN, self).__init__()
        self.linear = nn.Linear(in_dim, hidden_dim)
        self.relu = nn.ReLU()
        self.num_direction=2 if bidirectional else 1
        self.lstm = nn.LSTM(hidden_dim, hidden_dim, num_layers,
                            bidirectional=bidirectional, batch_first=True,
                            dropout=dropout)
        self.mdn = MDNLayer(self.num_direction*hidden_dim, out_dim, num_gaussians, dim_wise)

    def prediction_type(self):
        return PredictionType.PROBABILISTIC

    def forward(self, x, lengths):
        out = self.linear(x)
        sequence = pack_padded_sequence(self.relu(out), lengths, batch_first=True)
        out, _ = self.lstm(sequence)
        out, _ = pad_packed_sequence(out, batch_first=True)
        out = self.mdn(out)
        return out

    def inference(self, x, lengths=None):
        log_pi, log_sigma, mu = self.forward(x, lengths)
        sigma, mu = mdn_get_most_probable_sigma_and_mu(log_pi, log_sigma, mu)
        return mu, sigma

class MDN(BaseModel):
    def __init__(self, in_dim, hidden_dim, out_dim, num_layers=1, dropout=0.0,
            num_gaussians=8, dim_wise=False):
        super(MDN, self).__init__()
        model = [nn.Linear(in_dim, hidden_dim), nn.ReLU()]
        if num_layers > 1:
            for _ in range(num_layers - 1):
                model += [nn.Linear(hidden_dim, hidden_dim), nn.ReLU()]
        model += [MDNLayer(hidden_dim, out_dim, num_gaussians, dim_wise)]
        self.model = nn.Sequential(*model)

    def prediction_type(self):
        return PredictionType.PROBABILISTIC

    def forward(self, x, lengths=None):
        return self.model(x)

    def inference(self, x, lengths=None):
        log_pi, log_sigma, mu = self.forward(x, lengths)
        sigma, mu = mdn_get_most_probable_sigma_and_mu(log_pi, log_sigma, mu)
        return mu, sigma

class Conv1dResnetMDN(BaseModel):
    def __init__(self, in_dim, hidden_dim, out_dim, num_layers=4, dropout=0.0,
            num_gaussians=8, dim_wise=False):
        super().__init__()
        model = [Conv1dResnet(in_dim, hidden_dim, hidden_dim, num_layers, dropout),
                 nn.ReLU(),
                 MDNLayer(hidden_dim, out_dim, num_gaussians, dim_wise)]
        self.model = nn.Sequential(*model)

    def prediction_type(self):
        return PredictionType.PROBABILISTIC

    def forward(self, x, lengths=None):
        return self.model(x)

    def inference(self, x, lengths=None):
        log_pi, log_sigma, mu = self.forward(x, lengths)
        sigma, mu = mdn_get_most_probable_sigma_and_mu(log_pi, log_sigma, mu)
        return mu, sigma


import unittest
from almostlist import AlmostList
from os import sys

class AlmostSearch:
    @staticmethod
    def search(almostList, value, i=0):
        '''Search sorted AlmostList for value and return its index, or None if not there'''
        comp = almostList[i]
        if comp == None and i == 0:
            return None
        if comp == None or comp > value:
            # passed the value, so it is in between i and i/2
            return AlmostSearch.binSearch(almostList, value, i, i//2)

        elif comp == value:
            return i

        else:# almostList[i] < value:
            # increase the list index by double
            if i==0: # two times zero is still zero, avoid this by incrementing by one
                i+=1
            else:
                i=i*2
            return AlmostSearch.search(almostList, value, i)

    @staticmethod
    def binSearch(data, value, high, low):
        ''' This is binary search with recursion and avoids changing the array
        '''
        #create the midpoint
        m = int(((high-low)//2)+low)
        val=data[m]

        if val==None:
            # if the value is still past the end of the array
            return AlmostSearch.binSearch(data, value, m-1, low)
        if low > high:
            # if the value is not in the array
            return None
        elif value==val:
            # we found a match
            return m
        elif value < val:
            # choose the lower half of array
            return AlmostSearch.binSearch(data, value, m-1, low)
        else:
            # choose the upper half of the array
            return AlmostSearch.binSearch(data, value, high, m+1)

if __name__ == "__main__":
    print(AlmostSearch.search(AlmostList([-22, -11, 0, 11, 22, 33, 44, 55, 66, 77, 88, 99]), 44) == 4)
    print(AlmostSearch.search(AlmostList([-22, -11, 0, 11, 22, 33, 44, 55, 66, 77, 88, 99]), -22) == 0)
    print(AlmostSearch.search(AlmostList([-66, -55, -44, -33, -22, -11, 0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 88]), 88) == None)


while 1 == 1:
  print("введите ширину и высоту таблицы через пробел")
  x, y = map(int,input().split())
  print("введите изначальную расстановку на поле")
  if x < 1 or y < 1:
    print("данное поле введено некорректно")
  else:
    break
print("количество строк:", y,"количество символов в строке:", x)
print("введите начальную расстановку на поле (через пробел)")
print("условные обозначения: 0 - ничего, 1 - креветка, 2 - камень, 3 - рыба")
N = [] #вспомогательная матрица, содержащая кол-во соседей около клетки
while 1 == 1:
  L = [] #матрица, содержащая поле и пустые клетки вокруг (для лучшей работы)
  Q = "0"
  for i in range(y + 2):
    N.append([])
    for k in range(x + 2):
      N[-1].append(0)
    if i % (y + 1) == 0:
      L.append([])
      for k in range(x + 2):
        L[-1].append(0)
    elif Q == "0":
      l = list(map(int,input().split())) #мини-список для имплантации в L
      for j in range(len(l)):
        if l[j] > 3 or l[j] < 0:
          print("некорректный ввод; Введите начальную расстановку на поле заного")
          Q = "1"
          break
      l.insert(0,0)
      l.append(0)
      L.append(l)
  if Q == "0":
    break
print("для того, чтобы завершить работу программы, нужно ввести 'end' после очередного шага, либо не 'end' для вывода следующего шага. програма завершится автоматически если поле станет статичным")
while Q != "end":
  q = 0
  for i in range(1, y + 1):  #забиваем соседей в матрицу N
    for j in range(1, x + 1):
      if L[i][j] % 2 == 1:
        N[i][j-1] = L[i][j] * L[i][j] + N[i][j-1] #добавляю в каждую боковую клетку число, зависящее от того, рыба в этой клетке или креветка
        N[i][j+1] = L[i][j] * L[i][j] + N[i][j+1] #за каждую рыбу добавляется 9
        N[i-1][j] = L[i][j] * L[i][j] + N[i-1][j] #за каждую же креветку - 1
        N[i+1][j] = L[i][j] * L[i][j] + N[i+1][j] #т.к. "соседних" клеток 8 этого достаточно
        N[i-1][j-1] = L[i][j] * L[i][j] + N[i-1][j-1] #вычисление кол-ва соседей определённого типа
        N[i-1][j+1] = L[i][j] * L[i][j] + N[i-1][j+1] #будет производится путём мат.вычислений
        N[i+1][j-1] = L[i][j] * L[i][j] + N[i+1][j-1]
        N[i+1][j+1] = L[i][j] * L[i][j] + N[i+1][j+1] #я считаю что угловые клетки это тоже соседи, т.к. иного не найдено в условии
  for i in range(1, y + 1):  #изменяем матрицу L
    for j in range(1, x + 1):
      if L[i][j] == 0:
        if N[i][j] // 9 == 3: #проверка сначала на трёх рыб
          L[i][j] = 3
          q = 1 #индикатор статичности системы. если 0 то система статична
        elif N[i][j] % 9 == 3: #далее на 3 креветок
          L[i][j] = 1
          q = 1
      elif L[i][j] % 2 == 1: #уборка зажатых и одиноких
        if N[i][j] > 3 or N[i][j] < 2:
          L[i][j] = 0
          q = 1
  Q = input()
  for u in range(1, y+1): #вывод мастрицы L, с пробелами между цифр и пустой строкой в конце
    for g in range(1, x+1):
      print(L[u][g], end=" ")
    print("")
  if q == 0:
    Q = "end"
    print("система статична") #сначала выведется матрица, и если она не изменилась, то ещё и обьявление
  if Q == "end":
    break
  else:
    for i in range(y + 2): #обнуление кол-ва соседей у клетки
      for j in range(x + 2):
        N[i][j] = 0


from flask import Blueprint
from flask import render_template, url_for, flash, redirect, request, abort
from flask_login import current_user, login_required
from na_service import app, db, posts, logger, logger_post
from na_service.models import Post
from na_service.posts.forms import PostForm

posts = Blueprint("posts", __name__)

@posts.route("/post/new", methods=['GET', 'POST'])
@login_required
def new_post():
    form = PostForm()
    if form.validate_on_submit():
        post = Post(title=form.title.data, content=form.content.data, author=current_user)
        db.session.add(post)
        db.session.commit()
        logger_post.info(f"A post has been created by {current_user.username}, post_id = {post.id}")
        flash("Your post has been created!", "success")
        return redirect(url_for('main.home'))
    return render_template("create_post.html", title="New Post", form=form, legend="Create Post")

@posts.route("/post/<int:post_id>")
def post(post_id):
    post = Post.query.get_or_404(post_id)
    return render_template('post.html', title=post.title, post=post)

@posts.route("/post/update/<int:post_id>", methods=['GET', 'POST'])
@login_required
def update_post(post_id):
    post = Post.query.get_or_404(post_id)
    if post.author != current_user:
        abort(403)
    form = PostForm()
    if form.validate_on_submit():
        post.title = form.title.data
        post.content = form.content.data
        db.session.commit()
        logger_post.info(f"The post: {post.id} has been updated by {current_user.username}")
        flash("Your post has been updated!", "success")
        return redirect(url_for('posts.post', post_id=post.id))
    elif request.method == "GET":
        form.title.data = post.title
        form.content.data = post.content
    return render_template("create_post.html", title="Update Post", form=form, legend="Update Post")

@posts.route("/post/delete/<int:post_id>", methods=["POST"])
@login_required
def delete_post(post_id):
    post = Post.query.get_or_404(post_id)
    if post.author != current_user:
        abort(403)
    db.session.delete(post)
    db.session.commit()
    logger_post.warning(f"The post: title - {post.title} has been deleted by {current_user.username}")
    flash("Your post has been deleted!", "success")
    return redirect(url_for("main.home"))


import multiprocessing
import os.path as osp

import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

from .graph import Graph
from .ds_base import ds_base, get_dataset_base,generate_maps

def load_data():
    label_path = osp.abspath(osp.join(get_dataset_base(), "cirrhosis", "labels.txt"))
    y = pd.read_csv(label_path, sep='\t', header=None)
    y = y.T.iloc[0]
    label_encoder = LabelEncoder()
    y = label_encoder.fit_transform(y)

    data_path = osp.abspath(osp.join(get_dataset_base(), "cirrhosis", "count_matrix.csv"))
    X = pd.read_csv(data_path, sep=',', header=None)
    X = X.loc[(X != 0).any(axis=1)].as_matrix()
    # X = (X - X.min()) / (X.max() - X.min())

    train_idx, test_idx = train_test_split(range(len(X)), random_state=0, train_size=0.7, stratify=y)
    return (X[train_idx], y[train_idx]), (X[test_idx], y[test_idx])

def load_data_phy():
    my_x = []
    my_y = []

    data_path = osp.abspath(osp.join(get_dataset_base(), "cirrhosis", "count_matrix.csv"))
    my_x = np.loadtxt(data_path, dtype=np.float32, delimiter=',')

    label_path = osp.abspath(osp.join(get_dataset_base(), "cirrhosis", "labels.txt"))
    my_y = np.genfromtxt(label_path, dtype=np.str_, delimiter=',')

    otu_path = osp.abspath(osp.join(get_dataset_base(), "cirrhosis", "otu.csv"))
    features = np.genfromtxt(otu_path, dtype=np.str_, delimiter=',')

    my_ref = pd.factorize(my_y)[1]
    label_reference_path = osp.abspath(osp.join(get_dataset_base(), "cirrhosis", "label_reference.txt"))
    f = open(label_reference_path, 'w')
    f.write(str(my_ref))
    f.close()

    newick_path = osp.abspath(osp.join(get_dataset_base(), "cirrhosis", "newick.txt"))
    g = Graph()
    g.build_graph(newick_path)

    my_data = pd.DataFrame(my_x)
    my_data = np.array(my_data)
    my_lab = pd.factorize(my_y)[0]
    my_maps = []
    my_benchmark = []

    num_cores = multiprocessing.cpu_count()
    results = Parallel(n_jobs=num_cores)(delayed(generate_maps)(x, g, features) for x in my_data)
    my_maps.append(np.array(np.take(results, 1, 1).tolist()))
    my_benchmark.append(np.array(np.take(results, 0, 1).tolist()))

    my_maps = np.array(my_maps)
    map_rows = my_maps.shape[2]
    map_cols = my_maps.shape[3]

    train_idx, test_idx = train_test_split(range(len(my_maps[0])), random_state=0, train_size=0.7, stratify=my_lab)
    x_train, y_train, x_test, y_test = my_maps[0][train_idx], my_lab[train_idx], my_maps[0][test_idx], my_lab[test_idx]

    x_train = np.array(x_train).reshape(-1, map_rows, map_cols)
    x_test = np.array(x_test).reshape(-1, map_rows, map_cols)
    y_train = np.squeeze(np.array(y_train).reshape(1, -1), 0)
    y_test = np.squeeze(np.array(y_test).reshape(1, -1), 0)

    return x_train, y_train, x_test, y_test

class T2D(ds_base):
    def __init__(self, **kwargs):
        super(T2D, self).__init__(**kwargs)
        (X_train, y_train), (X_test, y_test) = load_data()
        if self.data_set == "train":
            X = X_train
            y = y_train
        elif self.data_set == "test":
            X = X_test
            y = y_test
        elif self.data_set == "all":
            X = np.vstack((X_train, X_test))
            y = np.vstack((y_train, y_test))
        else:
            raise ValueError("CIRRHOSIS Unsupported data_set: ", self.data_set)

        X = X[:, np.newaxis, :, np.newaxis]
        X = self.init_layout_X(X)
        y = self.init_layout_y(y)
        self.X = X
        self.y = y


#!/usr/bin/python2.7
# Copyright 2013 SoluRank Inc. All Rights Reserved.
# Author: andy@solurank.com (Andy Zhau)

import multiprocessing
import threading
import time

from datetime import datetime
from datetime import timedelta
from solurank.framework import db
from solurank.framework import model

STATUS_INVALID = 0;
STATUS_VALID = 1;
STATUS_RESERVED = 2;
STATUS_RUNNING = 3;
STATUS_SUCCESS = 4;
STATUS_ERROR = 5;
STATUS_TIMEOUT = 6;

PRIORITY_LOW = 0;
PRIORITY_NORMAL = 10;
PRIORITY_HIGH = 20;

class TaskConfigModel(model.Model):

  ModelPath = "task.config"
  Index = [
      model.Index("name_index", keys=[("name", 1)], unique=True),
  ]
  Rules = [
      model.Rule("name", type=unicode, required=True),
      model.Rule("period", type=int),
      model.Rule("priority", type=int, required=True),
      model.Rule("timeout", type=int, required=True),
      model.Rule("version", type=int, required=True),
  ]

  @classmethod
  def RegisterTask(cls, task_cls):
    ins = cls({
        "name": unicode(task_cls.__NAME__),
        "priority": task_cls.__PRIORITY__,
        "version": task_cls.__VERSION__,
        "timeout": task_cls.__TIMEOUT__,
        "last_scheduled": datetime.utcnow(),
    })
    if task_cls.__PERIOD__ is not None:
      ins["period"] = task_cls.__PERIOD__
    try:
      ins.Save(safe=True)
    except db.DuplicateKeyError as e:
      ins = cls.FindOne({"name": task_cls.__NAME__})
      if ins["version"] > task_cls.__VERSION__:
        task_cls.__ENABLED__ = False
      else:
        update = False
        if ins["priority"] != task_cls.__PRIORITY__:
          ins["priority"], update = task_cls.__PRIORITY__, True
        if ins["version"] != task_cls.__VERSION__:
          ins["version"], update = task_cls.__VERSION__, True
        if ins["timeout"] != task_cls.__TIMEOUT__:
          ins["version"], update = task_cls.__TIMEOUT__, True
        if update:
          ins.Save()
    return ins

  @classmethod
  def GetTaskConfig(cls, name):
    return cls.FindOne({"name": name})

  def ScheduleTask(self):
    if not self.Get("period"): return
    ins = self.FindAndModify({
        "name": self["name"],
        "period": { "$gt": 0 },
        "last_scheduled": { "$lt": datetime.utcnow(), },
    }, {
        "$set": {
            "last_scheduled": (
                datetime.utcnow() + timedelta(seconds=self["period"])),
        },
    }, new=True)
    if ins is not None:
      TaskModel({
          "name": ins["name"],
          "happens_after": ins["last_scheduled"],
          "priority": ins["priority"],
          "start": STATUS_VALID,
          "timeout": ins["timeout"],
          "version": ins["version"],
          "status": STATUS_VALID,
      }).Save()
      self.data = ins.data

class TaskModel(model.Model):

  ModelPath = "task"
  Index = [
      model.Index("task_index", keys=[
          ("status", 1), ("happens_after", 1), ("priority", -1)]),
  ]
  Rules = [
      model.Rule("name", type=unicode, required=True),
      model.Rule("happens_after", type=datetime, required=True),
      model.Rule("priority", type=int, required=True),
      model.Rule("status", type=int, required=True),
      model.Rule("timeout", type=int, required=True),
      model.Rule("version", type=int, required=True),
  ]

  @classmethod
  def TaskNew(cls, task_cls, happens_after=None, **args):
    ins = cls({
        "args": args,
        "name": task_cls.__NAME__,
        "happens_after": happens_after or datetime.utcnow(),
        "priority": task_cls.__PRIORITY__,
        "status": STATUS_VALID,
        "timeout": task_cls.__TIMEOUT__,
        "version": task_cls.__VERSION__,
    })
    ins.Save()
    return ins

  def Reserve(self):
    ins = self.FindAndModify({
        "_id": self["_id"],
        "status": STATUS_VALID,
    }, update={
        "$set": {
            "status": STATUS_RESERVED,
        },
    }, new=True)
    if ins is None:
      return False
    self.data = ins.data
    return True

  def Success(self, result):
    self["status"] = STATUS_SUCCESS
    self["result"] = result
    self.Save()

  def Running(self):
    self["status"] = STATUS_RUNNING
    self.Save()

  def Error(self, e):
    self["status"] = STATUS_ERROR
    self["error_message"] = unicode(e)
    self.Save()

class TaskType(type):

  _TaskTypeDict = {}
  _TaskTypeDictLock = multiprocessing.Lock()

  def __new__(cls, name, bases, dct):
    task_cls = type.__new__(cls, name, bases, dct)

    if task_cls.__NAME__:
      cls._TaskTypeDictLock.acquire()
      if task_cls.__NAME__ in cls._TaskTypeDict:
        raise AttributeError("Task %s already exists." % task_cls.__NAME__)
      cls._TaskTypeDict[task_cls.__NAME__] = task_cls
      cls._TaskTypeDictLock.release()
      TaskConfigModel.RegisterTask(task_cls)

    return task_cls

  @classmethod
  def GetTaskType(cls, name):
    cls._TaskTypeDictLock.acquire()
    ret = cls._TaskTypeDict.get(name)
    cls._TaskTypeDictLock.release()
    return ret

class Task(object):

  __metaclass__ = TaskType
  __NAME__ = None
  __PRIORITY__ = PRIORITY_NORMAL
  __VERSION__ = 0
  __PERIOD__ = None
  __TIMEOUT__ = 300
  __ENABLED__ = True
  __AUTOCLEAN__ = False

  @classmethod
  def TaskNew(cls, **task_args):
    return TaskModel.TaskNew(cls, **task_args)

  @classmethod
  def Run(cls, task):
    pass

class TaskScheduler(Task):

  __NAME__ = "TaskScheduler"
  __PRIORITY__ = PRIORITY_HIGH
  __VERSION__ = 0
  __PERIOD__ = 5
  __AUTOCLEAN__ = True

  @classmethod
  def Run(cls, task):
    for config in TaskConfigModel.Find({
        "period": { "$gt": 0 },
        "last_scheduled": { "$lt": datetime.utcnow() },
        }):
      config.ScheduleTask()

class TaskAutoCleaner(Task):

  __NAME__ = "TaskAutoCleaner"
  __VERSION__ = 0
  __PERIOD__ = 60
  __AUTOCLEAN__ = True

  @classmethod
  def Run(cls, task):
    TaskType._TaskTypeDictLock.acquire()
    tasks = [task_cls
             for name, task_cls in TaskType._TaskTypeDict.iteritems()
             if task_cls.__AUTOCLEAN__]
    TaskType._TaskTypeDictLock.release()

    for task in tasks:
      TaskModel.FindAndRemove({
          "name": task.__NAME__,
          "status": STATUS_SUCCESS,
      })

class TaskManager(threading.Thread):

  def run(self):
    task_scheduler_config = TaskConfigModel.GetTaskConfig("TaskScheduler")
    if task_scheduler_config is not None:
      task_scheduler_config.ScheduleTask()

    while True:
      task = TaskModel.FindOne({
          "status": STATUS_VALID,
          "happens_after": {
              "$lt": datetime.utcnow(),
          },
      }, sort=[("priority", -1)])
      if not task:
        time.sleep(0.5)
      else:
        self.RunTask(task)

  def RunTask(self, task):
    task_cls = TaskType.GetTaskType(task["name"])
    if not task_cls:
      return

    if task["version"] > task_cls.__VERSION__:
      task_cls.__ENABLED__ = False
    if not task_cls.__ENABLED__ or not task.Reserve():
      return

    task.Running()

    try:
      task.Success(task_cls.Run(task))
    except Exception as e:
      raise
      task.Error(e)

TaskManager().start()


import collections
from collections import OrderedDict
from tempfile import mkstemp

from frozendict import frozendict
from werkzeug.datastructures import MultiDict

import slivka
from slivka.db.documents import UploadedFile, JobRequest
from slivka.utils import Singleton
from .fields import *

class DeclarativeFormMetaclass(type):
    @classmethod
    def __prepare__(mcs, name, bases):
        return OrderedDict()

    def __new__(mcs, name, bases, attrs):
        fields = OrderedDict()
        for key, field in list(attrs.items()):
            if isinstance(field, BaseField):
                fields[field.name] = field
                attrs.pop(key)
        attrs['fields'] = fields
        return super().__new__(mcs, name, bases, attrs)

class BaseForm(metaclass=DeclarativeFormMetaclass):
    service = ''

    def __init__(self, data=None, files=None):
        self.is_bound = not (data is None and files is None)
        self.data = MultiDict() if data is None else data
        self.files = MultiDict() if files is None else files
        self._errors = None
        self._cleaned_data = frozendict()

    @property
    def errors(self) -> collections.Mapping:
        if self._errors is None:
            self.full_clean()
        return self._errors

    @property
    def cleaned_data(self) -> collections.Mapping:
        return self._cleaned_data

    def is_valid(self):
        return self.is_bound and not self.errors

    def full_clean(self):
        errors = {}
        if not self.is_bound:
            return
        cleaned_data = {}
        for name, field in self.fields.items():
            value = field.value_from_request_data(self.data, self.files)
            try:
                value = field.validate(value)
                cleaned_data[name] = value
                if value is not None and isinstance(field, FileField):
                    assert isinstance(value, FileWrapper)
                    # If no validator has checked for the type we need
                    # to check here whether it's allowed on the server.
                    media_type = value.get_detected_media_type()
                    if not media_type:
                        raise ValidationError(
                            'File content does not match %s '
                            'or this media type is not allowed' % value.media_type,
                            'invalid'
                        )
            except ValidationError as err:
                errors[field.name] = err
        self._errors = frozendict(errors)
        self._cleaned_data = frozendict(cleaned_data)

    def __iter__(self):
        return iter(self.fields.values())

    def __getitem__(self, item):
        return self.fields[item]

    def save(self, database):
        if not self.is_valid():
            raise RuntimeError(self.errors, 'invalid_form')
        inputs = {}
        uploaded_files = []
        for name, field in self.fields.items():
            value = self.cleaned_data[name]
            if value is not None and isinstance(field, FileField):
                assert isinstance(value, FileWrapper)
                # make sure that the type was detected by full_clean
                assert value.get_detected_media_type()
                if value.uuid is None:
                    (fd, path) = mkstemp(dir=slivka.settings.UPLOADS_DIR)
                    with open(fd, 'wb') as fp:
                        value.save_as(fp, path=path)
                    uploaded_files.append(UploadedFile(
                        title=value.title,
                        path=path,
                        media_type=value.get_detected_media_type()
                    ))
            inputs[name] = field.serialize_value(value)
        request = JobRequest(service=self.service, inputs=inputs)
        request.insert(database)
        for file in uploaded_files:
            file.insert(database)
        return request

class FormLoader(metaclass=Singleton):
    def __init__(self):
        self._forms = {
            service: self._build_form_class(service, conf.form)
            for service, conf in slivka.settings.service_configurations.items()
        }

    def get_form_class(self, service):
        return self[service]

    def __getitem__(self, item):
        return self._forms[item]

    def _build_form_class(self, service, json_data):
        attrs = OrderedDict((name, self._build_field(name, data))
                            for name, data in json_data.items())
        attrs['service'] = service
        cls = DeclarativeFormMetaclass(
            service.capitalize() + 'Form',
            (BaseForm,),
            attrs
        )
        return cls

    @staticmethod
    def _build_field(name, field_meta):
        value_meta = field_meta['value']
        field_type = value_meta['type']
        common_kwargs = {
            'name': name,
            'label': field_meta['label'],
            'description': field_meta.get('description'),
            'default': value_meta.get('default'),
            'required': value_meta.get('required', True),
            'multiple': value_meta.get('multiple', False)
        }
        if field_type == 'int':
            return IntegerField(
                **common_kwargs,
                min=value_meta.get('min'),
                max=value_meta.get('max')
            )
        elif field_type == 'float' or field_type == 'decimal':
            return DecimalField(
                **common_kwargs,
                min=value_meta.get('min'),
                max=value_meta.get('max'),
                min_exclusive=value_meta.get('min-exclusive', False),
                max_exclusive=value_meta.get('max-exclusive', False)
            )
        elif field_type == 'text':
            return TextField(
                **common_kwargs,
                min_length=value_meta.get('min-length'),
                max_length=value_meta.get('max-length')
            )
        elif field_type == 'boolean' or field_type == 'flag':
            return FlagField(
                **common_kwargs
            )
        elif field_type == 'choice':
            return ChoiceField(
                **common_kwargs,
                choices=value_meta.get('choices')
            )
        elif field_type == 'file':
            return FileField(
                **common_kwargs,
                extensions=value_meta.get('extensions', ()),
                media_type=value_meta.get('media-type')
            )
        else:
            raise RuntimeError('Invalid field type "%r"' % field_type)


# coding: utf8
from flask import request
from flask import jsonify
from flask import abort
from application import db
from application.models.memo import Memo
from application.models.relation_label_memo import RelationLabelMemo
from . import api

@api.route('/memos', methods=['POST'])
def create_memo():
    request_body_dic = request.get_json()
    selected_label_id = request_body_dic.get('selectedLabelId')

    try:
        # memo = Memo(**request_body_dic)
        memo = Memo()
        memo.update_data(**request_body_dic)
    except TypeError:
        return abort(400)
    else:
        db.session.add(memo)
        db.session.commit()
        return_dic = memo.serialize()
        label_ids = return_dic.setdefault('labelIds', [])
        if selected_label_id:
            relation = RelationLabelMemo(memo_id=memo.id, label_id=selected_label_id)
            db.session.add(relation)
            db.session.commit()
            label_ids.append(relation.label_id)

        return jsonify(data=return_dic), 201

@api.route('/memos', methods=['GET'])
def get_all_memos(memo_id=0):
    q = db.session.query(Memo, RelationLabelMemo) \
        .outerjoin(RelationLabelMemo, Memo.id == RelationLabelMemo.memo_id)

    if memo_id:
        q = q.filter(Memo.id == memo_id);

    result_dict = {}
    for memo, relation in q.all():
        if memo.id not in result_dict:
            memo_dic = result_dict.setdefault(memo.id, memo.serialize())
            memo_dic['labelIds'] = []

        if relation is not None:
            result_dict[memo.id]['labelIds'].append(relation.label_id)

    result_list = result_dict.values()
    result_list.sort(key=(lambda x: (x['updateTime'], x)))
    result_list.reverse()

    if memo_id:
        return jsonify(
            data=result_list[0]
        ), 200

    return jsonify(
        data=result_list
    ), 200

@api.route('/memos/<int:memo_id>', methods=['PUT'])
def update_memo(memo_id):
    memo = db.session.query(Memo).get(memo_id)
    if memo is None:
        return jsonify(
            userMessage="can not found memo"
        ), 404

    request_body_dic = request.get_json()
    memo.update_data(**request_body_dic)
    db.session.commit()
    return get_all_memos(memo_id=memo.id)

@api.route('/memos/<int:memo_id>', methods=['DELETE'])
def delete_memo(memo_id):
    memo = db.session.query(Memo).get(memo_id)
    if memo is None:
        return jsonify(
            userMessage="can not found label"
        ), 404
    db.session.delete(memo)
    db.session.commit()
    return "", 204

@api.route('/memos/delete', methods=['POST'])
def delete_memo_multiple():
    request_body_dic = request.get_json()
    delete_memo_ids = request_body_dic.get('ids', [])

    q = db.session.query(Memo).filter(Memo.id.in_(delete_memo_ids))
    map(db.session.delete, q.all())
    db.session.commit()
    return "", 204


from netapp.ntdtest.ntdtest_info import NtdtestInfo
from netapp.netapp_object import NetAppObject

class ActionTypedef2(NetAppObject):
    """
    Nested input typedef for action
    """

    _field_17 = None
    @property
    def field_17(self):
        """
        Dummy/Generic Action Field 17
        Attributes: required-for-create, non-modifiable
        """
        return self._field_17
    @field_17.setter
    def field_17(self, val):
        if val != None:
            self.validate('field_17', val)
        self._field_17 = val

    _field_6 = None
    @property
    def field_6(self):
        """
        Dummy/Generic Action Field 6
        Attributes: required-for-create, non-modifiable
        """
        return self._field_6
    @field_6.setter
    def field_6(self, val):
        if val != None:
            self.validate('field_6', val)
        self._field_6 = val

    _field_2 = None
    @property
    def field_2(self):
        """
        Dummy/Generic Action Field 2
        Attributes: key, required-for-create, non-modifiable
        """
        return self._field_2
    @field_2.setter
    def field_2(self, val):
        if val != None:
            self.validate('field_2', val)
        self._field_2 = val

    @staticmethod
    def get_api_name():
          return "action-typedef-2"

    @staticmethod
    def get_desired_attrs():
        return [
            'field-17',
            'field-6',
            'field-2',
        ]

    def describe_properties(self):
        return {
            'field_17': { 'class': NtdtestInfo, 'is_list': True, 'required': 'optional' },
            'field_6': { 'class': basestring, 'is_list': False, 'required': 'optional' },
            'field_2': { 'class': basestring, 'is_list': False, 'required': 'optional' },
        }


#!/usr/bin/env python
# coding: utf-8

# # Detect generations

# Natalia Vélez, May 2021
#
# This script goes through each family and assigns each avatar a generation number.

import json
import numpy as np
import pandas as pd
import networkx as nx
from tqdm import notebook

import sys
sys.path.append('..')
from utils import gsearch, str_extract, int_extract, load_json
_, sample = sys.argv # new: take sample as an argument

# ## Load data

# Family registry:
family_df = pd.read_csv('outputs/%s_family_playerID.tsv' % sample, sep='\t', index_col=0)
family_df = family_df.sort_values(by='avatar', ascending=True)
family_df.head(20)

# Lifelogs:
life_df = pd.read_csv('outputs/%s_lifelogs_compact.csv' % sample)
life_df = life_df[['avatar', 'parent', 'tBirth']]
life_df.head()

# Merged data:
merge_df = life_df.merge(family_df, on='avatar')
merge_df = merge_df.sort_values(by=['family', 'tBirth']).reset_index(drop=True)
merge_df.head()

# Family graphs:
graph_files = gsearch('outputs/%s_families/*.json' % sample)
graph_files.sort()
print('Found %i files' % len(graph_files))
print(*graph_files[:10], sep='\n')

# Remove duplicate files:
graph_path_df = pd.DataFrame({'path': graph_files})
graph_path_df['eve'] = graph_path_df.path.apply(lambda s: int_extract('(?<=eve-)[0-9]+', s))

print('Before removing duplicates: %i' % graph_path_df.shape[0])
graph_path_df = graph_path_df.drop_duplicates(subset='eve', keep='first')
print('After: %i' % graph_path_df.shape[0])

# ## Search through generations iteratively

gen_list = []

for idx, row in notebook.tqdm(graph_path_df.iterrows(), total=graph_path_df.shape[0]):
    # Family/eve names
    fam = row.path.replace('outputs/%s_families/families_' % sample, '').replace('.json', '')
    eve = row.eve

    # Load graph
    G_data = load_json(row.path)
    G = nx.json_graph.node_link_graph(G_data)
    G = nx.relabel.relabel_nodes(G, {n:int(n) for n in G.nodes()}) # make sure nodes are integers

    # Compute generation #s for each avatar
    dist_dict = nx.single_source_shortest_path(G, eve)
    dist_list = [(fam, k, len(v)-1) for k,v in dist_dict.items()]
    dist_df = pd.DataFrame(dist_list, columns=['family', 'avatar', 'gen'])

    gen_list.append(dist_df)

gen_df = pd.concat(gen_list)

# Save outputs to file
print('Saving dataframe of generation numbers:')
print(gen_df.head())

gen_df.to_csv('outputs/%s_family_generations.csv' % sample, index=False)


import re
import sys
import time
import requests
from tqdm import tqdm

class SearchByTitle:
    def __init__(self):
        self._regex = re.compile(r'href="(/subtitles/.+?)"')
        self._session = requests.session()
        self._url = "https://subscene.com/subtitles/searchbytitle"

    def search(self, title):
        count = 0
        while count < 3:
            data = {"query": title, "l": ""}
            r = self._session.post(self._url, data=data)
            if self._is_hammer(r.text):
                count += 1
                time.sleep(7)
                continue
            else:
                return list(set(re.findall(self._regex, r.text)))

    def _is_hammer(self, content):
        return "Please do not hammer on Subscene" in content

def main():
    searcher = SearchByTitle()
    for title in tqdm(sys.stdin):
        try:
            title = title.strip()
            for url_suffix in searcher.search(title):
                sys.stdout.write(url_suffix + "\n")
            sys.stdout.flush()
            time.sleep(7)
        except Exception:
            time.sleep(30)

if __name__ == "__main__":
    # Usage: cat titlelist.txt | python subscene_searchbytitle.py
    main()


a = input ("quel est votre chiffre d'affaire ? : ")
taux_charges = 0.2730
cotisation = int(a) * taux_charges
print("Le montant de vos charges s'élève à :",cotisation)
ca_charge = int(a) - cotisation
print("votre chiffre après chagrge est de :", ca_charge)
taux_imp = 0.18
montant_imp = ca_charge * taux_imp
print('le montant des impôts à payer est de :', montant_imp)
ca_net = ca_charge - montant_imp
print("votre chiffre net est de :", ca_net)


import mock

from django.core.urlresolvers import reverse
from django.conf import settings

from core.testing.test_base import CLATFrontEndTestCase

class RotaTests(CLATFrontEndTestCase):

    zone = "call_centre"

    @mock.patch("cla_auth.models.get_raw_connection")
    def __call__(self, result, mocked_get_raw_connection, *args, **kwargs):
        self.mocked_get_raw_connection = mocked_get_raw_connection

        self.connection = mock.MagicMock()

        # Mock get_raw_connection API
        self.connection.user.me.get.return_value = {
            "username": "test_operator",
            "first_name": "",
            "last_name": "",
            "email": "",
            "is_manager": False,
        }
        self.mocked_get_raw_connection.return_value = self.connection

        super(RotaTests, self).__call__(result, *args, **kwargs)

    def setUp(self):
        self.rota_url = reverse("call_centre:admin:rota")
        super(RotaTests, self).setUp()

    def test_get_not_logged_in(self):
        response = self.client.get(self.rota_url, follow=True)

        expected_url = u"%s?next=%s" % (reverse(settings.LOGIN_URL), self.rota_url)
        self.assertRedirects(response, expected_url)

    def test_get_logged_in_not_manager(self):
        self.login()

        self.connection.user.me.get.return_value["is_manager"] = False

        response = self.client.get(self.rota_url, follow=True)
        self.assertEqual(response.status_code, 404)

    def test_get_logged_in_manager(self):
        self.login()

        self.connection.user.me.get.return_value["is_manager"] = True

        response = self.client.get(self.rota_url, follow=True)
        self.assertEqual(response.status_code, 200)


"""
Copyright (c) 2004-Present Pivotal Software, Inc.

This program and the accompanying materials are made available under
the terms of the under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from collections import namedtuple
import sys
import re

import tinctest
from tinctest import TINCTestSuite, TINCTestLoader

class TINCLogFileParser(object):

    test_tuple = namedtuple('test_tuple', 'result duration test_case_object')
    def __init__(self, logfile):
        """
        The constructor requires a TINC log file to parse and the query handler to apply.
        """
        self.logfile = logfile

        # Key is test name, Value is test_tuple
        self.dict_of_test_tuples = {}
        self._populate_dict_of_test_tuples()

    def _populate_dict_of_test_tuples(self):
        """
        This hidden method will parse through the log file and find all the tests.
        All the information will be populated in self.dict_of_test_tuples
        """
        # All completed test cases will have the following line:
        # 20140417:08:34:13::[STATUS]::__init__.py:115::pid27994::MainThread:: Finished test: tinctest.test.sample.TINCTestCaseWithDataProviderComplicated.test_with_data_provider_complicated_type1_type6_type3 Result: PASS Duration: 0.39 ms Message: NONE
        re_compiled_object = re.compile(r"Finished test: (.*) Result: (.*) Duration: (.*) ms ")

        # We want to gather all of the tests in dict_of_test_tuples

        try:
            file_object = open(self.logfile, 'r')
            file_object.close()
        except:
            raise TINCLogFileParserException("Cannot open file %s" % self.logfile)

        tinctest.logger.debug("Parsing file %s" %self.logfile)
        with open(self.logfile, 'r') as file_object:
            for line in file_object:
                line = line.strip()
                if line.find('[STATUS]') == -1:
                    continue
                re_result_object = re_compiled_object.search(line)
                if re_result_object:
                    test_name = re_result_object.group(1)
                    test_result = re_result_object.group(2)
                    test_duration = re_result_object.group(3)
                    test_case_object = TINCTestLoader.findTINCTestFromName(test_name)
                    self.dict_of_test_tuples[test_name] = TINCLogFileParser.test_tuple(result=test_result, duration=test_duration, test_case_object=test_case_object)


# フォルダとそのサブフォルダ内にあるブックのパス情報を、一つのシートにまとめる
from pathlib import Path
from openpyxl import Workbook
from openpyxl.styles import Border, PatternFill, Side

wb = Workbook()
ws = wb.active
ws.column_dimensions['B'].width = 40
ws.column_dimensions['C'].width = 30

black_thin = Side(color='000000', border_style='thin')
border = Border(left=black_thin, right=black_thin, top=black_thin, bottom=black_thin)
green_fill = PatternFill(fgColor='C6E0B4', fill_type='solid')

for i, col_name in enumerate(('項番', 'パス', 'ブック名',)):
    col_no = i + 1
    ws.cell(1, col_no).value = col_name
    ws.cell(1, col_no).fill = green_fill
    ws.cell(1, col_no).border = border

path = Path('./books')
for i, file in enumerate(path.glob('**/*.xlsx')):
    row_no = i + 2
    ws.cell(row_no, 1).value = i + 1

    absolute_path = file.resolve()
    filepath = absolute_path.parent
    filename = absolute_path.name

    ws.cell(row_no, 2).value = str(filepath)
    ws.cell(row_no, 3).value = filename

    ws.cell(row_no, 1).border = border
    ws.cell(row_no, 2).border = border
    ws.cell(row_no, 3).border = border

wb.save('引き継ぎ資料.xlsx')


from pathlib import Path
import re

def tokenize_file(src, out, tokenizer=None, lang=None, html=False, unicode=False, keep_empty=True, lowercase=False,
                  verbose=False):
    unescape = None

    src_path = Path(src).resolve()
    out_path = Path(out).resolve()

    if html:
        from html import unescape

    if tokenizer is None or tokenizer == 'spacy':
        if lang is None:
            raise TypeError("When using spaCy, you must specify the source and target languages. If you want to"
                            " solely rely on NLTK, or even a naive tokenizer, specify 'nltk' or 'naive' as"
                            " the tokenizer")

    # Set up tokenizer
    spacy_tok, nltk_tok, naive_tok = _set_tokenizer(tokenizer, lang, verbose)

    line_idx = 0
    empty_lines = 0
    with open(str(src_path), 'r', encoding='utf-8') as fh_in, \
            open(str(out_path), 'w', encoding='utf-8') as fh_out:

        for line_idx, line in enumerate(fh_in):
            if verbose:
                print(f'Processing l. {line_idx+1}...', end="\r", flush=True)

            if html:
                line = unescape(line)

            if unicode:
                line = unicode_replace(line)

            if spacy_tok:
                doc = spacy_tok(line)
                line = ' '.join(str(token) for token in doc)
            elif nltk_tok:
                line = ' '.join(nltk_tok(line))
            elif naive_tok:
                line = re.sub(r"\b", " ", line)
                line = re.sub(r" {2,}", " ", line)

            line = line.strip()

            if lowercase:
                line = line.lower()

            if line != '':
                fh_out.write(line + '\n')
            else:
                if keep_empty:
                    fh_out.write('\n')
                empty_lines += 1

    if verbose:
        _print_file_tokenize_summary(out_path, line_idx, empty_lines, html, unicode, keep_empty, lowercase)

def tokenize_string(line, tokenizer=None, lang=None, html=False, unicode=False, lowercase=False, verbose=False):
    unescape = None

    if html:
        from html import unescape

    if tokenizer is None or tokenizer == 'spacy':
        if lang is None:
            raise TypeError("When using spaCy, you must specify the source and target languages. If you want to"
                            " solely rely on NLTK, or even a naive tokenizer, specify 'nltk' or 'naive' as"
                            " the tokenizer")

    # Set up tokenizer
    spacy_tok, nltk_tok, naive_tok = _set_tokenizer(tokenizer, lang, verbose)

    if html:
        line = unescape(line)

    if unicode:
        line = unicode_replace(line)

    if spacy_tok:
        doc = spacy_tok(line)
        line = ' '.join(str(token) for token in doc)
    elif nltk_tok:
        line = ' '.join(nltk_tok(line))
    elif naive_tok:
        line = re.sub(r"\b", " ", line)
        line = re.sub(r" {2,}", " ", line)

    line = line.strip()

    if lowercase:
        line = line.lower()

    return line

def unicode_replace(s):
    def repl(match):
        match = match.group()
        try:
            return match.encode('utf-8').decode('unicode-escape')
        except UnicodeDecodeError:
            return match

    return re.sub(r'(?<!\b[a-zA-Z]:)(\\u[0-9A-Fa-f]{4})', repl, s)

def _get_spacy(lang):
    import spacy
    nlp = spacy.load(lang, disable=['parser', 'ner', 'tagger', 'textcat'])

    return nlp

def _print_file_tokenize_summary(out_path, line_idx, empty_lines, html, unicode, keep_empty, lowercase):
    end_credits = 'Done tokenizing:'
    if html:
        end_credits += '\n\t- unescaped HTML entities;'
    if unicode:
        end_credits += '\n\t- replaced unicode sequences;'
    if lowercase:
        end_credits += '\n\t- lower-cased characters;'

    if empty_lines > 0:
        if keep_empty:
            end_credits += f'\n\t- kept {empty_lines} empty lines;'
        else:
            end_credits += f'\n\t- removed {empty_lines} empty lines;'

    end_credits += f'\n\t- wrote {line_idx - empty_lines} lines in total.'

    end_credits += f'\n\nResults in {str(out_path)}'

    print(end_credits)

def _set_tokenizer(tokenizer, lang, verbose):
    spacy_nlp = None
    word_tokenize = None
    naive_tokenizer = False

    try:
        if tokenizer is None:
            try:
                if verbose:
                    print('Using spaCy\'s tokenizer...', flush=True)
                spacy_nlp = _get_spacy(lang)
            except (ModuleNotFoundError, ImportError, AttributeError, OSError) as e:
                if verbose:
                    print(f'Failed to load spaCy: {e}...\nWill try to use NLTK instead', flush=True)
                try:
                    from nltk import word_tokenize
                except (ModuleNotFoundError, ImportError, AttributeError, OSError) as e:
                    if verbose:
                        print(f'Failed to load nltk: {e}...\nWill use naive tokenizer instead', flush=True)
                    naive_tokenizer = True
        else:
            if tokenizer == 'spacy':
                if verbose:
                    print('Using spaCy\'s tokenizer...', flush=True)
                    spacy_nlp = _get_spacy(lang)
            elif tokenizer == 'nltk':
                if verbose:
                    print('Using NLTK\'s tokenizer...', flush=True)
                from nltk import word_tokenize
            elif tokenizer == 'naive':
                if verbose:
                    print('Using naive tokenizer...', flush=True)
                naive_tokenizer = True
            else:
                raise ValueError('tokenizer has to be None (default), spacy, nltk, or naive')
    except (ModuleNotFoundError, ImportError) as e:
        error = 'To tokenize your input, you need the spaCy library or NLTK installed.'
        if tokenizer is not None:
            error += f' You chose to tokenize with {tokenizer}, which is not installed.'
        error += f'\nError message: {e}'
        raise type(e)(error)
    except (OSError, AttributeError) as e:
        raise type(e)(f'Something went wrong while loading a tokenizer.\nError message: {e}')

    return spacy_nlp, word_tokenize, naive_tokenizer


"""
Challange:

You have n coins and you want to build a staircase with these coins. The staircase consists of k rows where
the ith row has exactly i coins. The last row of the staircase may be incomplete.
Given the integer n, return the number of complete rows of the staircase you will build.
"""

class Solution:
    def arrangeCoins(self, n: int) -> int:
        count = 1
        while True:
            n -= count
            if n < 0:
                return count - 1
            count += 1

print(Solution.arrangeCoins(Solution, 8))


# -*- coding: utf-8 -*-
import unittest
import os

import dotdot
import fake_env

from pubs import content, filebroker, databroker, datacache
from pubs.config import conf

import str_fixtures
from pubs import endecoder

class TestDataBroker(fake_env.TestFakeFs):

    def test_databroker(self):

        ende = endecoder.EnDecoder()
        page99_metadata = ende.decode_metadata(str_fixtures.metadata_raw0)
        page99_bibentry = ende.decode_bibdata(str_fixtures.bibtex_raw0)

        for db_class in [databroker.DataBroker, datacache.DataCache]:
            self.reset_fs()

            db = db_class('tmp', 'tmp/doc', create=True)

            db.push_metadata('citekey1', page99_metadata)
            self.assertFalse(db.exists('citekey1', meta_check=True))
            self.assertFalse(db.exists('citekey1', meta_check=False))

            db.push_bibentry('citekey1', page99_bibentry)
            self.assertTrue(db.exists('citekey1', meta_check=False))
            self.assertTrue(db.exists('citekey1', meta_check=True))

            self.assertEqual(db.pull_metadata('citekey1'), page99_metadata)
            pulled = db.pull_bibentry('citekey1')['Page99']
            for key, value in pulled.items():
                self.assertEqual(pulled[key], page99_bibentry['Page99'][key])
            self.assertEqual(db.pull_bibentry('citekey1'), page99_bibentry)

    def test_existing_data(self):

        ende = endecoder.EnDecoder()
        page99_bibentry = ende.decode_bibdata(str_fixtures.bibtex_raw0)

        for db_class in [databroker.DataBroker, datacache.DataCache]:
            self.reset_fs()

            fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'testrepo'), 'repo')

            db = db_class('repo', 'repo/doc', create=False)

            self.assertEqual(db.pull_bibentry('Page99'), page99_bibentry)

            for citekey in ['10.1371_journal.pone.0038236',
                            '10.1371journal.pone.0063400',
                            'journal0063400']:
                db.pull_bibentry(citekey)
                db.pull_metadata(citekey)

            with self.assertRaises(IOError):
                db.pull_bibentry('citekey')
            with self.assertRaises(IOError):
                db.pull_metadata('citekey')

            db.add_doc('Larry99', 'docsdir://Page99.pdf')
            self.assertTrue(content.check_file('repo/doc/Page99.pdf', fail=False))
            self.assertTrue(content.check_file('repo/doc/Larry99.pdf', fail=False))

            db.remove_doc('docsdir://Page99.pdf')

if __name__ == '__main__':
    unittest.main()


import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM

path = 'C:/Users/Froilan/Desktop/myFiles/JupyterFiles/stock_correlation_prediction/portfolios/portfolio_1/ADS_ANTM.csv'
data = pd.read_csv(path)
data = data.set_index("Date").values

rnn_X= []
rnn_y = []
for i in range(len(data)):
    try:
        rnn_y.append(data[i+199][0])
        rnn_X.append(data[i:i+100].reshape(1,100)[0])
    except IndexError :
        rnn_X = np.array(rnn_X)
        rnn_y = np.array(rnn_y)
        break

rnn_X = np.reshape(rnn_X, (rnn_X.shape[0], 100, 1))
rnn_y = np.reshape(rnn_y, (rnn_y.shape[0], 1, 1))

model = Sequential()
model.add(LSTM(20, input_shape=(100,1)))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(rnn_X, rnn_y, epochs=100, batch_size=100)
print(model.predict(rnn_X))


import math
import random

def genArray(n):
    nextn = n
    count = 0
    while nextn>2:
        nextn = math.floor(math.sqrt(nextn))
        count=count+1
    print("Original n was:",n,"it took",count,"tries to get to:",nextn)
    return n

def function(arr):
    arr1 = arr[0:(len(arry)/2)-1]
    arr2 = arr[(len(arry)/2):len(arr)-1]

#Main code
if __name__=='__main__':
    test = genArray(100)

RPI-CSCI-2300 Introduction to Algorithms/labs/lab4/untitled-1.py


import pandas as pd
import numpy as np
from sklearn.externals import joblib
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, r2_score
import os
from flask import current_app

def predict(dataset_path, filename):
    #############################################
    # load and clean the data + add some features
    #############################################
    file_path = os.path.join(dataset_path, filename)

    df = pd.read_csv(file_path, sep=';')
    df['RSSI'] = [x.replace(',', '.') for x in df.RSSI]
    # missing values will be interpreted as Nan
    df['RSSI'] = pd.to_numeric(df.RSSI, errors='coerce')

    # fix code rate column
    df.rename(columns={' 4/5': 'codeRate'}, inplace=True)
    df['codeRate'] = [x.replace(' ', '') for x in df.codeRate]
    df['codeRate'] = [0.8 for x in df.codeRate if x == '4/5']

    # insted of the JSON calculate actual data rate value
    df.dataRate = df.SPF * df.BW * df.codeRate / (2**df.SPF) * 1000

    # write freq in mHz insted of Hz
    df.Freq = df.Freq / 10**6

    # add macPayloadLen column that contains length (in bits) of MAC Payload
    df['macPayloadLen'] = df.macPayload.str.len() * 4 # times 4 because of HEX

    ####################################################
    # create test set / if we are uploading full dataset
    ####################################################
    split = StratifiedShuffleSplit(n_splits=1, test_size=0.2,
                                    random_state=42)
    for train_index, test_index in split.split(df, df['SPF']):
        strat_train_set = df.loc[train_index]
        strat_test_set = df.loc[test_index]

    X_test = strat_test_set.drop('RSSI', axis=1)
    y_test = strat_test_set['RSSI'].copy()

    # cleaning missing values of numerical attributes in trainng dataset
    categorical_vals = ["Time", "DevAddr", "mType", "macPayload", "_id"]
    data_num = X_test.drop(categorical_vals, axis=1)

    imputer = SimpleImputer(strategy='median')
    imputer.fit(data_num)

    # pipeline
    num_pipeline = Pipeline([
        ('imputer', SimpleImputer(strategy='median')),
        ('std_scaler', StandardScaler()),
        ])

    num_attribs = list(data_num)
    cat_attribs = ['mType']

    full_pipeline = ColumnTransformer([
        ("num", num_pipeline, num_attribs),
        ("cat", OneHotEncoder(), cat_attribs),
    ])

    X_test_prepared = full_pipeline.fit_transform(X_test)

    grid_search = joblib.load(current_app.config['MODEL'])
    model = grid_search.best_estimator_

    predictions = model.predict(X_test_prepared)

    mse = mean_squared_error(y_test, predictions)
    rmse = np.sqrt(mse)

    r2 = r2_score(y_test, predictions)

    return predictions, rmse, r2


import re
from collections import Counter

import torch
from pytorch_lightning import LightningDataModule
from spacy.lang.en import English
from spacy.tokenizer import Tokenizer
from torch.utils.data.dataloader import DataLoader
from torch.utils.data.dataset import Dataset
from torchtext.vocab import Vocab

nlp = English()
tokenizer = Tokenizer(nlp.vocab)

class PadToken:
    def __init__(self, orth_):
        self.orth_ = orth_

class Word2VecDataset(Dataset):
    def __init__(self, tokens, context_size, pad_token="<pad>"):
        self.tokens = tokens
        self.context_size = context_size
        self.pad_token = PadToken(pad_token)

        self._ix = 0

    def __len__(self):
        return len(self.tokens)

    def __getitem__(self, ix):
        start = max(0, ix - self.context_size)
        end = min(len(self.tokens), ix + self.context_size + 1)
        pre_ctx = self.tokens[start:ix]
        post_ctx = self.tokens[ix + 1 : end]
        ctx = list(pre_ctx) + list(post_ctx)
        pads_needed = 2 * self.context_size - len(ctx)
        for _ in range(pads_needed):
            ctx.append(self.pad_token)

        return self.tokens[ix], ctx

class Word2VecDatamodule(LightningDataModule):
    def __init__(
        self,
        docs,
        vocab,
        context_size,
        negative_sampler,
        k: int,
        batch_size,
        num_workers=0,
    ):
        super().__init__()
        self.docs = docs
        self.vocab = vocab
        self.context_size = context_size
        self.negative_sampler = negative_sampler
        self.k = k
        self.batch_size = batch_size
        self.num_workers = num_workers

    def _collate(self, batch):
        return (
            torch.tensor([self.vocab.stoi[t[0].orth_] for t in batch]),
            torch.tensor([[self.vocab.stoi[t.orth_] for t in x[1]] for x in batch]),
            self.negative_sampler((len(batch), self.context_size, self.k)),
        )

    def train_dataloader(self):
        return DataLoader(
            Word2VecDataset(self.docs["train"], self.context_size),
            collate_fn=self._collate,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
        )

    def val_dataloader(self):
        return DataLoader(
            Word2VecDataset(self.docs["val"], self.context_size),
            collate_fn=self._collate,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
        )

    def test_dataloader(self):
        return DataLoader(
            Word2VecDataset(self.docs["test"], self.context_size),
            collate_fn=self._collate,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
        )

def get_data(
    train_path="data/wikitext-2/wiki.train.tokens",
    val_path="data/wikitext-2/wiki.valid.tokens",
    test_path="data/wikitext-2/wiki.test.tokens",
):
    paths = {"train": train_path, "val": val_path, "test": test_path}
    docs = {}
    vocab = None
    for stage, path in paths.items():
        txt = open(path).read().strip()
        txt = re.sub(r"\n\s*\n\s*", "\n", txt)
        doc = tokenizer(txt)
        docs[stage] = doc
        if stage == "train":
            vocab = Vocab(Counter(map(str, doc)))

    return docs, vocab


# import necessary libraries
from mpl_toolkits import mplot3d
from matplotlib.tri import Triangulation
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns

# initialize Seaborn
# sns.set()

# Z axis function
def f(x, y):
    return np.sin(
       np.sqrt(x**2 + y**2)
    )

# X and Y data
x = np.linspace(-6, 6, 30)
y = np.linspace(-6, 6, 30)
X, Y = np.meshgrid(x,y)  # generate a mesh grid of data coordinates in 2-D
Z = f(X, Y)

# Contour 3D Gray
fig = plt.figure()
ax = plt.axes(projection='3d')
ax.axis('off')
ax.grid(False)
ax.set_xticks([])
ax.set_yticks([])
ax.set_zticks([])
ax.contour3D(X, Y, Z, 80, cmap='viridis')
ax.view_init(60, 35)
fig
plt.show()

# Contour 3D Colored
fig = plt.figure()
ax = plt.axes(projection='3d')
ax.axis('off')
ax.grid(False)
ax.set_xticks([])
ax.set_yticks([])
ax.set_zticks([])
ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap='binary', edgecolor='none')
# ax.set_title('surface')
ax.view_init(60, 35)
fig
plt.show()

# Partial Polar Grid
fig = plt.figure()
ax = plt.axes(projection='3d')
ax.axis('off')
ax.grid(False)
ax.set_xticks([])
ax.set_yticks([])
ax.set_zticks([])
r = np.linspace(0, 6, 20)
theta = np.linspace(-0.9 * np.pi , 0.8 * np.pi, 40)
r, theta = np.meshgrid(r, theta)
X = r * np.sin(theta)
Y = r * np.cos(theta)
Z = f(X, Y)
ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap='viridis', edgecolor='none')
ax.view_init(35, 300)
fig
plt.show()

# Mobius Strip
fig = plt.figure()
ax = plt.axes(projection='3d')
ax.axis('off')
ax.grid(False)
ax.set_xticks([])
ax.set_yticks([])
ax.set_zticks([])
w = np.linspace(-0.25, 0.25, 8)
theta = np.linspace(0, 2 * np.pi, 30)
w, theta = np.meshgrid(w, theta)
phi = 0.5 * theta
r = 1 + w * np.cos(phi)  # radius in x-plane

X = np.ravel(r * np.cos(theta))
Y = np.ravel(r * np.sin(theta))
Z = np.ravel(w * np.sin(phi))

tri = Triangulation(np.ravel(w), np.ravel(theta))

ax.plot_trisurf(X, Y, Z, triangles=tri.triangles, cmap='viridis', linewidths=0.2)
ax.set_xlim(-1, 1)
ax.set_ylim(-1, 1)
ax.set_zlim(-1, 1)
ax.view_init(35, 300)
fig
plt.show()


# -*- coding: utf-8 -*-

"""Test Markdown plugin."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import json

from pytest import fixture
from CommonMark import Parser
from six import string_types

from podoc.ast import ASTNode, ast_from_pandoc
from podoc.utils import pandoc, PANDOC_MARKDOWN_FORMAT
from .._markdown import (CommonMarkToAST, ASTToMarkdown, MarkdownPlugin)

#------------------------------------------------------------------------------
# Fixtures
#------------------------------------------------------------------------------

@fixture
def commonmark():
    contents = 'hello *world*'
    parser = Parser()
    cm = parser.parse(contents)
    return cm

@fixture
def ast():
    # TODO: move this to conftest
    ast = ASTNode('root')

    # First block
    block = ASTNode(name='Para',
                    children=['hello '])
    inline = ASTNode(name='Emph')
    inline.add_child('world')
    block.add_child(inline)
    ast.add_child(block)
    return ast

@fixture
def markdown():
    return 'hello *world*'

#------------------------------------------------------------------------------
# Test CommonMark to AST
#------------------------------------------------------------------------------

def test_cm_to_ast(commonmark, ast):
    ast_t = CommonMarkToAST().transform_main(commonmark)
    ast.show()
    ast_t.show()
    assert ast == ast_t

#------------------------------------------------------------------------------
# Test AST to Markdown
#------------------------------------------------------------------------------

def test_ast_to_markdown(ast, markdown):
    md = ASTToMarkdown().transform(ast)
    assert md == markdown

#------------------------------------------------------------------------------
# Test Markdown plugin
#------------------------------------------------------------------------------

def test_markdown_read(ast, markdown):
    assert MarkdownPlugin().read(markdown) == ast

def test_markdown_write(ast, markdown):
    assert MarkdownPlugin().write(ast) == markdown

# -----------------------------------------------------------------------------
# Test Markdown renderer inline
# Check safe round-tripping on CommonMark -> AST -> CommonMark
# -----------------------------------------------------------------------------

def _tree_contains_nodes(ast, names):
    if isinstance(ast, string_types):
        return False
    if ast.name in names:
        return True
    return all(any(_tree_contains_nodes(child, (name,))
                   for child in ast.children)
               for name in names)

def _test_renderer(s, *contains_nodes):
    """Test the renderer on a string."""
    # Parse the string with CommonMark-py.
    ast = MarkdownPlugin().read(s)
    print()
    print('****** PODOC AST ******')
    ast.show()
    # Check that the tree contains a node.
    if contains_nodes:
        assert _tree_contains_nodes(ast, contains_nodes)

    # Check markdown =(podoc)=> AST =(podoc)=> markdown.
    contents = ASTToMarkdown().transform(ast)
    assert contents.strip() == s

    # Check markdown =(podoc)=> AST =(pandoc)=> markdown.
    pandoc_json = json.dumps(ast.to_pandoc(), indent=2)
    markdown_pandoc = pandoc(pandoc_json,
                             PANDOC_MARKDOWN_FORMAT, format='json')
    # NOTE: the pandoc-converted Markdown is not guaranteed to
    # be equal to the original Markdown document. For example * List
    # is transformed into - List, same with ATXHeader etc.
    # However, we do test that the generated JSON is compatible with pandoc.
    assert markdown_pandoc

    # Check markdown_pandoc =(podoc)=> (AST_pandoc == AST_podoc).
    ast_0 = MarkdownPlugin().read(markdown_pandoc)
    print('****** PODOC AST (from markdown_pandoc) ******')
    ast_0.show()
    assert ast_0 == ast

    # Check markdown =(pandoc)=> (AST_pandoc == AST_podoc).
    p = pandoc(s, 'json', format=PANDOC_MARKDOWN_FORMAT)
    # NOTE: we use the '-' bullet char in the tests, for better compatibility
    # with pandoc.
    ast_1 = ast_from_pandoc(json.loads(p),
                            bullet_char='-')
    print('****** PODOC AST (via pandoc) ******')
    ast_1.show()
    assert ast_1 == ast

def test_markdown_renderer_simple():
    _test_renderer('hello')
    _test_renderer('hello world')
    _test_renderer('hello *world*', 'Emph')
    _test_renderer('hello **world**', 'Strong')

def test_markdown_renderer_link():
    _test_renderer('[hello](world)', 'Link')
    _test_renderer('![hello](world)', 'Image')

def test_markdown_renderer_codeinline():
    _test_renderer('hello `world`', 'Code')

# -----------------------------------------------------------------------------
# Test Markdown renderer block
# -----------------------------------------------------------------------------

def test_markdown_renderer_header():
    _test_renderer('# Hello', 'Header')
    _test_renderer('## Hello world', 'Header')

def test_markdown_renderer_codeblock():
    _test_renderer('```\nhello world\n```', 'CodeBlock')
    _test_renderer('```python\nhello world\n```', 'CodeBlock')

def test_markdown_renderer_blockquote():
    _test_renderer('> hello world', 'BlockQuote')
    _test_renderer('> hello world\n> end', 'BlockQuote')

def test_markdown_renderer_bullet_list():
    _test_renderer('- Item 1')
    _test_renderer('- Item 1\n- Item 2')
    # TODO: fix the following test
    # _test_renderer('- Item 1\n  - Item 1.2')

def test_markdown_renderer_ordered_list():
    _test_renderer('1. Item 1')
    _test_renderer('1. Item 1\n2. Item 2')
    _test_renderer('2. Item 1\n3. Item 2')

# -----------------------------------------------------------------------------
# Test Markdown renderer multiple blocks
# -----------------------------------------------------------------------------

def test_markdown_renderer_paras():
    _test_renderer('hello\nworld')
    _test_renderer('hello\n\nworld')

def test_markdown_renderer_headers():
    _test_renderer('# 1\n\n# 2')
    _test_renderer('# 1\n\n## 2')
    _test_renderer('## 2\n\n# 1')

def test_markdown_renderer_codeblocks():
    _test_renderer('```python\nhello world\n```\n\n```\nother\n```')

def test_markdown_renderer_ordered_bullet():
    # NOTE: we use `-` bullets instead of `*` to ensure stable round-trip
    # with pandoc which uses - by default.
    _test_renderer('1. Item 1\n\n- Bullet',
                   'BulletList', 'OrderedList')
    _test_renderer('- Bullet\n\n1. Item 1',
                   'BulletList', 'OrderedList')
    _test_renderer('1. Item 1\n2. Item 2\n\n- Bullet',
                   'BulletList', 'OrderedList')
    _test_renderer('1. Item 1\n2. Item 2\n\n- Bullet\n\n3. Item 3',
                   'BulletList', 'OrderedList')

def test_markdown_math_inline():
    _test_renderer('a $x*x=y*y$ b', 'Math')

def test_markdown_math_block():
    _test_renderer(r'$$\int_a^b f_0(x) dx$$', 'MathBlock')
    _test_renderer(r'$$\begin{eqnarray}\nx &= y\n\end{eqnarray}$$',
                   'MathBlock')


from argparse import ArgumentParser
from typing import Type, Mapping

from analytics.misc.utils import load_entity_maps_pkl
from analytics.aggregations.aggregation import Aggregation
from analytics.aggregations.pipeline_runs import PipelineRunsAggregation
from analytics.aggregations.primitive_pairs import PrimitivePairComparisonAggregation
from analytics.aggregations.pipeline_dags import PipelineDAGsAggregation

aggregation_map: Mapping[str, Type[Aggregation]] = {
    "primitive_pairs": PrimitivePairComparisonAggregation,
    "pipeline_runs": PipelineRunsAggregation,
    "pipeline_dags": PipelineDAGsAggregation,
}

def get_parser() -> ArgumentParser:
    """
    Configures the argument parser for aggregating a pickled extraction of pipeline runs.
    """
    parser = ArgumentParser(
        description="Load and aggregate a pickled extraction of pipeline runs"
    )
    parser.add_argument(
        "aggregation",
        choices=aggregation_map.keys(),
        help="The name of the aggregation to conduct",
    )
    parser.add_argument(
        "--verbose",
        "-v",
        action="store_true",
        help=(
            "If present, the aggregation will output detailed information on its "
            "progress and intermediate results during computation, if applicable"
        ),
    )
    parser.add_argument(
        "--refresh",
        "-r",
        action="store_true",
        help=("If present, any caches that the aggregation uses will be refreshed."),
    )
    return parser

if __name__ == "__main__":
    parser = get_parser()
    args = parser.parse_args()
    entity_maps = load_entity_maps_pkl()
    aggregation_class: Type[Aggregation] = aggregation_map[args.aggregation]
    aggregation = aggregation_class()
    print(f"Now running aggregation...")
    aggregation.run(entity_maps, args.verbose, args.refresh)