import datetime
import glob
import os
import subprocess

import numpy as np
import pandas as pd
import pose
import poses
import rosbag

# Forward errors so we can recover failures
# even when running commands through multiprocessing
# pooling
def full_traceback(func):
    import functools
    import traceback

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception as e:
            msg = "{}\n\nOriginal {}".format(e, traceback.format_exc())
            raise type(e)(msg)

    return wrapper

def get_files(directory, file_string):
    return glob.glob(os.path.join(directory, file_string))

def get_files_recursive(directory, file_string):
    subdirectory_csv_files = []
    _, subdirectories, _ = next(os.walk(directory))
    for subdirectory in subdirectories:
        subdirectory_path = os.path.join(directory, subdirectory)
        for subdirectory_csv_file in get_files(subdirectory_path, file_string):
            subdirectory_csv_files.append(subdirectory_csv_file)
    return subdirectory_csv_files

def create_directory(directory=None):
    if directory == None:
        directory = os.path.join(
            os.getcwd(), datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        )
    if os.path.exists(directory):
        print((directory + " already exists!"))
        exit()
    os.makedirs(directory)
    return directory

def load_dataframe(files):
    dataframes = [pd.read_csv(file) for file in files]
    dataframe = pd.concat(dataframes)
    return dataframe

def run_command_and_save_output(command, output_filename, print_command=True):
    if print_command:
        print(command)
    with open(output_filename, "w") as output_file:
        subprocess.call(command, shell=True, stdout=output_file, stderr=output_file)

def basename(filename):
    return os.path.splitext(os.path.basename(filename))[0]

# TODO(rsoussan): Move these to different utilities file
def get_topic_rates(
    bag_name,
    topic,
    min_time_diff_for_gap,
    use_header_time=True,
    verbose=False,
    ignore_zero_time_diffs=True,
):
    with rosbag.Bag(bag_name, "r") as bag:
        last_time = 0.0
        gaps = 0
        time_diffs = []
        for _, msg, t in bag.read_messages([topic]):
            time = (
                msg.header.stamp.secs + msg.header.stamp.nsecs * 1.0e-9
                if use_header_time
                else t.secs + t.nsecs * 1.0e-9
            )
            time_diff = time - last_time
            if last_time != 0 and time_diff >= min_time_diff_for_gap:
                if verbose:
                    print(
                        (
                            topic
                            + " Gap: time: "
                            + str(time)
                            + ", last_time: "
                            + str(last_time)
                            + ", diff: "
                            + str(time_diff)
                        )
                    )
                gaps += 1
            if last_time != 0 and (time_diff != 0 or not ignore_zero_time_diffs):
                time_diffs.append(time_diff)
            last_time = time

        mean_time_diff = np.mean(time_diffs)
        min_time_diff = np.min(time_diffs)
        max_time_diff = np.max(time_diffs)
        stddev_time_diff = np.std(time_diffs)
        if verbose:
            if use_header_time:
                print("Using Header time.")
            else:
                print("Using Receive time.")
            print(
                (
                    "Found "
                    + str(gaps)
                    + " time diffs >= "
                    + str(min_time_diff_for_gap)
                    + " secs."
                )
            )
            print(("Mean time diff: " + str(mean_time_diff)))
            print(("Min time diff: " + str(min_time_diff)))
            print(("Max time diff: " + str(max_time_diff)))
            print(("Stddev time diff: " + str(stddev_time_diff)))

def make_absolute_poses_from_relative_poses(absolute_poses, relative_poses, name):
    starting_relative_time = relative_poses.times[0]
    np_times = np.array(absolute_poses.times)
    closest_index = np.argmin(np.abs(np_times - starting_relative_time))
    start_pose = absolute_poses.pose(closest_index)
    new_pose = start_pose
    new_poses_list = [start_pose]
    new_poses_times = [absolute_poses.times[closest_index]]
    for index in range(len(relative_poses.times)):
        relative_pose = relative_poses.pose(index)
        new_pose = new_pose * relative_pose
        new_poses_list.append(new_pose)
        new_poses_times.append(relative_poses.times[index])
    new_poses = poses.Poses(name, "")
    new_poses.init_from_poses(new_poses_list, new_poses_times)
    return new_poses

def integrate_velocities(localization_states):
    delta_times = [
        j - i
        for i, j in zip(localization_states.times[:-1], localization_states.times[1:])
    ]
    # Make sure times are same length as velocities, ignore last velocity
    delta_times.append(0)
    # TODO(rsoussan): Integrate angular velocities?
    # TODO(rsoussan): central difference instead?
    x_increments = [
        velocity * delta_t
        for velocity, delta_t in zip(localization_states.velocities.xs, delta_times)
    ]
    y_increments = [
        velocity * delta_t
        for velocity, delta_t in zip(localization_states.velocities.ys, delta_times)
    ]
    z_increments = [
        velocity * delta_t
        for velocity, delta_t in zip(localization_states.velocities.zs, delta_times)
    ]

    return add_increments_to_absolute_pose(
        x_increments,
        y_increments,
        z_increments,
        localization_states.positions.xs[0],
        localization_states.positions.ys[0],
        localization_states.positions.zs[0],
        localization_states.times,
        "Integrated Graph Velocities",
    )

def add_increments_to_absolute_pose(
    x_increments,
    y_increments,
    z_increments,
    starting_x,
    starting_y,
    starting_z,
    times,
    poses_name="Increment Poses",
):
    integrated_positions = poses.Poses(poses_name, "")
    cumulative_x_increments = np.cumsum(x_increments)
    integrated_positions.positions.xs = [
        starting_x + cumulative_x_increment
        for cumulative_x_increment in cumulative_x_increments
    ]
    cumulative_y_increments = np.cumsum(y_increments)
    integrated_positions.positions.ys = [
        starting_y + cumulative_y_increment
        for cumulative_y_increment in cumulative_y_increments
    ]
    cumulative_z_increments = np.cumsum(z_increments)
    integrated_positions.positions.zs = [
        starting_z + cumulative_z_increment
        for cumulative_z_increment in cumulative_z_increments
    ]

    # Add start positions
    integrated_positions.positions.xs.insert(0, starting_x)
    integrated_positions.positions.ys.insert(0, starting_y)
    integrated_positions.positions.zs.insert(0, starting_z)

    # Remove last elements (no timestamp for these)
    del integrated_positions.positions.xs[-1]
    del integrated_positions.positions.ys[-1]
    del integrated_positions.positions.zs[-1]

    integrated_positions.times = times
    return integrated_positions


import typing

from sqlalchemy import and_
from sqlalchemy import Boolean
from sqlalchemy import cast
from sqlalchemy import column
from sqlalchemy import DateTime
from sqlalchemy import false
from sqlalchemy import Float
from sqlalchemy import func
from sqlalchemy import Integer
from sqlalchemy import or_
from sqlalchemy import select
from sqlalchemy import String
from sqlalchemy import true

# builtin.pyi stubs define object.__eq__() as returning bool,  which
# can't be overridden (it's final).  So for us to type `__eq__()` and
# `__ne__()`, we have to use type: ignore[override].  Test if does this mean
# the typing tools don't know the type, or if they just ignore the error.
# (it's fortunately the former)
expr1 = column("x", Integer) == 10

c1 = column("a", String)

c2 = column("a", Integer)

expr2 = c2.in_([1, 2, 3])

expr2_set = c2.in_({1, 2, 3})

expr2_gen = c2.in_((x for x in (1, 2, 3)))

nexpr2 = c2.not_in([1, 2, 3])

nexpr2_set = c2.not_in({1, 2, 3})

nexpr2_gen = c2.not_in((x for x in (1, 2, 3)))

short_cir1 = and_(True, c2 == 5)
short_cir2 = or_(False, c2 == 5)

short_cir3 = and_(true(), c2 == 5)
short_cir4 = or_(false(), c2 == 5)

# EXPECTED_MYPY: Missing positional argument "initial_clause" in call to "and_"
no_empty_1 = and_()

# EXPECTED_MYPY: Missing positional argument "initial_clause" in call to "or_"
no_empty_2 = or_()

expr3 = c2 / 5

expr4 = -c2

expr5 = ~(c2 == 5)

q = column("q", Boolean)
expr6 = ~q

expr7 = c1 + "x"

expr8 = c2 + 10

stmt = select(column("q")).where(lambda: column("g") > 5).where(c2 == 5)

expr9 = c1.bool_op("@@")(func.to_tsquery("some & query"))

def test_issue_9418() -> None:
    and_(c1.is_(q))
    and_(c1.is_not(q))
    and_(c1.isnot(q))
    and_(c1.not_in(["x"]))
    and_(c1.notin_(["x"]))
    and_(c1.not_like("x"))
    and_(c1.notlike("x"))
    and_(c1.not_ilike("x"))
    and_(c1.notilike("x"))

def test_issue_9451() -> None:
    # issue #9451
    c1.cast(Integer)
    c1.cast(Float)
    c1.op("foobar")("operand").cast(DateTime)
    cast(c1, Float)
    cast(c1.op("foobar")("operand"), DateTime)

def test_issue_9650_char() -> None:
    and_(c1.contains("x"))
    and_(c1.startswith("x"))
    and_(c1.endswith("x"))
    and_(c1.icontains("x"))
    and_(c1.istartswith("x"))
    and_(c1.iendswith("x"))

def test_issue_9650_bitwise() -> None:
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_and(5))
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_or(5))
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_xor(5))
    # EXPECTED_TYPE: UnaryExpression[int]
    reveal_type(c2.bitwise_not())
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_lshift(5))
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_rshift(5))
    # EXPECTED_TYPE: ColumnElement[int]
    reveal_type(c2 << 5)
    # EXPECTED_TYPE: ColumnElement[int]
    reveal_type(c2 >> 5)

if typing.TYPE_CHECKING:
    # as far as if this is ColumnElement, BinaryElement, SQLCoreOperations,
    # that might change.  main thing is it's SomeSQLColThing[bool] and
    # not 'bool' or 'Any'.
    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.bool\]
    reveal_type(expr1)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnClause\[builtins.str.?\]
    reveal_type(c1)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnClause\[builtins.int.?\]
    reveal_type(c2)

    # EXPECTED_RE_TYPE: sqlalchemy..*BinaryExpression\[builtins.bool\]
    reveal_type(expr2)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[Union\[builtins.float, .*\.Decimal\]\]
    reveal_type(expr3)

    # EXPECTED_RE_TYPE: sqlalchemy..*UnaryExpression\[builtins.int.?\]
    reveal_type(expr4)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.bool.?\]
    reveal_type(expr5)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.bool.?\]
    reveal_type(expr6)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.str\]
    reveal_type(expr7)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.int.?\]
    reveal_type(expr8)

    # EXPECTED_TYPE: BinaryExpression[bool]
    reveal_type(expr9)


#### with lru caching method #########

from functools import lru_cache
@lru_cache(maxsize=16)
def fib(n):
    if n<=2 :
        return 1
    return fib(n-1) + fib(n-2)

print(fib(50))

######################### with memoization technique #########
def fib(n,memo):

    if (n in memo):
        return memo[n]
    if n<=2 :
        return 1
    memo[n]  = fib(n-1,memo) + fib(n-2,memo)

    return memo[n]
memo={}
print(fib(15,memo))


import sys

from cx_Freeze import setup, Executable

# Dependencies are automatically detected, but it might need
# fine tuning.
buildOptions = dict(
  packages = [], excludes = [],
  include_files = ['icon','toc'],
)

name = 'example'

if sys.platform == 'win32':
  name = name + '.exe'

base = None
if sys.platform == "win32":
    base = "Win32GUI"

executables = [
  Executable('main.py', base = base, targetName = name,
             compress = True,
            )
]

setup(name='Example',
      version = '1.0',
      description = 'An example program',
      options = dict(build_exe = buildOptions),
      executables = executables)


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from pytorch3d.ops import knn_points, ball_query
from .logger import logger

def index_points(points, idx):
    """
    Input:
        points: input points data, [B, N, C]
        idx: sample index data, [B, S, [K]]
    Return:
        new_points:, indexed points data, [B, S, [K], C]
    """
    raw_size = idx.size()
    idx = idx.reshape(raw_size[0], -1)
    res = torch.gather(points, 1, idx[..., None].expand(-1, -1, points.size(-1)))
    return res.reshape(*raw_size, -1)

def sample_points_from_ball_query(pt_xyz, pt_feats, center_point, k, radius):
    _, ball_idx, xyz = ball_query(center_point, pt_xyz, K=k, radius=radius, return_nn=True)
    invalid = torch.sum(ball_idx == -1) > 0
    if invalid:
        logger.warning(f"ball query returns {torch.sum(ball_idx == -1)} / {torch.numel(ball_idx)} -1 in its index, "
                       f"which means you need to increase raidus or decrease K")

    points = index_points(pt_feats, ball_idx).squeeze(1)
    xyz = xyz.squeeze(1)
    return xyz, points

def sample_points_from_knn(pt_xyz, pt_feats, center_point, k):
    _, knn_idx, xyz = knn_points(center_point, pt_xyz, K=k, return_nn=True)
    points = index_points(pt_feats, knn_idx).squeeze(1)
    xyz = xyz.squeeze(1)
    return xyz, points


import asyncio

from supybot import callbacks, httpserver, log

from .helpers import github

# Import files that will hook themself up when imported
from .events import (  # noqa
    commit_comment,
    discussion,
    issue,
    pull_request,
    push,
    tag,
)
from .patches import gidgethub  # noqa
from .protocols import irc as irc_protocols  # noqa
from .protocols import discord as discord_protocols  # noqa

class GitHubCallback(httpserver.SupyHTTPServerCallback):
    name = "GitHub"
    defaultResponse = "404: page not found"

    def doPost(self, handler, path, form):
        if path != "/":
            self.send_response(404)
            self.send_header("Content-type", "text/plain")
            self.end_headers()
            self.wfile.write(b"404: page not found")
            return

        headers = {key.lower(): value for key, value in dict(self.headers).items()}

        try:
            asyncio.run(github.dispatch(headers, form))

            self.send_response(200)
            self.send_header("Content-Type", "text/plain")
            self.end_headers()
            self.wfile.write(b"200: OK")
        except Exception:
            log.exception("Failed to handle GitHub event")
            self.send_response(403)
            self.send_header("Content-type", "text/plain")
            self.end_headers()
            self.wfile.write(b"403: failed processing event")

class GitHub(callbacks.Plugin):
    """Translates GitHub events to IRC messages."""

    def __init__(self, irc):
        self.__parent = super(GitHub, self)
        callbacks.Plugin.__init__(self, irc)

        httpserver.hook("github", GitHubCallback())

    def die(self):
        self.__parent.die()
        httpserver.unhook("github")

Class = GitHub


import pytest

import for_mocking

"""
1. Реализовать программу на Python. Программа может содержать любое количество методов и классов (>0), но обязательно должна иметь класс main.
2. Имитировать:
    a. Метод созданного класса (метод не должен являться генератором).
    b. Параметр внутри метода класса.
    c. Класс.
3. Спровоцировать имитированную ошибку.
4. Применить декоратор @patch на любом имитируемом объекте (классе, методе и т.д.).
Выбирается студентом самостоятельно.
5. Имитировать объект-генератор (см.пр. 26.6.3.3.[1]).
"""

def test_format_status(mocker):
    main_object = for_mocking.Main('Not OK')
    mocker.patch.object(main_object, 'format_status',
                        return_value=['Status: OK;'])
    assert main_object.format_status() == ['Status: OK;']
    assert main_object.status == 'Not OK'

def test_property_mocking(mocker):
    main_object = for_mocking.Main('Not OK')
    main_object = mocker.patch.object(main_object, 'format_status')
    main_object.format_status(mocker.ANY)
    main_object.format_status.assert_called_once_with(mocker.ANY)

def test_class_mocking(mocker):
    mocker.patch.object(for_mocking, 'Main')
    for_mocking.Main.format_status.return_value = 123
    mocked_instance = for_mocking.Main('wow')
    mocked_instance.format_status()
    mocked_instance.format_status.assert_called_once_with()  # pylint: disable=no-member

@pytest.mark.xfail(raises=Exception)
def test_generator(mocker):
    main_object = for_mocking.Main('Not OK')
    main_object = mocker.patch.object(main_object, 'format_status')
    main_object.format_status(123)

def test_letter_generator(mocker):
    main_object = for_mocking.Main('Not OK')
    mocker.patch.object(main_object, 'letter_generator',
                        return_value=iter('Status: OK;'))
    assert list(main_object.letter_generator()) == list('Status: OK;')


# Mathematics > Probability > Sherlock and Probability
# Help Sherlock in finding the probability.
#
# https://www.hackerrank.com/challenges/sherlock-and-probability/problem
# https://www.hackerrank.com/contests/infinitum-jul14/challenges/sherlock-and-probability
# challenge id: 2534
#

from fractions import Fraction

for _ in range(int(input())):
    n, k = map(int, input().split())
    bits = input()

    counter = [0] * (n + 1)
    for i, bit in enumerate(bits):
        counter[i + 1] = counter[i] + int(bit)

    p = 0
    for i, bit in enumerate(bits):
        if bit == "1":
            p += counter[min(n, i + k + 1)] - counter[max(0, i - k)]

    r = Fraction(p, n * n)
    print("{}/{}".format(r.numerator, r.denominator))


#!/usr/bin/env python3

import numpy as np
import h5py
import argparse
import matplotlib
import matplotlib.pyplot as plt
import csv
import glob
import os

def get_dataset_keys(f):
    keys = []
    f.visit(lambda key : keys.append(key) if isinstance(f[key], h5py.Dataset) else None)
    return keys

def plot(time, data, key, figure_name):
    os.system("mkdir -p {}min_max_mean/".format(figure_name))
    if "min" in key or "mean" in key or "max" in key:
        fig, (ax1) = plt.subplots(nrows=1, sharex=True)
        plt.subplots_adjust(hspace=0.1)
        ax1.plot(time, data[key], c="k")  # , s=0.1, alpha=0.3)
        ax1.set_title("Key: {}".format(key))
        # ax1.set_xlabel(r'$r$')
        # ax1.set_ylabel(r'$\rho$')
        # ax1.set_xlim(0, r_max)
        # ax1.set_ylim(0, 4.0)
        fig.tight_layout()
        plt.savefig("{}min_max_mean/min_max_mean_{}.png".format(figure_name, key))
    else:
        fig, (ax1) = plt.subplots(nrows=1, sharex=True)
        plt.subplots_adjust(hspace=0.1)
        # linestyle="dotted",
        # linestyle="dashed",
        # linestyle="dashdot"
        ax1.plot(time, data["{}_min".format(key)], c="k", linestyle="dotted", label="min")  # , s=0.1, alpha=0.3)
        ax1.plot(time, data["{}_max".format(key)], c="k", linestyle="dashed", label="max")
        ax1.plot(time, data["{}_mean".format(key)], c="k", linestyle="dashdot", label="mean")
        ax1.set_title("Key: {}".format(key))
        ax1.legend(loc="best")
        # ax1.set_xlabel(r'$r$')
        # ax1.set_ylabel(r'$\rho$')
        # ax1.set_xlim(0, r_max)
        # ax1.set_ylim(0, 4.0)
        fig.tight_layout()
        plt.savefig("{}min_max_mean/min_max_mean_{}.png".format(figure_name, key))

if __name__ == '__main__':

    plot_all = False

    parser = argparse.ArgumentParser(description="min/max/mean for all entries")
    parser.add_argument("--input", "-i", metavar="str", type=str, help="input file",
                        nargs="?", default="../output")
    parser.add_argument("--output", "-o", metavar="str", type=str, help="output directory",
                        nargs="?", default="-")
    parser.add_argument("--key", "-k", metavar="str", type=str, help="output directory",
                        nargs="?", default="-")
    parser.add_argument('--all', "-a", action='store_true')
    args = parser.parse_args()

    if args.all:
        plot_all = True

    header = []
    detailed_header = []
    data = {}

    with open(args.input, newline='\n') as csvfile:
        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for i_row, row in enumerate(reader):
            if i_row == 0:
                for elem in row:
                    header.append(elem)
            elif i_row == 1:
                for elem in row:
                    detailed_header.append(elem)
                    data[elem] = []
            else:
                for i_elem, elem in enumerate(row):
                    data[detailed_header[i_elem]].append(float(elem))

    # print("data[{}] = {}".format(args.key, data[args.key]))
    if args.key in header or args.key in detailed_header:
        plot(data["time_max"], data, args.key, args.output)

    if plot_all:
        for _header in header:
            print("plotting {} ...".format(_header))
            plot(data["time_max"], data, _header, args.output)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
from django.conf.urls import patterns, url

from .views import PromoDetail, PromoList, ChannelPromoList

urlpatterns = patterns(
    '',
    url(
        r'^$',
        PromoList.as_view(),
        name='list_promos'
    ),
    url(
        r'^channel/(?P<channel__long_slug>[\w//-]+)$',
        ChannelPromoList.as_view(),
        name='channel_promo'
    ),
    url(
        r'^(?P<slug>[\w-]+)/(?P<result>[\w-]+)$',
        PromoDetail.as_view(),
        name='result_promo'
    ),
    url(
        r'^(?P<slug>[\w-]+)\.html$',
        PromoDetail.as_view(),
        name='open_promo'
    ),
)


import decimal
import io
from nose.tools import assert_equal
import utcdatetime

from os.path import join as pjoin

from snipe import WatchListSnipesParser, parse_datetime

def test_get_snipes():
    with io.open(pjoin('sample_data', 'watch_list.html')) as f:
        parser = WatchListSnipesParser(f.read())

    assert_equal(
        list(parser.get_snipes()), [
            ('113143572176', decimal.Decimal('7.00')),
            ('292631197506', decimal.Decimal('44.00')),
        ]
    )

def test_parse_snipe_note():
    TEST_CASES = [
        ('snipe: 45', decimal.Decimal(45.00)),
        ('snipe: 45.00', decimal.Decimal(45.00)),
        ('snipe: £45.00', decimal.Decimal(45.00)),
    ]

    for note, expected_amount in TEST_CASES:
        yield assert_equal, WatchListSnipesParser._parse_snipe_note(note), expected_amount

def test_parse_datetime():
    TEST_CASES = [
        (
            '(11 Jul, 2018\n09:58:34 BST)',
            utcdatetime.utcdatetime(2018, 7, 11, 8, 58, 34)
        ),
        (
            '(11 Jan, 2018\n09:58:34 GMT)',
            utcdatetime.utcdatetime(2018, 1, 11, 9, 58, 34)
        ),
    ]

    for string, expected in TEST_CASES:
        yield assert_equal, parse_datetime(string), expected


import numpy as np
import pandas as pd
from sklearn.metrics import log_loss
from sklearn.model_selection import StratifiedKFold, StratifiedShuffleSplit
from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping
from data_loader import get_data, generator

def cross_validation(model, X_train, X_train_angle, Y_train, X_test, X_test_angle, K):
    folds = list(StratifiedKFold(n_splits=K, shuffle=True).split(X_train, Y_train))
    y_test_pred = 0
    y_train_pred = 0
    y_valid_pred = 0.0 * Y_train
    for j, (train_idx, test_idx) in enumerate(folds):
        print('\n===================FOLD=', j)
        X_cv = X_train[train_idx]
        Y_cv = Y_train[train_idx]
        X_angle_cv = X_train_angle[train_idx]
        X_hold = X_train[test_idx]
        Y_hold = Y_train[test_idx]
        X_angle_hold = X_train_angle[test_idx]

        # define file path and get callbacks
        model.reset()
        file_path = model.model_kind + "_model_weights_%s.hdf5" % j
        model.train(X_cv, X_angle_cv, Y_cv, X_hold, X_angle_hold, Y_hold, file_path)

        # Getting Training Score
        loss, acc = model.eval(X_cv, X_angle_cv, Y_cv)
        print('Train loss:', loss, ' Train accuracy:', acc)

        # Getting Test Score
        loss, acc = model.eval(X_hold, X_angle_hold, Y_hold)
        print('Test loss:', loss, ' Test accuracy:', acc)

        # Getting validation Score.
        y_valid_pred[test_idx] = model.predict(X_hold, X_angle_hold)
        y_test_pred += model.predict(X_test, X_test_angle)
        y_train_pred += model.predict(X_train, X_train_angle)

    y_test_pred = y_test_pred / K
    y_train_pred = y_train_pred / K

    print('\n Train Loss = ', log_loss(Y_train, y_train_pred))
    print(' Train Acc = ', np.sum(Y_train == (y_train_pred > 0.5)) / Y_train.shape[0])
    print(' Val Loss = ', log_loss(Y_train, y_valid_pred))
    print(' Val Acc  =', np.sum(Y_train == (y_valid_pred > 0.5)) / Y_train.shape[0])

    return y_test_pred


from django.urls import path, include
from website.views import home, blog, perfil, login, acessar, cadastrar

urlpatterns = [
    path('', home),
    path('blog', blog),
    path('login', login),
    path('acessar', acessar),
    path('perfil', perfil),
    path('cadastrar', cadastrar),
]


from msilib.schema import Error
import matplotlib.pyplot as plt
import numpy as np
import boto3
import os

def draw_chart(result, user_id, practice_id, gender):
    w_min_jitter = 1.599
    w_max_jitter = 2.310
    w_min_shimmer = 7.393
    w_max_shimmer = 12.221

    m_min_jitter = 2.159
    m_max_jitter = 3.023
    m_min_shimmer = 10.132
    m_max_shimmer = 13.526

    min_speed = 96
    max_speed = 124

    font_size = 12

    # 차트 그리기
    # 1) speed
    speed_x = ['slow', 'user', 'fast']
    speed_y = [min_speed, result['voice']['speed'], max_speed]
    speed_colors = ['lightgray', 'mediumpurple', 'silver']
    plt.xlabel('standard', size=font_size)
    plt.ylabel('Words Per Minute', size=font_size)
    bar_chart=plt.bar(speed_x, speed_y, color=speed_colors)

    for rect in bar_chart:
        height = rect.get_height()
        plt.text(rect.get_x()+rect.get_width()/2.0, height-font_size, height, ha='center', va='bottom', size=12)

    plt.savefig(f'./{user_id}_{practice_id}_speed.png')
    plt.clf()

    # 2) shimmer, jitter
    if gender == 'W':
        mins = [w_min_jitter, w_min_shimmer]
        maxs = [w_max_jitter, w_max_shimmer]
    else:
        mins = [m_min_jitter, m_min_shimmer]
        maxs = [m_max_jitter, m_max_shimmer]

    users = [result['voice']['jitter'], result['voice']['shimmer']]
    checks = ['jitter', 'shimmer']

    bar_width=0.25

    index = np.arange(2)

    b1 = plt.bar(index, mins, bar_width, color='lightgray', label='mins')
    b2 = plt.bar(index+bar_width, users, bar_width, color='mediumpurple', label='users')
    b3 = plt.bar(index+2*bar_width, maxs, bar_width, color='silver', label='maxs')

    plt.legend()
    plt.savefig(f'./{user_id}_{practice_id}_shimmer_jitter.png')
    plt.clf()

    # 3) closing remarks
    closing_remark_ratio = [result['voice']['closing_remarks'], 100-result['voice']['closing_remarks']]
    closing_remark_labels = ['correctly recognized', 'recognization failed']
    closing_remark_colors = ['thistle', 'mediumpurple']
    wedgeprops = {'width': 0.7, 'edgecolor': 'w', 'linewidth': 5}

    plt.pie(closing_remark_ratio, labels=closing_remark_labels, autopct='%.1f%%', startangle=180, colors = closing_remark_colors, wedgeprops=wedgeprops)
    plt.savefig(f'./{user_id}_{practice_id}_closing_remarks.png')
    plt.clf()

    # 4) movement
    move_duration = result['video']['pose']['inclined_duration']
    +result['video']['pose']['first_duration']
    +result['video']['pose']['second_duration']
    +result['video']['pose']['third_duration']
    +result['video']['eyes']['script_duration']
    + result['video']['eyes']['around_duration']
    + result['video']['eyes']['face_move_duration']
    move_ratio = [result['video']['pose']['total_duration'], move_duration]
    move_labels = ['normal', 'bad movement']
    move_colors = ['thistle', 'mediumpurple']

    plt.pie(move_ratio, labels=move_labels, autopct='%.1f%%', startangle=180, colors=move_colors, wedgeprops=wedgeprops)
    plt.savefig(f'./{user_id}_{practice_id}_movement.png')
    plt.clf()

    # 5) movement detail
    bad_x = ['around', 'inclined', 'first', 'second', 'third', 'script', 'face_move']
    bad_y = [
        result['video']['eyes']['around_duration'],
        result['video']['pose']['inclined_duration'],
        result['video']['pose']['first_duration'],
        result['video']['pose']['second_duration'],
        result['video']['pose']['third_duration'],
        result['video']['eyes']['script_duration'],
        result['video']['eyes']['face_move_duration']
    ]
    bad_colors = ['thistle', 'plum', 'mediumpurple', 'mediumslateblue', 'mediumorchid', 'darkorchid', 'rebeccapurple']

    plt.xlabel('sort', size=font_size)
    plt.ylabel('duration', size=font_size)
    bar_chart=plt.bar(bad_x, bad_y, color=bad_colors)
    plt.savefig(f'./{user_id}_{practice_id}_movement_detail.png')
    plt.clf()

def upload_chart_to_s3(user_id, practice_id):
    s3=boto3.client('s3')
    bucket_name = "sookpeech-wavfile"

    # chart 업로드하기
    files = {
        "closing_remarks" : f'./{user_id}_{practice_id}_closing_remarks.png',
        "movement" : f'./{user_id}_{practice_id}_movement.png',
        "movement_detail": f'./{user_id}_{practice_id}_movement_detail.png',
        "shimmer_jitter" : f'./{user_id}_{practice_id}_shimmer_jitter.png',
        "speed" : f'./{user_id}_{practice_id}_speed.png'
    }

    for key in files.keys():
        try:
            file = open(files[key], 'rb')
            s3.upload_fileobj(file, bucket_name, f'{user_id}/{practice_id}/{key}.png', ExtraArgs={'ContentType': 'image/png', 'ACL':'public-read'})
        except:
            print(f'failed to upload chart image = {key}')

def delete_image(user_id, practice_id):
    files = {
        "closing_remarks" : f'./{user_id}_{practice_id}_closing_remarks.png',
        "movement" : f'./{user_id}_{practice_id}_movement.png',
        "movement_detail": f'./{user_id}_{practice_id}_movement_detail.png',
        "shimmer_jitter" : f'./{user_id}_{practice_id}_shimmer_jitter.png',
        "speed" : f'./{user_id}_{practice_id}_speed.png'
    }

    for key in files.keys():
        if os.path.exists(files[key]):
            os.remove(files[key])


#!/usr/bin/env python
import time
import openstack
NODE_COUNT = 43
def get_connection():
    # openstack.enable_logging(debug=True)
    conn = openstack.connect()
    return conn
def main():
    conn = get_connection()
    for i in range(NODE_COUNT):
        name = "euclid-ral_compute_%d" % i
        conn.delete_server(name, wait=True)
    for i in range(NODE_COUNT):
        name = "euclid-ral_compute_%d" % i
        conn.delete_volume(name, wait=True)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Preppin' Data 2020: Week 9 - C&BS Co: Political Monitoring
https://preppindata.blogspot.com/2020/02/2020-week-9.html

- Input data
- Remove the Average Record for the polls
- Clean up your Dates
- Remove any Null Poll Results
- Form a Rank (modified competition) of the candidates per Poll based on their results
- Determine the spread for each poll from 1st Place to 2nd Place
- Rename Sample Types: RV = Registered Voter, LV = Likely Voter, null = Unknown
- Output the Data
- Optional: Build the Viz

Author: Kelly Gilbert
Created: 2022-02-02
Requirements:
  - input dataset:
      - PD 2020 Wk 9 Input - Sheet1.csv
  - output dataset (for results check only):
      - PD 2020 Wk 9 Output.csv
"""

from numpy import nan, where
import pandas as pd

#---------------------------------------------------------------------------------------------------
# input the data
#---------------------------------------------------------------------------------------------------

# read in the file, melt the candidates into rows, remove nulls, remove averages
df = pd.read_csv(r'.\inputs\PD 2020 Wk 9 Input - Sheet1.csv', na_values='--')\
       .melt(id_vars=['Poll', 'Date', 'Sample'], var_name='Candidate', value_name='Poll Results')\
       .dropna(subset=['Poll Results'])\
       .query("~Poll.str.contains('Average')", engine='python')

#---------------------------------------------------------------------------------------------------
# process the data
#---------------------------------------------------------------------------------------------------

# clean up end date
df['End Date'] = pd.to_datetime(df['Date'].str.extract('.*- (\d+/\d+)', expand=False) + '/2020')
df['End Date'] = where(df['End Date'].dt.month >= 7,
                       df['End Date'] + pd.DateOffset(years=-1),
                       df['End Date'])

# form a Rank (modified competition) of the candidates per Poll based on their results
df['Rank'] = df.groupby(['Poll', 'End Date', 'Sample'])['Poll Results'].rank(method='max', ascending=False)\
               .astype(int)

# difference in poll results between first and second
df['Spread from 1st to 2nd Place'] = \
    df.groupby(['Poll', 'End Date', 'Sample'], as_index=False)['Poll Results']\
      .transform(lambda x: x.max() - x.nlargest(2).min())

# rename sample types
sample_map = {'.*RV' : 'Registered Voter', '.*LV' : 'Likely Voter', nan : 'Unknown'}
df['Sample Type'] = df['Sample'].replace(sample_map, regex=True)

#---------------------------------------------------------------------------------------------------
# output the file
#---------------------------------------------------------------------------------------------------

out_cols = ['Candidate', 'Poll Results', 'Spread from 1st to 2nd Place', 'Rank', 'End Date',
            'Sample Type', 'Poll']
df.to_csv(r'.\outputs\output-2020-09.csv', index=False, columns=out_cols, date_format='%d/%m/%Y')

#---------------------------------------------------------------------------------------------------
# create chart
#---------------------------------------------------------------------------------------------------

from bokeh.io import output_file
from bokeh.layouts import row
from bokeh.models import CustomJS, Legend, DatetimeTickFormatter, Select, Title
from bokeh.plotting import figure, show

# color constants
color_selected = '#0066cc'
color_deselected = '#bab0ac'

# set the output file path
output_file('dimensions.html')

# subset of registered voters
df_rv = df.loc[df['Sample Type']=='Registered Voter']\
          .sort_values(by=['Candidate', 'End Date', 'Poll'])

# add a figure and format it
p = figure(width=900, height=500, x_axis_type='datetime')
p.add_layout(Title(text='Data from: realclearpolitics.com; Sample Type: Registered Voter',
                   text_font_size='9pt'), 'above')
p.add_layout(Title(text='2020 Democratic Presidential Nominations', text_font_size="24pt"), 'above')
p.xaxis.formatter=DatetimeTickFormatter(days=["%Y-%m-%d"])
p.y_range.flipped = True
p.add_layout(Legend(), 'right')

# add a line and circles for each candidate option
candidates = sorted(df_rv['Candidate'].unique())

line_dict = {}
circle_dict = {}
for i, c in enumerate(candidates):
    line_dict[c] = p.line(df_rv.loc[df['Candidate']==c]['End Date'],
                          df_rv.loc[df['Candidate']==c]['Rank'],
                          legend_label=c, line_width=2,
                          color=color_selected if i==0 else color_deselected)

    circle_dict[c] = p.circle(df_rv.loc[df['Candidate']==c]['End Date'],
                              df_rv.loc[df['Candidate']==c]['Rank'],
                              legend_label=c, size=7,
                              line_width=0,
                              fill_color=color_selected if i==0 else color_deselected)

# create a drop-down menu
menu = Select(options=candidates, value=candidates[0], title='Select an item:')

# link the plot and the button using a callback function

# cb_obj = the model that triggered the callback (e.g. button model, dropdown model)
# args = list of name=object values you want to have accessible inside the callback function
# can't assign cb_obj.value to a model property directly; you have to store it in a variable first

callback = CustomJS(args=dict(p=p, lines=line_dict, circles=circle_dict), code="""
const t = cb_obj.value;

// make the selected item's marks blue and the rest gray
for (let i in lines) {
  if (i == t) {
      lines[i].glyph.line_color = '""" + color_selected + """';
      circles[i].glyph.fill_color = '""" + color_selected + """';
  } else {
      lines[i].glyph.line_color = '""" + color_deselected + """';
      circles[i].glyph.fill_color = '""" + color_deselected + """';
  }
}
""")

menu.js_on_change('value', callback)

# display the layout
chart_layout = row(p, menu)
show(chart_layout)

#---------------------------------------------------------------------------------------------------
# check results
#---------------------------------------------------------------------------------------------------

solution_files = ['PD 2020 Wk 9 Output.csv']
my_files = ['output-2020-09.csv']
unique_cols = [['Candidate', 'Poll', 'End Date', 'Sample Type']]
col_order_matters = True
round_dec = 8

for i, solution_file in enumerate(solution_files):
    print('---------- Checking \'' + solution_file + '\' ----------\n')

    # read in the files
    df_sol = pd.read_csv('.\\outputs\\' + solution_file, encoding='utf-8')
    df_mine = pd.read_csv('.\\outputs\\' + my_files[i], encoding='utf-8')

    # are the columns the same?
    solution_cols = list(df_sol.columns)
    myCols = list(df_mine.columns)
    if not col_order_matters:
         solution_cols.sort()
         myCols.sort()

    col_match = False
    if solution_cols != myCols:
        print('*** Columns do not match ***')
        print('    Columns in solution: ' + str(list(df_sol.columns)))
        print('    Columns in mine    : ' + str(list(df_mine.columns)))
        print('\n\n')
    else:
        print('Columns match\n')
        col_match = True

    # are the values the same? (only check if the columns matched)
    if col_match:
        errors = 0
        df_compare = df_sol.merge(df_mine, how='outer', on=unique_cols[i],
                                  suffixes=['_sol', '_mine'], indicator=True)

        # extra/missing records
        if len(df_compare[df_compare['_merge'] != 'both']) > 0:
            print('*** Missing or extra records ***\n')
            print('In solution, not in mine:\n')
            print(df_compare[df_compare['_merge'] == 'left_only'][unique_cols[i]])
            print('\n\nIn mine, not in solution:\n')
            print(df_compare[df_compare['_merge'] == 'right_only'][unique_cols[i]])
            errors += 1

        # for the records that matched, check for mismatched values
        for c in [c for c in df_sol.columns if c not in unique_cols[i]]:
            if 'float' in df_compare[f'{c}_sol'].dtype.name:
                df_compare[f'{c}_sol'] = df_compare[f'{c}_sol'].round(round_dec)
                df_compare[f'{c}_mine'] = df_compare[f'{c}_mine'].round(round_dec)

            unmatched = df_compare[(df_compare['_merge']=='both')
                                   & (df_compare[f'{c}_sol'] != df_compare[f'{c}_mine'])]
            if len(unmatched) > 0:
                print(f'*** Values do not match: {c} ***\n')
                print(df_compare[(df_compare['_merge']=='both')
                                 & (df_compare[f'{c}_sol'] != df_compare[f'{c}_mine'])]\
                                [unique_cols[i] + [f'{c}_sol', f'{c}_mine']])
                print('\n')
                errors += 1

        if errors == 0:
            print('Values match')

    print('\n')


import sys
import pygame
import random
from src import hero
from src import enemy

class Controller:
    def __init__(self, width=640, height=480):
        """
        Initializes and sets up the game
        args: self.width (int) Width (left to right) of the screen
        self.height (int) Height (top to bottom) of the screen
        self.screen (display) The screen that is displayed
        self.background (surface) Background of the screen
        self.enemies (sprite) Group of enemies as sprites
        num_enemies (int) The number of enemies in the game
        x (int) Random x coordinate for each enemy
        y (int) Random y coordinate for each enemy
        self.hero (class)
        self.all_sprites (sprite) All the sprites, including the hero and the enemies
        self.state (str) String labeled "GAME"
        return: none
        """
        pygame.init()
        self.width = width
        self.height = height
        self.screen = pygame.display.set_mode((self.width, self.height))
        self.background = pygame.Surface(self.screen.get_size()).convert()
        self.background.fill((250, 250, 250))  # set the background to white
        pygame.font.init()  # you have to call this at the start, if you want to use this module.
        pygame.key.set_repeat(1, 50)  # initialize a held keey to act as repeated key strikes
        """Load the sprites that we need"""

        self.enemies = pygame.sprite.Group()
        num_enemies = 3
        for i in range(num_enemies):
            x = random.randrange(100, 400)
            y = random.randrange(100, 400)
            self.enemies.add(enemy.Enemy("Boogie", x, y, 'assets/enemy.png'))
        self.hero = hero.Hero("Conan", 50, 80, "assets/hero.png")
        self.all_sprites = pygame.sprite.Group((self.hero,) + tuple(self.enemies))
        self.state = "GAME"

    def mainLoop(self):
        """
        Controls the state of the game
        args: self.state (str) String labeled as either "GAME" or "GAMEOVER"
        return: none
        """
        while True:
            if(self.state == "GAME"):
                self.gameLoop()
            elif(self.state == "GAMEOVER"):
                self.gameOver()

    def gameLoop(self):
        """
        Enables the hero to move based on key inputs from the user
        args: self.state (str) String labeled as "GAME"
        event.type (text) The input of the event that is taking place
        event.key (int) The key that is inputted on the keyboard from the user
        fights (list) The hero and the enemies colliding as means of fighting
        e (int) The enemy that the fighter is fighting
        self.hero.health (int) The number of health points the hero has
        self.state (str) String labeled as "GAMEOVER"
        return: none
        """
        while self.state == "GAME":
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    sys.exit()
                if event.type == pygame.KEYDOWN:
                    if(event.key == pygame.K_UP):
                        self.hero.move_up()
                    elif(event.key == pygame.K_DOWN):
                        self.hero.move_down()
                    elif(event.key == pygame.K_LEFT):
                        self.hero.move_left()
                    elif(event.key == pygame.K_RIGHT):
                        self.hero.move_right()

            # check for collisions
            fights = pygame.sprite.spritecollide(self.hero, self.enemies, True)
            if(fights):
                for e in fights:
                    if(self.hero.fight(e)):
                        e.kill()
                        self.background.fill((250, 250, 250))
                    else:
                        self.background.fill((250, 0, 0))
                        self.enemies.add(e)

            # redraw the entire screen
            self.enemies.update()
            self.screen.blit(self.background, (0, 0))
            if(self.hero.health == 0):
                self.state = "GAMEOVER"
            self.all_sprites.draw(self.screen)

            # update the screen
            pygame.display.flip()

    def gameOver(self):
        """
        Ends the game and shows a gameover screen
        args: myfont (font) Changes the system font
        message (str) Writes out 'Game Over'
        event.type (text) The input of the event taking place
        return: none
        """
        self.hero.kill()
        myfont = pygame.font.SysFont(None, 30)
        message = myfont.render('Game Over', False, (0, 0, 0))
        self.screen.blit(message, (self.width / 2, self.height / 2))
        pygame.display.flip()
        while True:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    sys.exit()


# A very simple Flask Hello World app for you to get started with...
import logging, sys
from flask import Flask, request
from CrapBot import Bot

crap_bot = Bot()
app = Flask(__name__)

@app.route('/')
def hello_world():
    return 'Hello from Flask!'

@app.route('/bot', methods=['POST', 'GET'])
def bot():
    update = request.json
    if update:
        crap_bot.handle(update)
    return 'Ok'

if __name__ == '__main__':
    from CrapBot import Logger
    Logger.warn('bot started')
    #from CrapBot.Api import set_webhook
    #r = set_webhook()
    #print(r)
    crap_bot.listen()


from lyse import *
import matplotlib.pyplot as plt
import numpy as np
from scipy.optimize import curve_fit
import scipy.constants as constants
import AnalysisSettings
import SrConstants
from Subroutines.FitFunctions import gauss
from scipy import stats

camera = AnalysisSettings.Camera
pixelSize = SrConstants.pixelSizeDict[camera]

name = "Different_Loading_Curves_20_02_04"
path = "C:\\Experiments\\example_experiment\\grating_MOT\\AnalysisData\\" + name + ".h5"
run = Run(path)
df = data()

sequence_index = df["sequence_index"]
run_number = df["run number"]
filepath = df["filepath"]
norms = df['FitMOTLoad','fluorNorm']
BlueMOTLoadTime = df['BlueMOTLoadTime']
BlueMOTBeatnote = df['BlueMOTBeatnote']
BlueMOTPower_V = df['BlueMOTPower']
atomNumber = df["splice_gaussian_fit","atomNumber"]
loadingTimeConstant = df["FitMOTLoad", "loadingTimeConstant"]
loadingRate = df["FitMOTLoad", "loadingRate"]
widthX = df["splice_gaussian_fit","widthX"]
widthZ = df["splice_gaussian_fit","widthZ"]
TimeOfFlight = df["TimeOfFlight"]
avgWidth = df["splice_gaussian_fit","avgWidth"]
avgPeakOD = df["splice_gaussian_fit","avgPeakOD"]
avgNorm = 0.04
normFlrNum = loadingRate * loadingTimeConstant * avgNorm/ norms
avgNormRate = loadingRate * avgNorm / norms
SourceCurrent = df["SourceCurrent"]
runtime = df["run time"]

fluorCounts = []
fluorCountsCulled = range(len(filepath))
time = range(len(filepath))
timeCulled = range(len(filepath))

i = 0
for file in filepath:
    fluorCounts.append(np.zeros(1))
    with h5py.File(file,'a') as f:
        for result in f['results']['FitMOTLoad']['atomNummbers']:
            np.append(fluorCounts[i],result)
    i+= 1
print(fluorCounts)

FluorCounts = df["FitMOTLoad","atomNummbers"]
FluorCountsCulled = df["FitMOTLoad","atomNummbersCulled"]
time = df["FitMOTLoad", "time"]
timeCulled = df["FitMOTLoad", "timeCulled"]

def parabola(v,a,b,c):
    return a*(v-c)**2 +b

power_params = [-758.105, 92.576, 4.087]

gaussianWidth = 0.00914
Isat = 403
BlueMOTPower_mW = parabola(BlueMOTPower_V, *power_params)
I_Isat = (BlueMOTPower_mW*(10**-3)/(np.pi*gaussianWidth**2))/Isat

#Do Calculations

pOpt, pCov = curve_fit(gauss, BlueMOTBeatnote, atomNumber, p0 = (1000, 95, 10, 0))

PeakBeatnote = pOpt[1]

Detuning = BlueMOTBeatnote - PeakBeatnote

linewidth = 30.5

# SourceCurrentSing = [12,12.5,13,13.5,14]
DetuningSing = []
normFlrNumSing = []
normFlrNumSingDev =[]
#
for beatnote in Detuning:
    if beatnote not in DetuningSing:
        DetuningSing.append(beatnote)
#
#
atomNumberSing = []
atomNumberSingDev =[]
normTauSing = []
normTauSingDev =[]
normRateSing = []
normRateSingDev = []
#
for beatnote in DetuningSing:
    points1 = []
    #points2 = []
    #points3 = []
    for i in range(len(run_number)):
        if Detuning[i] == beatnote:
            points1.append(atomNumber[i])
            # points2.append(avgNormRate[i])
            # points3.append(normFlrNum[i])
    atomNumberSing.append(np.average(points1))
    atomNumberSingDev.append(stats.sem(points1))
    # normRateSing.append(np.average(points2))
    # normRateSingDev.append(np.std(points2))
    # normFlrNumSing.append(np.average(points3))
    # normFlrNumSingDev.append(np.std(points3))

corAtomNumberSing = np.array(atomNumberSing) * 1/(1 + 4*(np.array(DetuningSing)/linewidth)**2)
corAtomNumberSingDev = np.array(atomNumberSingDev) * 1/(1 + 4*(np.array(DetuningSing)/linewidth)**2)

# fig, ax1 = plt.subplots(1)
# #plot = ax.errorbar(BlueMOTPower_V, normFlrNum, fmt = 'bo', label = 'norms')
# #plot = plt.scatter(SourceCurrent, loadingTimeConstant)
# ax1.errorbar(DetuningSing, atomNumberSing, yerr = atomNumberSingDev,fmt = 'bo')
# ax1.errorbar(DetuningSing, corAtomNumberSing, yerr = corAtomNumberSingDev,fmt = 'go')
# ax1.plot(np.linspace(-18,9,100), gauss(np.linspace(-18,9,100) + PeakBeatnote,*pOpt), 'r-')

run.save_result('norms',norms)
run.save_result('atomNumber', atomNumber)
run.save_result("loadingRate", loadingRate)
run.save_result("loadingTimeConstant", loadingTimeConstant)
run.save_result("avgWidth", avgWidth)
run.save_result("avgPeakOD", avgPeakOD)
run.save_result("avgNorm", np.average(norms))
run.save_result("TimeOfFlight", TimeOfFlight)
run.save_result("widthX", widthX)
run.save_result("widthZ", widthZ)
run.save_result("FluorCounts",FluorCounts)
run.save_result("FluorCountsCulled", FluorCountsCulled)
run.save_result("time", time)
run.save_result("timeCulled", timeCulled)
#run.save_result("BlueMOTPower_V", BlueMOTPower_V)
# run.save_result("Detuning", Detuning)
# run.save_result("DetuningSing", DetuningSing)
# run.save_result("corAtomNumberSing", corAtomNumberSing)
# run.save_result("corAtomNumberSingDev",corAtomNumberSingDev)
# run.save_result("PeakBeatnote" ,PeakBeatnote)
# run.save_result("linewidth", linewidth)
# run.save_result("atomNumberSing",atomNumberSing)
# run.save_result("atomNumberSingDev",atomNumberSingDev)
# run.save_result("SourceCurrent", SourceCurrent)
run.save_result("normFlrNum", normFlrNum)
#run.save_result("normFlrNumSing", normFlrNumSing)
#run.save_result("normFlrNumSingDev", normFlrNumSingDev)
#run.save_result("normTauSing", normTauSing)
#run.save_result("normTauSingDev", normTauSingDev)
#run.save_result("normRateSing", normRateSing)
#run.save_result("normRateSingDev", normRateSingDev)
#run.save_result("SourceCurrentSing", SourceCurrentSing)
# run.save_result("gaussianWidth", gaussianWidth)
# run.save_result("Isat", Isat)
# run.save_result("BlueMOTPower_mW", BlueMOTPower_mW)
# run.save_result("I_Isat", I_Isat)
# run.save_result("avgNormRate", avgNormRate)
# run.save_result("avgNorm", avgNorm)

files = []
for i in range(len(sequence_index)):
    files.append(str(sequence_index[i]) + '_' + str(run_number[i]))
    run.save_result("files", files)
run.save_result("filepath", filepath)


"""
Harvester scripts

Currently only supports AVR atdf files
"""
# Python 3 compatibility for Python 2
from __future__ import print_function

import argparse
import textwrap
from xml.etree import ElementTree

from pymcuprog.deviceinfo.memorynames import MemoryNames
from pymcuprog.deviceinfo.deviceinfokeys import DeviceMemoryInfoKeys, DeviceInfoKeysAvr

def map_atdf_memory_name_to_pymcuprog_name(atdf_name):
    """
    Mapping a memory name in atdf files to the corresponding memory name used in the pymcuprog device models

    :param atdf_name: Name of memory in atdf files
    :return: Name of memory in pymcuprog device models
    """
    pymcuprog_name = 'unknown'
    if atdf_name == 'progmem':
        pymcuprog_name = MemoryNames.FLASH
    if atdf_name == 'user_signatures':
        # Datasheets actually use user_row for UPDI devices at least
        pymcuprog_name = MemoryNames.USER_ROW
    if atdf_name == 'eeprom':
        pymcuprog_name = MemoryNames.EEPROM
    if atdf_name == 'fuses':
        pymcuprog_name = MemoryNames.FUSES
    if atdf_name == 'lockbits':
        pymcuprog_name = MemoryNames.LOCKBITS
    if atdf_name == 'signatures':
        pymcuprog_name = MemoryNames.SIGNATURES
    if atdf_name == 'internal_sram':
        pymcuprog_name = MemoryNames.INTERNAL_SRAM

    return pymcuprog_name

def determine_chiperase_effect(name, architecture):
    if 'avr' in architecture:
        if name in ['user_signatures', 'fuses', 'signatures']:
            return('ChiperaseEffect.NOT_ERASED')
        elif name in ['internal_sram', 'lockbits', 'progmem']:
            return('ChiperaseEffect.ALWAYS_ERASED')
        elif name in ['eeprom']:
            return('ChiperaseEffect.CONDITIONALLY_ERASED_AVR')
    else:
        return('# To be filled in manually')

def determine_isolated_erase(name, architecture):
    if 'avr' in architecture:
        if name in ['user_signatures', 'eeprom']:
            return('True')
        elif name in ['internal_sram', 'lockbits', 'progmem', 'fuses', 'signatures']:
            return('False')
    else:
        return('# To be filled in manually')

def determine_write_size(memory_name, page_size, device_name):
    write_size = 1
    device_name = device_name.lower()
    if memory_name is 'flash':
        if (device_name.find('avr') != -1 and ((device_name.find('da') != -1) or (device_name.find('db') != -1) or (device_name.find('dd') != -1))):
            write_size = 2
        else:
            write_size = page_size
    elif memory_name is 'signatures':
        write_size = 0
    return write_size

def capture_memory_attribute(attribute, architecture, device_name):
    """
    Capture the memory attribute

    :param attribute: memory attribute to capture
    :return: attributes found as a string
    """
    name = attribute['name'].lower()
    size = attribute['size']
    start = attribute['start']

    try:
        pagesize = attribute['pagesize']
    except KeyError:
        pagesize = '1'
    # For some AVRs the ATDF gives a pagesize of fuses and lockbits equal to flash or EEPROM page size but fuses and lockbits are always byte accessible.
    if name in ['fuses', 'lockbits']:
        pagesize = '1'
    # Read size is always 1 byte except for flash that can only read complete words
    readsize = 1
    if name in ['progmem']:
        readsize = 2
    output = ""
    # These names are the names used in the atdf files and might differ from the pymcuprog MemoryNames
    if name in ['progmem', 'eeprom', 'user_signatures', 'fuses', 'lockbits', 'signatures', 'internal_sram']:
        print_name = map_atdf_memory_name_to_pymcuprog_name(name)
        output += "\n    # {}\n".format(print_name)
        output += capture_field('{}_{}_byte'.format(print_name, DeviceMemoryInfoKeys.ADDRESS), start)
        output += capture_field('{}_{}_bytes'.format(print_name, DeviceMemoryInfoKeys.SIZE), size)
        output += capture_field('{}_{}_bytes'.format(print_name, DeviceMemoryInfoKeys.PAGE_SIZE), pagesize)
        # These are the same for all AVRs
        output += "    '{}_{}_bytes': {},\n".format(print_name, DeviceMemoryInfoKeys.READ_SIZE, readsize)
        output += "    '{}_{}_bytes': {},\n".format(print_name, DeviceMemoryInfoKeys.WRITE_SIZE, determine_write_size(print_name, pagesize, device_name))
        output += "    '{}_{}': {},\n".format(print_name, DeviceMemoryInfoKeys.CHIPERASE_EFFECT, determine_chiperase_effect(name, architecture))
        output += "    '{}_{}': {},\n".format(print_name, DeviceMemoryInfoKeys.ISOLATED_ERASE, determine_isolated_erase(name, architecture))
    return output

def capture_register_offset(name, offset):
    """
    Wrapper to create a string definition

    :param name: register name
    :param offset: register offset
    :return: string of register and offset
    """
    return capture_field("{}_base".format(name.lower()), offset)

def capture_field(field, value):
    """
    Macro to create text format field

    :param field: register name
    :param value: register value
    :return: string of definition
    """
    try:
        _test_value = int(value, 16)
    except (ValueError, AttributeError):
        # Can't convert string to int, assumed to be string
        return "    '{}': '{}',\n".format(field, value)
    return "    '{}': {},\n".format(field, value)

def capture_device_element(element, device_name):
    """
    Capture data from a device element

    :param element: element with tag='device'
    :return: captured data from the device element as a string
    """
    architecture = element.attrib['architecture'].lower()
    output = capture_field('name', element.attrib['name'].lower())
    output += capture_field('architecture', architecture)
    for i in element.iterfind("address-spaces/address-space/memory-segment"):
        output += capture_memory_attribute(i.attrib, architecture, device_name)
    output += "\n    # Some extra AVR specific fields\n"
    return output

def capture_module_element(element):
    """
    Capture data from a module element

    This function will return data captured from the module element but will also check if the module
    element contains info about an UPDI fuse (fuse to configure a shared UPDI pin)
    :param element: element with tag='module'
    :return output, found_updi_fuse
        output: captured module element data as a string
        found_updi_fuse: True if the module element contained info about an UPDI fuse
    """
    output = ""
    found_updi_fuse = False
    for i in element.iterfind("instance/register-group"):
        name = i.attrib['name']
        offset = "0x{:08X}".format(int(i.attrib['offset'], 16))
        if i.attrib['name'] == 'SYSCFG':
            output += capture_register_offset(name, offset)
            output += capture_register_offset('OCD', "0x{:08X}".format(int(offset, 16) + 0x80))
        if i.attrib['name'] == 'NVMCTRL':
            output += capture_register_offset(name, offset)
    for i in element.iterfind("instance/signals/signal"):
        if i.attrib['group'] == 'UPDI' and i.attrib['pad'] is not None:
            output += capture_field('prog_clock_khz', '900')
            found_updi_fuse = True
    return output, found_updi_fuse

#This dict serves as a temporary workaround for an xml-parsing bug, where the device signature can't be extracted for some devices
spare_signatures = {
    'atmega16':	bytearray(b'\x1e\x94\x03'),
    'atmega1609': bytearray(b'\x1e\x94\x26'),
    'atmega168a': bytearray(b'\x1e\x94\x06'),
    'atmega3209': bytearray(b'\x1e\x95\x31'),
    'atmega328': bytearray(b'\x1e\x95\x14'),
    'atmega4809': bytearray(b'\x1e\x96\x51'),
    'atmega48p': bytearray(b'\x1e\x92\x0a'),
    'atmega809': bytearray(b'\x1e\x93\x2a'),
    'atmega88a': bytearray(b'\x1e\x93\x0a'),
    'attiny202': bytearray(b'\x1e\x91\x23'),
    'attiny402': bytearray(b'\x1e\x92\x27'),
    'avr128da32': bytearray(b'\x1e\x97\x09'),
    'avr32da32': bytearray(b'\x1e\x95\x33'),
    'avr64da32': bytearray(b'\x1e\x96\x14')
}

def capture_signature_from_property_groups_element(element, device_name):
    """
    Capture signature (Device ID) data from a property-group element

    :param element: element with tag='property-groups'
    :return bytearray with 3 bytes of Device ID data
    """
    signature = bytearray(3)
    success = False
    for i in element.iterfind("property-group/property"):
        if i.attrib['name'] == 'SIGNATURE0':
            signature[0] = int(i.attrib['value'], 16)
        if i.attrib['name'] == 'SIGNATURE1':
            signature[1] = int(i.attrib['value'], 16)
        if i.attrib['name'] == 'SIGNATURE2':
            signature[2] = int(i.attrib['value'], 16)
            success = True
    if success is False:
        signature = spare_signatures[device_name.lower()]
        success = True
    return signature, success

#This dict serves as a temporary workaround for an xml-parsing bug, where the flash_offset can't be extracted for some devices
spare_flash_offset = {
    'atmega809': '0x00004000',
    'atmega1609': '0x00004000',
    'attiny202': '0x00008000',
    'attiny402': '0x00008000',
    'avr32da32': '0x00800000',
    'avr64da32': '0x00800000',
    'avr128da32': '0x00800000',
}

def correct_flash_offset(element, device_name, data_string):
    corrected_data_string = ''
    device_name = device_name.lower()
    success = False
    for i in element.iterfind("property-group/property"):
        if i.attrib['name'] == 'PROGMEM_OFFSET':
            value = i.attrib['value']
            success = True
    if success is False:
        try:
            value = spare_flash_offset[device_name.lower()]
            success = True
        except:
            success = False
    if success is True:
        ind = data_string.find("'flash_address_byte': ")
        corrected_data_string += data_string[:ind+22]
        corrected_data_string += value
        corrected_data_string += data_string[ind+22+10:]
        return corrected_data_string

    return data_string

def determine_address_size(device_name):
    address_size = '16-bit'
    device_name = device_name.lower()
    if (device_name.find('avr') != -1 and ((device_name.find('da') != -1) or (device_name.find('db') != -1))):
        address_size = '24-bit'
    return address_size

def harvest_from_file(filename):
    """
    Harvest parameters from a file

    :param filename: path to file to parse
    :return: list of parameters
    """
    xml_iter = ElementTree.iterparse(filename, events=('start', 'end'))
    output = ""
    shared_updi = False
    for event, elem in xml_iter:
        if event == 'start':
            if elem.tag == 'device':
                device_name = elem.attrib['name']
                output += capture_device_element(elem, device_name)
            if elem.tag == 'module':
                module_output, found_updi_fuse = capture_module_element(elem)
                output += module_output
                if found_updi_fuse:
                    shared_updi = True
            if elem.tag == 'interface':
                output += capture_field(elem.tag, elem.attrib['name'])
            if elem.tag == 'property-groups':
                signature, signature_harvest_success = capture_signature_from_property_groups_element(elem, device_name)
                    # signature_harvest_success provides a temporary workaround for an xml-parsing bug
                output = correct_flash_offset(elem, device_name, output)

    output += capture_field('address_size', determine_address_size(device_name))
    if not shared_updi:
        output += capture_field(DeviceInfoKeysAvr.PROG_CLOCK_KHZ, '1800')
    output += capture_field(DeviceInfoKeysAvr.DEVICE_ID,
                            "0x{:02X}{:02X}{:02X}".format(signature[0], signature[1], signature[2]))

    return output, signature_harvest_success

def main():
    """
    Main function for the harvest utility
    """
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent('''\
    Harvests device data from a device data file (.atdf) for one device.

    The harvested data can be used to populate a device file in deviceinfo.devices
        '''))

    parser.add_argument("filename",
                        help="name (and path) of file to harvest data from"
                        )

    arguments = parser.parse_args()

    dict_content = harvest_from_file(arguments.filename)
    content = "\nfrom pymcuprog.deviceinfo.eraseflags import ChiperaseEffect\n\n"
    content += "DEVICE_INFO = {{\n{}}}".format(dict_content)
    print(content)

if __name__ == "__main__":
    main()


from django.db.models import F
from django.contrib.auth.models import User
from lunchclub.models import AccessToken

class TokenBackend(object):
    def authenticate(self, token=None):
        try:
            token = AccessToken.objects.get(token=token)
        except AccessToken.DoesNotExist:
            return None
        return token.person.get_or_create_user()

    def get_user(self, user_id):
        try:
            return User.objects.get(pk=user_id)
        except User.DoesNotExist:
            return None


import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

# Global variables
csv_file = 'RealtimePlot.csv'  # Replace with the path to your CSV file
update_interval = 1000  # Update plot every 1000 milliseconds (1 second)
fig, ax = plt.subplots()
ax2 = ax.twinx()

# Function to update the plot
def update_plot():
    df = pd.read_csv(csv_file)
    plt.cla()  # Clear the current plot
    ax.plot(df['Time'], df['Distance'], 'b-', label='Distance')
    # ax2.plot(dataTime, dataVR, 'r--', label='Velocity')
    ax2.plot(df['Time'], df['Speed'], 'g-.', label='Speed Command')
    plt.axis('on')  # Turn off axis labels and ticks

    ax.set_xlabel("Time (s)")
    ax.set_ylabel("Distance (mm)")
    ax2.set_ylabel("Speed (mm/s)")

    plt.title('Distance and Speed Command Real-Time Plot')
    # Adjust plot limits if needed
    ax.relim()
    ax.autoscale_view()
    # Redraw the plot
    plt.draw()

if __name__ == '__main__':
    # Main loop to update the plot periodically
    while True:
        update_plot()
        plt.pause(0.1)  # Pause for a short duration to allow the plot to update


# @time     : 2020/7/12 16:35
# @author  : HerbLee
# @file    : finance.py

from sanic import Blueprint
from sanic.response import text
from models.funddb import FundDb, CurrentFund

fund = Blueprint("fund", url_prefix="/fund")

@fund.route("/get_data")
async def get_v2_data(request):
    return text("it is finance")

@fund.route("/add", methods=["POST",])
async def add_fund(request):
    datas = request.form
    # await FundDb.create(name=datas['name'][0], code=datas['code'][0],
    #                     price=float(datas['price'][0]), cost=float(datas['cost'][0]), nums=float(datas['nums'][0]))
    res = await FundDb.filter(code=datas['code'][0])
    if not res:
        fdb = await FundDb.create(name=datas['name'][0], code=datas['code'][0])

    await CurrentFund.create(code=res[0] if res else fdb, price=float(datas['price'][0]), cost=float(datas['cost'][0]), nums=float(datas['nums'][0]))
    return text("success")


import numpy as np
import pandas as pd
import seaborn as sns; sns.set()
import matplotlib.pyplot as plt

from multiprocessing import Pool
from functools import partial

from sklearn.model_selection import KFold
from MatrixFactorization import FactorizeMatrix, GetRepresentationError, CreateLatentVariables
from FeatureSimilarity import GetTopGenes

def RandomParams(eta_low, eta_high, lamb1_low, lamb1_high, lamb2_low, lamb2_high, num_reps=20):
    hyper_params = np.zeros((num_reps, 3)).astype(np.float32)
    hyper_params[:, 0] = np.random.uniform(low=eta_low, high=eta_high, size=(num_reps,))
    hyper_params[:, 1] = np.random.uniform(low=lamb1_low, high=lamb1_high, size=(num_reps,))
    hyper_params[:, 2] = np.random.uniform(low=lamb2_low, high=lamb2_high, size=(num_reps,))
    return hyper_params

def TrainOnParams(params, X, k, neighbors, train_indices, test_indices):
    print('.', end='')
    n, g = X.shape
    eta, lamb1, lamb2 = params
    U, V = CreateLatentVariables(n, g, k)
    U, V = FactorizeMatrix(X, U, V, neighbors, eta=eta, lamb1=lamb1, lamb2=lamb2, trainIndices=train_indices)
    paramError = GetRepresentationError(X, U, V, known_indices=test_indices)
    return paramError

def TrainVerboseOnParams(params, X, k, neighbors, train_indices, test_indices):
    print('.', end='')
    n, g = X.shape
    eta, lamb1, lamb2 = params
    U, V = CreateLatentVariables(n, g, k)
    U, V, trainError, testError = FactorizeMatrix(X, U, V, neighbors, eta=eta, lamb1=lamb1, lamb2=lamb2, trainIndices=train_indices, returnErrorVectors=True)
    paramError = GetRepresentationError(X, U, V, known_indices=test_indices)
    return paramError, trainError, testError

def CrossValidation(X, k, hyper_params, neighbors=None, foldcount=5, returnVectorDF=False, numProcesses=20):
    '''
    Runs the matrix factorization algorithm for each specified value of eta and lambda
    and computes the reconstruction errors for each run.

    Args:
        X: An n x g, possibly sparse numpy matrix, where missing entries are indicated by np.nan values,
           where n represents the number of samples and g represents the number of genes, or items.
        k: The latent dimension of the factorization. Typically, k < min(n, g).
        hyper_params: A list of tuples, each corresponding to a setting of hyper parameters (eta, lamb1, lamb2).
        foldcount: An integer denoting the number of folds for cross validation.

    Returns:
        A len(etas) x len(lambs) x foldcount tensor denoting the reconstruction error for each
        setting of eta and lambda on each fold.
    '''
    n, g = X.shape

    kf = KFold(n_splits=foldcount, shuffle=True)
    errorsDF = pd.DataFrame(np.zeros((len(hyper_params) * foldcount, 5)))
    errorsDF.columns = ['eta', 'lamb1', 'lamb2', 'error', 'fold']

    #Okay not to shuffle because kf shuffles for you
    known_indices = np.argwhere(~np.isnan(X)).astype(np.int32)
    np.random.shuffle(known_indices)

    if returnVectorDF:
        trainErrorDF = pd.DataFrame()
        testErrorDF  = pd.DataFrame()

    fold = 0
    df_index = 0

    p = Pool(numProcesses)

    for train_index, test_index in kf.split(known_indices):
        print('Training fold {}'.format(fold))
        if returnVectorDF:
            foldTrainDF = pd.DataFrame()
            foldTestDF  = pd.DataFrame()

        train_indices = known_indices[train_index].astype(np.int32)
        test_indices  = known_indices[test_index].astype(np.int32)

        if (returnVectorDF):
            errorVec = p.map(partial(TrainVerboseOnParams, X=X, k=k, neighbors=neighbors,
                                     train_indices=train_indices, test_indices=test_indices), hyper_params)
            for i in range(len(hyper_params)):
                eta, lamb1, lamb2 = hyper_params[i]
                paramError, trainError, testError = errorVec[i]
                foldTrainDF = pd.concat([foldTrainDF,
                                      pd.DataFrame({
                                          'eta{:.5f}_lamb1{:.5f}_lamb2{:.5f}'.format(eta, lamb1, lamb2): trainError
                                      })
                                    ], axis=1)
                foldTestDF  = pd.concat([foldTestDF,
                                          pd.DataFrame({
                                          'eta{:.5f}_lamb1{:.5f}_lamb2{:.5f}'.format(eta, lamb1, lamb2): testError
                                      })
                                    ], axis=1)
                errorsDF.iloc[df_index] = np.array([eta, lamb1, lamb2, paramError, fold])
                df_index += 1
        else:
            errorVec = p.map(partial(TrainOnParams, X=X, k=k, neighbors=neighbors,
                                     train_indices=train_indices, test_indices=test_indices), hyper_params)
            for i in range(len(hyper_params)):
                eta, lamb1, lamb2 = hyper_params[i]
                paramError = errorVec[i]
                errorsDF.iloc[df_index] = np.array([eta, lamb1, lamb2, paramError, fold])
                df_index += 1

        if returnVectorDF:
            foldTrainDF['fold'] = fold
            foldTestDF['fold']  = fold
            maxEpochs, _ = foldTrainDF.shape
            foldTrainDF['epochs']  = np.arange(maxEpochs).astype(np.float32)
            foldTestDF['epochs']   = np.arange(maxEpochs).astype(np.float32)

            trainErrorDF = pd.concat([trainErrorDF, foldTrainDF])
            testErrorDF  = pd.concat([testErrorDF, foldTestDF])

        fold = fold + 1

    p.close()
    p.join()

    if returnVectorDF:
        return errorsDF, trainErrorDF, testErrorDF
    else:
        return errorsDF

def PlotErrorDF(errorDF, id_vars=['epochs', 'fold'], ax=None):
    data = pd.melt(errorDF, id_vars=id_vars, value_name='error', var_name='run')
    if ax is not None:
        ax = sns.lineplot(x='epochs', y ='error', hue='run', data=data, ax=ax, legend=False)
    else:
        ax = sns.lineplot(x='epochs', y ='error', hue='run', data=data, legend='brief')
        plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    return ax

def PlotParamDF(paramDF, id_vars=['error', 'fold'], ax=None):
    data = pd.melt(paramDF, id_vars=id_vars, value_name='param_value', var_name='param_type')
    if ax is not None:
        ax = sns.lineplot(x='param_value', y='error', hue='param_type', data=data, ax=ax)
    else:
        ax = sns.lineplot(x='param_value', y='error', hue='param_type', data=data)
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    return ax


'''
print out cmds for training and inference
'''

import argparse
import os
from DPR.dpr.utils.tasks import task_map, train_cluster_map, test_cluster_map
import random
import textwrap
from tqdm import tqdm

def wrap(cmd):
    '''
    wrap cmd
    '''
    bs = ' \\\n\t '
    return bs.join(textwrap.wrap(cmd,break_long_words=False,break_on_hyphens=False))

def get_cmds(args):

    # ================================== Train Stage ===================================
    # 1. random sample prompts and score data
    prompt_pool_dir = os.path.join(args.output_dir, 'prompt_pool')
    random_sample_dir = os.path.join(args.output_dir, 'find_random')
    scored_dir = os.path.join(args.output_dir, 'scored')

    exp_name = f'train_{args.train_clusters}_test_{args.test_clusters}'
    exp_path = os.path.join(args.output_dir, 'experiment', exp_name)
    os.makedirs(exp_path, exist_ok=True)

    random_port = random.randint(21966,25000)

    if args.train_clusters is None:
        clusters = list(train_cluster_map.keys())
    else:
        clusters = args.train_clusters.split('+')
    train_cmd_list=[]
    for cluster in tqdm(clusters):
        for task in train_cluster_map[cluster]:
            echo_cmd = f'echo "scoring {task} task of {cluster} cluster..."'
            task_cls = task_map.cls_dic[task]()
            prompt_pool_path = os.path.join(prompt_pool_dir, cluster, task+'_prompts.json')
            random_sample_path = os.path.join(random_sample_dir, cluster, task+'_random_samples.json')
            find_random_cmd=\
                f'python find_random.py output_path=$PWD/{random_sample_path} \
                task_name={task} +ds_size={args.ds_size} L={task_cls.finder_L} \
                prompt_pool_path=$PWD/{prompt_pool_path} cache_dir=$PWD/{args.cache_dir}\
                hydra.run.dir=$PWD/{exp_path}'

            scored_train_path = os.path.join(scored_dir, cluster, task+'_scored_train.json')
            scored_valid_path = os.path.join(scored_dir, cluster, task+'_scored_valid.json')
            run_scorer_cmd = \
                f'accelerate launch --multi_gpu --num_processes {args.gpus} --main_process_port {random_port} \
                scorer.py example_file=$PWD/{random_sample_path} \
                output_train_file=$PWD/{scored_train_path} \
                output_valid_file=$PWD/{scored_valid_path} \
                batch_size={task_cls.run_scorer_bsz} task_name={task}  \
                model_name={args.scr_model} \
                prompt_pool_path=$PWD/{prompt_pool_path} cache_dir=$PWD/{args.cache_dir} \
                hydra.run.dir=$PWD/{exp_path}'

            train_cmd_list += [echo_cmd, find_random_cmd, run_scorer_cmd]

    # 2. train a retriever:
    echo_cmd = f'echo "start training the retriever..."'
    train_retriever_cmd = \
        f'python DPR/train_dense_encoder.py train_datasets=[uprise_dataset] dev_datasets=[uprise_valid_dataset] \
        train=biencoder_uprise output_dir=$PWD/{exp_path} \
        datasets.train_clusters={args.train_clusters} \
        datasets.train_file=$PWD/{scored_dir} \
        datasets.valid_file=$PWD/{scored_dir} \
        datasets.hard_neg=true datasets.multi_task={args.multi_task} \
        datasets.top_k={args.retriever_top_k} train.hard_negatives={args.retriever_top_k} \
        train.batch_size={args.retriever_bsz} \
        train.num_train_epochs={args.retriever_epoch} \
        datasets.prompt_pool_path=$PWD/{prompt_pool_dir} \
        datasets.prompt_setup_type={args.retriever_prompt_setup} \
        datasets.task_setup_type=q encoder.cache_dir=$PWD/{args.cache_dir}\
        hydra.run.dir=$PWD/{exp_path}'

    train_cmd_list += [echo_cmd, train_retriever_cmd]

    # write train cmds in train.sh
    train_cmd_list = [wrap(cmd) for cmd in train_cmd_list]

    # write run.sh
    with open(f"{exp_path}/train.sh","w") as f:
        f.write("\n\n".join(train_cmd_list))
    print('saved training cmds to: ', f"{exp_path}/train.sh")

    # ================================== Inference Stage ===================================
    inference_cmd_list = []
    # 1. encode the whole prompt pool, using prompt encoder of the trained retriever
    echo_cmd = f'echo "encoding the whole prompt pool..."'
    gen_emb_cmd = \
        f"python DPR/generate_dense_embeddings.py model_file=$PWD/{exp_path}/dpr_biencoder.best_valid \
        ctx_src=dpr_uprise shard_id=0 num_shards=1 \
        out_file=$PWD/{exp_path}/dpr_enc_index \
        ctx_sources.dpr_uprise.train_clusters={args.train_clusters} \
        ctx_sources.dpr_uprise.prompt_pool_path=$PWD/{prompt_pool_dir} \
        ctx_sources.dpr_uprise.prompt_setup_type={args.retriever_prompt_setup} \
        encoder.cache_dir=$PWD/{args.cache_dir} \
        hydra.run.dir=$PWD/{exp_path}"

    inference_cmd_list += [echo_cmd, gen_emb_cmd]

    def get_inference_cmd(num_prompts=3, retriever='uprise'):

        assert retriever in [None, 'Random', 'Bm25', 'Sbert', 'Uprise']
        random = True if retriever == "random" else False

        echo_cmd = f'echo "running inference on {task} task of {cluster} cluster with {retriever} retriever..."'
        pred_outpath = os.path.join(exp_path, f'preds_for_{cluster}', f'{task}_prompts{args.num_prompts}_retriever{retriever}_preds.json')

        run_inference_cmd = \
            f"accelerate launch --num_processes 1  --main_process_port {random_port} \
            inference.py prompt_file=$PWD/{retrieve_prompts_outpath} \
            task_name={task} \
            output_file=$PWD/{pred_outpath} \
            res_file=$PWD/{eval_res_outpath} \
            batch_size={args.inference_bsz} \
            train_clusters={args.train_clusters} \
            model_name={args.inf_model} \
            prompt_pool_path=$PWD/{prompt_pool_dir} \
            num_prompts={num_prompts} \
            random_sample={random} random_seed=42 \
            cache_dir=$PWD/{args.cache_dir} \
            hydra.run.dir=$PWD/{exp_path}"

        return [echo_cmd, run_inference_cmd]

    # 2. retrieve positive prompts from the prompt pool, for each task in the testing clusters:
    test_clusters = args.test_clusters.split('+')
    for cluster in test_clusters:
        eval_res_outpath = os.path.join(exp_path, f'eval_res_for_{cluster}.txt')
        for task in test_cluster_map[cluster]:
            echo_cmd = f'echo "uprise retrieves on {task} task of {cluster} cluster..."'
            retrieve_prompts_outpath = os.path.join(exp_path, f'uprise_prompts_for_{cluster}', f'{task}_prompts.json')
            retrieve_prompts_cmd = \
                f'python DPR/dense_retriever.py model_file=$PWD/{exp_path}/dpr_biencoder.best_valid \
                qa_dataset=qa_uprise ctx_datatsets=[dpr_uprise] \
                encoded_ctx_files=["$PWD/{exp_path}/dpr_enc_index_*"]\
                out_file=$PWD/{retrieve_prompts_outpath} \
                datasets.qa_uprise.task_name={task} \
                datasets.qa_uprise.task_setup_type=q  \
                datasets.qa_uprise.cache_dir=$PWD/{args.cache_dir} \
                n_docs={args.num_prompts} \
                ctx_sources.dpr_uprise.prompt_pool_path=$PWD/{prompt_pool_dir} \
                ctx_sources.dpr_uprise.train_clusters={args.train_clusters} \
                ctx_sources.dpr_uprise.prompt_setup_type={args.retriever_prompt_setup} \
                encoder.cache_dir=$PWD/{args.cache_dir} \
                hydra.run.dir={exp_path}'
            inference_cmd_list += [echo_cmd, retrieve_prompts_cmd]

            # vanilla zero shot
            inference_cmd_list += get_inference_cmd(num_prompts=0, retriever=None)

            # uprise zero shot
            inference_cmd_list += get_inference_cmd(num_prompts=args.num_prompts, retriever='Uprise')

            # Ablations: replace uprise retriever with random, bm25 and sbert
            if args.retrieve_random:
                inference_cmd_list += get_inference_cmd(num_prompts=args.num_prompts, retriever='Random')
            if args.retrieve_bm25:
                echo_cmd = f'echo "bm25 retrieves on {task} task of {cluster} cluster..."'
                retrieve_prompts_outpath = os.path.join(exp_path, f'bm25_prompts_for_{cluster}', f'{task}_prompts.json')
                retrieve_bm25_prompts_cmd = \
                    f'python retrieve_bm25.py \
                    train_clusters={args.train_clusters} \
                    task_name={task} cache_dir=$PWD/{args.cache_dir} \
                    prompt_pool_path=$PWD/{prompt_pool_dir} \
                    out_file=$PWD/{retrieve_prompts_outpath} \
                    prompt_setup_type={args.retriever_prompt_setup} n_docs={args.num_prompts} \
                    hydra.run.dir=$PWD/{exp_path} '
                inference_cmd_list += [echo_cmd, retrieve_bm25_prompts_cmd]
                inference_cmd_list += get_inference_cmd(num_prompts=args.num_prompts, retriever='Bm25')
            if args.retrieve_sbert:
                echo_cmd = f'echo "sbert retrieves on {task} task of {cluster} cluster..."'
                retrieve_prompts_outpath = os.path.join(exp_path, f'sbert_prompts_for_{cluster}', f'{task}_prompts.json')
                retrieve_sbert_prompts_cmd = \
                    f'python retrieve_sbert.py \
                    train_clusters={args.train_clusters} \
                    task_name={task} cache_dir=$PWD/{args.cache_dir} \
                    prompt_pool_path=$PWD/{prompt_pool_dir} \
                    out_file=$PWD/{retrieve_prompts_outpath} \
                    prompt_setup_type={args.retriever_prompt_setup} n_docs={args.num_prompts} \
                    hydra.run.dir=$PWD/{exp_path} '
                inference_cmd_list += [echo_cmd, retrieve_sbert_prompts_cmd]
                inference_cmd_list += get_inference_cmd(num_prompts=args.num_prompts, retriever='Sbert')

    inference_cmd_list = [wrap(cmd) for cmd in inference_cmd_list]

    # write run.sh
    with open(f"{exp_path}/inference.sh","w") as f:
        f.write("\n\n".join(inference_cmd_list))
    print('saved inference cmds to: ', f"{exp_path}/inference.sh")

    return

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--output_dir',
                        type=str, help='Directory for saving all the intermediate and final outputs.',
                        default='my_data')
    parser.add_argument('--cache_dir',
                        type=str, help='Directory for caching the huggingface models and datasets.',
                        default='../cache')
    parser.add_argument('--gpus',
                        type=int, help='number of gpus to use',
                        default=8)

    # training
    parser.add_argument('--train_clusters',
                        type=str,
                        help='a string concatenating task clusters for training, \
                            e.g., `nli+common_reason` means nli and common_reason task clusters \
                            all supoorted clusters are in DPR.dpr.utils.tasks.train_cluster_map \
                            clusters=`all supported clsuters` when the passed value is None',
                        default=None)
    parser.add_argument('--retriever_prompt_setup',
                        type=str,
                        help='setup type of prompt, recommend setting as `qa` for cross-task training \
                            and `q` for task-specific training',
                        default="qa")
    parser.add_argument('--ds_size',
                        type=int,
                        help='number of maximum data examples sampled from each training dataset',
                        default=10000)
    parser.add_argument('--scr_model',
                        type=str,
                        help='Huggingface model for scoring data',
                        default="EleutherAI/gpt-neo-2.7B")
    parser.add_argument("--multi_task",
                        action="store_true",
                        help="True for multi-task and False for task-specific, \
                            the difference reflects on the sampling of negative prompts ONLY \
                            refer to `UpriseDataset` in `DPR/dpr/data/biencoder_data.py` for details")
    parser.add_argument('--retriever_top_k',
                        type=int,
                        help='number of k (hard) negatives for training the retriever',
                        default=20)
    parser.add_argument('--retriever_bsz',
                        type=int,
                        help='sum of batch size of all gpus, NOT per gpu',
                        default=16)
    parser.add_argument('--retriever_epoch',
                        type=int,
                        help='maximum training epoch, recommend setting as `3` when cross-task training, \
                             and `10` when task-specific training',
                        default=3)

    # inference
    parser.add_argument('--inf_model',
                        type=str,
                        help='Huggingface model for inference',
                        default="EleutherAI/gpt-neo-2.7B")
    parser.add_argument('--test_clusters',
                        type=str,
                        help='a string concatenating task clusters for training, \
                            e.g., `nli+common_reason` means nli and common_reason task clusters \
                            all supoorted clusters are in DPR.dpr.utils.tasks.test_cluster_map',
                        default="nli+common_reason")
    parser.add_argument('--num_prompts',
                        type=int,
                        help='maximum number of retrieved prompts to be concatenated before the task input',
                        default=3)
    parser.add_argument('--retrieve_random',
                        action="store_true",
                        help='whether to random retrieve from our prompt pool, and run a baseline')
    parser.add_argument('--retrieve_bm25',
                        action="store_true",
                        help='whether to use bm25 retriever to retrieve from our prompt pool, and run a baseline')
    parser.add_argument('--retrieve_sbert',
                        action="store_true",
                        help='whether to use sbert to retrieve from our prompt pool, and run a baseline')
    parser.add_argument('--inference_bsz',
                        type=int,
                        help='sum of batch size of all gpus, NOT per gpu',
                        default=1)

    args = parser.parse_args()

    get_cmds(args)


import os
import PIL
import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.axes_grid1 import ImageGrid

imgs = []
directory = "output/grids"
# for filename in os.listdir(directory):
#     print(filename)
#     image = PIL.Image.open(os.path.join(directory, filename))
#     imgs.append(np.array(image))

for row in range(9):
    for col in range(9):
        image = PIL.Image.open(f"output/grids/dim_{row}_{col}.jpg")
        imgs.append(np.array(image))

fig = plt.figure(figsize=(9,9))
grid = ImageGrid(fig, 111,
        nrows_ncols=(9,9),
        axes_pad=0.1,
        )

for ax, im in zip(grid, imgs):
    ax.imshow(im)

plt.savefig("grid.jpg")


import logging
import os
import signal

import watchdog.events
import watchdog.observers.polling
import watchdog_gevent

# https://github.com/Bogdanp/dramatiq/blob/master/dramatiq/__main__.py
def setup_file_watcher(path, callback, use_polling=False):
    """Sets up a background thread that watches for source changes and
    automatically sends SIGHUP to the current process whenever a file
    changes.
    """
    if use_polling:
        observer_class = watchdog.observers.polling.PollingObserver
    else:
        observer_class = watchdog_gevent.Observer

    file_event_handler = watchdog.events.PatternMatchingEventHandler(patterns=['*.py'])
    # file_event_handler = watchdog.events.FileSystemEventHandler()
    # monkey patching is perfectly fine.
    file_event_handler.on_any_event = callback

    # start the watcher
    file_watcher = observer_class()
    file_watcher.schedule(file_event_handler, path, recursive=True)
    print("000000000000000000000000")
    file_watcher.start()
    print("&&&&&&&&&&&&&&&&&&&&&&&&")
    return file_watcher

if __name__ == "__main__":
    """runs this in debugger to see file changes."""
    import gevent

    def cb(event):
        print(event)

    file_watcher = setup_file_watcher('.', cb)
    while True:
        gevent.sleep(1)

    file_watcher.stop()
    file_watcher.join()