#!/usr/bin/python3
import numpy as np
import gym
from gym import spaces
from gym.utils import seeding
from gym.envs.classic_control import rendering
import math
from pdb import set_trace
import reeds_shepp

try:
    # When running as a package
    from gym.envs.classic_control.demo import DemoTrack
    from gym.envs.classic_control.path_planner import TruckPlanner
    from gym.envs.classic_control.path_planner import ExcavatorPlanner
    from gym.envs.classic_control.collision_check import Rectangle
    from gym.envs.classic_control.collision_check import CollisionCheck
    from gym.envs.classic_control.collision_check import Contains
except BaseException:
    # When running current file as a file
    from demo import DemoTrack
    from path_planner import TruckPlanner
    from path_planner import ExcavatorPlanner
    from collision_check import Rectangle
    from collision_check import CollisionCheck
    from collision_check import Contains

class MiningEnv(gym.Env):
    """
    Description:
        A truck and an excavator is tasked to move a pile of minerals out of
        the load site. The truck and the excavator both starts with a random
        position and orientation in a designated region. The goal is to
        coordinate both the truck and the excavator to reach a position where
        the hand of the excavator is right above the container of the truck.

    Observations:
        Type: Box(14)
        Num        Observation                              Min            Max
        0          Truck x-position for truck               50             650
        1          Truck y-position for truck               250            700
        2          Truck orientation for truck(deg)         0              360
        3          Excav. x-position for truck              300            600
        4          Excav. y-position for truck              100            200
        5          Excav. orientation for truck(deg)        -90            90
        6          Excav. hand orientation for truck(deg)   -90            90
        7          Truck x-position for excav.              50             650
        8          Truck y-position for excav.              250            700
        9          Truck orientation for excav.(deg)        0              360
        10         Excav. x-position for excav.             300            600
        11         Excav. y-position for excav.             100            200
        12         Excav. orientation for excav.(deg)       -90            90
        13         Excav. hand orientation for excav.(deg)  -90            90

    Actions:
        Type: Box(7)
        Num        Action                              Min            Max
        0          Truck loading x-position            250            650
        1          Truck loading y-position            300            480
        2          Truck loading orientation(deg)      135            225
        3          Excav. loading x-position           300            600
        4          Excav. loading y-position           100            200
        5          Excav. loading orientation(deg)     -90            90
        6          Excav. hand orientation(deg)        -90            90

    Reward:
        Reward is -reeds_shepp_curve_length for every step taken, including
        the termination step, and 7000 for successfully loading on the truck.

    Starting State:
        All observations are assigned a uniform random value corresponding to
        the random initial state.

    Episode Termination:
        The hand of the excavator is above the truck bin. Then the system
        executes dumping and signals the truck to exit the loading site.

        or

        Episode length is greater than 20

    Solved Requirements(beta):
        Considered solved when the average reward is greater than or equal to
        0 over 100 consecutive trials.

    Confusions:
        1. Does the excavator and the truck operate in completely different
           regions of the loading site, thus diminishes the possibility of them
           colliding.
        2. How should I construct the path planner.

    Places that may cause confusions:
        1. The truck in gym environment is pointing down when no rotation
           (0 deg) is added, however the excavator is pointing up when (0 deg)
           no rotation is added. The code for Reeds-Shepp path generation uses
           a different set of angle definitions, in their definition 0 deg is
           pointing right.
    """

    metadata = {
        'render.modes': ['human', 'rgb_array'],
        'video.frames_per_second': 30
    }

    def __init__(self):

        # [truck_x, truck_y, truck_sin, truck_cos]
        # truck_low = np.array([250, 300, math.sqrt(3)/2, -1/2])
        # truck_high = np.array([650, 480, 1, 1/2])
        # truck_range = truck_high - truck_low

        # [e_x, e_y, ep_sin, ep_cos, ea_sin, ea_cos]
        # excvtr_low = np.array([300, 100, -1/2, math.sqrt(3)/2, math.sqrt(2)/2, -math.sqrt(2)/2])
        # excvtr_high = np.array([600, 200, 1/2, 1, 1, math.sqrt(2)/2])
        # excvtr_range = excvtr_high - excvtr_low

        # [truck_x, truck_y, truck_sin, truck_cos, e_x, e_y, ep_sin, ep_cos, ea_sin, ea_cos]
        # e = excavator, ep = excavator platform, ea = excavator arm
        self.observation_low = np.array([50, 250, -1, -1, 300, 100, -1, -1, -1, -1])
        self.observation_high = np.array([650, 700, 1, 1, 600, 200, 1, 1, 1, 1])
        self.observation_range = self.observation_high - self.observation_low
        self.observation_space = [spaces.Box(np.zeros(10), np.ones(10), dtype=np.float32),
                                  spaces.Box(np.zeros(10), np.ones(10), dtype=np.float32)]

        # [truck_x, truck_y]
        self.truck_low = np.array([250, 300])
        self.truck_high = np.array([650, 320])
        self.truck_range = self.truck_high - self.truck_low
        # [e_x, e_y]
        self.excvtr_low = np.array([300, 180])
        self.excvtr_high = np.array([350, 200])
        self.excvtr_range = self.excvtr_high - self.excvtr_low
        self.action_space = [spaces.Box(np.zeros(2), np.ones(2), dtype=np.float32),
                             spaces.Box(np.zeros(2), np.ones(2), dtype=np.float32)]

        self.seed()
        self.viewer = None
        self.state = None
        self.if_render = 0

        # Demo track
        self.track = DemoTrack()
        self.n = 2
        self.truck_planner = TruckPlanner()
        self.excvtr_planner = ExcavatorPlanner()
        self.success_counter = 0

    def seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

    def step(self, action):
        """
        Input: list(np.array([1x2]), np.array([1x2]))
        Output: None
        Use: Perform a step in the simulation environment.
        """
        # scaled target state
        scl_truck_trgt, scl_excvtr_trgt = action

        # unscaled target state
        truck_trgt = self.truck_low + np.multiply(scl_truck_trgt, self.truck_range)
        excvtr_trgt = self.excvtr_low + np.multiply(scl_excvtr_trgt, self.excvtr_range)

        # radian state
        rd_state = self._get_radian_state()
        truck_state = rd_state[:3].tolist()
        truck_trgt = truck_trgt.tolist()
        excvtr_state = rd_state[3:-1].tolist()
        excvtr_trgt = excvtr_trgt.tolist()

        # get reeds-shepp path
        self.truck_path = self.truck_planner.get_path(truck_state, truck_trgt)
        self.excvtr_path = self.excvtr_planner.get_path(excvtr_state, excvtr_trgt)
        # set_trace()

        self.step_counter = 0
        # self.counter += 1
        info = [None, None]

        # execute first 60 points on the path if collides terminate the episode
        for _ in range(60):
            if self.if_render:
                self.render()
            obsv, flag = self._update_state()
            if flag:
                # set_trace()
                # print('collision occurred')
                return obsv, [-1000, -1000], [True, True], info

        # get current scaled state
        scaled_state = self._get_scaled_state()
        observation = [scaled_state, scaled_state]

        # get current radian state
        rd_state = self._get_radian_state()
        truck_state = rd_state[:3].tolist()

        excvtr_pos = rd_state[3:5]
        arm_ori = rd_state[-1]
        arm_pos = excvtr_pos + np.array([130*math.cos(arm_ori), 130*math.sin(arm_ori)])
        arm_state = np.concatenate((arm_pos, np.pi/2), axis=None)

        # calculate reward
        reward = -reeds_shepp.path_length(tuple(truck_state), tuple(
            arm_state), self.truck_planner.rho)

        # check for termination
        done = False
        if self._can_load_on_truck():
            done = True
            reward = 8000
            # self.success_counter += 1
            # print('loads on truck, count: {}'.format(self.success_counter))

        return observation, [reward, reward], [done, done], info

    def reset(self):
        # Define unscaled initial state region
        # Initial state for truck is between [50, 650 -100deg] ~ [100, 700, -80deg]
        # For the excavator it is [300, 100, 80deg, 170deg] ~ [350, 120, 100deg, 190deg]
        low = np.array([50, 650, -1, math.cos(math.pi*(-100)/180),
                        300, 100, math.sin(math.pi*(80)/180),
                        math.cos(math.pi*(100)/180),
                        math.sin(math.pi*(190)/180), -1])
        high = np.array([100, 700, math.sin(math.pi*(-80)/180),
                         math.cos(math.pi * (-80)/180), 350, 120, 1,
                         math.cos(math.pi*(80)/180),
                         math.sin(math.pi*(170)/180),
                         math.cos(math.pi*(190)/180)])

        # Sample initial state
        self.state = self.np_random.uniform(low=low, high=high)
        # Scale sampled state
        scaled_state = self._get_scaled_state()
        obsv = [scaled_state, scaled_state]

        # records step number for termination check
        # self.counter = 0
        self.i = 0

        return obsv

    def render(self, mode='human', demo=0, render=1):
        screen_width = 600
        screen_height = 800

        truck_container_width = 60
        truck_container_length = 100
        truck_head_width = 50
        truck_head_length = 20

        excvtr_platform_width = 80
        excvtr_platform_length = 100
        excvtr_track_width = 15
        excvtr_track_length = 80
        excvtr_bin_size = 40  # Square
        excvtr_arm_width = 12
        excvtr_arm_length = 100
        excvtr_hand_width = 40
        excvtr_hand_length = 20

        if self.viewer is None:
            self.viewer = rendering.Viewer(screen_width, screen_height)

            minerals = rendering.make_circle(10)
            minerals.set_color(1, 171/255, 64/255)
            minerals.add_attr(rendering.Transform(translation=(150, 100)))
            self.viewer.add_geom(minerals)

            self.trucktrans = rendering.Transform()
            self.excvtrplattrans = rendering.Transform()
            self.excvtrbintrans = rendering.Transform()

            cntnr = self._render_polygon(truck_container_length, truck_container_width)
            cntnr.set_color(102/255, 102/255, 102/255)
            cntnr.add_attr(rendering.Transform(translation=(0, -truck_container_width/2)))
            cntnr.add_attr(self.trucktrans)
            self.viewer.add_geom(cntnr)

            hd = self._render_polygon(truck_head_length, truck_head_width)
            hd.set_color(191/255, 144/255, 0)
            hd.add_attr(rendering.Transform(translation=(truck_head_length /
                                                         2 + truck_container_length/2, -truck_head_width/2)))
            hd.add_attr(self.trucktrans)
            self.viewer.add_geom(hd)

            excvtr_plat = self._render_polygon(excvtr_platform_length, excvtr_platform_width)
            excvtr_plat.set_color(241/255, 194/255, 50/255)
            excvtr_plat.add_attr(rendering.Transform(
                translation=(excvtr_bin_size/2, -excvtr_platform_width/2)))
            excvtr_plat.add_attr(self.excvtrplattrans)
            self.viewer.add_geom(excvtr_plat)

            excvtr_track_l = self._render_polygon(excvtr_track_length, excvtr_track_width)
            excvtr_track_l.set_color(67/255, 67/255, 67/255)
            excvtr_track_l.add_attr(rendering.Transform(translation=(
                excvtr_bin_size/2,
                excvtr_platform_width/2)))
            excvtr_track_l.add_attr(self.excvtrplattrans)
            self.viewer.add_geom(excvtr_track_l)

            excvtr_track_r = self._render_polygon(excvtr_track_length, excvtr_track_width)
            excvtr_track_r.set_color(67/255, 67/255, 67/255)
            excvtr_track_r.add_attr(rendering.Transform(translation=(
                excvtr_bin_size/2,
                -excvtr_platform_width/2-excvtr_track_width)))
            excvtr_track_r.add_attr(self.excvtrplattrans)
            self.viewer.add_geom(excvtr_track_r)

            excvtr_bin = self._render_polygon(excvtr_bin_size, excvtr_bin_size)
            excvtr_bin.set_color(164/255, 194/255, 244/255)
            excvtr_bin.add_attr(rendering.Transform(translation=(0, -excvtr_bin_size/2)))
            excvtr_bin.add_attr(self.excvtrbintrans)
            self.viewer.add_geom(excvtr_bin)

            excvtr_arm = self._render_polygon(excvtr_arm_length, excvtr_arm_width)
            excvtr_arm.set_color(204/255, 204/255, 204/255)
            excvtr_arm.add_attr(rendering.Transform(
                translation=(excvtr_arm_length/2+excvtr_bin_size/2, -excvtr_arm_width/2)))
            excvtr_arm.add_attr(self.excvtrbintrans)
            self.viewer.add_geom(excvtr_arm)

            excvtr_hand = self._render_polygon(excvtr_hand_length, excvtr_hand_width)
            excvtr_hand.set_color(90/255, 90/255, 90/255)
            excvtr_hand.add_attr(rendering.Transform(
                translation=(excvtr_arm_length+excvtr_bin_size/2+excvtr_hand_length/2, -excvtr_hand_width/2)))
            excvtr_hand.add_attr(self.excvtrbintrans)
            self.viewer.add_geom(excvtr_hand)

        if render:
            # Render planned path
            truck_xys, truck_trgt = self._get_trajectory_list(self.truck_path)
            excvtr_xys, excvtr_trgt = self._get_trajectory_list(self.excvtr_path)

            self.truck_track = rendering.make_polyline(truck_xys)
            self.truck_track.set_color(0, 0, 1)
            self.truck_track.set_linewidth(4)
            self.viewer.add_geom(self.truck_track)

            truck_destination = rendering.make_circle(10)
            truck_destination.set_color(0, 0, 1)
            truck_destination.add_attr(rendering.Transform(translation=(
                truck_trgt[0], truck_trgt[1])))
            self.viewer.add_geom(truck_destination)

            self.excvtr_track = rendering.make_polyline(excvtr_xys)
            self.excvtr_track.set_color(1, 0, 0)
            self.excvtr_track.set_linewidth(4)
            self.viewer.add_geom(self.excvtr_track)

            excvtr_destination = rendering.make_circle(10)
            excvtr_destination.set_color(1, 0, 0)
            excvtr_destination.add_attr(rendering.Transform(translation=(
                excvtr_trgt[0], excvtr_trgt[1])))
            self.viewer.add_geom(excvtr_destination)

        rd_state = self._get_radian_state()

        self.trucktrans.set_translation(rd_state[0], rd_state[1])
        self.trucktrans.set_rotation(rd_state[2])  # in radian
        self.excvtrplattrans.set_translation(rd_state[3], rd_state[4])
        self.excvtrplattrans.set_rotation(rd_state[5])
        self.excvtrbintrans.set_translation(rd_state[3], rd_state[4])
        self.excvtrbintrans.set_rotation(rd_state[6])

        return self.viewer.render(return_rgb_array=mode == 'rgb_array')

    def close(self):
        if self.viewer:
            self.viewer.close()
            self.viewer = None

    def _render_polygon(self, width, length):
        l, r, t, b = -width/2, width/2, length, 0
        return rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)])

    def _get_trajectory_list(self, trajectory):
        x = []
        y = []

        for point in trajectory:
            x.append(point[0])
            y.append(point[1])

        xys = list(zip(x, y))
        destination = [x[-1], y[-1]]
        return xys, destination

    def _update_state(self):
        old_state = self._get_radian_state()
        try:
            rd_state = np.asarray(self.truck_path[self.step_counter])
        except BaseException:
            rd_state = np.asarray(self.truck_path[-1])

        try:
            rd_state = np.concatenate((rd_state, np.asarray(
                self.excvtr_path[self.step_counter])), axis=None)
        except BaseException:
            rd_state = np.concatenate((rd_state, np.asarray(
                self.excvtr_path[-1])), axis=None)

        if old_state[6] > np.pi/2:
            rd_state = np.concatenate((rd_state, old_state[6]-np.pi/200), axis=None)
        else:
            rd_state = np.concatenate((rd_state, np.pi/2), axis=None)

        self.state = self._get_sin_cos_state(rd_state)
        scaled_state = self._get_scaled_state()
        obsv = [scaled_state, scaled_state]
        flag = self._check_collision()

        self.step_counter += 1

        return obsv, flag

    def _can_load_on_truck(self):
        """
        Input: None
        Output: bool(1x1)
        Use: Gives a true-false response for whether the current state of the
             truck and the excavator is suitable for loading.
        """

        rd_state = self._get_radian_state()
        truck_pos = rd_state[:2]
        truck_ori = rd_state[2]
        rect = Rectangle(truck_pos, 60, 40, truck_ori)
        excvtr_pos = rd_state[3:5]
        arm_ori = rd_state[-1]
        arm_pos = excvtr_pos + np.array([130*math.cos(arm_ori), 130*math.sin(arm_ori)])
        contains = Contains(arm_pos[0], arm_pos[1], rect)
        return contains.contains()

    def _get_radian_state(self):
        """
        Input: None
        Output: np.array([1x7])
        Use: Returns the state of the environment where the orientation of the
             truck and the excavator are given in radians.
        """

        truck_pos = [self.state[0], self.state[1]]
        truck_ang_rad = math.atan2(self.state[2], self.state[3])
        excvtr_pos = [self.state[4], self.state[5]]
        excvtr_plat_ang_rad = math.atan2(self.state[6], self.state[7])
        excvtr_arm_ang_rad = math.atan2(self.state[8], self.state[9])

        radian_state = truck_pos + [truck_ang_rad] + \
            excvtr_pos + [excvtr_plat_ang_rad, excvtr_arm_ang_rad]
        return np.array(radian_state)

    def _get_scaled_state(self):
        """
        Input: None
        Output: np.array([1x10])
        Use: Returns the state of the environment after scaling them all
             between 0 and 1.
        """
        scaled_state = (self.state - self.observation_low)/self.observation_range
        return scaled_state

    def _get_sin_cos_state(self, rd_state):
        """
        Input: np.array([1x7])
        Output: np.array([1x10])
        Use: Returns the state of the environment after changing the radian
             states into a sin-cos representation.
        """
        truck_pos = rd_state[:2]
        truck_sin_cos = np.array([np.sin(rd_state[2]), np.cos(rd_state[2])])
        excvtr_pos = rd_state[3:5]
        excvtr_plat_sin_cos = np.array([np.sin(rd_state[5]), np.cos(rd_state[5])])
        excvtr_arm_sin_cos = np.array([np.sin(rd_state[6]), np.cos(rd_state[6])])
        sin_cos_state = np.concatenate(
            (truck_pos, truck_sin_cos, excvtr_pos, excvtr_plat_sin_cos, excvtr_arm_sin_cos), axis=None)
        return sin_cos_state

    def _check_collision(self):
        rd_state = self._get_radian_state()
        rect1 = Rectangle(rd_state[:2], 120, 60, rd_state[2], 10)
        rect2 = Rectangle(rd_state[3:5], 100, 80, rd_state[5], 20)
        ccheck = CollisionCheck(rect1, rect2)
        return ccheck.if_collide()

def main():
    # Test rendering effect
    m = MiningEnv()

    for i in range(60000):
        done = False
        rewards = 0
        m.reset()
        while not done:
            truck_action = m.action_space[0].sample()
            excvtr_action = m.action_space[1].sample()
            action = [truck_action, excvtr_action]
            # action = [np.array([0.5, 0]), np.array([0.5, 0.8])]
            observation, reward, done_n, info = m.step(action)
            rewards += reward[0]
            done = all(done_n)
        # print('Episode {}, Reward {}'.format(i, rewards))
    m.close()

if __name__ == "__main__":
    main()


#!/usr/bin/env python

from setuptools import setup
from setuptools.command.test import test as TestCommand

# get version
from memsql import __version__

class PyTest(TestCommand):
    user_options = [
        ('watch', 'w',
         "watch tests for changes"),
        ('scan-directory=', 'd',
         "only search for tests in the specified directory"),
    ]
    boolean_options = ['watch']

    def initialize_options(self):
        self.watch = False
        self.scan_directory = None
        self.test_suite = None
        self.test_module = None
        self.test_loader = None

    def finalize_options(self):
        TestCommand.finalize_options(self)

        self.test_suite = True
        self.test_args = []
        if self.watch:
            self.test_args.append('-f')
        if self.scan_directory is not None:
            self.test_args.append(self.scan_directory)

    def run_tests(self):
        import os, sys, glob

        MY_PATH = os.path.dirname(__file__)
        sys.path.append(MY_PATH)
        os.environ['PYTHONPATH'] = os.environ.get('PYTHONPATH', '') + ':' + MY_PATH

        egg_dirs = glob.glob('*.egg')
        ignore_args = ['--ignore=%s' % d for d in egg_dirs]

        import pytest
        errno = pytest.main(ignore_args + self.test_args)
        raise sys.exit(errno)

setup(
    name='memsql',
    version=__version__,
    author='MemSQL',
    author_email='support@memsql.com',
    url='http://github.com/memsql/memsql-python',
    license='LICENSE.txt',
    description='Useful utilities and plugins for MemSQL integration.',
    long_description=open('README.rst').read(),
    classifiers=[
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
    ],
    packages=[
        'memsql',
        'memsql.common',
        'memsql.perf',
    ],
    zip_safe=False,
    install_requires=['ordereddict', 'MySQL-python>=1.2.4', 'wraptor', 'simplejson'],
    tests_require=['pytest', 'mock', 'pytest-xdist'],
    cmdclass={ 'test': PyTest },
)


# Deze file download en laad de De amazon_us_reviews dataset aangeboden door tesorflow in en vormd deze om het erna weg te scrijven in een ".csv" bestand met 2 colommen genaamd "review_body" en "star_rating"
# Anders dan de normale "LoadToCsv.py" file verwijderd deze alle reviews met een star rating van 5
from __future__ import absolute_import, division, print_function, unicode_literals
import os
import sys
import csv
import string
from string import digits
import numpy as np
import pandas as pd
import sklearn as sk
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow import keras
import tensorflow_hub as hub
import tensorflow_datasets as tfds
from sklearn.model_selection import train_test_split

print(f"Tensor Flow Version: {tf.__version__}")
print(f"Keras Version: {tf.keras.__version__}")
print()
print(f"Python {sys.version}")
print(f"Pandas {pd.__version__}")
print(f"Scikit-Learn {sk.__version__}")
print("GPU is", "available" if tf.test.is_gpu_available() else "NOT AVAILABLE")
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

def remove_numbers(x):
    x = re.sub('[0-9]{5,}', '#####', x)
    x = re.sub('[0-9]{4}', '####', x)
    x = re.sub('[0-9]{3}', '###', x)
    x = re.sub('[0-9]{2}', '##', x)
    return x

print("Loading Dataset: amazon_us_reviews/PC_v1_00 from Tensorflow")

Data, info = tfds.load(name="amazon_us_reviews/PC_v1_00", with_info=True, split="train")
assert isinstance(Data, tf.data.Dataset)

print("Dataset info: ", info)
csv_file = "amazon_pc_user_reviews_no_5.csv"
try:
    with open(csv_file, 'w', encoding="utf-8") as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(["review_body","star_rating"])
        for data in Data.take(100000):
            review, stars = data["data"]["review_body"], data["data"]["star_rating"]
            review = review.numpy().decode("utf-8")
            stars = stars.numpy()
            if stars == 5:
                continue
            review = review.translate(str.maketrans('', '', string.punctuation)).lower()
            review = review.translate(str.maketrans('', '', digits))
            writer.writerow([review,stars])
        print("Dataset saved with name: ", csv_file)
except IOError:
    print("I/O error")


import math
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np

class EquationsMixin:
    def leonard_jones_potential(self, distance):
        """
        eq (1) in paper, error in paper? Following cited paper.
        Possibly lennard jones.
        """
        # ignore divide by 0 and nan subtraction
        with np.errstate(divide='ignore', invalid='ignore'):
            div = (self.SIGMA / distance)
            div6 = div ** 6
            result = self.e * (div6 ** 6 - 2 * div6)
            return np.nan_to_num(result, 0)

    def bond_stretching_potential(self, distance):
        """
        eq (2) in paper, only applies if the atoms are neighbors
        is_next should be 0-1 matrix
        """
        return self.BETA * (distance - self.bond_length) ** 2

    def total_potential(self, distance, is_next):
        """
        total potential in terms of distance
        """
        lj = self.leonard_jones_potential(distance)
        bond = self.bond_stretching_potential(distance, is_next)
        return np.nan_to_num(bond, 0)

    def potential_from_indices(self, i, j, k, l):
        # j and l are the spot index, change to coordinates
        j_loc = self.index_to_location(j)
        l_loc = self.index_to_location(l)

        # distance between j and l
        distance = self.distance(j_loc, l_loc)

        # boolean matrices or different properties
        is_adjacent_atom = (np.abs(k - i) == 1)
        is_the_same_atom = i == k

        full_lj = self.leonard_jones_potential(distance)
        full_bond = self.bond_stretching_potential(distance)

        bond = (1 * is_adjacent_atom) * full_bond
        lj = 1 * np.logical_not(is_adjacent_atom | is_the_same_atom) * full_lj
        return lj + bond

    def objective_value(self, solution):
        total = 0
        ones = filter(lambda t: t[1] == 1, enumerate(solution))
        for (ij, one) in ones:
            [i, j] = self.q_to_ij(ij)
            for (kl, one) in ones:
                [k, l] = self.q_to_ij(kl)
                total += self.potential_from_indices(i, j, k, l)
        return total

class UtilsMixin:
    def distance(self, x1, x2):
        """
        Computes the euclidean distance between points in R^3
        """
        return self.CELL_LENGTH * np.sum((x1 - x2) ** 2, 0) ** 0.5

    def ij_to_q(self, i, j):
        """
        Returns the corresponding index in the hamiltonian of the binary variable
        corresponding to atom i being in spot j
        """
        # flattens matrix of binary vars to array
        return i * self.N_CELLS + j

    def q_to_ij(self, q_index):
        """
        Given a binary variable index returns i, j corresponding to atom i being in spot j
        """
        return (np.floor(q_index / self.N_CELLS), q_index % self.N_CELLS)

    def index_to_location(self, i):
        """
        Returns the euclidean coordinates for spot i
        """
        div = np.array([i / self.LATTICE_LENGTH ** d for d in range(3)])
        mod = np.floor(div) % self.LATTICE_LENGTH
        return mod

    def is_solution_valid(self, X):
        """
        Input: np solution matrix
        Returns a boolean if the solution fulfills basic
        problem constraints
        """
        every_atom_in_one_spot = (np.sum(X, 1) == 1).all()
        spot_has_zero_or_one_atoms = (np.sum(X, 0) <= 1).all()
        correct_number_of_atoms = np.sum(X) == self.N_ATOMS
        return every_atom_in_one_spot and spot_has_zero_or_one_atoms and correct_number_of_atoms

    def sample_to_x_ij_matrix(self, sample):
        """
        Turns a sample from the form of a dict {(i,j): 0|1, ...} into a np
        0-1 matrix of size N_ATOMS x N_CELLS
        """
        X = np.zeros([self.N_ATOMS, self.N_CELLS])
        ones = filter(lambda i: i[1] == 1, dict(sample).items())
        for qi, _ in ones:
            (i, j) = self.q_to_ij(qi)
            X[math.floor(i), j] = 1
        return X

    def sample_to_positions(self, sample):
        """
        Turns a sample from the form of a dict {(i,j): 0|1, ...} into a np
        matrix of size N_ATOMS x 3 which is a list of the grid positions in R^3
        """
        positions = np.zeros([self.N_ATOMS, 3])
        ones = filter(lambda i: i[1] == 1, dict(sample).items())
        for qi, _ in ones:
            (i, j) = self.q_to_ij(qi)
            positions[math.floor(i), :] = self.index_to_location(j)
        return positions

    def plot_3d(self, positions):
        fig = plt.figure()
        ax = fig.add_subplot(111, projection='3d')

        ax.scatter(positions[:, 0], positions[:, 1],
                   positions[:, 2], s=250, c='r')
        ax.plot(positions[:, 0], positions[:, 1], positions[:, 2])

        print("DISTANCES:")
        for i in range(self.N_ATOMS - 1):
            print(f"{i} to {i+1}",
                  self.distance(positions[i, :], positions[i + 1, :]))

        atoms = f'{self.N_ATOMS} atoms'
        lattice = f'{self.LATTICE_LENGTH}x{self.LATTICE_LENGTH} lattice'
        plt.title(f'{atoms} in a {lattice}')
        plt.show()


import telegram
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, BaseFilter, CallbackContext
import logging
import requests
import time

# class
class FilterStop(BaseFilter):
    def filter(self, message):
        return 'stop receiving btc' in message.text

def crawl_currency():
    current_time = str(time.time_ns())[0:12]
    req_url = requests.get('https://www.bitoex.com/api/v0/rate/' + current_time)
    response = req_url.json()
    return response['BTC']

# value
isRunning = False
userID = ''

# start up
updater = Updater('1041411939:AAFE7sIstwya3cxb9MksuOnBfE6dUwPnBHI', use_context=True)
job = updater.job_queue
print("Start Telegram Bot...")

# for detect error
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)
dispatcher = updater.dispatcher

# function here
def start(update, context):
    context.bot.send_message(chat_id=update.effective_chat.id, text="/BTC /stop")

def get_btc(update, context):
    price = crawl_currency()
    context.bot.send_message(chat_id=update.message.chat_id, text="Buy = " + price[0] + ", Sell = " + price[1])

def loopGetBTC(context: telegram.ext.CallbackContext):
    price = crawl_currency()
    context.bot.send_message(chat_id=1108395640, text="Buy = " + price[0] + ", Sell = " + price[1])

def setReceiveBTC(update, context):
    context.bot.send_message(chat_id=update.message.chat_id, text="test")
    # print(update.message.chat_id)
    # global isRunning
    # if isRunning:
    #     job.stop()
    #     isRunning = False
    # else:
    #     job.run_repeating(loopGetBTC(context), interval=20, first=0)
    #     isRunning = True

# start main
# filter

# handler
start_handler = CommandHandler('start', start)
get_btc_handler = CommandHandler('BTC', get_btc)
test_handler = CommandHandler('test', setReceiveBTC)

dispatcher.add_handler(start_handler)
dispatcher.add_handler(test_handler)
dispatcher.add_handler(get_btc_handler)

updater.start_polling()


import turtle as t
from shapely.geometry import mapping
from shapely.wkt import loads
import utilities.read_from_file_or_net as rfn
import utilities.generic_useful_parameters as gp

def calc_ratios(screen_width,screen_height,data):
    """
    dist_x = max easting - min easting
    dist_y = max northing - min northing

    # Scaling ratio each axis
    # to map points from world to screen
    x_ratio = screen width / dist_x
    y_ratio = screen height / dist_y

    :param data: List of rows representing query result set.
    :return: x and y screen ratios, bounding box for collection (result set)
    """
    bbox = [9999999, 9999999, 0, 0]

    for row in data:
        coords = loads(row['geom_str'])
        bbox[0] = min(coords.bounds[0], bbox[0])
        bbox[1] = min(coords.bounds[1], bbox[1])
        bbox[2] = max(coords.bounds[2], bbox[2])
        bbox[3] = max(coords.bounds[3], bbox[3])

    dist_x = bbox[2] - bbox[0]
    dist_y = bbox[3] - bbox[1]

    return screen_width / dist_x, screen_height / dist_y, bbox

def convert_point(screen_width, screen_height,ratios, point):
    """
    Function to convert easting/northing to screen coordinates.
    Remember screen coordinates are necessary for rendering.
    :param point: The point in easting/northing to convert
    :return: x,y pair in screen coordinates
    """

    # ratios[2] is the bounding box. Its elements 2 & 3 are max x and y
    x = screen_width - ((ratios[2][2] - point[0]) * ratios[0])
    y = screen_height - ((ratios[2][3] - point[1]) * ratios[1])

    # Python turtle graphics start in the middle of the screen so we must offset the points so they are centered
    x = x - (screen_width / 2)
    y = y - (screen_height / 2)

    return x, y

def draw_data(screen_Width, screen_height, data):
    """
    Draw map based on coordinates from query result set. 'World' coordinates must be converted to screen (pixel)
    coordinates.

    :param data: List of rows representing query result set.
    :return: None. Function draws map in window.
    """

    ratios = calc_ratios(screen_Width,screen_height,data)

    for row in data:
        t.up()

        # Convert WKT representation to geometry object.
        coords = loads(row['geom_str'])

        centroid = (coords.centroid.bounds[0], coords.centroid.bounds[1])

        # Make 'World' point into screen coordinate pair.
        t.goto(convert_point(screen_Width,screen_height,ratios, centroid))

        # Write row name and population at geometry centroid (approximates to the 'middle' of the row multipolygon..
        legend = "{} (pop: {})".format(row['countyname'], row['total2011'])
        t.write(legend, align="center", font=("Arial", 10))

        # For each polygon in Multpolygon geometry, get each point in polygon, convert it to screen point and draw it.
        for geom in coords.geoms:
            for point in list(geom.exterior.coords):
                t.goto(convert_point(screen_Width,screen_height,ratios, point))
                t.down()
            t.up()

    t.hideturtle()
    t.exitonclick()

def main():
    SCREEN_WIDTH = 800
    SCREEN_HEIGHT = 600
    SQL = "select st_astext(geom) as geom_str, * from \"Counties\" where countyname like '%erry%' or countyname like '%imerick%' or countyname like '%ork%'"
    CONN_STRING = gp.DB_CONN_STRING

    my_data = rfn.get_data_from_postgres(CONN_STRING, SQL)
    if my_data:
        draw_data(SCREEN_WIDTH, SCREEN_HEIGHT, my_data)
    else:
        print("Nothing to draw.")

if __name__ == "__main__":
    main()


import csv
import datetime
import logging
import os.path
import re
import statistics
import sys

from bs4 import BeautifulSoup
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models import Prefetch
from django.utils import timezone
from guardian.core import ObjectPermissionChecker
from jsonfield import JSONField

from .utils import get_turkle_template_limit

logger = logging.getLogger(__name__)

# The default field size limit is 131072 characters
csv.field_size_limit(sys.maxsize)

class Task(models.Model):
    """Human Intelligence Task
    """
    class Meta:
        verbose_name = "Task"

    batch = models.ForeignKey('Batch', on_delete=models.CASCADE)
    completed = models.BooleanField(default=False)
    input_csv_fields = JSONField()

    def __str__(self):
        return 'Task id:{}'.format(self.id)

    def populate_html_template(self):
        """Return HTML template for this Task's project, with populated template variables

        Returns:
            String containing the HTML template for the Project associated with
            this Task, with all template variables replaced with the template
            variable values stored in this Task's input_csv_fields.
        """
        result = self.batch.project.html_template
        for field in self.input_csv_fields.keys():
            result = result.replace(
                r'${' + field + r'}',
                self.input_csv_fields[field]
            )
        return result

class TaskAssignment(models.Model):
    """Task Assignment
    """
    class Meta:
        verbose_name = "Task Assignment"

    answers = JSONField(blank=True)
    assigned_to = models.ForeignKey(User, db_index=True, null=True, on_delete=models.CASCADE)
    completed = models.BooleanField(db_index=True, default=False)
    created_at = models.DateTimeField(auto_now_add=True)
    expires_at = models.DateTimeField(null=True)
    task = models.ForeignKey(Task, on_delete=models.CASCADE)
    updated_at = models.DateTimeField(auto_now=True)

    @classmethod
    def expire_all_abandoned(cls):
        result = cls.objects.\
            filter(completed=False).\
            filter(expires_at__lt=timezone.now()).\
            delete()
        logger.info('Expired %i task assignments', result[0])
        return result

    def save(self, *args, **kwargs):
        self.expires_at = timezone.now() + \
            datetime.timedelta(hours=self.task.batch.allotted_assignment_time)

        if 'csrfmiddlewaretoken' in self.answers:
            del self.answers['csrfmiddlewaretoken']
        super().save(*args, **kwargs)

        # Mark Task as completed if all Assignments have been completed
        if self.task.taskassignment_set.filter(completed=True).count() >= \
           self.task.batch.assignments_per_task:
            self.task.completed = True
            self.task.save()

    def work_time_in_seconds(self):
        """Return number of seconds elapsed between Task assignment and submission

        We compute the time elapsed in Python instead of in SQL
        because "there are no native date/time fields in SQLite and
        Django currently emulates these features using a text field,"
        per the Django Docs:
          https://docs.djangoproject.com/en/2.1/ref/models/querysets/#aggregation-functions

        Returns:
            Integer for seconds elapsed between Task assignment and submission

        Raises:
            ValueError if TaskAssignment is not completed
        """
        if self.completed:
            return int((self.updated_at - self.created_at).total_seconds())
        else:
            raise ValueError(
                'Cannot compute work_time_in_seconds for incomplete TaskAssignment %d' %
                self.id)

class Batch(models.Model):
    class Meta:
        verbose_name = "Batch"
        verbose_name_plural = "Batches"

    active = models.BooleanField(db_index=True, default=True)
    allotted_assignment_time = models.IntegerField(default=24)
    assignments_per_task = models.IntegerField(default=1, verbose_name='Assignments per Task')
    created_at = models.DateTimeField(auto_now_add=True)
    created_by = models.ForeignKey(User, null=True)
    filename = models.CharField(max_length=1024)
    project = models.ForeignKey('Project', on_delete=models.CASCADE)
    name = models.CharField(max_length=1024)

    def assignments_completed_by(self, user):
        """
        Returns:
            QuerySet of all TaskAssignments completed by specified user
            that are part of this Batch
        """
        return TaskAssignment.objects.\
            filter(completed=True).\
            filter(assigned_to_id=user.id).\
            filter(task__batch=self)

    def available_tasks_for(self, user):
        """Retrieve a list of all Tasks in this batch available for the user.

        This list DOES NOT include Tasks in the batch that have been assigned
        to the user but not yet completed.

        Args:
            user (User|AnonymousUser):

        Returns:
            QuerySet of Task objects
        """
        if not user.is_authenticated and self.project.login_required:
            return Task.objects.none()

        hs = self.task_set.filter(completed=False)

        if self.assignments_per_task > 1:
            # Exclude Tasks that have already been assigned to this user.
            if user.is_authenticated:
                # If the user is not authenticated, then user.id is None,
                # and the query below would exclude all uncompleted Tasks.
                hs = hs.exclude(taskassignment__assigned_to_id=user.id)

            # Only include Tasks when # of (possibly incomplete) assignments < assignments_per_task
            hs = hs.annotate(ac=models.Count('taskassignment')).\
                filter(ac__lt=self.assignments_per_task)
        elif self.assignments_per_task == 1:
            # Only returns Tasks that have not been assigned to anyone (including this user)
            hs = hs.filter(taskassignment=None)

        return hs

    def available_task_ids_for(self, user):
        return self.available_tasks_for(user).values_list('id', flat=True)

    def clean(self):
        # Without this guard condition for project_id, a
        # RelatedObjectDoesNotExist exception is thrown before a
        # ValidationError can be thrown.  When this model is edited
        # using a form, this causes the server to generate an HTTP 500
        # error due to the uncaught RelatedObjectDoesNotExist
        # exception, instead of catching the ValidationError and
        # displaying a form with a "Field required" warning.
        if self.project_id:
            if not self.project.login_required and self.assignments_per_task != 1:
                raise ValidationError('When login is not required to access a Project, ' +
                                      'the number of Assignments per Task must be 1')

    def csv_results_filename(self):
        """Returns filename for CSV results file for this Batch
        """
        batch_filename, extension = os.path.splitext(os.path.basename(self.filename))

        # We are following Mechanical Turk's naming conventions for results files
        return "{}-Batch_{}_results{}".format(batch_filename, self.id, extension)

    def create_tasks_from_csv(self, csv_fh):
        """
        Args:
            csv_fh (file-like object): File handle for CSV input

        Returns:
            Number of Tasks created from CSV file
        """
        header, data_rows = self._parse_csv(csv_fh)

        logger.info('Creating tasks for Batch(%i) %s', self.id, self.name)
        num_created_tasks = 0
        for row in data_rows:
            if not row:
                continue
            task = Task(
                batch=self,
                input_csv_fields=dict(zip(header, row)),
            )
            task.save()
            num_created_tasks += 1
        logger.info('Created %i tasks for Batch(%i) %s', num_created_tasks, self.id, self.name)

        return num_created_tasks

    def finished_tasks(self):
        """
        Returns:
            QuerySet of all Task objects associated with this Batch
            that have been completed.
        """
        return self.task_set.filter(completed=True).order_by('-id')

    def finished_task_assignments(self):
        """
        Returns:
            QuerySet of all Task Assignment objects associated with this Batch
            that have been completed.
        """
        return TaskAssignment.objects.filter(task__batch_id=self.id)\
                                     .filter(completed=True)

    def mean_work_time_in_seconds(self):
        """
        Returns:
            Float for mean work time (in seconds) for completed Tasks in this Batch
        """
        finished_assignments = self.finished_task_assignments()
        if finished_assignments.count() > 0:
            return statistics.mean(
                [ta.work_time_in_seconds() for ta in self.finished_task_assignments()])
        else:
            return 0

    def median_work_time_in_seconds(self):
        """
        Returns:
            Integer for median work time (in seconds) for completed Tasks in this Batch
        """
        finished_assignments = self.finished_task_assignments()
        if finished_assignments.count() > 0:
            # np.median returns float but we convert back to int computed by work_time_in_seconds()
            return int(statistics.median(
                [ta.work_time_in_seconds() for ta in self.finished_task_assignments()]
            ))
        else:
            return 0

    def next_available_task_for(self, user):
        """Returns next available Task for the user, or None if no Tasks available

        Args:
            user (User):

        Returns:
            Task|None
        """
        return self.available_tasks_for(user).first()

    def total_assignments_completed_by(self, user):
        """
        Returns:
            Integer of total number of TaskAssignments completed by
            specified user that are part of this Batch
        """
        return self.assignments_completed_by(user).count()

    def total_available_tasks_for(self, user):
        """Returns number of Tasks available for the user

        Args:
            user (User):

        Returns:
            Number of Tasks available for user
        """
        return self.available_tasks_for(user).count()

    def total_finished_tasks(self):
        return self.finished_tasks().count()
    total_finished_tasks.short_description = 'Total finished Tasks'

    def total_finished_task_assignments(self):
        return self.finished_task_assignments().count()
    total_finished_task_assignments.short_description = 'Total finished Task Assignments'

    def total_task_assignments(self):
        return self.assignments_per_task * self.total_tasks()

    def total_tasks(self):
        return self.task_set.count()
    total_tasks.short_description = 'Total Tasks'

    def total_users_that_completed_tasks(self):
        """
        Returns:
            Integer counting number of Users that have completed
            TaskAssignments that are part of this Batch
        """
        return self.users_that_completed_tasks().count()

    def total_work_time_in_seconds(self):
        """
        Returns:
            Integer sum of work_time_in_seconds() for all completed
            TaskAssignments in this Batch
        """
        return sum([ta.work_time_in_seconds() for ta in self.finished_task_assignments()])

    def to_csv(self, csv_fh, lineterminator='\r\n'):
        """Write CSV output to file handle for every Task in batch

        Args:
            csv_fh (file-like object): File handle for CSV output
        """
        fieldnames, rows = self._results_data(self.task_set.all())
        writer = csv.DictWriter(csv_fh, fieldnames, lineterminator=lineterminator,
                                quoting=csv.QUOTE_ALL)
        writer.writeheader()
        for row in rows:
            writer.writerow(row)

    def to_input_csv(self, csv_fh, lineterminator='\r\n'):
        """Write (reconstructed) CSV input to file handle for every Task in Batch

        PLEASE NOTE: The column order in the reconstructed CSV file
        may not match the column order in the original CSV file.
        """
        tasks = self.task_set.all()
        if not tasks.exists():
            return

        # Some rows may (theoretically) be missing fields
        fieldnames = set()
        for task in tasks:
            fieldnames.update(task.input_csv_fields.keys())

        writer = csv.DictWriter(csv_fh, list(fieldnames), lineterminator=lineterminator,
                                quoting=csv.QUOTE_ALL)
        writer.writeheader()
        for task in tasks:
            writer.writerow(task.input_csv_fields)

    def unfinished_tasks(self):
        """
        Returns:
            QuerySet of all Task objects associated with this Batch
            that have NOT been completed.
        """
        return self.task_set.filter(completed=False).order_by('id')

    def users_that_completed_tasks(self):
        """
        Returns:
            QuerySet of all Users who have completed TaskAssignments
            that are part of this Batch
        """
        return User.objects.\
            filter(taskassignment__task__batch=self).\
            filter(taskassignment__completed=True).\
            distinct()

    def _parse_csv(self, csv_fh):
        """
        Args:
            csv_fh (file-like object): File handle for CSV output

        Returns:
            A tuple where the first value is a list of strings for the
            header fieldnames, and the second value is an iterable
            that returns a list of values for the rest of the row in
            the CSV file.
        """
        rows = csv.reader(csv_fh)
        header = next(rows)
        return header, rows

    def _get_csv_fieldnames(self, task_queryset):
        """
        Args:
            task_queryset (QuerySet):

        Returns:
            A tuple of strings specifying the fieldnames to be used in
            in the header of a CSV file.
        """
        input_field_set = set()
        answer_field_set = set()
        task_assignments = TaskAssignment.objects.\
            filter(task__in=task_queryset).\
            prefetch_related(Prefetch('task', queryset=task_queryset))
        for task_assignment in task_assignments:
            input_field_set.update(task_assignment.task.input_csv_fields.keys())

            # If the answers JSONField is empty, it evaluates as a string instead of a dict
            if task_assignment.answers != '':
                answer_field_set.update(task_assignment.answers.keys())
        return tuple(
            ['HITId', 'HITTypeId', 'Title', 'CreationTime', 'MaxAssignments',
             'AssignmentDurationInSeconds', 'AssignmentId', 'WorkerId',
             'AcceptTime', 'SubmitTime', 'WorkTimeInSeconds'] +
            ['Input.' + k for k in sorted(input_field_set)] +
            ['Answer.' + k for k in sorted(answer_field_set)] +
            ['Turkle.Username']
        )

    def _results_data(self, task_queryset):
        """
        All completed Tasks must come from the same project so that they have the
        same field names.

        Args:
            task_queryset (QuerySet):

        Returns:
            A tuple where the first value is a list of fieldname strings, and
            the second value is a list of dicts, where the keys to these
            dicts are the values of the fieldname strings.
        """
        rows = []
        time_format = '%a %b %d %H:%M:%S %Z %Y'
        task_assignments = TaskAssignment.objects.\
            filter(task__in=task_queryset).\
            filter(completed=True).\
            prefetch_related(Prefetch('task', queryset=task_queryset))
        for task_assignment in task_assignments:
            task = task_assignment.task
            batch = task.batch
            project = task.batch.project

            if task_assignment.assigned_to:
                username = task_assignment.assigned_to.username
            else:
                username = ''

            row = {
                'HITId': task.id,
                'HITTypeId': project.id,
                'Title': project.name,
                'CreationTime': batch.created_at.strftime(time_format),
                'MaxAssignments': batch.assignments_per_task,
                'AssignmentDurationInSeconds': batch.allotted_assignment_time * 3600,
                'AssignmentId': task_assignment.id,
                'WorkerId': task_assignment.assigned_to_id,
                'AcceptTime': task_assignment.created_at.strftime(time_format),
                'SubmitTime': task_assignment.updated_at.strftime(time_format),
                'WorkTimeInSeconds': task_assignment.work_time_in_seconds(),
                'Turkle.Username': username,
            }
            row.update({'Input.' + k: v for k, v in task.input_csv_fields.items()})
            row.update({'Answer.' + k: v for k, v in task_assignment.answers.items()})
            rows.append(row)

        return self._get_csv_fieldnames(task_queryset), rows

    def __str__(self):
        return 'Batch: {}'.format(self.name)

class TurklePermissionChecker(ObjectPermissionChecker):
    """
    Wrapper for django-guardian's permissions checker.

    Handles projects that don't have custom permissions.
    """

    def has_perm(self, perm, obj):
        """
        Checks if user/group has given permission for object.
        :param perm: permission as string, may or may not contain app_label
          prefix (if not prefixed, we grab app_label from ``obj``)
        :param obj: Django model instance for which permission should be checked
        """
        if self.user and not self.user.is_active:
            return False
        elif self.user and self.user.is_superuser:
            return True
        elif not self.user.is_authenticated and obj.login_required:
            return False
        elif obj.custom_permissions:
            if '.' in perm:
                _, perm = perm.split('.', 1)
            return perm in self.get_perms(obj)
        else:
            return True

class Project(models.Model):
    class Meta:
        permissions = (
            ('can_work_on', 'Can work on Tasks for this Project'),
        )
        verbose_name = "Project"

    active = models.BooleanField(db_index=True, default=True)
    assignments_per_task = models.IntegerField(db_index=True, default=1)
    created_at = models.DateTimeField(auto_now_add=True)
    created_by = models.ForeignKey(User, null=True, related_name='created_projects')
    custom_permissions = models.BooleanField(default=False)
    filename = models.CharField(max_length=1024, blank=True)
    html_template = models.TextField()
    html_template_has_submit_button = models.BooleanField(default=False)
    login_required = models.BooleanField(db_index=True, default=True)
    name = models.CharField(max_length=1024)
    updated_at = models.DateTimeField(auto_now=True)
    updated_by = models.ForeignKey(User, null=True, related_name='updated_projects')

    # Fieldnames are automatically extracted from html_template text
    fieldnames = JSONField(blank=True)

    @classmethod
    def all_available_for(cls, user):
        """Retrieve the Projects that the user has permission to access

        Args:
            user (User):

        Returns:
            QuerySet of Project objects this user can access
        """
        projects = cls.objects.filter(active=True)
        if not user.is_authenticated:
            projects = projects.filter(login_required=False)

        # Can implement our own prefetch_perms() in future for efficiency
        checker = TurklePermissionChecker(user)
        checker.prefetch_perms(projects)

        return [p for p in projects if checker.has_perm('can_work_on', p)]

    def available_for(self, user):
        """
        Returns:
            Boolean indicating if this Project is available for the user
        """
        if not user.is_authenticated and self.login_required:
            return False
        elif self.custom_permissions:
            return user.has_perm('can_work_on', self)
        else:
            return True

    def batches_available_for(self, user):
        """Retrieve the Batches that the user has permission to access

        Args:
            user (User):

        Returns:
            QuerySet of Batch objects this user can access
        """
        batches = self.batch_set.filter(active=True)
        if not user.is_authenticated:
            batches = batches.filter(project__login_required=False)
        return batches

    def clean(self):
        super().clean()
        if len(self.html_template) > get_turkle_template_limit(True):
            raise ValidationError({'html_template': 'Template is too large'}, code='invalid')
        if not self.login_required and self.assignments_per_task != 1:
            raise ValidationError('When login is not required to access the Project, ' +
                                  'the number of Assignments per Task must be 1')
        self.process_template()

    def process_template(self):
        soup = BeautifulSoup(self.html_template, 'html.parser')
        self.html_template_has_submit_button = bool(soup.select('input[type=submit]'))

        # Extract fieldnames from html_template text, save fieldnames as keys of JSON dict
        unique_fieldnames = set(re.findall(r'\${(\w+)}', self.html_template))
        self.fieldnames = dict((fn, True) for fn in unique_fieldnames)

        # Matching mTurk we confirm at least one input, select, or textarea
        if soup.find('input') is None and soup.find('select') is None \
                and soup.find('textarea') is None:
            msg = "Template does not contain any fields for responses. " + \
                  "Please include at least one field (input, select, or textarea)." + \
                  "This usually means you are generating HTML with JavaScript." + \
                  "If so, add an unused hidden input."
            raise ValidationError({'html_template': msg}, code='invalid')

    def to_csv(self, csv_fh, lineterminator='\r\n'):
        """
        Writes CSV output to file handle for every Task associated with project

        Args:
            csv_fh (file-like object): File handle for CSV output
        """
        batches = self.batch_set.all()
        if batches:
            fieldnames = self._get_csv_fieldnames(batches)
            writer = csv.DictWriter(csv_fh, fieldnames, lineterminator=lineterminator,
                                    quoting=csv.QUOTE_ALL)
            writer.writeheader()
            for batch in batches:
                _, rows = batch._results_data(batch.finished_tasks())
                for row in rows:
                    writer.writerow(row)

    def _get_csv_fieldnames(self, batches):
        """
        Args:
            batches (List of Batch objects)

        Returns:
            A tuple of strings specifying the fieldnames to be used in
            in the header of a CSV file.
        """
        input_field_set = set()
        answer_field_set = set()
        for batch in batches:
            for task in batch.task_set.all():
                for task_assignment in task.taskassignment_set.all():
                    input_field_set.update(task.input_csv_fields.keys())
                    answer_field_set.update(task_assignment.answers.keys())
        return tuple(
            ['HITId', 'HITTypeId', 'Title', 'CreationTime', 'MaxAssignments',
             'AssignmentDurationInSeconds', 'AssignmentId', 'WorkerId',
             'AcceptTime', 'SubmitTime', 'WorkTimeInSeconds'] +
            ['Input.' + k for k in sorted(input_field_set)] +
            ['Answer.' + k for k in sorted(answer_field_set)] +
            ['Turkle.Username']
        )

    def __str__(self):
        return self.name


import numpy as np
import matplotlib
matplotlib.use('WXAgg')
import pylab as pl
import time
import equipment.LabJack.u3 as u3

lj = u3.U3()

N = 100
REFRESH_MS = 100
CLOCK_PER_MS = 4000 # clock is 4MHz

try:

    lj.configIO(EnableCounter0=True, EnableCounter1=True, NumberOfTimersEnabled = 2, TimerCounterPinOffset=6, FIOAnalog=0x0F)
    #timer 0 on FIO6
    #timer 1 on FIO7
    #counter0 on EIO0
    #counter1 on EIO2
    lj.getFeedback(u3.TimerConfig(timer = 0, TimerMode = 10)) # system timer LSW
    lj.getFeedback(u3.TimerConfig(timer = 1, TimerMode = 11)) # system timer MSW

    feedback_req = [
        u3.Timer(timer=0, UpdateReset = False, Value = 0, Mode = None),
        u3.Timer(timer=1, UpdateReset = False, Value = 0, Mode = None),
        u3.Counter(counter = 0, Reset=True),
        u3.Counter(counter = 1, Reset=False)
        ]

    fig = pl.figure(1)
    ax  = fig.add_subplot(111)

    ax.set_xlabel("time (ms)")
    ax.set_ylabel("Counts (Hz)")

    T  = np.zeros(N, dtype=np.uint64) # clock times
    C0 = np.zeros(N, dtype=np.uint32) # raw counts between measurements
    C0_rate = np.zeros(N, dtype=np.float) # rate in Hz

    c0 = 0
    c1 = 0
    t  = 0 # in clock ticks
    t_prev = -1*REFRESH_MS*CLOCK_PER_MS # in clock ticks

    ii = 0

    plot_line, = ax.plot(np.arange(0,N)*REFRESH_MS, C0_rate)
    plot_current_pos = ax.axvline(ii)

    def update_graph(ax):
        global T, C0, ii, t, t_prev

        ii += 1
        ii %= N

        tLSW, tMSW, c0, c1 = lj.getFeedback(feedback_req)

        #t = tMSW << 32 + tLSW
        t_prev = t
        t = tLSW

        dt = np.abs(t - t_prev) #% 0xFFFF

        #c0_rate = c0*1e3/REFRESH_MS
        c0_rate = c0 * CLOCK_PER_MS * 1.0e3 / dt  # Hz

        #T[ii] = t
        C0[ii] = c0
        C0_rate[ii] = c0_rate

        plot_line.set_ydata(C0_rate)
        plot_current_pos.set_xdata( (ii*REFRESH_MS,ii*REFRESH_MS) )
        #self.hist_i += 1
        #self.hist_i %= HIST_LEN

        ax.set_title("%i, %i, %e, dt=%g ms (%i ticks)" % (t, c0, c0_rate, dt*1.0/CLOCK_PER_MS, dt))

        if (ii % 10) == 0:
            ax.relim()
            ax.autoscale_view(scalex=False, scaley=True)

        fig.canvas.draw()

    timer = fig.canvas.new_timer(interval=REFRESH_MS)
    #timer.set_interval(REFRESH_MS)
    #._timer_set_interval(REFRESH_MS)
    timer.add_callback(update_graph, ax)
    timer.start()

    pl.show()

finally:
    lj.close()


"""
This project aims to introduce my mentees basic of python with all fundamental methods.

Game Concept is Rock Paper Scissors and everyone plays this game even one time in life.

Rock smashes scissors.
Paper covers rock.
Scissors cut paper.

You play this game aganist computer and who will the 3 as score will be winner!
"""

import random

def main():
    """Runs Main Function"""  # docstring
    user_score = 0
    computer_score = 0

    # Game Mechanism on Terminal
    while True:
        user = get_user_choice()

        computer = get_computer_choice()

        user_score, computer_score = compare(
            user, computer, user_score, computer_score)

        score_control(user_score, computer_score)

def score_control(user_s, computer_s):
    """Gets user and computer scores as integer and if one of them reach 3, complete the game"""
    if user_s == 3 or computer_s == 3:
        if user_s == 3:
            print("You Win the Game!")
            exit()
        else:
            print("Computer Wins the Game!")
            exit()

def compare(user_val, computer_val, user_score, computer_score):
    """Get user_val and computer_val, and evalutes game rule. Gets user_score and computer score and return both in every case.
        - Rock smashes scissors.
        - Paper covers rock.
        - Scissors cut paper.
    """

    game_choice_dict = {0: "Rock", 1: "Paper", 2: "Scissors"}

    if user_val != computer_val:

        if (user_val == 1 and computer_val == 0) or (user_val == 2 and computer_val == 1) or (user_val == 0 and computer_val == 2):
            print("User Wins")
            user_score += 1

        elif (user_val == 0 and computer_val == 1) or (user_val == 1 and computer_val == 2) or (user_val == 2 and computer_val == 0):
            print("Computer Wins")
            computer_score += 1
    else:
        print("Draw! You choose the same!")

    print("User showed:{} | Computer shoved:{}".format(
        game_choice_dict.get(user_val), game_choice_dict.get(computer_val)))
    print("User Score:{} | Computer Score:{}\n-----".format(user_score, computer_score))
    return user_score, computer_score

def get_computer_choice():
    """Return value between 0 to 2 for computer choice of the game"""
    computer_choice = random.randint(0, 2)
    return computer_choice

def get_user_choice():
    """Get user input and return it as integer value"""
    user_choice = int(input("Rock[0], Paper[1], Scisscors[2]: "))
    return user_choice

if __name__ == "__main__":

    main()


import logging
import spotipy
import spotipy.util as util

import requests
from io import BytesIO
from PIL import Image

from transferImage import display

def getSongInfo(username):
  scope = 'user-read-currently-playing'
  token = util.prompt_for_user_token(username, scope)

  if token:
      sp = spotipy.Spotify(auth=token)
      result = sp.current_user_playing_track()

      if result is None:
        logging.warning("No song is currently playing")
      else:
        song = result["item"]["name"]
        artist = result["item"]["album"]["artists"][0]["name"]
        imageURL = result["item"]["album"]["images"][0]["url"]
        # Remaining seconds in the song
        timeLeft = (result["item"]["duration_ms"]/1000) - (result["progress_ms"]/1000)
        response = requests.get(imageURL)
        image = Image.open(BytesIO(response.content))

        logging.info("Song: %s ", song)
        logging.info("Artist: %s", artist)
        logging.info("Image URL: %s ", imageURL)
        logging.info("Time Left: %ss ", timeLeft)
        display(image, timeLeft)
  else:
      print("Can't get token for", username)


###
### Config data
###

import os
from math import *
from copy import deepcopy
#import geometria_linea

example_test= """5
6 1 1000000000000000
1 1 1 1 0 100 0
6 1 -100
1 1 1 1 0 100 0
10 1 8
4 3 4 1 5 20 -10
10 2 8
4 3 4 1 5 20 -10
10 1 8
4 3 4 1 5 20 -19"""

filename = __file__[:-3]
__script_dir__ = os.path.realpath(__file__)
__script_path__ = os.path.dirname(__script_dir__) + os.sep

def input_iter(has_finish_flag = False, flag = ""):
	while True:
		try: # Instead of using a conditional, we use a try/except to catch other exceptions
			 # created by input finishing.
			i = input()
			assert(not(has_finish_flag and i==flag))
		except:
			break
		yield(i)

def string_iter(string=example_test):
    for i in string.split("\n"):
        yield(i)

def file_iter(filepath = "input.txt"):
    _file = open(__script_path__ + filepath, "r")
    i_buffer = _file.readline()
    for i in _file.readlines():
        yield(i_buffer[:-1])
        i_buffer = i
    _file.close()
    yield(i_buffer)

#line = string_iter()
#line = file_iter(filename[-1] +"-small-attempt1.in")
#line = file_iter("B-small-attempt3.in")
line = file_iter("A-small-practice.in")

#fobject = None
fobject = open(__script_path__ + filename[-1] + 'output.txt', 'w')
pend = None

### Run
###

T = int(next(line))

for case in range(T):

	answer = "IMPOSSIBLE"

	N,O,D,x = *map(int,next(line).split()), [0,0]
	x[0], x[1], A, B, C, M, L = map(int,next(line).split())
	minimum = min(x[0] + L,x[1] + L)
	for i in range(2, N):
		x.append(((A * x[i-1]) + (B * x[i-2]) + C)%(M))
		#minimum = min(minimum, x[-1] + L)

	for i in range(len(x)):
			x[i] += L

	treshold = lambda c_d, c_o: (c_d > D or c_o > O)

	p1 = 0
	p2 = 0
	smallest = min(*x)
	largest = smallest - 1
	odds = 0
	tsum = 0
	while p1 < len(x) and p2 <len(x):
		tsum += x[p2]
		odds += 1 if x[p2] % 2 == 1 else 0

		while (treshold(tsum, odds)) and p1 <= p2:
			tsum -= x[p1]
			odds -= 1 if x[p1] % 2 == 1 else 0
			p1 += 1

		if(not treshold(tsum, odds)):
			largest = max(largest, tsum)

		p2 += 1

	if largest >= smallest:
		answer = largest

	print("Case #{}: {}".format(case+1,answer), file= None)
	print("Case #{}: {}".format(case+1,answer), file= fobject)


"""
Author @ Mihir_Srivastava
Dated - 17-07-2020
File - Sentiment_analysis_of_tweets_NB
Aim - To do the sentiment analysis of tweets from the twitter_samples DataSet available in the nltk library and classify
them as positive or negative tweets using sklearn's Multinomial Naive Bayes.
"""

# Import necessary libraries
import numpy as np
import matplotlib.pyplot as plt
import random
import re
import string
import warnings
from nltk.corpus import twitter_samples
from nltk.tokenize import TweetTokenizer
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from sklearn import naive_bayes

# Collect positive and negative tweets
positive_tweets = twitter_samples.strings('positive_tweets.json')
negative_tweets = twitter_samples.strings('negative_tweets.json')

# Append positive and negative tweets to get all the tweets
all_tweets = positive_tweets + negative_tweets

# Split the data into two pieces, one for training and one for testing (validation set)
test_pos = positive_tweets[4000:]
train_pos = positive_tweets[:4000]
test_neg = negative_tweets[4000:]
train_neg = negative_tweets[:4000]

train_x = train_pos + train_neg
test_x = test_pos + test_neg

train_y = np.append(np.ones(len(train_pos)), np.zeros(len(train_neg)))
test_y = np.append(np.ones(len(test_pos)), np.zeros(len(test_neg)))

# A function to visualize the classes
def visualize(all_positive_tweets, all_negative_tweets):
    # Declare a figure with a custom size
    fig = plt.figure(figsize=(5, 5))

    # labels for the two classes
    labels = 'Positives', 'Negative'

    # Sizes for each slide
    sizes = [len(all_positive_tweets), len(all_negative_tweets)]

    # Declare pie chart, where the slices will be ordered and plotted counter-clockwise:
    plt.pie(sizes, labels=labels, autopct='%1.1f%%', shadow=True, startangle=90)

    # Equal aspect ratio ensures that pie is drawn as a circle.
    plt.axis('equal')

    # Display the chart
    plt.show()

# A function to print a sample tweet from each class
def sample_tweets():
    print('\033[92m' + positive_tweets[random.randint(0, 5000)])
    print('\033[91m' + negative_tweets[random.randint(0, 5000)])

# A function to preprocess a tweet before feeding to our ML model
def process_tweet(tweet):
    # remove stock market tickers like $GE
    tweet2 = re.sub(r'\$\w*', '', tweet)

    # remove old style retweet text "RT"
    tweet2 = re.sub(r'^RT[\s]+', '', tweet2)

    # remove hyperlinks
    tweet2 = re.sub(r'https?:\/\/.*[\r\n]*', '', tweet2)

    # remove hashtags (only removing the hash # sign from the word)
    tweet2 = re.sub(r'#', '', tweet2)

    # Create Tweet Tokenizer object
    token = TweetTokenizer(preserve_case=False, reduce_len=False, strip_handles=True)

    # Tokenize the tweet
    tweet_tokenized = token.tokenize(tweet2)

    # Create Porter Stemmer object
    stem = PorterStemmer()

    # Create stop words object
    st = stopwords.words('english')
    tweet_cleaned = []

    # Remove stopwords and punctuations and apply stemming
    for word in tweet_tokenized:
        if word not in st and word not in string.punctuation:
            stemmed_word = stem.stem(word)
            tweet_cleaned.append(stemmed_word)

    return tweet_cleaned

# A function to create a dictionary mapping {(word, label) --> frequency of that word in that label}
def build_freq(tweets, ys):
    yslist = np.squeeze(ys).tolist()
    frequency = {}
    for y, tweet in zip(yslist, tweets):
        for word in process_tweet(tweet):
            pair = (word, y)
            frequency[pair] = frequency.get(pair, 0) + 1
    return frequency

freqs = build_freq(train_x, train_y)

# A function to extract the necessary features from a tweet [positive frequency sum, negative frequency sum]
def extract_features(tweet, freqs):
    # process_tweet tokenizes, stems, and removes stopwords
    word_list = process_tweet(tweet)

    # 2 elements in the form of a 1 x 2 vector
    x = np.zeros((1, 2))

    # loop through each word in the list of words
    for word in word_list:
        # increment the word count for the positive label 1
        x[0, 0] += freqs.get((word, 1.0), 0)
        # increment the word count for the negative label 0
        x[0, 1] += freqs.get((word, 0.0), 0)

    return x

warnings.filterwarnings('ignore')

# collect the features 'x' and stack them into a matrix 'X'
X_train = np.zeros(((len(train_x)), 2))
for i in range(len(train_x)):
    X_train[i, :] = extract_features(train_x[i], freqs)

# training labels corresponding to X
y_train = np.array(train_y)

# Create Multinomial Naive Bayes object
model = naive_bayes.MultinomialNB(alpha=1)

# Train the model
model.fit(X_train, y_train)

# Extract features of the test set
X_test = np.zeros(((len(test_x)), 2))
for i in range(len(test_x)):
    X_test[i, :] = extract_features(test_x[i], freqs)

y_test = np.array(test_y)

tweet = input("Enter your tweet: ")
tweet_final = extract_features(tweet, freqs)

print(model.predict(tweet_final))

if model.predict(tweet_final) == 1.:
    print("Positive sentiment")
else:
    print("Negative sentiment")

accuracy = model.score(X_test, y_test)
print("accuracy: " + str(accuracy))


# -*- coding: utf-8 -*-
# 参数定义的顺序必须是：必选参数、默认参数、可变参数、命名关键字参数和关键字参数。
import os
l=[]
for root, dirs, files in os.walk('D:\\aliDoc'):
    #print(root, "consumes", end=" ")
    if len(files)>0:
        l=l+list([(name,os.path.getsize(os.path.join(root, name))) for name in files])
    #print("bytes in", len(files), "non-directory files")
print('总共有',len(l),'个文件')
l.sort()
l.reverse()
for i in l:
    print(i)


import os
import random
import shutil

#TRAIN
TRAIN_START = 1000
TRAIN_END = 2999
TRAIN_QUANTITY = 700

#NOISE
NOISE_START = 0
NOISE_END = 999
NOISE_QUANTITY = 10

TRAIN_SRC = '/home/siri/PycharmProjects/MNIST/train_data/'
TRAIN_DEST = './data/mnist_10/train_data/'

#VALID
VALID_START = 0
VALID_END = 500
VALID_QUANTITY = 300

VALID_SRC = '/home/siri/PycharmProjects/MNIST/valid_data/'
VALID_DEST = './data/mnist_10/valid_data/'

#TEST
TEST_START = 3000
TEST_END = 5000
TEST_QUANTITY = 500

def string2Number(img_name):
    img_name = str(img_name).replace('.png','')
    return int(img_name[2:])

def sample_selector(src,dest,train=False):
    for i in os.listdir(src):
        src2 = src+i

        if train:
            train_arr = []
            noise_arr = []
            #test_arr = []

            for j in os.listdir(src2):
                if string2Number(j)>TRAIN_START and string2Number(j)<TRAIN_END:
                    train_arr.append(j)
                elif string2Number(j)>NOISE_START and string2Number(j)<NOISE_END:
                    noise_arr.append(j)
                #else:
                    #test_arr.append(j)

            train_sample = random.sample(train_arr,TRAIN_QUANTITY)
            noise_sample = random.sample(noise_arr,NOISE_QUANTITY)
            #test_sample = random.sample(test_arr,TEST_QUANTITY)

            for each in train_sample:
                shutil.copy(src2+'/'+each,dest+i)
            for each in noise_sample:
                shutil.copy(src2+'/'+each,dest+i)
            #for each in test_sample:
                #shutil.copy(src2+'/'+each,TEST_DIR+i)

        else:
            valid_arr = []

            for j in os.listdir(src2):
                if string2Number(j) > VALID_START and string2Number(j) < VALID_END:
                    valid_arr.append(j)
            valid_sample = random.sample(valid_arr, VALID_QUANTITY)

            for each in valid_sample:
                shutil.copy(src2 + '/' + each, dest + i)

sample_selector(TRAIN_SRC,TRAIN_DEST,True)


#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created on Thu Jan 20 15:36:37 2011
@ author:                  Sat Kumar Tomer (modified by Hylke Beck)
@ author's webpage:        http://civil.iisc.ernet.in/~satkumar/
@ author's email id:       satkumartomer@gmail.com
@ author's website:        www.ambhas.com

A library with Python functions for calculating several objective functions commonly used in hydrological sciences.
Inputs consist of two equal sized arrays representing modeled and observed time series, and an integer specifying the
number of days to ignore in the beginning of the time series.

Example usage:
correlation = HydroStats.correlation(s=Qsim,o=Qobs,365)

Functions:
    RSR :     RMSE-observations standard deviation ratio
    br :      bias ratio
    pc_bias : percentage bias
    pc_bias2: percentage bias 2
    apb :     absolute percent bias
    apb2 :    absolute percent bias 2
    rmse :    root mean square error
    mae :     mean absolute error
    bias :    bias
    NS :      Nash Sutcliffe Coefficient
    NSlog :   Nash Sutcliffe Coefficient from log-transformed data
    correlation: correlation
    KGE:      Kling Gupta Efficiency
    vr :      variability ratio

"""

# import required modules
import numpy as np
from random import randrange
import matplotlib.pyplot as plt

def filter_nan(s,o):
    """
    this functions removed the data  from simulated and observed data
    whereever the observed data contains nan

    this is used by all other functions, otherwise they will produce nan as
    output
    """
    data = np.array([s.flatten(),o.flatten()])
    data = np.transpose(data)
    data = data[~np.isnan(data).any(1)]

    #mask = ~np.isnan(s) & ~np.isnan(o)
    #o_nonan = o[mask]
    #s_nonan = s[mask]

    #return o_nonan,s_nonan
    return data[:,0],data[:,1]

def RSR(s,o,warmup):
    """
    RMSE-observations standard deviation ratio
    input:
        s: simulated
        o: observed
    output:
        RSR: RMSE-observations standard deviation ratio
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    RMSE = np.sqrt(np.sum((s-o) ** 2))
    STDEV_obs = np.sqrt(np.sum((o-np.mean(o)) ** 2))
    return RMSE/STDEV_obs

def br(s,o,warmup):
    """
    Bias ratio
    input:
        s: simulated
        o: observed
    output:
        br: bias ratio
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return 1 - abs(np.mean(s)/np.mean(o) - 1)

def pc_bias(s,o,warmup):
    """
    Percent Bias
    input:
        s: simulated
        o: observed
    output:
        pc_bias: percent bias
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return 100.0*sum(s-o)/sum(o)

def pc_bias2(s,o,warmup):
    """
    Percent Bias 2
    input:
        s: simulated
        o: observed
    output:
        apb2: absolute percent bias 2
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return 100*(np.mean(s)-np.mean(o))/np.mean(o)

def apb(s,o,warmup):
    """
    Absolute Percent Bias
    input:
        s: simulated
        o: observed
    output:
        apb: absolute percent bias
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return 100.0*sum(abs(s-o))/sum(o)

def apb2(s,o,warmup):
    """
    Absolute Percent Bias 2
    input:
        s: simulated
        o: observed
    output:
        apb2: absolute percent bias 2
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return 100*abs(np.mean(s)-np.mean(o))/np.mean(o)

def rmse(s,o,warmup):
    """
    Root Mean Squared Error
    input:
        s: simulated
        o: observed
    output:
        rmses: root mean squared error
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return np.sqrt(np.mean((s-o)**2))

def mae(s,o,warmup):
    """
    Mean Absolute Error
    input:
        s: simulated
        o: observed
    output:
        maes: mean absolute error
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return np.mean(abs(s-o))

def bias(s,o,warmup):
    """
    Bias
    input:
        s: simulated
        o: observed
    output:
        bias: bias
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return np.mean(s-o)

def NS(s,o,warmup):
    """
    Nash-Sutcliffe efficiency coefficient
    input:
        s: simulated
        o: observed
    output:
        NS: Nash-Sutcliffe efficient coefficient
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return 1 - sum((s-o)**2)/sum((o-np.mean(o))**2)

def NSlog(s,o,warmup):
    """
    Nash-Sutcliffe efficiency coefficient from log-transformed data
    input:
        s: simulated
        o: observed
    output:
        NSlog: Nash-Sutcliffe efficient coefficient from log-transformed data
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    s = np.log(s)
    o = np.log(o)
    return 1 - sum((s-o)**2)/sum((o-np.mean(o))**2)

def correlation(s,o,warmup):
    """
    correlation coefficient
    input:
        s: simulated
        o: observed
    output:
        correlation: correlation coefficient
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    if s.size == 0:
        corr = np.NaN
    else:
        corr = np.corrcoef(o, s)[0,1]

    return corr

def index_agreement(s,o,warmup):
    """
	index of agreement
	input:
        s: simulated
        o: observed
    output:
        ia: index of agreement
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    ia = 1 -(np.sum((o-s)**2))/(np.sum(
    			(np.abs(s-np.mean(o))+np.abs(o-np.mean(o)))**2))
    return ia

def KGE(s,o,warmup):
    """
	Kling Gupta Efficiency (Kling et al., 2012, http://dx.doi.org/10.1016/j.jhydrol.2012.01.011)
	input:
        s: simulated
        o: observed
    output:
        KGE: Kling Gupta Efficiency
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    B = np.mean(s) / np.mean(o)
    y = (np.std(s) / np.mean(s)) / (np.std(o) / np.mean(o))
    r = np.corrcoef(o, s)[0,1]

    KGE = 1 - np.sqrt((r - 1) ** 2 + (B - 1) ** 2 + (y - 1) ** 2)

    return KGE

def vr(s,o,warmup):
    """
	Variability ratio
	input:
        s: simulated
        o: observed
    output:
        vr: variability ratio
    """
    s = s[warmup+1:]
    o = o[warmup+1:]
    s,o = filter_nan(s,o)
    return 1 - abs((np.std(s) / np.mean(s)) / (np.std(o) / np.mean(o)) - 1)


import torch
from torch import nn

def backward_hook(module, grad_input, grad_output):
    #print(module, gradInput, gradOutput)
    #print(copy.deepcopy('{0}'.format(str(i))), module)
    # for i in grad_input:
    #     print(i.shape)
    #print(module, grad_input[0].shape, grad_input[1].shape, grad_output[0].shape)
    module.grad_inputs = grad_input[0].data.clone()
    module.grad_outputs = grad_output[0].data.clone()

def forward_hook(module, m_input, m_output):
    #print(module, gradInput, gradOutput)
    #print(copy.deepcopy(name.format()), module)
    module.m_inputs = m_input[0].data.clone()
    module.m_outputs = m_output[0].data.clone()

def add_hooks(net):
    for name, module in net.named_modules():
        if 'layer' in name and 'conv' in name:
            module.register_forward_hook(forward_hook)
            module.register_backward_hook(backward_hook)

def init_cond_stats(net):
    stats = {
        'forwards_cond': {},
        'backwards_cond': {},
        'forwards_fbnorm': {},
        'backwards_fbnorm': {},
        'forwards_shape': {},
        'backwards_shape': {}
    }
    for name, _ in net.named_modules():
        if 'layer' in name and 'conv' in name:
            for key in stats:
                stats[key][name] = []

    return stats

import numpy as np
from collections import OrderedDict

def init_grad_stats(net):
    stats = OrderedDict()
    stats['forwards_cond'] = OrderedDict()
    stats['backwards_cond'] = OrderedDict()

    for name, _ in net.named_modules():
        if 'layer' in name and 'conv' in name:
            for key in stats:
                stats[key][name] = []

    return stats

def add_grad_stats(net, stats):
    with torch.no_grad():
        for name, module in net.named_modules():
            if 'layer' in name and 'conv' in name:
                stats['forwards_cond'][name] += module.forwards_cond
                stats['backwards_cond'][name] += module.backwards_cond

                # g = module.weight.grad1.flatten(1)
                # g = (g - g.mean(dim=1, keepdim=True)) / (g.std(dim=1, keepdim=True) + 1e-5)
                # corr = g.mm(g.T) / g.shape[-1]
                # lower_corr = []
                # for i in range(corr.shape[0]):
                #     lower_corr.append(corr[i][i + 1:])

                # lower_corr = torch.cat(lower_corr)

                # results['corr'] = lower_corr

def add_cond_stats(net, stats):
    with torch.no_grad():
        for name, module in net.named_modules():
            if 'layer' in name and 'conv' in name and module.kernel_size == (1, 1):
                inputs = module.m_inputs.flatten(2)
                grads = module.grad_outputs.flatten(2)

                # I = inputs.bmm(inputs.transpose(1, 2))
                # G = grads.bmm(grads.transpose(1, 2))

                _, s1, _ = inputs.svd(compute_uv=False)
                _, s2, _ = grads.svd(compute_uv=False)

                forward_cond_nums = (
                    s1[:, 0].log() - s1[:, -1].log()).detach().cpu()
                backward_cond_nums = (
                    s2[:, 0].log() - s2[:, -1].log()).detach().cpu()
                forwards_fbnorm = torch.norm(inputs, p='fro', dim=(1, 2))
                backwards_fbnorm = torch.norm(grads, p='fro', dim=(1, 2))

                #     for g in grads:
                #         _, s, _ = g.svd(compute_uv=False)
                #         s = s.detach().cpu()
                #         backward_cond_nums.append(s[0].log() - s[-1].log())

                stats['forwards_cond'][name].append(forward_cond_nums)
                stats['backwards_cond'][name].append(backward_cond_nums)
                stats['forwards_fbnorm'][name].append(forwards_fbnorm)
                stats['backwards_fbnorm'][name].append(backwards_fbnorm)
                stats['forwards_shape'][name] = inputs.size()[1:]
                stats['backwards_shape'][name] = grads.size()[1:]


#Jake Parker 09/25/2016
#Section 005
#ParkerJake_assign2_problem3.py
'''This program converts size in KB to bits, bytes, MB, and GB.
First, the conversion factors are stored in variables. Then,
the user is prompted to enter the size in KB. Then, the conversions
are printed out in tabular format. Note that the values are calculated
with the conversion values in the actual print statements. Memory size
in bytes acts as the base value'''

#bits in 1 byte
byte = 8 #bits
#bytes in 1 kB
kilo = 1024 #bytes
#bytes in 1 MB
mega = 1024 ** 2 #bytes
#bytes in 1 GB
giga = 1024 ** 3 #bytes
#ERROR EXAMPLE 1
#writing the following line instead would cause a logic error
#because the multiplication operator is used instead of the exponent
#operator
#giga = 1024 * 3

#ask user for size in KB
size_KB = int(input('Enter a file size, in kilobytes (KB): '))
#ERROR EXAMPLE 2
#writing the following line instead would cause a runtime error
#because the functions are nested incorrectly
#size_KB = input(int('Enter a file size, in kilobytes (KB): '))

#calculate size in bytes
size_bytes = size_KB * kilo

print() #print empty line
print(size_KB, 'equals ...\n')
#ERROR EXAMPLE 3
#writing the following line would cause a syntax error
#because the numerical variable and string aren't seperated
#print(size_KB 'equals ...\n')

#calculate number of bits from bytes and print in formatted line
print(format('... in bits', '<20s'),\
      format(size_bytes * byte, '20,.2f'), 'bits')
#ERROR EXAMPLE 4
#writing the following line would cause a logic error because
#the float number would not be truncated to 2 decimal places
#   format(size_bytes *byte, '20,f'), 'bits')

#print number of bytes and print in formatted line
print(format('... in bytes', '<20s'),\
      format(size_bytes, '20,.2f'), 'bytes')
#calculate number of MB from bytes and print in formatted line
print(format('... in megabytes', '<20s'),\
      format(size_bytes / mega, '20,.2f'), 'MB')
#calculate number of GB from bytes and print in formatted line
print(format('... in gigabytes', '<20s'),\
      format(size_bytes / giga, '20,.2f'), 'GB')
#ERROR EXAMPLE 5
#writing the following line would cause a runtime error because
#the input into the format function would not work for a string
#print(format('... in gigabytes', '<20f'),\


# -*- coding: utf-8 -*-
# @Time    : 2017/7/30 20:33
# @Author  : lileilei
# @Site    :
# @File    : setting.py
# @Software: PyCharm
from os import path
setting={
    'DEBUG': True,
    'xsrf_cookies': True,
    'static_path': path.join(path.dirname(__file__), 'static'),
    'template_path': path.join(path.dirname(__file__), 'templates'),
    'cookie_secret':'aiuasdhflashjdfoiuashdfiuh',
    'debug':True
}


import re
p = re.compile(r"(\w+)\s+\d+[-]\d+[-]\d+")
m = p.search("park 010-1234-1234")
print(m.group(1))

'''
text = "문의사항이 있으면 032-232-3245 으로 연락주시기 바랍니다."

regex = re.compile(r'(\d{3})-(\d{3}-\d{4})')
matchobj = regex.search(text)
areaCode = matchobj.group(1)
num = matchobj.group(2)
fullNum = matchobj.group()
print(areaCode, num) # 032 232-3245
'''


import a1_task1 as a1t1
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import os
import scipy.stats as stats
from pyfinance.ols import *

def ff_3factor_rolling_beta(tmp_etf_daily_ret, tmp_ff_daily, window):
    new_beta_dict = {}
    new_alpha_dict = {}
    new_residual_dict = {}

    if window != tmp_etf_daily_ret.shape[0]:
        for tmp_ff in list(tmp_ff_daily):
            new_beta_dict[tmp_ff] = pd.DataFrame()

        for tmp_etf in list(tmp_etf_daily_ret):
            tmp_roll_beta = PandasRollingOLS(y=tmp_etf_daily_ret[tmp_etf], x=tmp_ff_daily, window=window)

            for tmp_ff in list(tmp_ff_daily):
                new_beta_dict[tmp_ff][tmp_etf] = tmp_roll_beta.beta[tmp_ff]

            new_alpha_dict[tmp_etf] = tmp_roll_beta.alpha
            new_residual_dict[tmp_etf] = tmp_roll_beta.resids

    else:
        for tmp_ff in list(tmp_ff_daily):
            new_beta_dict[tmp_ff] = {}

        for tmp_etf in list(tmp_etf_daily_ret):
            tmp_roll_beta = OLS(y=tmp_etf_daily_ret[tmp_etf], x=tmp_ff_daily)

            for tmp_ff in list(tmp_ff_daily):
                new_beta_dict[tmp_ff][tmp_etf] = tmp_roll_beta.beta[list(tmp_ff_daily).index(tmp_ff)]

            new_alpha_dict[tmp_etf] = tmp_roll_beta.alpha
            new_residual_dict[tmp_etf] = tmp_roll_beta.resids

    return new_beta_dict, new_alpha_dict, new_residual_dict

if __name__ == '__main__':

    # (a)

    print()
    print("Task 1(a)\n")

    ff_daily = pd.read_csv("data/F-F_Research_Data_Factors_daily.CSV")
    ff_daily.rename(columns={'Unnamed: 0': 'Date'}, inplace=True)
    num = ff_daily.isna().sum()
    print("Empty value number:")
    print(num)
    ff_daily["Date"] = pd.to_datetime(ff_daily["Date"], format="%Y%m%d")
    ff_daily.dropna(axis=0, how='any', inplace=True)
    ff_daily.set_index(["Date"], inplace=True)
    ff_daily = ff_daily.loc[pd.to_datetime("2010-01-01", format="%Y-%m-%d"):, ]
    print("Total Day Number: ", ff_daily.shape[0])

    work_dir = os.getcwd()
    data_path = work_dir + "/data/ETF"

    index_dic = a1t1.clean_data(data_path)
    index_name = list(index_dic.keys())
    index_num = len(index_dic.keys())

    for tmp_index_name in index_name:
        index_dic[tmp_index_name] = index_dic[tmp_index_name].loc[
                                    pd.to_datetime("2010-01-01", format="%Y-%m-%d"):ff_daily.index[-1], ]
        print(index_dic[tmp_index_name].shape[0])

    print()

    # (b)

    print("Task 1(b)\n")

    ff_regress_name = ["Mkt-RF", "SMB", "HML"]

    etf_daily_adj_close = pd.DataFrame()
    for tmp_index_name, tmp_index_data in index_dic.items():
        etf_daily_adj_close[tmp_index_name] = tmp_index_data["Adj Close"]

    etf_daily_ret = a1t1.gen_daily_ret(etf_daily_adj_close)
    etf_name = list(etf_daily_ret)

    ff_daily = ff_daily.iloc[1:, ]
    ff_cov = np.cov(np.mat(ff_daily[ff_daily.columns[:-1]]).T)
    sns.set(font_scale=0.5)
    # a1t1.plot_sns(ff_cov, "Task1(b) Covariance Matrix of FF Factors", ff_daily.columns[:-1])
    # plt.savefig("a2_report/t1(b)_cov_ff.pdf")
    # plt.show()

    ff_corr = ff_daily[ff_daily.columns[:-1]].corr()
    # a1t1.plot_sns(ff_corr, "Task1(b) Correlation Matrix of FF Factors", ff_daily.columns[:-1])
    # plt.savefig("a2_report/t1(b)_corr_ff.pdf")
    # plt.show()

    print()

    # (c)

    print("Task 1(c)\n")

    # ninety_fac_corr = pd.DataFrame()
    # ninety_fac_corr["Mkt-RF VS SMB"] = ff_daily["Mkt-RF"].rolling(90).corr(ff_daily["SMB"]).dropna()
    # ninety_fac_corr["Mkt-RF VS HML"] = ff_daily["Mkt-RF"].rolling(90).corr(ff_daily["HML"]).dropna()
    # ninety_fac_corr["SMB VS HML"] = ff_daily["SMB"].rolling(90).corr(ff_daily["HML"]).dropna()

    ninety_fac_corr_dict={}

    for tmp_fac in ff_regress_name:
        ninety_fac_corr_dict[tmp_fac]=ff_daily[tmp_fac].rolling(90).corr(etf_daily_ret).dropna()
        ninety_fac_corr_dict[tmp_fac].plot()
        plt.ylabel("Correlation")
        plt.title("Task1(c) 90 Day Rolling Correlation between ETFs and "+tmp_fac)
        plt.savefig("a2_report/t1(c)_90roll_fac_corr_"+tmp_fac+".pdf")
        plt.show()
        print("Curve plot of 90 Day rolling correlation of factor returns successfully "
              "saved as 't1(c)_90roll_fac_corr_"+tmp_fac+".pdf' under directory 'a2_report'")

    # ninety_fac_corr.plot()
    # plt.legend(prop={'size': 6})
    # plt.ylabel("Correlation")
    # plt.title("Task1(c) 90 Day Rolling Correlation of Factor Returns")
    # plt.savefig("a2_report/t1(c)_90roll_fac_corr.pdf")
    # plt.show()
    # print("Curve plot of 90 Day rolling correlation of factor returns successfully "
    #       "saved as 't1(c)_90roll_fac_corr.pdf' under directory 'a2_report'")

    print()

    # (d)

    print("Task 1(d)\n")

    kstest_pval_dict = {}
    for tmp_factor_name in list(ff_daily):
        if tmp_factor_name != "RF":
            tmp_normaltest_pval = stats.normaltest(ff_daily[tmp_factor_name])[1]
            kstest_pval_dict[tmp_factor_name] = tmp_normaltest_pval

    print(kstest_pval_dict)
    print()

    # (e)

    print("Task 1(e)\n")

    ff_daily["Mkt"] = ff_daily["Mkt-RF"] + ff_daily["RF"]
    for tmp_etf_name in list(etf_daily_ret):
        etf_daily_ret[tmp_etf_name] -= ff_daily["RF"]
    entire_beta_dict, entire_alpha_dict, entire_residual_dict = ff_3factor_rolling_beta(etf_daily_ret,
                                                                                ff_daily[ff_regress_name],
                                                                                etf_daily_ret.shape[0])

    roll_beta_dict, roll_alpha_dict, roll_residual_dict = ff_3factor_rolling_beta(etf_daily_ret,
                                                                                ff_daily[ff_regress_name], 90)

    # alpha_plus_RF = roll_alpha_dict["XLI"] + ff_daily["RF"]
    # alpha_plus_RF.dropna(axis=0, how='any', inplace=True)
    # alpha_plus_RF.plot(ylim=[-0.05, 0.05])

    # roll_residual_dict["XLI"].plot()
    # plt.show()

    for tmp_new_ff in list(ff_regress_name):
        print(tmp_new_ff)
        print(entire_beta_dict[tmp_new_ff])

    for tmp_beta_df_name in roll_beta_dict.keys():
        roll_beta_dict[tmp_beta_df_name].plot()
        plt.ylabel("beta")
        plt.title("Task1(e) 90 Day Rolling Beta "+tmp_beta_df_name)
        plt.legend(prop={'size': 3})
        plt.savefig("a2_report/t1(e)_90roll_beta_"+tmp_beta_df_name+".pdf")
        plt.show()
        # plt.show()

    print()

    # (f)

    print("Task 1(f)\n")

    residual_mean_dict = {}
    residual_var_dict = {}
    residual_normaltest_pval_dict = {}

    for tmp_etf_name in etf_name:
        residual_mean_dict[tmp_etf_name] = np.mean(entire_residual_dict[tmp_etf_name])
        residual_var_dict[tmp_etf_name] = np.var(entire_residual_dict[tmp_etf_name])
        residual_normaltest_pval_dict[tmp_etf_name] = stats.normaltest(entire_residual_dict[tmp_etf_name])[1]

    print(residual_mean_dict)
    print(residual_var_dict)
    print(residual_normaltest_pval_dict)


# STEP 4

import pandas as pd

def _classify_file(filename, template_list):
    df = pd.read_csv(filename, dtype=str)
    for template in template_list:
        df_t = df.ix[df['EventId'] == template]
        df_t.to_csv(filename[: -4] + '_' + template + '.csv', index=False)

def _template_count(filename):
    df = pd.read_csv(filename, dtype=str)
    return df['EventId'].value_counts().to_dict()

def classify(train_file, validate_file, test_file, tot_syms, train_min, validate_min, test_min, dataset='HDFS'):
    if dataset == 'HDFS':
        print("== STEP 4 ==")
    else:
        print("== STEP 2 ==")
    train_tc = _template_count(train_file)
    validate_tc = _template_count(validate_file)
    test_tc = _template_count(test_file)

    valid_templates = []
    for i in range(tot_syms):
        template = 'E%d' % i
        if template in train_tc.keys() and train_tc[template] >= train_min \
            and template in validate_tc.keys() and validate_tc[template] >= validate_min \
            and template in test_tc.keys() and test_tc[template] >= test_min:
            valid_templates.append(template)

    _classify_file(train_file, valid_templates)
    _classify_file(validate_file, valid_templates)
    _classify_file(test_file, valid_templates)

    return valid_templates


import random

def num_generator():
    # returns 10 numbers between 1 and 100
    for i in range(6):
        yield random.randint(1, 100)

    # returns a 7th number between 10 and 20
    yield random.randint(10,20)

for random_number in num_generator():
       print("And the next number is... %d!" %(random_number))