from django.urls import path

from staff.views import members
#from staff.views import stripe

app_name = 'staff'
urlpatterns = [
     path('members/', members.members, name='members'),
     path('members/<group>/', members.members, name='member_group'),
     path('bcc/', members.bcc_tool, name='bcc_tool'),
     path('bcc/<group>/', members.bcc_tool, name='group_bcc'),
     path('deposits/', members.security_deposits, name='deposits'),
     path('export/', members.export_users, name='export_users'),
     path('search/', members.member_search, name='search'),
     path('new_user/', members.new_user, name='new_user'),
     path('user_reports/', members.view_user_reports, name='user_reports'),
     path('slack_users/', members.slack_users, name='slack_users'),
     path('membership/<username>/', members.membership, name='membership'),
     path('confirm/<username>/<package>/<end_target>/<start_target>/<new_subs>/', members.confirm_membership, name='confirm'),
     path('edit_bill_day/<username>/', members.edit_bill_day, name='edit_bill_day'),
     path('organizations/', members.org_list, name='organizations'),
     path('organization/<int:org_id>/', members.org_view, name='organization'),
     path('files/<username>/', members.files, name='files'),
     path('detail/<username>/', members.detail, name='detail'),
     #path('detail/<username>/stripe/', stripe.Checkout.as_view(), name='stripe-checkout'),
]

# Copyright 2021 Office Nomads LLC (https://officenomads.com/) Licensed under the AGPL License, Version 3.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.gnu.org/licenses/agpl-3.0.html. Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="textfield_parser",
    version="0.0.29",
    author="Thorsten Hapke",
    author_email="thorsten.hapke@sap.com",
    description="Parsing Textfields",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/thhapke/textfield_parser",
    keywords = ['textfield parsing'],
    packages=setuptools.find_packages(),
    install_requires=[],
    classifiers=[
    	'Programming Language :: Python :: 3.5',
    	'Programming Language :: Python :: 3.6',
    	'Programming Language :: Python :: 3.7',
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
)


# PygameProject.py

import pygame
import random
pygame.font.init()

# Global constants
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
GREEN = (0, 255, 0)
RED = (255, 0, 0)
BLUE = (0, 0, 255)
WIDTH = 800
HEIGHT = 600
FONT = pygame.font.SysFont("Comic Sans", 50)
HEART = pygame.transform.scale(pygame.image.load("./assets/heart.png"), (35, 35))

class Player(pygame.sprite.Sprite):
    """" This class represents the sprite that the player controls. """

    def __init__(self):
        super().__init__()

        # Set the image of the sprite and scale it
        self.image = pygame.image.load("./assets/kirby.png")
        self.image = pygame.transform.scale(self.image, (40, 40))

        # Make a hitbox
        self.rect = self.image.get_rect()

        # Set the speed vector of player
        self.change_x = 0
        self.change_y = 0

        # Set the starting point
        self.rect.x = (WIDTH/2)
        self.rect.y = (HEIGHT/2)

    def update(self):
        """" Moves the player (arrow keys). """

        # Move the player according to its speed/vectors
        self.rect.x += self.change_x
        self.rect.y += self.change_y

        if self.rect.x < 0:
            self.rect.x = WIDTH
        if self.rect.x > WIDTH:
            self.rect.x = 0
        if self.rect.y < 0:
            self.rect.y = HEIGHT
        if self.rect.y > HEIGHT:
            self.rect.y = 0

    # Player-controlled movement:
    def go_left(self):
        """ Called when the user hits the left arrow. """
        self.change_x = -6

    def go_right(self):
        """ Called when the user hits the right arrow. """
        self.change_x = 6

    def go_up(self):
        """ Called when the user hits the up arrow. """
        self.change_y = -6

    def go_down(self):
        """ Called when the user hits the down arrow. """
        self.change_y = 6

    def stop_horizontal(self):
        """ Called when the user lets go of the left or right. """
        self.change_x = 0

    def stop_vertical(self):
        """ Called when the user lets go of up or down. """
        self.change_y = 0

class Bullet(pygame.sprite.Sprite):
    """" This class represents the bullets that will be launched at the player"""

    def __init__(self):
        super().__init__()

        # Set the image of the sprite and scale it
        self.image = pygame.image.load("./assets/bullet.png")
        self.image = pygame.transform.scale(self.image, (21, 21))

        # Make a hitbox
        self.rect = self.image.get_rect()

        # Set the speed vector of bullet
        self.change_x = random.randrange(-4, 4)
        self.change_y = random.randrange(-4, 4)

        if self.change_x == 0:
            self.change_x += 1
        elif self.change_y == 0:
            self.change_y += 1

        # Set the start point
        if self.change_x > 0:
            self.rect.x = random.randrange(0, 50)
        else:
            self.rect.x = random.randrange(WIDTH - 50, WIDTH)

        if self.change_y > 0:
            self.rect.y = random.randrange(0, 50)
        else:
            self.rect.y = random.randrange(HEIGHT - 50, HEIGHT)

    def update(self):
        """" Moves the bullet """

        # Move the bullet according to its speed/vectors
        self.rect.x += self.change_x
        self.rect.y += self.change_y

def main():
    """ Main Program. """
    pygame.init()

    # ----- SCREEN PROPERTIES -----
    size = (WIDTH, HEIGHT)
    screen = pygame.display.set_mode(size)
    pygame.display.set_caption("Dodgeball")

    # ----- LOCAL VARIABLES -----
    done = False
    clock = pygame.time.Clock()
    num_bullets = 15
    lives = 3
    score = 0

    # Create sprite groups and fill them
    all_sprites_group = pygame.sprite.Group()
    bullet_sprites_group = pygame.sprite.Group()
    player = Player()
    all_sprites_group.add(player)

    for i in range(num_bullets):
        bullet = Bullet()
        all_sprites_group.add(bullet)
        bullet_sprites_group.add(bullet)

    # ----- MAIN LOOP -----
    while not done:

        # -- Event Handler -----
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                done = True

            # Make the player move around depending on the arrow keys pressed down
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT:
                    player.go_left()
                if event.key == pygame.K_RIGHT:
                    player.go_right()
                if event.key == pygame.K_UP:
                    player.go_up()
                if event.key == pygame.K_DOWN:
                    player.go_down()

            if event.type == pygame.KEYUP:
                if event.key == pygame.K_LEFT and player.change_x < 0:
                    player.stop_horizontal()
                if event.key == pygame.K_RIGHT and player.change_x > 0:
                    player.stop_horizontal()
                if event.key == pygame.K_UP and player.change_y < 0:
                    player.stop_vertical()
                if event.key == pygame.K_DOWN and player.change_y > 0:
                    player.stop_vertical()

        # ----- LOGIC -----
        all_sprites_group.update()

        # Collide the player with any bullets
        collisions = pygame.sprite.spritecollide(player, bullet_sprites_group, dokill=True)

        # take away a life for every collision and lower the current number of bullets
        for collision in collisions:
            num_bullets -= 1
            lives -= 1

        # If a bullet goes out of bounds, delete it and lower the current number of bullets
        for bullet in bullet_sprites_group:
            if bullet.rect.x < 0 or bullet.rect.x > WIDTH:
                bullet.kill()
                num_bullets -= 1

            if bullet.rect.y < 0 or bullet.rect.y > HEIGHT:
                bullet.kill()
                num_bullets -= 1

        # Make sure there are always ten bullets
        if num_bullets < 15:

            # Replace the bullet
            bullet = Bullet()
            all_sprites_group.add(bullet)
            bullet_sprites_group.add(bullet)
            num_bullets += 1

        # ----- RENDER -----
        screen.fill(BLACK)
        all_sprites_group.draw(screen)

        # If the player is still alive
        if lives > 0:
            # Add 1 to the score every tick
            score += 1

            # Display a timer at the bottom right as the score
            screen.blit(FONT.render(f'Score: {score}', True, GREEN), (20, HEIGHT - 80))

        # If the player has three lives
        if lives == 3:
            # Display three hearts
            screen.blit(HEART, (0, 0))
            screen.blit(HEART, (45, 0))
            screen.blit(HEART, (90, 0))

        # If the player has two lives
        if lives == 2:
            # Display two hearts
            screen.blit(HEART, (0, 0))
            screen.blit(HEART, (45, 0))

        # If the player has one life left
        if lives == 1:
            # Display one heart
            screen.blit(HEART, (0, 0))

        # If the player runs out of lives
        if lives <= 0:
            # Kill the player
            player.kill()

            # Display a Game Over Screen and the final score
            screen.blit(FONT.render('Game Over', True, GREEN), ((WIDTH/2) - 120, (HEIGHT/2) - 50))
            screen.blit(FONT.render(f'Score: {score}', True, GREEN), (20, HEIGHT - 80))

        # ----- UPDATE DISPLAY -----
        pygame.display.flip()
        clock.tick(60)

    pygame.quit()

if __name__ == "__main__":
    main()


import asyncio
from itertools import chain

from httpx import AsyncClient, Timeout
from loguru import logger

from documents import upload_documents
from scrapy import ChankedQueue, get_all_url_from_html, scrape
from config import Config
from postprocess import build_bigram_words_dictionary

def get_new_urls(paths: set[str], visited: set[str]) -> set[str]:
    return {
        f'{Config.url}{path}'
        for path in paths
        if path.startswith('/wiki/') and path not in visited
    }

async def empty_coroutine() -> None:
    return

async def run():
    visited_paths = {f'{Config.url}{Config.start_path}'}
    urls_to_process = ChankedQueue()
    urls_to_process.append((f'{Config.url}{Config.start_path}',))

    upload_documents_task = empty_coroutine()

    session = AsyncClient(timeout=Timeout(100, connect=60))

    while len(visited_paths) < Config.count_documents:

        url_chunk = urls_to_process.popleft()
        scrape_tasks = [scrape(url, session=session, throttler=Config.throttler) for url in url_chunk]
        _, *results = await asyncio.gather(upload_documents_task, *scrape_tasks)

        visited_paths.update(url_chunk)
        paths_in_results = set(
            chain.from_iterable(
                get_all_url_from_html(result.text)
                for result in results
                if result
            )
        )
        urls_to_process.extend(get_new_urls(paths_in_results, visited_paths))

        upload_documents_task = upload_documents(results)

    await upload_documents_task
    try:
        await session.aclose()
    except Exception as e:
        logger.error('Wtff {}', e)

    logger.debug('finished downloading, start build dictionary and bigrams')

    await build_bigram_words_dictionary()

    logger.debug("finished scraping")

if __name__ == "__main__":
    asyncio.run(run())


from django.core import mail
from django.test import TestCase, SimpleTestCase, Client

# A test case for the home page.
class HomePageTestCase(SimpleTestCase):
    # Set up the class for testing.
    @classmethod
    def setUpClass(cls):
        # Call the superclass' setUpClass() method.
        super().setUpClass()

        # Create a client object for the class to make requests.
        cls.client = Client()

    # Test get requests for the home page.
    def test_get(self):
        # Send a get request without a trailing forward slash.
        response = self.client.get('')

        # Ensure that the response has the correct status code and the proper templates were used.
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'lens/base.html')
        self.assertTemplateUsed(response, 'lens/home.html')

        # Send a get request with a trailing forward slash.
        response = self.client.get('/')

        # Ensure that the response has the correct status code and the proper template was rendered.
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'lens/base.html')
        self.assertTemplateUsed(response, 'lens/home.html')

# A test case which tests whether emails can successfully be sent with AWS SES.
class SendEmailTestCase(TestCase):
    # Set up the class for testing.
    @classmethod
    def setUpClass(cls):
        # Call the superclass' setUpClass() method.
        super().setUpClass()

        # Add test data to the class.
        cls.email_subject = 'Lens Email Test'
        cls.email_body = 'This is a test email for programmatic email delivery from Lens with AWS SES'
        cls.email_sender = 'Lens Email Test Service <email-test@epm0dev.me>'
        cls.email_recipients = ['epmancin@ncsu.edu']

    # Test whether or not an email can be sent successfully and whether or not it's contents are correct in the outbox.
    def test_send_email(self):
        # Send an email with the class' test data and store the status that is returned.
        status = mail.send_mail(
            self.email_subject,
            self.email_body,
            self.email_sender,
            self.email_recipients,
            fail_silently=False,
        )

        # Ensure that the returned status code is not 0 (i.e. the email was successfully sent).
        self.assertNotEqual(status, 0)

        # Ensure that the length of the mail outbox is correct.
        self.assertEqual(len(mail.outbox), 1)

        # Ensure that the contents of the email are correct.
        self.assertEqual(mail.outbox[0].subject, self.email_subject)
        self.assertEqual(mail.outbox[0].body, self.email_body)


import sys
import time
import random
import pickle
import itertools
import logging
import argparse
import uuid
from math import exp
from multiprocessing import Pool
from utils import format_xyz, tmer2_gmtkn_parser, get_charges_multiplicities
from plots import learning_curve, shap_analysis
from hyperopt import hp, Trials, fmin, tpe, STATUS_OK
from hyperopt.pyll import scope
from pyscf import dft
from pyscf import gto

def calc_E(params, geom, basis, charge, multiplicity, tmp_id):

    mol = gto.M(atom=geom, basis=basis, charge=charge, spin=multiplicity)
    mr = dft.UKS(mol)
    mr.init_guess = tmp_id
    mr.max_cycle = 1000
    mr.xc = '{}*SR_HF({}) + {}*LR_HF({}) + {}*XC_LDA_X + ' \
            '{}*XC_GGA_X_B88 + {}*XC_GGA_X_PBE + {}*XC_GGA_X_B86 + ' \
            '{}*XC_GGA_X_PW91 + {}*XC_GGA_X_SOGGA, {}*XC_LDA_C_XALPHA + ' \
            '{}*XC_LDA_C_VWN + {}*XC_LDA_C_RPA + {}*XC_GGA_C_PBE + ' \
            '{}*XC_GGA_C_PW91 + {}*XC_GGA_C_LYP'.format(
        params['param_00'], params['param_01'], params['param_02'],
        params['param_01'], params['param_03'], params['param_04'],
        params['param_05'], params['param_06'], params['param_07'],
        params['param_08'], params['param_09'], params['param_10'],
        params['param_11'], params['param_12'], params['param_13'],
        params['param_14'], params['param_15'],
    )

    E = mr.kernel()

    return E

def calc_mol(geom, params, charge, multiplicity):
    tmp_id = 'tmp/' + str(uuid.uuid4())

    mol = gto.M(atom=geom, basis='ccpvdz', charge=charge, spin=multiplicity)
    mf = dft.UKS(mol)
    mf.xc = 'PBE'
    mf.chkfile = tmp_id
    mf.kernel()

    basis2 = 'augccpvdz'
    basis3 = 'augccpvtz'

    E2 = calc_E(params, geom, basis2, charge, multiplicity, tmp_id)
    E3 = calc_E(params, geom, basis3, charge, multiplicity, tmp_id)

    if E3 < E2:
        E_total = E3 - 0.42*(E2 - E3)
        logging.info('D-zeta E = {0:7.2f}, T-zeta E = {1:7.2f}, CBS E = {2:7.2f}'.format(E2, E3, E_total))

    else:
        logging.warning("No CBS here")
        E_total = E2

    return E_total

def calc_reaction(list_of_geoms, list_of_coefficients, E_real, charges, multiplicities, reaction, params):

    E = 0
    pool = Pool(4)
    list_of_energies = list(pool.starmap(
        calc_mol, zip(list_of_geoms, itertools.repeat(params), charges, multiplicities)))
    pool.close()
    pool.join()

    for i in range(len(list_of_energies)):
        E += list_of_coefficients[i] * list_of_energies[i]

    dE = abs(E - E_real)

    logging.info("Reaction: {}".format(reaction))
    logging.info("Calculated value: {:.4f}".format(E))
    logging.info("Reference value: {:.4f}".format(E_real))

    return dE, reaction

def hyperopt_search(dataset, param_grid, n_iter, sample, trials_init):

    if trials_init:
        with open(trials_init, 'rb') as f:
            trials = pickle.load(f)
            n_trials_init = len(trials.trials)
    else:
        trials = Trials()
        n_trials_init = 0

    def objective(params):
        all_data = tmer2_gmtkn_parser("datasets/{}".format(dataset))
        sampled_data = random.sample(all_data, sample)
        sampled_systems = [k["atoms"] for k in sampled_data]
        sampled_stoichiometry = [k["stoichiometry"] for k in sampled_data]
        sampled_reference_value = [k["reference_value"] / 627.509 for k in sampled_data]
        sampled_charges = [k["charges"] for k in sampled_data]
        sampled_multiplicities = [k["multiplicities"] for k in sampled_data]
        sampled_reactions = [k["reaction"] for k in sampled_data]

        logging.info("New iteration...")
        start = time.time()

        losses = []
        reactions = []

        for syst, stoich, ref_v, charges, multi, reacs in zip(sampled_systems, sampled_stoichiometry,
                                                              sampled_reference_value, sampled_charges,
                                                              sampled_multiplicities, sampled_reactions):
            output = calc_reaction(syst, stoich, ref_v, charges, multi, reacs, params)
            losses.append(output[0])
            reactions.append(output[1])

        reactions_losses = dict(zip(reactions, losses))

        logging.info("Total loss: {:.4f}".format(sum(losses)))
        logging.info("Iteration took {:.1f} s\n".format(time.time() - start))

        return {'loss': sum(losses) / sample, 'params': params, 'reactions_losses': reactions_losses, 'status': STATUS_OK}

    current_time = time.strftime("%Y_%m_%d__%H_%M_%S", time.localtime())
    trials_filename = "{0}_{1}_iters_{2}_in_batch_{3}.pickle".format(dataset, n_iter, sample, current_time)

    for n in range(n_trials_init + 1, n_iter + 1):
        try:
            fmin(objective, param_grid, algo=tpe.suggest, trials=trials, max_evals=n)
            with open(trials_filename, 'wb') as f:
                pickle.dump(trials, f)
        except IndexError as err:
            print(err)
            with open(trials_filename, 'rb') as f:
                trials = pickle.load(f)

    return trials_filename

HYPERPARAMETERS = {
    'param_00': hp.uniform('param_00', 0.0, 1.0),
    'param_01': hp.uniform('param_01', 0.0, 1.0),
    'param_02': hp.uniform('param_02', 0.0, 1.0),
    'param_03': hp.uniform('param_03', -1.0, 1.0),
    'param_04': hp.uniform('param_04', -1.0, 1.0),
    'param_05': hp.uniform('param_05', -1.0, 1.0),
    'param_06': hp.uniform('param_06', -1.0, 1.0),
    'param_07': hp.uniform('param_07', -1.0, 1.0),
    'param_08': hp.uniform('param_08', -1.0, 1.0),
    'param_09': hp.uniform('param_09', -1.0, 1.0),
    'param_10': hp.uniform('param_10', -1.0, 1.0),
    'param_11': hp.uniform('param_11', 0.0, 1.0),
    'param_12': hp.uniform('param_12', -1.0, 1.0),
    'param_13': hp.uniform('param_13', -1.0, 1.0),
    'param_14': hp.uniform('param_14', -1.0, 1.0),
    'param_15': hp.uniform('param_15', -1.0, 1.0),
}

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('dataset', type=str)
    parser.add_argument('-i', '--iters', type=int, default=100)
    parser.add_argument('-b', '--batch', type=int, default=1)
    parser.add_argument('-t', '--trials', default=None)
    args = parser.parse_args()

    logfile = "{0}_{1}_iters_{2}_in_batch_{3}.log".format(
        args.dataset, args.iters, args.batch, time.strftime("%Y_%m_%d__%H_%M_%S", time.localtime()))

    logging.basicConfig(
        filename=logfile, filemode='w', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        level=logging.INFO
    )

    trials_filename = hyperopt_search(args.dataset, HYPERPARAMETERS, args.iters, args.batch, args.trials)
    learning_curve(trials_filename, title=args.dataset)
    shap_analysis(trials_filename, title=args.dataset)


from django.shortcuts import render

# Create your views here.
def home(request):
    person=[{'name':'vikash','age':23},{'name':'vikash','age':23}]
    for persons in person:
        print(persons)
    return render(request ,"index.html",context={'person':person})
def number(request):
    list=[1,2,3,4,5,6]
    for lists in list:
        print(lists)
    return render(request , "index.html", context={'list':list})


import json
import os
import random
from argparse import ArgumentParser

import torch
from tqdm import tqdm

from shadow_program_inversion.experiments.contact.common import ScenarioFactory
from shadow_program_inversion.priors.path_generator import PathGeneratorMoveLinear
from shadow_program_inversion.priors.static_prior import StaticPrior
from shadow_program_inversion.utils.io import save_simulation_data_to_file
from shadow_program_inversion.utils.sequence_utils import chunk
import shadow_program_inversion.utils.config as cfg

TRAJECTORY_LENGTH = 250

def main(args):
    scenario = ScenarioFactory.make_scenario(args.material)
    config_path = os.path.join(cfg.DATA_DIR, "config", f"config_urscript_{args.material}.json")
    ml_simulator = StaticPrior(PathGeneratorMoveLinear(), sampling_interval=0.016, multiproc=True)
    with open(config_path) as config_file:
        config = json.load(config_file)
    batch_size = (torch.multiprocessing.cpu_count() - 1) * 8
    for kind in ["train", "test"]:
        output_filepath = os.path.join(cfg.DATA_DIR, "urscript", args.material, f"sim_{kind}",
                                       f"urscript_{args.material}_{kind}.h5")
        num_data = int(0.8 * args.num_data) if kind == "train" else int(0.2 * args.num_data)
        for batch in tqdm(chunk(range(num_data), batch_size), total=int(num_data / batch_size)):
            inputs = []
            points_from = []
            for _ in batch:
                approach_pose = scenario.approach_pose()
                vel = random.uniform(*config["velocity"]["limits"]["cartesian"])
                acc = random.uniform(*config["acceleration"]["limits"]["cartesian"])
                z_offset = random.uniform(*config["point_to_offset"]["limits"]["z"])
                goal_pose = scenario.goal_pose()
                goal_pose.position.z += z_offset
                input_tensor = torch.cat((torch.as_tensor(goal_pose.parameters()), torch.as_tensor([vel, acc])), dim=-1)
                inputs.append(input_tensor)
                point_from_tensor = torch.as_tensor(approach_pose.parameters())
                points_from.append(point_from_tensor)
            input_batch = torch.stack(inputs)
            points_from_batch = torch.stack(points_from)
            simulations = ml_simulator.simulate(input_batch, points_from_batch,
                                                max_trajectory_len=TRAJECTORY_LENGTH, cache=False)
            save_simulation_data_to_file(output_filepath, input_batch, points_from_batch, simulations, None)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument("material", type=str, choices=["foam", "pcb", "rubber"])
    parser.add_argument("num_data", type=int)
    main(parser.parse_args())


import logging
from typing import List

import anndata as ad
import pandas as pd
import scanpy as sc
from sklearn.cluster import MiniBatchKMeans, SpectralClustering, AgglomerativeClustering

import config

def split_to_major_cluster(adata_with_scvi_emb: ad.AnnData, emb_col_name: str,
                           clustering_model_name: str, num_clusters: int,
                           col_name_to_split_by:str="major_clusters") -> List[ad.AnnData]:
    logging.info(f"spliting adata to {num_clusters} clusters, using {clustering_model_name}")
    if clustering_model_name == "kmeans":
        clustering_model = MiniBatchKMeans(n_clusters=num_clusters, verbose=1)
        adata_with_scvi_emb.obs[col_name_to_split_by] = clustering_model.fit(
            adata_with_scvi_emb.obsm[emb_col_name]).labels_
    elif clustering_model_name == "spectral":
        clustering_model = SpectralClustering(n_clusters=num_clusters, affinity='nearest_neighbors', n_jobs=10)
        adata_with_scvi_emb.obs[col_name_to_split_by] = clustering_model.fit(
            adata_with_scvi_emb.obsm[emb_col_name]).labels_
    elif clustering_model_name == "agglomerative":
        clustering_model = AgglomerativeClustering(n_clusters=num_clusters, linkage='ward')
        adata_with_scvi_emb.obs[col_name_to_split_by] = clustering_model.fit(
            adata_with_scvi_emb.obsm[emb_col_name]).labels_
    elif clustering_model_name == "leiden_and_agglomerative":
        clustering_model = AgglomerativeClustering(n_clusters=num_clusters, linkage='ward')

        sc.pp.neighbors(adata_with_scvi_emb, use_rep=emb_col_name)
        leiden_col_name = "leiden_for_major_cluster"
        sc.tl.leiden(adata_with_scvi_emb, resolution=config.TL_LEIDEN_RESOLUTION * 1.5, key_added=leiden_col_name)

        all_cells_df = pd.DataFrame(index=adata_with_scvi_emb.obs_names,
                                    data=adata_with_scvi_emb.obsm[emb_col_name])
        all_cells_df[leiden_col_name] = adata_with_scvi_emb.obs[leiden_col_name]
        df_of_leiden_means = all_cells_df.groupby(by=leiden_col_name).mean()
        leiden_labels_col_name = "leiden_labels"
        df_of_leiden_means[leiden_labels_col_name] = clustering_model.fit(df_of_leiden_means).labels_
        cells_cluster = pd.merge(left=all_cells_df, right=df_of_leiden_means, how="left",
                                 left_on=leiden_col_name, right_index=True)[leiden_labels_col_name]
        adata_with_scvi_emb.obs[col_name_to_split_by] = cells_cluster
    else:
        raise NotImplementedError

    adatas = []
    for group, idx in adata_with_scvi_emb.obs.groupby(col_name_to_split_by).indices.items():
        sub_adata = adata_with_scvi_emb[idx].copy()
        adatas.append(sub_adata)

    return adatas


import PyPDF2

# Open the PDF file in read-only mode
with open(r"C:\Users\nharw\Desktop\PDF2Anki Project\Cho Chikun LD1.pdf", 'rb') as file:
    # Create a PDF object
    pdf = PyPDF2.PdfReader(file)

    # Iterate over every page
    for page in range(len(pdf.pages)):
        # Extract the text from the page
        text = pdf.pages[page].extract_text()

        # Save the text to a file
        with open(f'page_{page}.txt', 'w') as output:
            output.write(text)


"""not_selected_last_time

Revision ID: 29e59a93a93a
Revises: f5ef084d728b
Create Date: 2019-07-01 02:02:51.131299

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '29e59a93a93a'
down_revision = 'f5ef084d728b'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('raffle_configuration', sa.Column('not_selected_last_time_guaranteed_entry', sa.Boolean(), nullable=False))
    op.add_column('raffle_configuration', sa.Column('not_selected_last_time_increased_chance', sa.Boolean(), nullable=False))
    op.add_column('system_configuration', sa.Column('additional_teamcaptain_large_teams', sa.Boolean(), nullable=False))
    op.add_column('system_configuration', sa.Column('additional_teamcaptain_large_teams_cutoff', sa.Integer(), nullable=False))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('system_configuration', 'additional_teamcaptain_large_teams_cutoff')
    op.drop_column('system_configuration', 'additional_teamcaptain_large_teams')
    op.drop_column('raffle_configuration', 'not_selected_last_time_increased_chance')
    op.drop_column('raffle_configuration', 'not_selected_last_time_guaranteed_entry')
    # ### end Alembic commands ###


from odoo import api, fields, models, exceptions
from datetime import date

# some attributes can impact the database schema
# some attributes can impact the view

def default_available_from(arg):

	today_date = date.today()
	if (today_date.month >= 9):
		return date(today_date.year + 1, 9, 1)
	else:
		return date(today_date.year, 9, 1)

# the dependency triggers a recomputation when one of the fields is changed
@api.depends('available_to', 'available_from')
def _compute_total_price(self):

	for record in self:
		if (self.available_from is not None and self.available_to is not None):
			daily_price = (self.monthly_price * 12) / 365
			self.total_price = daily_price * (self.available_to - self.available_from).days
		else:
			self.total_price = None

@api.depends('offer_ids.monthly_price')
def _compute_best_monthly_price(self):

	if (self.offer_ids):
		self.best_monthly_price = min(self.offer_ids.mapped('monthly_price'))
	else:
		self.best_monthly_price = 0

def default_available_to(arg):

	today_date = date.today()
	if (today_date.month >= 9):
		return date(today_date.year + 2, 9, 1)
	else:
		return date(today_date.year + 1, 9, 1)

# modifies one record at a time in the form view when grass is checked

class SportsField(models.Model):

	_name = 'sports_field'
	_description = 'Properties of the potential sports fields to rent'
	_order = 'id desc'

	name = fields.Char(size=50, required=True)
	description = fields.Text()
	postcode = fields.Char(size=10)
	yearly_days_off = fields.Integer(default=0)
	available_from = fields.Date(required=True, default=default_available_from)
	available_to = fields.Date(required=True, default = default_available_to)
	monthly_price = fields.Float(required=True)
	grass = fields.Boolean()
	booked_from = fields.Date(copy=False, readonly=True)
	booked_to = fields.Date(copy=False, readonly=True)
	total_price = fields.Float(compute=_compute_total_price)
	final_total_price = fields.Float(copy=False, readonly=True)
	active = fields.Boolean(default=True)
	state = fields.Selection(
		selection=[('new', 'New'), ('offer_sent', 'Offer Sent'),
					('offer_accepted', 'Offer Accepted'), ('signed', 'Signed'),
					('canceled', 'Canceled')],
		default='new',
		required=True,
		copy=False
	)
	best_monthly_price = fields.Float(compute=_compute_best_monthly_price)
	_sql_constraints = [
		('check_monthly_price', 'CHECK(monthly_price >= 0)',
			'Monthly price should be greater or equal to 0.'),
	]

	### relations

	sports_field_type_id = fields.Many2one(
		'sports_field_type',
		string="Sports field type",
	)
	owner_id = fields.Many2one(
			'res.partner', string="Owner",
			help='The owner of the field',
			copy=False
	)
	# finder: defaults to the current user
	finder_id = fields.Many2one(
			'res.users', string='Finder',
			help="The club member that found the lead",
			default=lambda self: self.env.user
	)
	tag_ids = fields.Many2many('sports_field_tag', string="Tags")
	# one field can have many offers
	offer_ids = fields.One2many('sports_field_offer', 'field_id', string="Offers")

	@api.onchange('offer_ids')
	def _onchange_offer_ids(self):

		if (self.state == 'new'):
			self.state = 'offer_sent'
		if (len(self.offer_ids) == 0):
			self.state = 'new'

	@api.onchange('grass')
	def _onchange_grass(self):
		self.yearly_days_off = 10

	def action_sold(self):

		for record in self:
			if (record.state != 'canceled'):
				record.state = 'signed'
			else:
				raise exceptions.UserError('Canceled fields cannot be set as signed.')

	def action_cancel(self):

		for record in self:
			if (record.state != 'signed'):
				record.state = 'canceled'
			else:
				raise exceptions.UserError('Signed fields cannot be set as canceled.')


import base64
import time
from ast import literal_eval
from base64 import b64decode, b64encode
from hashlib import md5
import serial
import tensorflow as tf
import numpy as np
from Cryptodome.Protocol.KDF import PBKDF2
from Cryptodome.Hash import SHA512
from flask import Flask, request, render_template, session
from Cryptodome.Cipher import AES
from Cryptodome.Random import get_random_bytes
from Cryptodome.Util.Padding import pad, unpad
from datetime import datetime
import pandas as pd
import bchlib
import matplotlib.pyplot as plt
import numpy as np
import math
from ModelV3 import AgentRestore

app = Flask(__name__)
tf.random.set_seed(1)
np.random.seed(1)

key_length = 10
attribute_length = key_length
num_agent = 2
action_bound = 1
app.secret_key = '101'

class AESCipher:
    def __init__(self, key):
        self.key = md5(key.encode('utf8')).digest()

    def encrypt(self, data):
        iv = get_random_bytes(AES.block_size)
        cipher = AES.new(self.key, AES.MODE_CBC, iv)
        return b64encode(iv + cipher.encrypt(pad(data.encode('utf-8'), AES.block_size))).decode('utf-8')

    def decrypt(self, data):
        raw = b64decode(data.encode('utf-8'))
        cipher = AES.new(self.key, AES.MODE_CBC, raw[:AES.block_size])
        return unpad(cipher.decrypt(raw[AES.block_size:]), AES.block_size).decode('utf-8')

def get_keys(obs, action):
    feature_vector = np.zeros(len(action))
    key = np.zeros((len(feature_vector), 2))

    for i in range(len(feature_vector)):
        if action[i] >= 0:
            feature_vector[i] = 1

        if feature_vector[i] == 1:
            if obs[i] >= np.mean(obs) + np.std(obs):
                key[i][0] = 1
                key[i][1] = 1
            elif np.mean(obs) + np.std(obs) > obs[i] > np.mean(obs) - np.std(obs):
                key[i][0] = 0
                key[i][1] = 1
            elif obs[i] <= np.mean(obs) - np.std(obs):
                key[i][0] = 1
                key[i][1] = 0

    key = np.array(key, dtype=int)
    key = np.reshape(key, (1, -1))[0]

    return key, feature_vector

def get_physical_dynamics(data, idx):
    pressure = np.zeros(attribute_length)

    if data.shape[1] > 1:
        for i in range(len(data)):
            pressure[i] = float(data[i][2])
    else:
        for i in range(len(data)):
            row = str(data[i]).split(';')
            pressure[i] = float(row[2])

    try:
        min_pressure = min(pressure)
        max_pressure = max(pressure)
        if max_pressure - min_pressure != 0:
            pressure = (pressure - min_pressure) / (max_pressure - min_pressure)
    except:
        pressure[:] = np.nan

    return pressure

def calculate_entropy(key):
    num_possible_combinations = 2 ** len(key)
    return len(key) * math.log2(num_possible_combinations)

def plot_entropy_graph(key_name, entropies):
    plt.hist(entropies, bins=20, color='blue', edgecolor='black')
    plt.xlabel('Entropy (bits)')
    plt.ylabel('Frequency')
    plt.title(f'Entropy Distribution of {key_name} Key')
    plt.grid(True)
    plt.show()

@app.route('/', methods=['GET', 'POST'])
def process_data():
    encrypted_info = ''
    key_alice = ''
    decrypted_info = ''
    key_bob = ''
    BCH_POLYNOMIAL = 8219
    BCH_BITS = 10
    bch = bchlib.BCH(BCH_POLYNOMIAL, BCH_BITS)

    epoch = 15000
    alice = AgentRestore(name='./R1/Alice', num_obs=attribute_length, num_act=key_length, num_agent=num_agent,
                         epoch=epoch)
    bob = AgentRestore(name='./R1/Bob', num_obs=attribute_length, num_act=key_length, num_agent=num_agent, epoch=epoch)

    sender = serial.Serial("COM9", 9600)
    receiver = serial.Serial("COM11", 9600)
    # Get the current value of idx from the session, default to 1 if not set
    idx = session.get('idx', 1)
    if request.method == 'POST':
        if 'message' in request.form:
            # Encryption
            data = request.form['message']
            current_time = datetime.now().strftime("%H:%M:%S")
            time.sleep(15)
            df = pd.read_csv('uP428.csv')

            for i in range(len(df)):
                row = df.values[i][0]
                row = str(row).split(';')[0:6]
                time_idx = row[0]
                if current_time == time_idx:
                    break
            pressure = get_physical_dynamics(df.values[idx:idx + attribute_length, :], idx)
            obs_alice = pressure
            print(obs_alice)
            act_alice = tf.squeeze(alice.actor(tf.expand_dims(tf.convert_to_tensor(obs_alice), 0))).numpy()
            key_alice, _ = get_keys(obs_alice, act_alice)
            key_alice_bytearray = bytearray(key_alice)
            key_alice_ecc = bch.encode(key_alice_bytearray)
            key_alice_ecc_temp = [i for i in key_alice_ecc]
            key_alice = ''.join(str(key_alice[i]) for i in range(len(key_alice)))
            print(key_alice)
            key_alice.encode('utf-8')
            salt = bytes.fromhex("10101010101010101010")
            # Convert the integer (idx) to its binary representation
            idx_binary = bin(int(idx))[2:]  # [2:] to remove the '0b' prefix
            # Convert the binary string to bytes
            idx_binary_bytes = int(idx_binary, 2).to_bytes((len(idx_binary) + 7) // 8, byteorder='big')
            salt = salt + idx_binary_bytes
            key_alice_stretched = PBKDF2(key_alice, salt, 16, count=10000, hmac_hash_module=SHA512)
            key_alice_binary = b''.join(format(byte, '08b').encode('utf-8') for byte in key_alice_stretched)
            key_alice_binary_str = key_alice_binary.decode('utf-8')  # Convert bytes to string
            key_alice = key_alice_binary_str
            encrypted_info = AESCipher(key_alice).encrypt(data)
            send_info = str.encode(str(current_time) + ';' + str(key_alice_ecc_temp) + ';' + encrypted_info + '\n')
            sender.write(send_info)
            idx = idx + attribute_length

            # Calculate entropy for Alice's key
            entropy_alice = calculate_entropy(key_alice)

            # Plot the entropy graph for Alice's key
            plot_entropy_graph("Alice's", entropy_alice)

            # Decryption
            return_val = receiver.readline().decode().strip()
            return_val = return_val.split(';')
            current_time = str(return_val[0])
            ecc = literal_eval(return_val[1])
            ecc = bytearray(ecc)
            received_info = return_val[2]

            df = pd.read_csv('uP428.csv')
            for i in range(len(df)):
                row = df.values[i][0]
                row = str(row).split(';')[0:6]
                time_idx = row[0]
                if current_time == time_idx:
                    break

            pressure = get_physical_dynamics(df.values[idx:idx + attribute_length, :], idx)
            obs_bob = pressure
            act_bob = tf.squeeze(bob.actor(tf.expand_dims(tf.convert_to_tensor(obs_bob), 0))).numpy()
            key_bob, _ = get_keys(obs_bob, act_bob)
            key_bob_bytearray = bytearray(key_bob)
            bitflips = bch.decode_inplace(key_bob_bytearray, ecc)

            if bitflips <= BCH_BITS and bitflips >= 0:
                key_bob_final = bch.decode(key_bob, ecc)
                key_bob_final = key_bob_final[1]
                key_bob_final = np.array(key_bob_final)
                key_bob_final = np.reshape(key_bob_final, (-1, 4))
                key_bob_final = key_bob_final[:, 0]
                key_bob = ''.join(str(key_bob_final[i]) for i in range(len(key_bob_final)))
                print(key_bob)
                key_bob_stretched = (PBKDF2(key_bob, salt, 16, count=10000, hmac_hash_module=SHA512))
                key_bob_binary = b''.join(format(byte, '08b').encode('utf-8') for byte in key_bob_stretched)
                key_bob_binary_str = key_bob_binary.decode('utf-8')  # Convert bytes to string
                key_bob = key_bob_binary_str
                decrypted_info = AESCipher(key_bob).decrypt(received_info)
            session['idx'] = idx + attribute_length

    return render_template('index.html', encrypted_data=encrypted_info, key_alice=key_alice,
                           decrypted_data=decrypted_info, key_bob=key_bob)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=80, debug=True)


from kivy.logger import Logger as log

G = globals()
if 'store' in G:
    store = G['store']
else:
    store = {}

def get_store() -> dict:
    return store

def add(key, value) -> bool:
    if key in store:
        log.warn(f'The {value} was not added, because key {key} already exists.')
        return False
    else:
        store[key] = value
        log.info(f'Added {value} to store under key {key}.')
        return True

def add_or_update(key, value):
    if key in store:
        store.update({key: value})
        log.info(f'Update key {key} in store with value {value}.')
    else:
        store[key] = value
        log.info(f'Added new value {value} to store under key {key}.')

def delete(key) -> bool:
    if key in store:
        del store[key]
        log.info(f'Delete key {key} from store')
        return True
    else:
        log.warn(f'There is no key {key} in store.')
        return False

def update(key, value) -> bool:
    if key in store:
        store.update({key: value})
        log.info(f'Update key {key} in store with value {value}.')
        return True
    else:
        log.warn(f'The key {key} was not update, because there is no such a key in store.')
        return False

def get(key):
    if key in store:
        log.info(f"Getting value from store[{key}].")
        return store[key]
    else:
        return None


"""
This file will contain general information and operations we need
for our game.
  1. SCORES_FILE_NAME - A string representing a file name. By default “Scores.txt”
  2. BAD_RETURN_CODE - A number representing a bad return code for a function.
  3. Screen_cleaner - A function to clear the screen (useful when playing memory game or
before a new game starts).

"""
import subprocess
from time import sleep
from random import uniform
import os, platform, sys
import MainScores
from datetime import datetime
import threading

##############
# Globals
##############
MAIN_DIRECTORY = os.path.dirname(__file__)
SCORES_FILE_NAME=f"{MAIN_DIRECTORY}/scores.txt"
USER_NAME="Shay"
BAD_RETURN_CODE=0
HOST_IP="127.0.0.1"
HOST_FLASK_PORT=5000
HOME_URL=f"http://{HOST_IP}:{HOST_FLASK_PORT}/"
HOME_PATH = os.path.abspath("./")
REPLAY = True

# datetime object containing current date and time
NOW = datetime.now()
# dd/mm/YY H:M:S
DT_STRING = NOW.strftime("%d/%m/%Y %H:%M:%S")

# Generic Error message
ERROR_MSG = "No Errors"

def reset_error_msg():
  ERROR_MSG = "No Errors"

# WEB PAGE ELEMENT
ELEMENT_ID_SCORE="score"
def screen_cleaner():
  # Clearing the Screen
  os.system('cls')

def start_web_server():
  #web_app = f"py.exe {MAIN_DIRECTORY}\MainScores.py"
  web_app = f"{MAIN_DIRECTORY}\MainScores.py"
  print(f"\nStarting the web app from: {web_app}")
  #thr =  threading.Thread(target=MainScores.main())
  #thr.start()
  #threading.Thread(MainScores.main())
  #os.system(f"py.exe {web_app}")
  #subprocess.Popen([os.system(web_app)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  subprocess.Popen(["py.exe", web_app], stdin=None, stdout=None, stderr=None)

def generic_error():
    print("Not a good selection, please enter a valid option! Try again.")
    return

def print_effect(input_line):
  for x in input_line:
    print(x, end='')
    sys.stdout.flush()
    sleep(uniform(0, 0.1))

def find_os():
  system = platform.system()
  return system

  #if system == "Windows":
  #    # code to run on Windows
  #elif system == "Linux":
  #    # code to run on Linux
  #elif system == "Darwin":
  #    # code to run on macOS
  #else:
  #    # code to run on other platforms


from pathlib import Path
import os
import json
import csv
import pickle

def folder_Size(path):
    folder_size = 0
    for file in Path(path).rglob('*'):
        if (os.path.isfile(file)):
            folder_size += os.path.getsize(file)
    return folder_size

def recursive_folder_traversal(path, level = 1):
    for j in (os.listdir(path)):
        file_size = os.path.getsize(path + '\\' + j)
        folder_size = folder_Size(path + '\\' + j)
        path_file_or_folder = path + '\\' + j
        if os.path.isfile(path_file_or_folder):
            list_data.append({
                'type': 'file',
                'name': j,
                'path': path,
                'size': file_size
            })

        else:
            list_data.append({
                'type': 'folder',
                'name': j,
                'path': path,
                'size': folder_size
            })

    for i in os.listdir(path):
        if os.path.isdir(path + '\\' + i):
            recursive_folder_traversal(path + '\\' + i, level + 1)

def write_To_Files(list_data):
    with open('test.json', 'w') as f_json:
        json.dump(list_data, f_json, indent=2, ensure_ascii=False)
    with open('test.csv', 'w', newline='', encoding='utf-8') as f_csv:
        csv_write = csv.DictWriter(f_csv, fieldnames=[*list_data[0]])
        csv_write.writeheader()
        csv_write.writerows(list_data)
    with open('test.pickle', 'wb') as f_pickle:
        pickle.dump(list_data, f_pickle)

if __name__ == '__main__':
    path = 'E:\Geekbrains\Immersion_in_python\DZ8'
    list_data = []
    recursive_folder_traversal(path)
    write_To_Files(list_data)


######################
## DEPRICATED - repairFacilityData.py
##
## The AMPD interface occasionally reutrns facility data files that are
## not valid CSVs. Usually there is a misalignment in some rows.
## This script reapirs corrupted CSVs by loading them and resaving
## as a gzipped tab-separated value file. The python readers appear
## to be more robust that those in readr to the errors and create
## readable output files.
##
## With the full release of the CAMPD API this error no longer occurs
## and this script is depricated.
#######################
import zipfile
import gzip
import os
import io
import yaml
import csv
import datetime as dt

#########################
## Load and register the configuration
#########################
#Determine the path to this script
scriptPath, _ = os.path.split(os.path.realpath(__file__))
projectRoot = os.path.join(scriptPath,"..","..")

#Load the config.yaml and config_local.yaml configuration Files
with open(os.path.join(projectRoot,"config.yaml")) as yamlin :
    projectConfig = yaml.safe_load(yamlin)

with open(os.path.join(projectRoot,"config_local.yaml")) as yamlin :
    projectLocalConfig = yaml.safe_load(yamlin)

outputRoot = projectLocalConfig["output"]["path"]
outPathBase = os.path.join(outputRoot,"data","source","EPA-CEMS","facility_data")

#Construct the directory tree if it doesn't already exist
if os.path.isdir(outputRoot) :
    pass
else :
    raise FileNotFoundError(
    "The data output defined path in the configuration file does not exist\n" +
    "Expecing path: " + outputRoot
    )

#Path to EPA hourly CEMS data. Subfolders should be years
baseURL = projectConfig["sources"]["EPA-CEMS"]["URL-hourly"]

#Years to process
startYear = int(projectConfig["sources"]["EPA-CEMS"]["start-year"])
endYear = projectConfig["sources"]["EPA-CEMS"]["end-year"]
if endYear is None or endYear.strip() == "" :
    endYear = dt.date.today().year
else :
    endYear = int(endYear)

with zipfile.ZipFile(os.path.join(outPathBase,"FacilityData.zip")) as zipin :
    filename = [x for x in zipin.namelist() if "facility" in x][0]
    #Use the CSV reader to parse the file
    with io.TextIOWrapper(zipin.open(filename,"r"),encoding='utf=8') as filein :
        #Open a gzipped TSV file for output
        with io.TextIOWrapper(gzip.open(os.path.join(outPathBase,"FacilityData.txt.gz"),"w"),encoding='utf-8',newline='') as fout :
            reader = csv.DictReader(filein,dialect='excel')
            writer = csv.DictWriter(fout,fieldnames=reader.fieldnames,delimiter="\t",quoting=csv.QUOTE_MINIMAL)
            writer.writeheader()
            for row in reader :
                #Remove the terminal key in row
                row.pop(None)
                writer.writerow(row)

    #with io.TextIOWrapper(zipin.open(filename,"r"),encoding="utf-8") as filein :
    #    for row in filein.readlines() :
    #        print(row)


from functools import partial

import numpy as np

from ...utils import common_utils
from . import augmentor_utils

class DataAugmentor(object):
    def __init__(self, root_path, augmentor_configs, class_names, logger=None):

        self.root_path = root_path
        self.class_names = class_names
        self.logger = logger

        self.data_augmentor_queue = []
        aug_config_list = augmentor_configs if isinstance(augmentor_configs, list) \
            else augmentor_configs.AUG_CONFIG_LIST

        for cur_cfg in aug_config_list:
            if not isinstance(augmentor_configs, list):

                if cur_cfg.NAME in augmentor_configs.DISABLE_AUG_LIST:
                    continue

            cur_augmentor = getattr(self, cur_cfg.NAME)(config=cur_cfg)
            self.data_augmentor_queue.append(cur_augmentor)

    def __getstate__(self):
        d = dict(self.__dict__)
        del d['logger']
        return d

    def __setstate__(self, d):
        self.__dict__.update(d)

    def random_world_flip(self, data_dict=None, config=None):

        if data_dict is None:
            return partial(self.random_world_flip, config=config)

        points = data_dict['points']

        for cur_axis in config['ALONG_AXIS_LIST']:
            assert cur_axis in ['x', 'y']

            points = getattr(augmentor_utils, 'random_flip_along_%s' % cur_axis)(
                points,
            )

        data_dict['points'] = points
        return data_dict

    def random_world_rotation(self, data_dict=None, config=None):

        if data_dict is None:
            return partial(self.random_world_rotation, config=config)

        rot_range = config['WORLD_ROT_ANGLE'] # [-0.78539816, 0.78539816]

        if not isinstance(rot_range, list):
            rot_range = [-rot_range, rot_range]

        points = augmentor_utils.global_rotation(
            data_dict['points'], rot_range=rot_range
        )

        data_dict['points'] = points
        return data_dict

    def random_world_scaling(self, data_dict=None, config=None):
        """
        随机缩放
        """
        if data_dict is None:
            return partial(self.random_world_scaling, config=config)
        # 调用augmentor_utils中的函数缩放box和点云
        points = augmentor_utils.global_scaling(
            data_dict['points'], config['WORLD_SCALE_RANGE'] # [0.95, 1.05]
        )

        data_dict['points'] = points
        return data_dict

    def forward(self, data_dict):
        """
        Args:
            data_dict:
                points: (N, 3 + C_in)
                gt_boxes: optional, (N, 7) [x, y, z, dx, dy, dz, heading]
                gt_names: optional, (N), string
                ...

        Returns:
        """
        # 遍历增强队列，逐个增强器做数据增强
        for cur_augmentor in self.data_augmentor_queue:
            data_dict = cur_augmentor(data_dict=data_dict)

        return data_dict


#!/usr/bin/env python3
"""Extremely primitive MIDI synthesizer"""
import sys
import argparse
from dataclasses import dataclass, field, replace
import time
from typing import Dict

import numpy as np
import sounddevice as sd
import mido

def note_frequency(note):
    return 440.0 * 2**((note - 69) / 12)

@dataclass
class Note:
    frequency: float = 0.0
    velocity: int = 0
    ontime: float = 0.0
    offtime: float = np.inf

@dataclass
class Channel:
    notes: Dict[int, Note] = field(default_factory=dict)
    program: int = 0
    pitch_bend: float = 0.0

class MidiSynth:
    def __init__(self, args):
        input_port = args.input_port
        if isinstance(input_port, int) and not args.virtual_port:
            input_port = mido.get_input_names()[input_port]
        self.input = mido.open_input(input_port, virtual=args.virtual_port)
        self.channels = [Channel() for i in range(16)]

        dev_info = sd.query_devices(args.output_device, 'output')
        self.sample_rate = dev_info['default_samplerate']
        self.start_idx = 0
        self.start_time = 0.0
        self.last_time = 0.0

        self.waveform = getattr(self, 'waveform_' + args.waveform)
        self.envelope = getattr(self, 'envelope_' + args.envelope)
        self.attack = args.attack
        self.decay = args.decay
        self.sustain = args.sustain
        self.release = args.release

        self.stream = sd.OutputStream(
            device=args.output_device, samplerate=self.sample_rate, channels=1,
            latency=args.latency, callback=self.callback)

    def close(self):
        self.input.close()
        self.stream.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def callback(self, outdata, frames, time, status):
        if status:
            print(status, file=sys.stderr)
        if self.start_idx == 0:
            self.start_time = time.currentTime
        t = (self.start_idx + np.arange(frames)) / self.sample_rate + self.start_time
        t = t.reshape(-1, 1)
        self.synthesize(outdata, t)
        self.start_idx += frames
        # print(f'{time.currentTime:f} {self.stream.time - time.currentTime:f} '
        #       f'{time.outputBufferDacTime - time.currentTime:f} '
        #       f'{time.currentTime - self.last_time:f} {frames}')
        # self.last_time = time.currentTime

    def synthesize(self, output, t):
        output[:] = 0.0
        for channel in self.channels:
            for note in channel.notes.values():
                output += self.synthesize_note(t, note, channel)
        # output.clip(-1, 1, out=output)
        # np.tanh(output, out=output)
        output /= max(abs(output).max(), 1)

    def synthesize_note(self, t, note, channel):
        # print(note.ontime, note.offtime, t[0,0])
        # if note.ontime <= t[0,0] < note.offtime:
        amp = note.velocity
        freq = note.frequency * 2**(channel.pitch_bend / 6)
        envelope = self.envelope(t - note.ontime, t - note.offtime)
        return amp * envelope * self.waveform(freq * t)

    def envelope_flat(self, t_on, t_off):
        return np.float32((t_on >= 0) & (t_off < 0))

    def envelope_asr(self, t_on, t_off):
        y = t_on/self.attack
        np.minimum(y, 1-t_off/self.release, out=y)
        np.clip(y, 0, 1, out=y)
        return y

    def envelope_adsr(self, t_on, t_off):
        y = t_on/self.attack
        dec_sus = np.maximum(1-(1-self.sustain)/self.decay*(t_on-self.attack),
                             self.sustain)
        np.minimum(y, dec_sus, out=y)
        np.minimum(y, self.sustain*(1-t_off/self.release), out=y)
        np.clip(y, 0, 1, out=y)
        return y

    def envelope_asr_exp(self, t_on, t_off):
        y = 1. - np.exp(-4./self.attack*t_on)
        y *= np.where(t_off < 0, 1, np.exp(-4./self.release*t_off))
        # np.minimum(y, np.exp(-4./self.release*t_off), out=y)
        np.clip(y, 0, 1, out=y)
        return y

    def envelope_adsr_exp(self, t_on, t_off):
        y = 1. - np.exp(-4./self.attack*t_on)
        y *= np.where(
            t_on < self.attack, 1,
            np.exp(-4./self.decay*(t_on-self.attack))*(1-self.sustain)+self.sustain)
        y *= np.where(t_off < 0, 1, np.exp(-4./self.release*t_off))
        np.clip(y, 0, 1, out=y)
        return y

    def waveform_sine(self, t):
        return np.sin(2*np.pi * t)

    def waveform_square(self, t):
        return 1 - 2*np.floor(2*t % 2)

    def waveform_saw(self, t):
        return (t - .5) % 1 * 2 - 1

    def waveform_tri(self, t):
        return abs((4*t - 1) % 4 - 2) - 1

    def process_msg(self, msg):
        channel = self.channels[msg.channel]
        if msg.type in ('note_on', 'note_off'):
            notes = channel.notes.copy()
            if msg.type == 'note_on' and msg.velocity:
                freq = note_frequency(msg.note)
                notes[msg.note] = Note(freq, msg.velocity/127, self.stream.time)
            else:
                # notes.pop(msg.note, None)
                note = notes.get(msg.note)
                if note:
                    notes[msg.note] = replace(note, offtime=self.stream.time)
            channel.notes = notes
        elif msg.type == 'program_change':
            channel.program = msg.program
        elif msg.type == 'pitchwheel':
            channel.pitch_bend = msg.pitch / 8192

    def flush_notes(self):
        for ch in range(16):
            channel = self.channels[ch]
            channel.notes = {n: note for n, note in channel.notes.items()
                             if self.stream.time - note.offtime < self.release}

    def run(self):
        with self.stream:
            self.stream.start()
            while True:
                for msg in self.input.iter_pending():
                    self.process_msg(msg)
                self.flush_notes()
                time.sleep(.01)

def main(args):
    try:
        with MidiSynth(args) as synth:
            print('Receiving midi messages... Press Ctr+C to quit')
            synth.run()
    except KeyboardInterrupt:
        pass

def int_or_str(text):
    """Helper function for argument parsing."""
    try:
        return int(text)
    except ValueError:
        return text

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('-l', '--list-ports', action='store_true',
                        help='show list of midi input ports and exit')
    parser.add_argument('-L', '--list-outputs', action='store_true',
                        help='show list of audio output devices and exit')
    parser.add_argument('-i', '--input-port', type=int_or_str,
                        help='midi input port (numeric ID or string)')
    parser.add_argument('-o', '--output-device', type=int_or_str,
                        help='output device (numeric ID or string)')
    parser.add_argument('-v', '--virtual-port', action='store_true',
                        help='open a virtual midi input port')
    parser.add_argument('-t', '--latency', type=float, default=.5,
                        help='latency for audio output')
    parser.add_argument('-w', '--waveform', default='tri',
                        choices=['sine', 'square', 'saw', 'tri'],
                        help='set the shape of the wave function')
    parser.add_argument('-e', '--envelope', default='adsr',
                        choices=['flat', 'asr', 'adsr', 'asr_exp', 'adsr_exp'],
                        help='set the envelope type')
    parser.add_argument('-a', '--attack', type=float, default=.05,
                        help='ADSR attack time')
    parser.add_argument('-d', '--decay', type=float, default=.2,
                        help='ADSR decay time')
    parser.add_argument('-s', '--sustain', type=float, default=.8,
                        help='ADSR sustain level')
    parser.add_argument('-r', '--release', type=float, default=.5,
                        help='ADSR release time')
    args = parser.parse_args()
    if args.list_outputs:
        # Some audio devices on windows have weird names with line breaks
        print(str(sd.query_devices()).replace('\r\n', ''))
        parser.exit()
    if args.list_ports:
        print('\n'.join(mido.get_input_names()))
        parser.exit()
    main(args)


from    chain_day   import  chain_day
from    chain_def   import  chain_def
from    chain_set   import  chain_set
from    json        import  loads
import  polars      as      pl
from    structs     import  opt_row
from    time        import  time
from    typing      import  List

###########################################################
## THIS CODE IS TOTALLY BROKEN DUE TO DB V3 MIGRATION... ##
###########################################################

CONFIG  = loads(open("./config.json").read())
DB      = pl.SQLContext()

DB.register("cme_opts", pl.read_parquet(CONFIG["opts_path"]).lazy())
DB.register("ohlc", pl.read_parquet(CONFIG["futs_path"]).lazy())

# see list_defs.py for usage

def get_chain_defs_by_date(symbol: str):

    rows = DB.query(
        f'''
            SELECT DISTINCT date, name, expiry, underlying_id
            FROM            cme_opts
            WHERE           underlying_symbol = '{symbol}'
            ORDER BY        date ASC, expiry ASC;
        '''
    ).rows()

    # weird hack, "DISTINCT" not working in query above... need to fix

    rows = sorted(list(set(rows)), key = lambda x: (x[0], x[2]))

    by_date = {}

    for row in rows:

        date = row[0]

        if row[0] not in by_date:

            by_date[date] = []

        by_date[date].append(
            chain_def(
                row[1],
                row[3],
                row[2]
            )
        )

    return by_date

# see daily.py for usage

def get_chain_day(definition: chain_def, date: str):

    ul_row = None

    try:

        ul_row = DB.query(
            f'''
                SELECT  date, contract_id, year, month, settle
                FROM    ohlc
                WHERE   contract_id = '{definition.underlying_id}' AND date = '{date}'
            '''
        ).rows()[0]

    except IndexError:

        # no data for underlying on given date

        return None

    opt_rows = DB.query(
        f'''
            SELECT  *
            FROM    cme_opts
            WHERE   date            = '{date}'
            AND     name            = '{definition.opt_class}'
            AND     expiry          = '{definition.expiry}'
            AND     underlying_id   = '{definition.underlying_id}'
        '''
    ).rows()

    cd = chain_day(
        date,
        definition.opt_class,
        opt_rows[0][opt_row.expiry],
        ul_row[3],
        ul_row[2],
        ul_row[4],
        ul_row[1]
    )

    cd.set_opt_rows(opt_rows)

    return cd

# see history.py for usage

def get_expiries_for_underlying_and_class(
    underlying_id:  str,
    opt_class:      str,
    date:           str
):

    exps = DB.query(
        f'''
            SELECT      expiry
            FROM        cme_opts
            WHERE       date            = '{date}'
            AND         name            = '{opt_class}'
            AND         underlying_id   = '{underlying_id}'
            GROUP BY    expiry
        '''
    ).rows()

    flat = [ exp[0] for exp in exps ]

    return flat

# assumes no two options for a symbol expire on the same day
# if this is not correct, choose opt classes to exclude with "exclude_classes" until the assumption is valid

def get_chain_set(
    symbol:             str,
    start:              str,
    end:                str,
    excluded_classes:   List[str]
):

    cs = chain_set(symbol)

    t1 = time()

    opt_rows = DB.query(
        f'''
            SELECT  *
            FROM    cme_opts
            WHERE   date BETWEEN '{start}' AND '{end}'
            AND     underlying_symbol = '{symbol}'
            AND     name NOT IN ({', '.join(excluded_classes)})
        '''
    ).rows()

    # print(f"cs.get_chain_set:query: {time() - t1:0.1f}")

    dates = { row[opt_row.date]: {} for row in opt_rows }

    for row in opt_rows:

        date            = row[opt_row.date]
        symbol          = row[opt_row.symbol]
        ul_id           = row[opt_row.underlying_id]
        expiry          = row[opt_row.expiry]
        chain_def_id    = (symbol, ul_id, expiry)

        if expiry not in dates[date]:

            ch_def              = chain_def(*chain_def_id)
            dates[date][expiry] = get_chain_day(ch_def, date)

    cs.set_data(dates)

    return cs


import pandas as pd
import pymongo
from pymongo import MongoClient
import requests
from lxml import etree
import os

def get_html(url):
    """
    获取页面数据(文本)
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
    }
    resp = requests.get(url, headers=headers)
    if resp.status_code == 200:
        return resp.text
    else:
        return None

def read_article_url():
    """
    读取MongoDB数据库中的文章链接（article_url）
    """
    client = MongoClient()
    db = client.JIANSHU
    collection = db.WEEKLY
    data = [i['article_url'] for i in collection.find()]
    return data

def parse_article_url_data(fileName, artiicle_url_data, db):
    """
    解释文章页面数据
    """
    root  = etree.HTML(artiicle_url_data)
    article_title = root.xpath('//div[@class="article"]/h1[@class="title"]/text()')[0] if root.xpath('//div[@class="article"]/h1[@class="title"]/text()') else ''
    article_content_selector = root.xpath('//div[@class="show-content"]')[0] if root.xpath('//div[@class="show-content"]') else ''
    article_content = article_content_selector.xpath('string()').strip() if article_content_selector else ""
    print(article_title + '\n', article_content)
    # save_data(db, artiicle_content)
    with open(fileName + '.txt',encoding="utf-8", mode='a') as f:
        f.write(article_title)
        f.write('\n')
        f.writelines(article_content)

def init_sql():
    """
    初始化数据库
    """
    client = MongoClient('localhost', 27017)
    db = client.JIANSHU
    return db

# def save_data(db, data):
#     """
#     保存数据到数据库中
#     """
#     db.ARTICLE.insert(data)

def main():
    article_urls = read_article_url()
    print(article_urls)

def test(db):
    article_urls = set(read_article_url())
    i = 0
    for article_url in article_urls:
        print(article_url)
        article_data = get_html(article_url)
        if article_data:
            parse_article_url_data(str(i), article_data, db)
        i += 1

if __name__ == '__main__':
    db = init_sql()
    test(db)


"""
This module contains various statistical hypothesis tests.

Contains:
- Kwiatkowski–Phillips–Schmidt–Shin (KPSS) test and augmented Dickey–Fuller
  (ADF) test to test the stationarity of a time-series.
- Ljung–Box test to test the autocorrelation of a time series.
"""

from typing import NamedTuple

import statsmodels.tsa.api as tsa
from tabulate import tabulate

class TestResults(NamedTuple):
    """NamedTuple to hold tests results."""

    test_name: str
    test_stat: float
    p_value: float
    lags: int = None
    crit_values: dict = None

    def format(self):
        """Format results for printing."""
        tabular_data = [
            ["Test", self.test_name],
            ["Value of test-statistic", self.test_stat],
            ["p-value", self.p_value],
            ["Lags", self.lags],
            ["Critical values", self.crit_values],
        ]
        formatted_table = tabulate(tabular_data, colalign=("right",))
        table_width = formatted_table.index("\n")
        print("-" * table_width)
        title = "{} test results".format(self.test_name)
        padding = (table_width - len(title)) // 2
        print(" " * padding + title)
        print(formatted_table)

def kpss(x, **kwargs):
    """
    Compute KPSS test for stationarity.

    H0: data are stationary.
    """
    if "nlags" not in kwargs.keys():
        kwargs["nlags"] = "auto"
    res = tsa.kpss(x, **kwargs)
    test_results = TestResults(
        test_name="KPSS",
        test_stat=res[0],
        p_value=res[1],
        lags=res[2],
        crit_values=res[3]
    )
    return test_results

def adf(x, **kwargs):
    """
    Compute ADF test for stationarity.

    H0: a unit root is present in data.
    """
    res = tsa.adfuller(x, **kwargs)
    test_results = TestResults(
        test_name="ADF",
        test_stat=res[0],
        p_value=res[1],
        lags=res[2],
        crit_values=res[4]
    )
    return test_results

def box_ljung(x, **kwargs):
    """Compute Box-Ljung Q statistic for autocorrelation."""
    res = tsa.acf(x, qstat=True, fft=True, ** kwargs)
    test_results = TestResults(
        test_name="Box-Ljung",
        test_stat=res[1],
        p_value=res[2]
    )
    return test_results

if __name__ == '__main__':
    import numpy as np
    np.random.seed(42)
    x = np.random.randn(100)

    # import pandas as pd
    # df = pd.read_csv("~/goog.csv", usecols=[1])
    # x = df.x[:200]

    kpss_results = kpss(x)
    adf_results = adf(x, maxlag=40)
    box_ljung_results = box_ljung(x, nlags=20)

    # pprint(kpss_results)
    # pprint(adf_results)
    # pprint(box_ljung_results)

    # kpss_results.format()
    # adf_results.format()
    # box_ljung_results.format()

    b = box_ljung_results
    b.format()
    # print("")
    #
    # for i in range(20):
    #     print("{: 12.5f} {}".format(b.test_stat[i], b.p_value[i]))


#!/usr/bin/env python

from optparse import OptionParser
import os
import sys
import re
import subprocess
import uuid
import datetime

"""
Due to old problem in visual studio many test frameworks wont run tests from static libraries.
see http://social.msdn.microsoft.com/forums/en-US/vclanguage/thread/2aa2e1b7-6677-4986-99cc-62f463c94ef3
This program will generate a header file from a lib file, including the appropriate references.

   Copyright 2011 Nikolas Bowe
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
   limitations under the License.
"""

regexes = map(	re.compile ,[
				# GTest
				# TEST(MathTest, TwoPlusTwoEqualsFour) {
				# 	EXPECT_EQ(2 + 2, 4);
				# }
				# produces a symbol like the following (ctor for generated test class)
				# (note TEST_F also generates similar code)
				# "  27993A0 ??0MathTest_TwoPlusTwoEqualsFour_Test@@QAE@XZ"
				r".*(\?\?0.+_.+_Test@@QAE@XZ)",

				# TODO: add more test frameworks regexes
			])

def main():
	parser = OptionParser(
		description="Generates a header from a msvc static library to force a program to reference test cases.",
		epilog = " ".join(["When linking a static library into a program, msvc will not run static initializers",
			"for objects in the library unless they are in an obj file that gets linked into the main program.\n",
			"This causes many test frameworks to not run some (or all) tests that are in static libs.\n",
			"This currently generates headers for Google Test, but its easy to extend to other frameworks\n",
			"Requires dumpbin to be in your path",
			]),
		usage="usage: %prog lib header",
		version="%prog 1.0",
	)
	(options, args) = parser.parse_args()
	if len(args) != 2:
		parser.print_help()
		sys.exit(1)
	libfile = args[0]
	headerfile = args[1]
	generate_header_for_lib(headerfile, libfile)

def generate_header_for_lib(headerfile, libfile):
	(lines,_) = command(["dumpbin", "/linkermember:1", libfile])
	test_symbols = find_matches(lines, regexes)

	with open(headerfile, 'w') as f:
		header_guard = "generated_"+str(uuid.uuid4()).replace('-','_')
		time = datetime.datetime.now().isoformat()
		f.write("#ifndef "+header_guard + '\n')
		f.write("#define "+header_guard + '\n')
		f.write("// Generated by gen_msvc_test_header at " + time + "\n")
		f.write("// do not modify \n\n")
		for symbol in test_symbols:
			pragma_line = '#pragma comment(linker, "/include:' + symbol + '")'
			f.write(pragma_line + '\n')
		f.write("\n#endif // "+header_guard + '\n')

def find_matches(lines, regexes):
	""" returns the first matchs group 1 for each line. if a line does not match any regex it is skipped """
	def match(line):
		for r in regexes:
			m = r.match(line)
			if m:
				return m.group(1)
		return None
	matches = filter(None, map(match,lines))
	return matches

def command(cmd, input=None):
	""" easy function for runnin simple commands.
	quite like subprocess.communicate but doesnt require 2.7 (a lot of people still run 2.6 or even 2.5)
	"""
	p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	(stdoutdata, stderrdata) = p.communicate(input=input)
	if p.returncode != 0:
			raise subprocess.CalledProcessError(p.returncode, cmd)
	stdout_lines = stdoutdata.split()
	stderr_lines = stderrdata.split()
	return (stdout_lines, stderr_lines)

def find_matches_test():
	# sample lines from dumpbin output. some matching GTests regex
	# I should really do a proper test suite but this is enough for now
	lines = ["  27993A0 ??0MathTest_TwoPlusTwoEqualsFour_Test@@QAE@XZ",
			"  27993A0 ??1MathTest_TwoPlusTwoEqualsFour_Test@@UAE@XZ",
			"  279A992 ??0FooTest@@QAE@XZ",
			"  279A992 ??0FooTest_test1_Test@@QAE@XZ",
			"  279A992 ??0FooTest_test2_Test@@QAE@XZ",
	]
	assert(find_matches(lines, regexes) == ["??0MathTest_TwoPlusTwoEqualsFour_Test@@QAE@XZ","??0FooTest_test1_Test@@QAE@XZ","??0FooTest_test2_Test@@QAE@XZ"])

find_matches_test()

if __name__ == "__main__":
	main()


#!/usr/bin/python3

"""
v3 - smarter storage/search of the cubes
"""

from stevepe import getIntPermutations

import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-l", "--log", help="logging level; one of DEBUG, INFO, WARNING", type=str)
args = parser.parse_args()

import logging
if args.log:
    loglevel = getattr(logging, args.log.upper())
else:
    loglevel = logging.WARN
logging.basicConfig(level=loglevel)

import time
tStart = time.time()

def store_cube(d, cube):
    high = cube // 10000
    low = cube % 10000
    if high not in d:
        d[high] = []
    d[high].append(low)
    return d

def cube_in_d(d, cube):
    high = cube // 10000
    low = cube % 10000
    if high not in d:
        return False
    return low in d[high]

seed = 345
cubes = {}
for i in range(seed+1):
    cube = pow(seed, 3)
    store_cube(cubes, cube)

max_cube = cube
max_cube_seed = seed

seed += 1

while True:
    cube = seed*seed*seed
    logging.debug(f"seed = {seed}; cube = {cube}")

    cubes_from_perms = 0

    deduped_perms = getIntPermutations(cube)
    for perm in deduped_perms:
        #logging.debug(f"testing perm {perm}")
        if perm > max_cube:
            while perm > max_cube:
                max_cube_seed += 1
                max_cube = pow(max_cube_seed, 3)
                #logging.debug(f"extending cubes by appending {max_cube}")
                store_cube(cubes, max_cube)
        if cube_in_d(cubes, perm):
            logging.debug(f"{perm} is a cube")
            cubes_from_perms += 1

    logging.debug(f"found {cubes_from_perms} cubes")
    if cubes_from_perms == 5:
        print("{}: answer: {!s}".format(__file__, cube))
        print("Run time: " + '{:.20f}'.format(time.time() - tStart) + " sec")

    seed += 1


import datetime
from concurrent.futures import ThreadPoolExecutor

import numpy as np
from multiprocessing import Pool
import matplotlib.pyplot as plt

class k_means_serial(object):
    def __init__(self, n_clusters, max_iter):
        self.num_of_cluster = n_clusters
        self.max_iter = max_iter

    def min_cluster_dist(self, clusters, x):
        return np.argmin([self.euc_dist(x, c) for c in clusters])

    def euc_dist(self, a, b):
        return np.sqrt(((a - b) ** 2).sum())

    def points_clustering(self, blobs):     # clustering
        self.cluster_prediction = [self.min_cluster_dist(self.cluster_centers, x) for x in blobs]
        ind = []
        for j in range(self.num_of_cluster):
            current_cluster = []
            for i, k in enumerate(self.cluster_prediction):
                if k == j:
                    current_cluster.append(i)
            ind.append(current_cluster)
        points_for_cluster = [blobs[i] for i in ind]
        return points_for_cluster

    def random_centroids(self, blobs):
        centroid = np.random.permutation(blobs.shape[0])[:self.num_of_cluster]
        return blobs[centroid]

    def fit(self, blobs):
        self.cluster_centers = self.random_centroids(blobs)
        #print(self.cluster_centers)

        for i in range(self.max_iter):
            #X_by_cluster = self.plot_clustering(blobs) #to plot
            executor = ThreadPoolExecutor(max_workers=50)
            X_by_cluster = executor.submit(self.points_clustering, blobs).result()
            new_cl_centers = [c.sum(axis=0) / len(c) for c in X_by_cluster] # new clsuter centers
            new_cl_centers = [arr.tolist() for arr in new_cl_centers]
            self.cluster_centers = new_cl_centers  # keep iter
        return self

    def plot_clustering(self, blobs):
        plt.xlim(-8, 8)
        plt.ylim(-8, 8)
        plt.scatter(self.cluster_centers[0][0], self.cluster_centers[0][1], color='red', zorder=1)
        plt.scatter(self.cluster_centers[1][0], self.cluster_centers[1][1], color='red', zorder=1)
        plt.scatter(self.cluster_centers[2][0], self.cluster_centers[2][1], color='red', zorder=1)
        X_by_cluster = self.points_clustering(blobs)
        plt.scatter(*zip(*X_by_cluster[0]), color='blue', s=3, zorder=0)
        plt.scatter(*zip(*X_by_cluster[1]), color='orange', s=3, zorder=0)
        plt.scatter(*zip(*X_by_cluster[2]), s=3, color='purple', zorder=0)
        plt.savefig('imgs/plt_S/'+ str(datetime.datetime.now())[6:]+'.png')
        plt.show()
        return X_by_cluster

class k_means_parallel(k_means_serial):
    def __init__(self, n_clusters, max_iter, num_of_cores):
        super().__init__(n_clusters, max_iter)
        self.num_cores = num_of_cores

    def shuffle(self, list_in, n):
        temp = np.random.permutation(list_in)
        result = [temp[i::n] for i in range(n)]
        return result

    def fit(self, blobs):
        self.cluster_centers = self.random_centroids(blobs)
        for i in range(self.max_iter):

            splitted_data = self.shuffle(blobs, self.num_cores)

            pool = Pool()
            executor = ThreadPoolExecutor(max_workers=50)
            result = executor.submit(pool.map, self.points_clustering, splitted_data).result()
            pool.close()
            pool.join()
            print('ciao matte')
            # splitted in più processi e riunito

            points_for_cluster = []
            for k in range(0, self.num_of_cluster):
                points = []
                for pool in range(0, self.num_cores):
                    tmp = result[pool][k].tolist()
                    points = sum([points, tmp], [])
                points_for_cluster.append(np.array(points))

            new_cl_centers = [k.sum(axis=0) / len(k) for k in points_for_cluster]
            new_cl_centers = [np.array(arr) for arr in new_cl_centers]
            self.cluster_centers = new_cl_centers

        self.number_of_iter = i
        return self


import sys
from os.path import abspath, join, dirname

sys.path.insert(0, join(abspath( dirname(__file__)), '../'))
from model.dataModel import HSISingleData

import h5py
import numpy as np
DATASETNAME="Harvard"

def cuthalf():
    rfile = f'./data/Train/Training_Data_{DATASETNAME}_48.mat'
    wfile = f'./data/Train/Training_Data_{DATASETNAME}_24.mat'
    data = h5py.File(rfile, 'r')
    label = data['label']
    n, c, w, h = label.shape
    halfW = w//2
    halfH = h//2
    lu = label[:, :, 0:halfW, 0:halfH]
    ru = label[:, :, 0:halfW, halfH:h]
    ld = label[:, :, halfW:w, 0:halfH]
    rd = label[:, :, halfW:w, halfH:h]
    # Version 2
    nn = np.zeros((len(lu) * 4, c , halfW, halfH))
    for i in range(0,len(lu)):
        nn[4 * i] = lu[i]
        nn[4 * i + 1] = ru[i]
        nn[4 * i + 2] = ld[i]
        nn[4 * i + 3] = rd[i]
        if i % 400 == 0:
            print(i)
    # Version 1
    # nn = np.stack([lu[0],ru[0],ld[0],rd[0]])
    # for i in range(1,len(lu)):
    #     nn = np.concatenate([nn, np.stack([lu[i], ru[i], ld[i], rd[i]])], axis=0)
    with h5py.File(wfile, 'w') as new_data:
        new_data.create_dataset('label', data=nn, compression='gzip')
        new_data.create_dataset('set', data=np.expand_dims(np.repeat(data['set'], 4),axis=1) , compression='gzip')
        # new_data["label"] = nn
        # new_data["set"] = np.expand_dims(np.repeat(data['set'], 4),axis=1)
        new_data.flush()

def cut(num = 4):
    rfile = f'./data/Train/Training_Data_{DATASETNAME}_48.mat'
    wfile = f'./data/Train/Training_Data_{DATASETNAME}_test.mat'
    data = h5py.File(rfile, 'r')
    label = data['label']
    new_data = h5py.File(wfile, 'w')
    # new_data.create_dataset("label",dtype=label.dtype)
    # new_data.create_dataset("set",dtype=data['set'].dtype)

    # new_data.create_dataset("PHITPHI",dtype=data['PHITPHI'].dtype)
    # new_data.create_dataset("SZ",dtype=data['SZ'].dtype)
    new_data.create_dataset('label', data=data['label'][0:num], compression='gzip')
    new_data.create_dataset('set', data=data['set'][0:num], compression='gzip')
    # new_data["label"] = data['label'][0:num]
    # new_data["set"] = data['set'][0:num]
    new_data.flush()

if __name__ == "__main__":
    # cuthalf()
    # cut()
    # rfile = f'./data/Train/Training_Data_{DATASETNAME}_48.mat'
    # data = h5py.File(rfile, 'r')
    # label = data['set']
    # print(np.array(label))
    dataset = HSISingleData(f'./data/Train/Training_Data_{DATASETNAME}_24.mat', (0, 3, 2, 1))
    print(len(dataset))
    print(dataset[0][0].shape)


import dash
import dash_auth

def extract_creds(file='hist_auth.txt'):
    with open(file, 'r') as f:
        return [f.read().split(',')]

creds = extract_creds()
app = dash.Dash('auth')
auth = dash_auth.BasicAuth(app, creds)

server = app.server
app.config.suppress_callback_exceptions = True


# stdlib imports
from typing import Optional

# Dependency imports
from jinja2 import Template

# Project imports
from ..templating import Templator
from ..templating import TemplateType
from ..templating import TemplatedData
from ..templating import ResumeData

class PlainTextBiographyTemplator(Templator):
    def __init__(self, template: Optional[Template]):
        self.loaded_template = template

    def template(self, data: ResumeData) -> TemplatedData:
        templated_data = self.loaded_template.render(biography=data.biography)

        return TemplatedData(
            data=templated_data,
            file_name="biography",
            type=self.type()
        )

    def type(self) -> TemplateType:
        return TemplateType.PLAINTEXT

def load_templator() -> Templator:
    return PlainTextBiographyTemplator(Template("{{ biography }}"))


import torch
from torch.autograd import Function

def weight_init_glorot_uniform(m):
    classname = m.__class__.__name__
    if classname.find("Linear") != -1:
        torch.nn.init.xavier_uniform_(m.weight)
        m.bias.data.fill_(0.01)
    if classname.find("Conv") != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find("BatchNorm") != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

class ReverseLayerF(Function):
    @staticmethod
    def forward(ctx, x, alpha):
        ctx.alpha = alpha

        return x.view_as(x)

    @staticmethod
    def backward(ctx, grad_output):
        output = grad_output.neg() * ctx.alpha
        return output, None


from bs4 import BeautifulSoup
import requests
import time

'''
source link:
    https://weheartit.com/inspirations/taylorswift
Aim:
    craw images from 20 pages to download local

demo:

url = 'https://data.whicdn.com/images/338888035/original.jpg?t=1577683048'

pic = requests.get(url)
with open('images/1.jpg', 'wb') as f:
    print(pic.content)
    f.write(pic.content)

'''

def getImageLinks(urls,data=[]):
    for url in urls:
        print(url)
        wb_data = requests.get(url)
        soup = BeautifulSoup(wb_data.text, 'lxml')
        imgs = soup.select('#main-container > div.grid-responsive > div.col.span-content > div > div > div > div > div > a > img')
        for img in imgs:
            data.append(img.get('src'))
    return data

urls = ['https://weheartit.com/inspirations/taylorswift'] + ['https://weheartit.com/inspirations/taylorswift?page={}'.format(i) for i in range(2,21)]

# print(urls)

# imgLinks = getImageLinks(urls)
# with open('data/project2Links','w') as f:
#     for link in imgLinks:
#         f.write(str(link)+'\n')

with open('data/project2Links','r') as f:
    links = f.readlines()
    i = 0
    for link in links:
        i += 1
        print('%s:%s' % (i,link))
        pic = requests.get(link)
        dir = 'images/'+str(i)+'.jpg'
        with open(dir,'wb') as img:
            img.write(pic.content)


import frame.rootManager as rootManager
import robot.robotManager as robotManager

from frame.iFrame import IFrame
from frame.messagesHelp import HELP_FDELETEROBOT

import tkinter as tk
from tkinter import messagebox
import utils.tkinter.tkUtils as tkUtils

class FDeleteRobot(IFrame):
    """
    Propose à l'utilisateur de supprimer un robot
    """

    def __init__(self, previousFrame):
        super().__init__(previousFrame, HELP_FDELETEROBOT)

    def draw(self):
        root = rootManager.getRoot()

        super().createButtonHelp()

        labelTitle = super().createLabel(master=root, text="Suppression de robot", fontSize=30)
        labelTitle.pack(side="top")

        frameMain = super().createFrame(root)
        frameMain.pack(pady=tkUtils.ratioHeight(0.04, root))

        # Nom du robot
        textName = super().createLabel(master=frameMain, text="Nom du robot:")
        textName.pack(anchor="n")

        # Liste de nom des robots
        self.comboboxName = super().createComboBox(list=robotManager.getLoadedRobots())
        self.comboboxName.pack(anchor="n")

        # Confirm
        buttonConfirm = super().createButton(text="Supprimer", cmd=lambda:self.confirmButtonAction())
        buttonConfirm.pack(pady=tkUtils.ratioHeight(0.01, root))

        # Retour
        buttonBack = super().createButton(text="Retour", cmd=lambda:super(FDeleteRobot, self).reopenLastFrame())
        super().modifyButton(buttonBack ,bg = "darkred", ab = "red")
        buttonBack.pack(side="bottom", anchor="w", padx=10, pady=10)

    def confirmButtonAction(self):
        """
        Méthode appelée lorsque l'utilisateur lorsqu'il appuie
        sur le bouton de confirmation de suppression d'un robot
        """
        selectedRobotName = self.comboboxName.get()

        # Ne rien faire si aucun robot entré
        if (selectedRobotName == ""):
            return

        messagebox.showinfo(title="Information", message=f"Le robot {selectedRobotName} a bien été supprimé")
        robotManager.deleteRobot(selectedRobotName)

        super(FDeleteRobot, self).reopenLastFrame()


"""
Configure gh-pipelines virtual enviroments
"""
import threading
import logging

from argparse import ArgumentParser
from utils.venv_tool import VenvTool, VenvTag
from typing import List

def setup_env(venv_tag: VenvTag) -> None:
    ''' Configure a single environment '''
    env_tool = VenvTool(venv_tag)
    env_tool.configure()

def parse(args: List[str]) -> None:
    ''' Configure gh-pipline environments '''
    parser = ArgumentParser(
        prog="configure",
        description="Configure gh-pipeline virtual enviroments"
    )

    parser.add_argument(
        'selected_tags',
        help="environment options",
        nargs='*'
    )

    command_args = parser.parse_args(args)

    venv_tags = [t for t in VenvTag]

    if command_args.selected_tags:
        tag_strings = [t.name for t in venv_tags]
        actual_set = set(tag_strings)
        selected_set = set(command_args.selected_tags)
        invalid_tags = selected_set - actual_set
        valid_tags = selected_set.intersection(actual_set)
        for tag in invalid_tags:
            message = '{} is not valid environment option.'
            logging.warning(message.format(tag))
        venv_tags = [VenvTag[st] for st in valid_tags]

    threads = []
    for tag in venv_tags:
        config_thread = threading.Thread(
            target=setup_env,
            args=(tag,),
            daemon=True
        )
        threads.append(config_thread)
        config_thread.start()

    for thread in threads:
        thread.join()


import numpy as np
import matplotlib as mpl
mpl.use('TkAgg')
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.decomposition.pca import PCA

def plot_embeddings(embeddings, labels, protecteds,
                    plot3d=False,
                    subsample=False,
                    label_names=None,
                    protected_names=None):
    if protected_names is None:
        protected_names = ["A0", "A1"]
    if label_names is None:
        label_names = ["L0", "L1"]
    n = embeddings.shape[0]
    if not subsample:
        subsample = n
    inds = np.random.permutation(n)[:subsample]
    pca = PCA(n_components= 3 if plot3d else 2)
    labels = labels.astype(bool)[inds]
    protecteds = protecteds.astype(bool)[inds]
    pca.fit(embeddings)
    embs = pca.transform(embeddings)[inds, :]
    fig = plt.figure()
    if plot3d:
        ax = fig.add_subplot(111, projection='3d')
    else:
        ax = fig.add_subplot(111)
    for l in [False, True]:# labels
        for p in [False, True]: # protecteds
            idxs = np.logical_and(labels == l, protecteds == p)
            embs_slice = embs[idxs, :]
            data_vectors = [embs_slice[:, 0], embs_slice[:, 1]]
            if plot3d: data_vectors.append(embs_slice[:,2])
            color = "b" if p else "r"
            marker = "o" if l else "x"
            name = "{} {}".format(protected_names[p],
                                  label_names[l])
            ax.scatter(*data_vectors,
                        edgecolors=color,
                        marker=marker,
                        facecolors=[color, 'none'][l], # only leave circles unfilled
                        label=name)
    ax.legend(fontsize="small")
    plt.show()


from datetime import date
import random

class User:

    def __init__(self, first_name, last_name, location) -> str:
        self.first_name = first_name.title()
        self.last_name = last_name.title()
        self.location = location.title()
        self.last_access = date.today()
        self.account_number = random.randint(2000, 10000)
        self.login_attempts = 0  # initializing login atempts

    def describe_user(self):
        print(
            f"first name is: {self.first_name}\n"\
            f"last name is: {self.last_name}\n"\
            f"{self.first_name}'s last access is: {self.last_access}\n"\
            f"{self.first_name}'s account number is {self.account_number}\n"\
            f"{self.first_name} is from {self.location}\n"
        )

    def greet_user(self):
        print(f"Hello {self.first_name}, glad you are here with ust today!!!")

    def increment_login_attempts(self):
        self.login_attempts += 1

    def reset_login_attempts(self):
        self.login_attempts = 0

alex = User("alex", "garcia", "Guadalajara, Mexico")
alex.describe_user()
alex.greet_user()

# increment several times, print and then reset, print again

for i in range (0, 10):
    alex.increment_login_attempts()
    print(f"login attempts: {alex.login_attempts}")

alex.reset_login_attempts()
print(f"\nlogin attempts: {alex.login_attempts}")


import os
import logging
import json
import time
import urllib.request
import shutil
import hashlib
from collections import OrderedDict
import cfutil.config as config
from . import settings

log = logging.getLogger(__name__)

class Device:
    """Represents a single CAN-FIX device type"""
    def __init__(self, name, dtype, model, version):
        self.name = name
        self.deviceType = dtype
        self.modelNumber = model
        self.version = version
        self.fwUpdateCode = None
        self.fwDriver = None
        self.parameters = []
        self.configuration = []

    def __str__(self):
        return "{} type=0x{:02X}, model={:06X}, version={}".format(self.name, self.deviceType, self.modelNumber, self.version)

# The key to the devices dictionary is a tuple made up of
# the device id, model number and version
devices = {}

dirlist = os.listdir(config.datapath + "/eds")
log.debug("Loading Devices")

def getParameterList(pl):
    l = []
    for p in pl:
        if(type(p) == str):
            l.append(int(p, 0))
        else:
            l.append(int(p))
    l.sort()
    return l

# Downloads a file from the URI given and stores it in the filename given.  The filename
# should be a complete path to the file.
def download_file(uri, filename):
    with urllib.request.urlopen(uri) as response, open(filename, 'wb') as out_file:
        shutil.copyfileobj(response, out_file)

# Here we decide if it's time to download and check a new data file index.  Then we loop through
# the EDS file list and compare the SHA256 hashes against the files that we already have in the
# data directory.  If any of thme need to be downloaded then we do that now.
last_start = 0 if settings.get("last_start") is None else settings.get("last_start")
if config.data_download_interval != -1 and (time.time() - last_start) > config.data_download_interval:
    log.info(f"Retrieving Data Index from {config.data_index_uri}")

    with urllib.request.urlopen(config.data_index_uri) as response, open(config.datapath + "/cfdataindex.json", 'wb') as out_file:
        shutil.copyfileobj(response, out_file)

    # Open and load the index file
    with open(config.datapath + "/cfdataindex.json", 'r') as file:
        index = json.load(file)

    # Check the SHA256 Hash against the existing files
    for eds in index['eds_index']:
        if eds['filename'] in dirlist:
            with open(f"{config.datapath}/eds/{eds['filename']}") as json_file:
                sha = hashlib.sha256()
                while True:
                    data = json_file.read(65536)
                    if not data:
                        break
                    sha.update(data.encode())
                # If the SHA hash doesn't match the index then we need to download the file
                if sha.hexdigest() != eds['sha256']:
                    log.info(f"Updating EDS file {config.datapath}/eds/{eds['filename']}")
                    download_file(eds['uri'], f"{config.datapath}/eds/{eds['filename']}")
        else:
            log.info(f"Downloading EDS file {config.datapath}/eds/{eds['filename']}")
            download_file(eds['uri'], f"{config.datapath}/eds/{eds['filename']}")

dirlist = os.listdir(config.datapath + "/eds")
for filename in dirlist:
    if filename[-5:] == ".json":
        with open(config.datapath + "/eds/" + filename) as json_file:
            log.debug(f"Loading device file {filename}")
            d = json.load(json_file, object_pairs_hook=OrderedDict)
        try: # These are required
            name = d["name"]
            if isinstance(d["type"], str):
                dtype = int(d["type"], 0)
            if isinstance(d["model"], str):
                model = int(d["model"], 0)
            if isinstance(d["version"], str):
                version = int(d["version"], 0)
        except KeyError as e:
            log.warn(f"Problem with device file {filename}:{e}")
        newdevice = Device(name, dtype, model, version)
        newdevice.fwUpdateCode = int(d.get("firmware_code"),0)
        newdevice.fwDriver = d.get("firmware_driver")
        newdevice.parameters = getParameterList(d.get("parameters", []))
        # TODO: Sanitize configuration list
        newdevice.configuration = d.get("configuration", [])

        devices[(dtype, model, version)] = newdevice
        log.info(str(newdevice))

def findDevice(device, model, version):
    return devices.get((device, model, version), None)

# TODO: Write a function to verify the configuration and other parts of the file.
#          and some unit tests to test it all