"""Implement scratch behavior where Reachy use its left arm to scratch its right forearm."""
import asyncio
import numpy as np

from reachy_sdk.trajectory import goto_async, InterpolationMode

from . import Behavior

class Scratch(Behavior):
    """
    Scratch class.

    Makes Reachy scratch its forearm.

    Uses: right_arm, left_arm, head
    Dependencies to other behaviors: none
    """

    def __init__(self, name: str, reachy, sub_behavior: bool = False) -> None:
        """Initialize the behavior."""
        super().__init__(name, reachy, sub_behavior=sub_behavior)

        self.scratch_arm = np.load('movements/scratch.npy')

        self.sampling_frequency = 100

        self.recorded_joints = [
            self.reachy.r_arm.r_shoulder_pitch,
            self.reachy.r_arm.r_shoulder_roll,
            self.reachy.r_arm.r_arm_yaw,
            self.reachy.r_arm.r_elbow_pitch,
            self.reachy.r_arm.r_forearm_yaw,
            self.reachy.r_arm.r_wrist_pitch,
            self.reachy.r_arm.r_wrist_roll,
            self.reachy.l_arm.l_shoulder_pitch,
            self.reachy.l_arm.l_shoulder_roll,
            self.reachy.l_arm.l_arm_yaw,
            self.reachy.l_arm.l_elbow_pitch,
            self.reachy.l_arm.l_forearm_yaw,
            self.reachy.l_arm.l_wrist_pitch,
            self.reachy.l_arm.l_wrist_roll,
            self.reachy.l_arm.l_gripper,
        ]

    async def run(self):
        """Implement the Scratch behavior."""
        for j in self.reachy.r_arm.joints.values():
            j.torque_limit = 100.0
        for j in self.reachy.l_arm.joints.values():
            j.torque_limit = 100.0

        look_down = self.reachy.head.look_at_async(
            0.5,
            -0.1,
            -0.5,
            duration=1.0,
            starting_positions={
                self.reachy.head.neck_roll: self.reachy.head.neck_roll.goal_position,
                self.reachy.head.neck_pitch: self.reachy.head.neck_pitch.goal_position,
                self.reachy.head.neck_yaw: self.reachy.head.neck_yaw.goal_position,
            })

        traj_antennas = goto_async(
            goal_positions={
                self.reachy.head.r_antenna: -30,
                self.reachy.head.l_antenna: 5,
            },
            duration=1.0,
            interpolation_mode=InterpolationMode.MINIMUM_JERK
        )

        first_point = dict(zip(self.recorded_joints, self.scratch_arm[50]))
        # Goes to the start of the trajectory in 1s
        first_pos = goto_async(first_point, duration=1.0)

        await asyncio.gather(
            look_down,
            first_pos,
            traj_antennas,
        )

        for jp_arms in self.scratch_arm[50:]:
            for joint, pos in zip(self.recorded_joints, jp_arms):
                joint.goal_position = pos

            await asyncio.sleep(1 / (self.sampling_frequency*2))

        traj_antennas = goto_async(
            goal_positions={
                self.reachy.head.r_antenna: -10,
                self.reachy.head.l_antenna: 65,
            },
            duration=1.0,
            interpolation_mode=InterpolationMode.MINIMUM_JERK
        )
        hands_back = goto_async({
            self.reachy.l_arm.l_shoulder_pitch: 0.0,
            self.reachy.l_arm.l_shoulder_roll: 0.0,
            self.reachy.l_arm.l_arm_yaw: 0.0,
            self.reachy.l_arm.l_elbow_pitch: 0.0,
            self.reachy.l_arm.l_forearm_yaw: 0.0,
            self.reachy.l_arm.l_wrist_pitch: 0.0,
            self.reachy.l_arm.l_wrist_roll: 0.0},
            duration=1.2,
        )
        watch_arm_head = self.reachy.head.look_at_async(
            0.5,
            -0.1,
            -0.3,
            duration=0.5,
            starting_positions={
                self.reachy.head.neck_roll: self.reachy.head.neck_roll.goal_position,
                self.reachy.head.neck_pitch: self.reachy.head.neck_pitch.goal_position,
                self.reachy.head.neck_yaw: self.reachy.head.neck_yaw.goal_position,
            })
        pose_watch_right_arm = [-21, 7, 38, -102, 27, -10, -4]
        goto_dic = {j: pos for j, pos in zip(self.reachy.r_arm.joints.values(), pose_watch_right_arm)}
        watch_arm_arm = goto_async(goal_positions=goto_dic, duration=0.5)

        await asyncio.gather(
            watch_arm_head,
            watch_arm_arm,
            hands_back,
            traj_antennas,
        )

        await asyncio.sleep(0.3)

        traj_antennas = goto_async(
            goal_positions={
                self.reachy.head.r_antenna: 0,
                self.reachy.head.l_antenna: 0,
            },
            duration=1.0,
            interpolation_mode=InterpolationMode.MINIMUM_JERK
        )

        look_back = self.reachy.head.look_at_async(
            0.5,
            0.0,
            0.0,
            duration=1.5,
            starting_positions={
                self.reachy.head.neck_roll: self.reachy.head.neck_roll.goal_position,
                self.reachy.head.neck_pitch: self.reachy.head.neck_pitch.goal_position,
                self.reachy.head.neck_yaw: self.reachy.head.neck_yaw.goal_position,
            })
        hands_back = goto_async({
            self.reachy.r_arm.r_shoulder_pitch: 0.0,
            self.reachy.r_arm.r_shoulder_roll: 0.0,
            self.reachy.r_arm.r_arm_yaw: 0.0,
            self.reachy.r_arm.r_elbow_pitch: 0.0,
            self.reachy.r_arm.r_forearm_yaw: 0.0,
            self.reachy.r_arm.r_wrist_pitch: 0.0,
            self.reachy.r_arm.r_wrist_roll: 0.0},
            duration=1.2,
        )

        await asyncio.gather(
            traj_antennas,
            look_back,
            hands_back,
        )

        self.reachy.turn_off_smoothly('r_arm')
        self.reachy.turn_off_smoothly('l_arm')

    async def teardown(self):
        """Use the teardown method from parent class."""
        return await super().teardown()


#!/usr/bin/env python
#-*- coding: UTF-8 -*-
import pygame
from pygame.locals import *

from cnst import *

import sprite
import player

import math
import random

class Boss(sprite.Sprite3):
    def __init__(self, game, rect, name, *args):
        sprite.Sprite3.__init__(self, game, rect, 'boss/left-0',
            (54, 124, 150, 78))
        self.rect.centery = rect.centery
        self.rect.centerx = rect.centerx
        self.groups.add('boss')
        self.hit_groups.add('capsule')
        self.game.sprites.append(self)
        self.game.boss = self

        self.vx = -1.4
        self.vy = 0

        self.auto_gc = False

        self.phase = 1
        self.phase_frames = 0

        self.taking_damage = 0

        self.dying = None
        self.dead = False

        self.drop = 120

        self.facing = 'right'

        if self.vx < 0:
            self.facing = 'left'

        self._prev = None
        self.strength = 6

    def loop(self):
        if self.dying != None:
            self.image = 'boss/%s-%d' % (self.facing,
                (self.game.frame / (FPS / 8)) % 8)
            if self.dying % 15 == 0:
                self.game.game.sfx['boss_explode'].play()
            if self.dying % 15 > 10:
                self.image = 'boss/%s-damage-%d' % (self.facing,
                    (self.game.frame / (FPS / 8)) % 8)
            self.dying += 1

            mid_frame = FPS * 3 / 2
            end_frame = FPS * 6 / 2

            if self.dying == mid_frame:
                # explode
                self.game.game.sfx['pop'].play()
                for n in xrange(128):
                    rect = pygame.Rect(random.randint(self.rect.left,
                        self.rect.right), random.randint(self.rect.top,
                        self.rect.bottom), 1, 1)
                    Bub(self.game, rect)

            if self.dying >= mid_frame:
                self.image = None

            if self.dying > end_frame:
                self.dead = True

            return

        if sprite.get_code(self.game, self, sign(self.vx), 0) == CODE_BOSS_TURN:
            self.vx = - self.vx

        if self.phase == 2:
            if self.phase_frames == 240 + 32:
                self.vy = 0
                self.vx = 1.2

        if self.vx > 0:
            self.facing = 'right'
        else:
            self.facing = 'left'

        self.rect.x += sprite.myinc(self.game.frame, self.vx)
        self.rect.y += sprite.myinc(self.game.frame, self.vy)

        self.image = 'boss/%s-%d' % (self.facing,
            (self.game.frame / (FPS / 8)) % 8)

        if self.taking_damage > 0:
            if self.taking_damage % 20 > 10:
                self.image = 'boss/%s-damage-%d' % (self.facing,
                (self.game.frame / (FPS / 8)) % 8)
            self.taking_damage -= 1

        self.phase_frames += 1

        if self.drop == 0:
            if self.vy == 0: # no bombing when flying up to the second phase
                Fireball(self.game, self.rect, self)
            if self.phase == 1:
                self.drop = 120
            if self.phase == 2:
                self.drop = 90
        s.drop -= 1

    def hit(self, a, b):
        self.game.game.sfx['boss_explode'].play()
        a.strength -= 1
        a.taking_damage = 60
        b.active = False
        if a.strength == 3:
            a.vy = -1
            a.vx = 0
            a.phase = 2
            a.phase_frames = 0
        if a.strength == 0:
            a.dying = 0

class Bub(sprite.Sprite3):
    def __init__(self, game, rect):
        sprite.Sprite3.__init__(self, game, rect, 'big-bubble', (0, 0, 16, 16))

        self.rect.centerx = r.centerx
        self.rect.centery = r.centery
        self.game.sprites.append(self)

        speed = random.randint(16, 64)
        angle = random.randint(0, 360) * 6.28 / 360
        self.vx = math.sin(angle) * speed
        self.vy = math.cos(angle) * speed
        self.frame = random.randint(0, FPS * 2)

    def loop(self):
        self.vx *= 0.98
        self.vy *= 0.98
        self.rect.x += sprite.myinc(self.game.frame, self.vx)
        self.rect.y += sprite.myinc(self.game.frame, self.vy)
        self.frame += 1
        if self.frame >= FPS * 2:
            self.active = False
            if random.randint(0, 3) == 1:
                self.game.game.sfx['pop'].play()


from datetime import datetime, timedelta
from django.db.models import Avg
import copy

from ..enums.definitions import platform_map
from ..analyzer.models import Analysis

def combine_search_scores(search_id):
    result = Analysis.objects.filter(search_id=search_id).aggregate(Avg('compound'))
    return result['compound__avg']

# expects list of analysis models
def combine_analysis_scores(analysis_list, start, end, query=""):
    buckets = {}
    avg_results = []
    delta = timedelta(days=1)
    start = datetime.combine(start, datetime.min.time())
    end = datetime.combine(end, datetime.min.time())

    bucket_dates = []
    bucket_filled = False
    original_start = start

    platforms = set([])
    for analysis in analysis_list:
        key = query + ' (' + platform_map['reverse'][analysis.collector_type] + ')'
        platforms.add(key)

    for platform in platforms:
        buckets[platform] = {}
        while start <= end:
            buckets[platform][start] = []
            if not bucket_filled:
                bucket_dates.append(start)
            start += delta
        start = original_start
        bucket_filled = True

    for analysis in analysis_list:
        floor = floor_by_day(analysis.date)
        key = query + ' (' + platform_map['reverse'][analysis.collector_type] + ')'
        buckets[key][floor].append(analysis.compound)

    for date_bucket in bucket_dates:
        result_obj = {'time': date_bucket}
        for platform in platforms:
            scores = buckets[platform][date_bucket]
            avg_score = 0
            if len(scores) != 0:
                avg_score = round(sum(scores)/len(scores), 2)
            result_obj[platform] = avg_score
        avg_results.append(result_obj)

    return avg_results

# datescores should look like [{'time': DATETIME, platform1: COMPOUNDSCORE, platform2: COMPOUNDSCORE, ...}, ...]
def union_scores(datescores1, datescores2, startdate, enddate):
    # variable number of platforms in each datescore
    platforms_all = set([])
    platforms1 = set([])
    platforms2 = set([])
    for item in datescores1:
        platforms1 = platforms1.union(item.keys())
    for item in datescores2:
        platforms2 = platforms2.union(item.keys())
    if 'time' in platforms1: platforms1.remove('time')
    if 'time' in platforms2: platforms2.remove('time')
    platforms_all = platforms1.union(platforms2)
    empty_platforms_obj = {platform: None for platform in platforms_all}
    time_to_scores = {}
    for item in datescores1:
        datekey = datetime.combine(item['time'], datetime.min.time())
        time_to_scores[datekey] = copy.deepcopy(empty_platforms_obj)
        for platform in platforms1:
            time_to_scores[datekey][platform] = item[platform]

    for item in datescores2:
        datekey = datetime.combine(item['time'], datetime.min.time())
        if datekey not in time_to_scores:
            time_to_scores[datekey] = copy.deepcopy(empty_platforms_obj)
        for platform in platforms2:
            time_to_scores[datekey][platform] = item[platform]

    # bucketing by days only for now
    delta = timedelta(days=1)
    date = datetime.combine(startdate, datetime.min.time())
    limit = datetime.combine(enddate, datetime.min.time())
    union = []
    while date <= limit:
        pt = {'time': date}
        pt.update(empty_platforms_obj)
        if date in time_to_scores:
            for platform in platforms_all:
                pt[platform] = time_to_scores[date][platform]
        union.append(pt)
        date += delta
    return union

#returns item in dates that is closest to date
def nearest(dates, date):
    date = date.replace(tzinfo=None)
    return min(dates, key=lambda x: abs(x - date))

def floor_by_day(dt):
    return datetime.combine(dt.date(), datetime.min.time())


#词性标注
'''jieba.posseg.POSTokenizer(tokenizer=None)新建自定义分词，
tokenizer参数可指定内部使用的jieba.Tokenizer分词器。jieba
.posseg.dt为默认词性标注分词器。
标注句子分词后每个词的词性，采用ictclas兼容的标记法。'''

#官方历程
import jieba.posseg as pseg

words=pseg.cut("我爱学习蟒蛇")
#words类别为：generator

for word,flag in words:
    print('%s %s'%(word,flag))

Python 数据分析/结巴分词和词云/词性标注.py


import torch
import torch.utils.tensorboard
import random
import time
import os
import logging
import argparse
import sys
import numpy as np

# sys.path.append('/home/alta/BLTSpeaking/exp-ytl28/local-ytl/grammatical-error-correction/')
from utils.misc import set_global_seeds, print_config, save_config, check_srctgt
from utils.misc import validate_config, get_memory_alloc, convert_dd_att_ref
from utils.misc import _convert_to_words_batchfirst, _convert_to_words, _convert_to_tensor, _del_var
from utils.dataset import Dataset
from utils.config import PAD, EOS
from modules.loss import NLLLoss, BCELoss, CrossEntropyLoss
from modules.optim import Optimizer
from modules.checkpoint import Checkpoint
from models.recurrent import Seq2Seq

logging.basicConfig(level=logging.DEBUG)
device = torch.device('cpu')

MAX_COUNT_NO_IMPROVE = 5
MAX_COUNT_NUM_ROLLBACK = 5
KEEP_NUM = 1

def load_arguments(parser):

	""" Seq2Seq model """

	# paths
	parser.add_argument('--train_path_src', type=str, required=True, help='train src dir')
	parser.add_argument('--train_path_tgt', type=str, required=True, help='train tgt dir')
	parser.add_argument('--path_vocab_src', type=str, required=True, help='vocab src dir')
	parser.add_argument('--path_vocab_tgt', type=str, required=True, help='vocab tgt dir')
	parser.add_argument('--dev_path_src', type=str, default=None, help='dev src dir')
	parser.add_argument('--dev_path_tgt', type=str, default=None, help='dev tgt dir')
	parser.add_argument('--save', type=str, required=True, help='model save dir')
	parser.add_argument('--load', type=str, default=None, help='model load dir')
	parser.add_argument('--load_embedding_src', type=str, default=None, help='pretrained src embedding')
	parser.add_argument('--load_embedding_tgt', type=str, default=None, help='pretrained tgt embedding')

	# model
	parser.add_argument('--use_bpe', type=str, default='False', help='use byte-pair emcoding or not')
	parser.add_argument('--embedding_size_enc', type=int, default=200, help='encoder embedding size')
	parser.add_argument('--embedding_size_dec', type=int, default=200, help='decoder embedding size')
	parser.add_argument('--hidden_size_enc', type=int, default=200, help='encoder hidden size')
	parser.add_argument('--num_bilstm_enc', type=int, default=2, help='number of encoder bilstm layers')
	parser.add_argument('--num_unilstm_enc', type=int, default=0, help='number of encoder unilstm layers')
	parser.add_argument('--hidden_size_dec', type=int, default=200, help='encoder hidden size')
	parser.add_argument('--num_unilstm_dec', type=int, default=2, help='number of encoder bilstm layers')
	parser.add_argument('--hard_att', type=str, default='False', help='use hard attention or not')
	parser.add_argument('--att_mode', type=str, default='bahdanau', \
							help='attention mechanism mode - bahdanau / hybrid / dot_prod')
	parser.add_argument('--hidden_size_att', type=int, default=1, \
							help='hidden size for bahdanau / hybrid attention')
	parser.add_argument('--hidden_size_shared', type=int, default=200, \
							help='transformed att output hidden size (set as hidden_size_enc)')
	parser.add_argument('--additional_key_size', type=int, default=0, \
							help='additional attention key size: keys = [values, add_feats]')

	# train
	parser.add_argument('--random_seed', type=int, default=666, help='random seed')
	parser.add_argument('--max_seq_len', type=int, default=32, help='maximum sequence length')
	parser.add_argument('--batch_size', type=int, default=64, help='batch size')
	parser.add_argument('--embedding_dropout', type=float, default=0.0, help='embedding dropout')
	parser.add_argument('--dropout', type=float, default=0.0, help='dropout')
	parser.add_argument('--teacher_forcing_ratio', type=float, default=0.0, help='ratio of teacher forcing')
	parser.add_argument('--num_epochs', type=int, default=10, help='number of training epoches')
	parser.add_argument('--learning_rate', type=float, default=0.001, help='learning rate')
	parser.add_argument('--residual', type=str, default='False', help='residual connection')
	parser.add_argument('--max_grad_norm', type=float, default=1.0, help='optimiser gradient norm clipping: max grad norm')
	parser.add_argument('--batch_first', type=str, default='True', help='batch as the first dimension')
	parser.add_argument('--use_gpu', type=str, default='False', help='whether or not using GPU')
	parser.add_argument('--eval_with_mask', type=str, default='True', help='calc loss excluding padded words')
	parser.add_argument('--scheduled_sampling', type=str, default='False', \
					 		help='gradually turn off teacher forcing \
					 		(if True, use teacher_forcing_ratio as the starting point)')
	parser.add_argument('--seqrev', type=str, default='False', help='reverse src, tgt sequence')

	# save and print
	parser.add_argument('--checkpoint_every', type=int, default=10, help='save ckpt every n steps')
	parser.add_argument('--print_every', type=int, default=10, help='print every n steps')

	# -- inactive arguments --
	parser.add_argument('--train_attkey_path', type=str, default=None, help='train set additional attention key')
	parser.add_argument('--dev_attkey_path', type=str, default=None, help='dev set additional attention key')
	parser.add_argument('--ptr_net', type=str, default='null', \
							help='whether or not to use pointer network - use attention weights to directly map embedding \
							comb | pure | none: comb combines posterior and att weights (combination ratio is learnt); \
							pure use att weights only; none use posterior only')
	parser.add_argument('--ddatt_loss_weight', type=float, default=0.0, help='determines attloss weight in sgd [0.0~1.0]')
	parser.add_argument('--ddattcls_loss_weight', type=float, default=0.0,
							help='determines attcls loss weight in sgd [0.0~1.0] (att column wise regularisation)')
	parser.add_argument('--att_scale_up', type=float, default=0.0,
							help='scale up att scores before cross entropy loss in regularisation')

	return parser

class Trainer(object):

	def __init__(self, expt_dir='experiment',
		load_dir=None,
		loss=NLLLoss(),
		batch_size=64,
		random_seed=None,
		checkpoint_every=100,
		print_every=100,
		use_gpu=False,
		learning_rate=0.001,
		max_grad_norm=1.0,
		eval_with_mask=True,
		scheduled_sampling=False,
		teacher_forcing_ratio=0.0,
		ddatt_loss_weight=0.0,
		ddattcls_loss_weight=0.0,
		att_scale_up=0.0
		):

		self.random_seed = random_seed
		if random_seed is not None:
			set_global_seeds(random_seed)

		self.loss = loss
		self.optimizer = None
		self.checkpoint_every = checkpoint_every
		self.print_every = print_every
		self.use_gpu = use_gpu
		self.learning_rate = learning_rate
		self.max_grad_norm = max_grad_norm
		self.eval_with_mask = eval_with_mask
		self.scheduled_sampling = scheduled_sampling
		self.teacher_forcing_ratio = teacher_forcing_ratio

		self.ddatt_loss_weight = ddatt_loss_weight
		self.ddattcls_loss_weight = ddattcls_loss_weight
		self.att_scale_up = att_scale_up

		if not os.path.isabs(expt_dir):
			expt_dir = os.path.join(os.getcwd(), expt_dir)
		self.expt_dir = expt_dir
		if not os.path.exists(self.expt_dir):
			os.makedirs(self.expt_dir)
		self.load_dir = load_dir

		self.batch_size = batch_size
		self.logger = logging.getLogger(__name__)
		self.writer = torch.utils.tensorboard.writer.SummaryWriter(log_dir=self.expt_dir)

	def _evaluate_batches(self, model, batches, dataset):

		model.eval()

		loss = NLLLoss()
		loss.reset()

		match = 0
		total = 0

		out_count = 0
		with torch.no_grad():
			for batch in batches:

				src_ids = batch['src_word_ids']
				src_lengths = batch['src_sentence_lengths']
				tgt_ids = batch['tgt_word_ids']
				tgt_lengths = batch['tgt_sentence_lengths']
				src_probs = None
				if 'src_ddfd_probs' in batch and model.additional_key_size > 0:
					src_probs =  batch['src_ddfd_probs']
					src_probs = _convert_to_tensor(src_probs, self.use_gpu).unsqueeze(2)
				src_labs = None
				if 'src_ddfd_labs' in batch:
					src_labs =  batch['src_ddfd_labs']
					src_labs = _convert_to_tensor(src_labs, self.use_gpu).unsqueeze(2)

				src_ids = _convert_to_tensor(src_ids, self.use_gpu)
				tgt_ids = _convert_to_tensor(tgt_ids, self.use_gpu)

				non_padding_mask_tgt = tgt_ids.data.ne(PAD)
				non_padding_mask_src = src_ids.data.ne(PAD)

				decoder_outputs, decoder_hidden, other = model(src_ids, tgt_ids,
					is_training=False, att_key_feats=src_probs)

				# Evaluation
				logps = torch.stack(decoder_outputs, dim=1).to(device=device)
				if not self.eval_with_mask:
					loss.eval_batch(logps.reshape(-1, logps.size(-1)),
						tgt_ids[:, 1:].reshape(-1))
				else:
					loss.eval_batch_with_mask(logps.reshape(-1, logps.size(-1)),
						tgt_ids[:, 1:].reshape(-1), non_padding_mask_tgt[:,1:].reshape(-1))

				seqlist = other['sequence']
				seqres = torch.stack(seqlist, dim=1).to(device=device)
				correct = seqres.view(-1).eq(tgt_ids[:,1:].reshape(-1))\
					.masked_select(non_padding_mask_tgt[:,1:].reshape(-1)).sum().item()
				match += correct
				total += non_padding_mask_tgt[:,1:].sum().item()

				if not self.eval_with_mask:
					loss.norm_term = 1.0 * tgt_ids.size(0) * tgt_ids[:,1:].size(1)
				else:
					loss.norm_term = 1.0 * torch.sum(non_padding_mask_tgt[:,1:])
				loss.normalise()

				if out_count < 3:
					srcwords = _convert_to_words_batchfirst(src_ids, dataset.tgt_id2word)
					refwords = _convert_to_words_batchfirst(tgt_ids[:,1:], dataset.tgt_id2word)
					seqwords = _convert_to_words(seqlist, dataset.tgt_id2word)
					outsrc = 'SRC: {}\n'.format(' '.join(srcwords[0])).encode('utf-8')
					outref = 'REF: {}\n'.format(' '.join(refwords[0])).encode('utf-8')
					outline = 'GEN: {}\n'.format(' '.join(seqwords[0])).encode('utf-8')
					sys.stdout.buffer.write(outsrc)
					sys.stdout.buffer.write(outref)
					sys.stdout.buffer.write(outline)
					out_count += 1

		att_resloss = 0
		attcls_resloss = 0
		resloss = loss.get_loss()

		if total == 0:
			accuracy = float('nan')
		else:
			accuracy = match / total
		torch.cuda.empty_cache()

		losses = {}
		losses['att_loss'] = att_resloss
		losses['attcls_loss'] = attcls_resloss

		return resloss, accuracy, losses

	def _train_batch(self, src_ids, tgt_ids, model, step, total_steps, src_probs=None, src_labs=None):

		"""
			Args:
				src_ids 		=     w1 w2 w3 </s> <pad> <pad> <pad>
				tgt_ids 		= <s> w1 w2 w3 </s> <pad> <pad> <pad>
			(optional)
				src_probs 		=     p1 p2 p3 0    0     ...
			Others:
				internal input 	= <s> w1 w2 w3 </s> <pad> <pad>
				decoder_outputs	= 	  w1 w2 w3 </s> <pad> <pad> <pad>
		"""

		# define loss
		loss = NLLLoss()

		# scheduled sampling
		if not self.scheduled_sampling:
			teacher_forcing_ratio = self.teacher_forcing_ratio
		else:
			# use self.teacher_forcing_ratio as the starting point
			progress = 1.0 * step / total_steps
			teacher_forcing_ratio = 1.0 - progress

		# get padding mask
		non_padding_mask_src = src_ids.data.ne(PAD)
		non_padding_mask_tgt = tgt_ids.data.ne(PAD)

		# Forward propagation
		decoder_outputs, decoder_hidden, ret_dict = model(src_ids, tgt_ids,
			is_training=True,teacher_forcing_ratio=teacher_forcing_ratio,att_key_feats=src_probs)

		# Get loss
		loss.reset()
		# import pdb; pdb.set_trace()

		logps = torch.stack(decoder_outputs, dim=1).to(device=device)
		if not self.eval_with_mask:
			loss.eval_batch(logps.reshape(-1, logps.size(-1)),
				tgt_ids[:, 1:].reshape(-1))
		else:
			loss.eval_batch_with_mask(logps.reshape(-1, logps.size(-1)),
				tgt_ids[:,1:].reshape(-1), non_padding_mask_tgt[:,1:].reshape(-1))

		if not self.eval_with_mask:
			loss.norm_term = 1.0 * tgt_ids.size(0) * tgt_ids[:,1:].size(1)
		else:
			loss.norm_term = 1.0 * torch.sum(non_padding_mask_tgt[:,1:])
		loss.normalise()

		# Backward propagation
		model.zero_grad()
		resloss = loss.get_loss()
		att_resloss = 0
		dsfclassify_resloss = 0

		loss.backward()
		self.optimizer.step()

		return resloss, att_resloss, dsfclassify_resloss

	def _train_epoches(self, train_set, model, n_epochs, start_epoch, start_step, dev_set=None):

		log = self.logger

		print_loss_total = 0  # Reset every print_every
		epoch_loss_total = 0  # Reset every epoch
		att_print_loss_total = 0  # Reset every print_every
		att_epoch_loss_total = 0  # Reset every epoch
		attcls_print_loss_total = 0  # Reset every print_every
		attcls_epoch_loss_total = 0  # Reset every epoch

		step = start_step
		step_elapsed = 0
		prev_acc = 0.0
		count_no_improve = 0
		count_num_rollback = 0
		ckpt = None

		# ******************** [loop over epochs] ********************
		for epoch in range(start_epoch, n_epochs + 1):

			for param_group in self.optimizer.optimizer.param_groups:
				print('epoch:{} lr: {}'.format(epoch, param_group['lr']))
				lr_curr = param_group['lr']

			# ----------construct batches-----------
			# allow re-shuffling of data
			if type(train_set.attkey_path) == type(None):
				print('--- construct train set ---')
				train_batches, vocab_size = train_set.construct_batches(is_train=True)
				if dev_set is not None:
					print('--- construct dev set ---')
					dev_batches, vocab_size = dev_set.construct_batches(is_train=False)
			else:
				print('--- construct train set ---')
				train_batches, vocab_size = train_set.construct_batches_with_ddfd_prob(is_train=True)
				if dev_set is not None:
					print('--- construct dev set ---')
					assert type(dev_set.attkey_path) != type(None), 'Dev set missing ddfd probabilities'
					dev_batches, vocab_size = dev_set.construct_batches_with_ddfd_prob(is_train=False)

			# --------print info for each epoch----------
			steps_per_epoch = len(train_batches)
			total_steps = steps_per_epoch * n_epochs
			log.info("steps_per_epoch {}".format(steps_per_epoch))
			log.info("total_steps {}".format(total_steps))

			log.debug(" ----------------- Epoch: %d, Step: %d -----------------" % (epoch, step))
			mem_kb, mem_mb, mem_gb = get_memory_alloc()
			mem_mb = round(mem_mb, 2)
			print('Memory used: {0:.2f} MB'.format(mem_mb))
			self.writer.add_scalar('Memory_MB', mem_mb, global_step=step)
			sys.stdout.flush()

			# ******************** [loop over batches] ********************
			model.train(True)
			for batch in train_batches:

				# update macro count
				step += 1
				step_elapsed += 1

				# load data
				src_ids = batch['src_word_ids']
				src_lengths = batch['src_sentence_lengths']
				tgt_ids = batch['tgt_word_ids']
				tgt_lengths = batch['tgt_sentence_lengths']

				src_probs = None
				src_labs = None
				if 'src_ddfd_probs' in batch and model.additional_key_size > 0:
					src_probs =  batch['src_ddfd_probs']
					src_probs = _convert_to_tensor(src_probs, self.use_gpu).unsqueeze(2)
				if 'src_ddfd_labs' in batch:
					src_labs = batch['src_ddfd_labs']
					src_labs = _convert_to_tensor(src_labs, self.use_gpu).unsqueeze(2)

				# sanity check src-tgt pair
				if step == 1:
					print('--- Check src tgt pair ---')
					log_msgs = check_srctgt(src_ids, tgt_ids, train_set.src_id2word, train_set.tgt_id2word)
					for log_msg in log_msgs:
						sys.stdout.buffer.write(log_msg)

				# convert variable to tensor
				src_ids = _convert_to_tensor(src_ids, self.use_gpu)
				tgt_ids = _convert_to_tensor(tgt_ids, self.use_gpu)

				# Get loss
				loss, att_loss, attcls_loss = self._train_batch(src_ids, tgt_ids,
					model, step, total_steps, src_probs=src_probs, src_labs=src_labs)

				print_loss_total += loss
				epoch_loss_total += loss
				att_print_loss_total += att_loss
				att_epoch_loss_total += att_loss
				attcls_print_loss_total += attcls_loss
				attcls_epoch_loss_total += attcls_loss

				if step % self.print_every == 0 and step_elapsed > self.print_every:
					print_loss_avg = print_loss_total / self.print_every
					att_print_loss_avg = att_print_loss_total / self.print_every
					attcls_print_loss_avg = attcls_print_loss_total / self.print_every
					print_loss_total = 0
					att_print_loss_total = 0
					attcls_print_loss_total = 0

					log_msg = 'Progress: %d%%, Train nlll: %.4f, att: %.4f, attcls: %.4f' % (
								step / total_steps * 100,
								print_loss_avg, att_print_loss_avg, attcls_print_loss_avg)
					# print(log_msg)
					log.info(log_msg)
					self.writer.add_scalar('train_loss', print_loss_avg, global_step=step)
					self.writer.add_scalar('att_train_loss', att_print_loss_avg, global_step=step)
					self.writer.add_scalar('attcls_train_loss', attcls_print_loss_avg, global_step=step)

				# Checkpoint
				if step % self.checkpoint_every == 0 or step == total_steps:

					# save criteria
					if dev_set is not None:
						dev_loss, accuracy, dev_attlosses = \
							self._evaluate_batches(model, dev_batches, dev_set)
						dev_attloss = dev_attlosses['att_loss']
						dev_attclsloss = dev_attlosses['attcls_loss']
						log_msg = 'Progress: %d%%, Dev loss: %.4f, accuracy: %.4f, att: %.4f, attcls: %.4f' % (
									step / total_steps * 100, dev_loss, accuracy,
									dev_attloss, dev_attclsloss)
						log.info(log_msg)
						self.writer.add_scalar('dev_loss', dev_loss, global_step=step)
						self.writer.add_scalar('dev_acc', accuracy, global_step=step)
						self.writer.add_scalar('att_dev_loss', dev_attloss, global_step=step)
						self.writer.add_scalar('attcls_dev_loss', dev_attclsloss, global_step=step)

						# save
						if prev_acc < accuracy:
							# save best model
							ckpt = Checkpoint(model=model,
									   optimizer=self.optimizer,
									   epoch=epoch, step=step,
									   input_vocab=train_set.vocab_src,
									   output_vocab=train_set.vocab_tgt)

							saved_path = ckpt.save(self.expt_dir)
							print('saving at {} ... '.format(saved_path))
							# reset
							prev_acc = accuracy
							count_no_improve = 0
							count_num_rollback = 0
						else:
							count_no_improve += 1

						# roll back
						if count_no_improve > MAX_COUNT_NO_IMPROVE:
							# resuming
							latest_checkpoint_path = Checkpoint.get_latest_checkpoint(self.expt_dir)
							if type(latest_checkpoint_path) != type(None):
								resume_checkpoint = Checkpoint.load(latest_checkpoint_path)
								print('epoch:{} step: {} - rolling back {} ...'
									.format(epoch, step, latest_checkpoint_path))
								model = resume_checkpoint.model
								self.optimizer = resume_checkpoint.optimizer
								# A walk around to set optimizing parameters properly
								resume_optim = self.optimizer.optimizer
								defaults = resume_optim.param_groups[0]
								defaults.pop('params', None)
								defaults.pop('initial_lr', None)
								self.optimizer.optimizer = resume_optim\
									.__class__(model.parameters(), **defaults)
								# start_epoch = resume_checkpoint.epoch
								# step = resume_checkpoint.step

							# reset
							count_no_improve = 0
							count_num_rollback += 1

						# update learning rate
						if count_num_rollback > MAX_COUNT_NUM_ROLLBACK:

							# roll back
							latest_checkpoint_path = Checkpoint.get_latest_checkpoint(self.expt_dir)
							if type(latest_checkpoint_path) != type(None):
								resume_checkpoint = Checkpoint.load(latest_checkpoint_path)
								print('epoch:{} step: {} - rolling back {} ...'
									.format(epoch, step, latest_checkpoint_path))
								model = resume_checkpoint.model
								self.optimizer = resume_checkpoint.optimizer
								# A walk around to set optimizing parameters properly
								resume_optim = self.optimizer.optimizer
								defaults = resume_optim.param_groups[0]
								defaults.pop('params', None)
								defaults.pop('initial_lr', None)
								self.optimizer.optimizer = resume_optim\
									.__class__(model.parameters(), **defaults)
								start_epoch = resume_checkpoint.epoch
								step = resume_checkpoint.step

							# decrease lr
							for param_group in self.optimizer.optimizer.param_groups:
								param_group['lr'] *= 0.5
								lr_curr = param_group['lr']
								print('reducing lr ...')
								print('step:{} - lr: {}'.format(step, param_group['lr']))

							# check early stop
							if lr_curr < 0.000125:
								print('early stop ...')
								break

							# reset
							count_no_improve = 0
							count_num_rollback = 0

						model.train(mode=True)
						if ckpt is None:
							ckpt = Checkpoint(model=model,
									   optimizer=self.optimizer,
									   epoch=epoch, step=step,
									   input_vocab=train_set.vocab_src,
									   output_vocab=train_set.vocab_tgt)
							saved_path = ckpt.save(self.expt_dir)
						ckpt.rm_old(self.expt_dir, keep_num=KEEP_NUM)
						print('n_no_improve {}, num_rollback {}'
							.format(count_no_improve, count_num_rollback))
					sys.stdout.flush()

			else:
				if dev_set is None:
					# save every epoch if no dev_set
					ckpt = Checkpoint(model=model,
							   optimizer=self.optimizer,
							   epoch=epoch, step=step,
							   input_vocab=train_set.vocab_src,
							   output_vocab=train_set.vocab_tgt)
					# saved_path = ckpt.save(self.expt_dir)
					saved_path = ckpt.save_epoch(self.expt_dir, epoch)
					print('saving at {} ... '.format(saved_path))
					continue

				else:
					continue
			# break nested for loop
			break

			if step_elapsed == 0: continue
			epoch_loss_avg = epoch_loss_total / min(steps_per_epoch, step - start_step)
			epoch_loss_total = 0
			log_msg = "Finished epoch %d: Train %s: %.4f" % (epoch, self.loss.name, epoch_loss_avg)

			log.info('\n')
			log.info(log_msg)

	def train(self, train_set, model, num_epochs=5, resume=False, optimizer=None, dev_set=None):

		"""
			Run training for a given model.
			Args:
				train_set: dataset
				dev_set: dataset, optional
				model: model to run training on, if `resume=True`, it would be
				   overwritten by the model loaded from the latest checkpoint.
				num_epochs (int, optional): number of epochs to run (default 5)
				resume(bool, optional): resume training with the latest checkpoint, (default False)
				optimizer (seq2seq.optim.Optimizer, optional): optimizer for training
				   (default: Optimizer(pytorch.optim.Adam, max_grad_norm=5))

			Returns:
				model (seq2seq.models): trained model.
		"""

		log = self.logger.info('MAX_COUNT_NO_IMPROVE: {}'.format(MAX_COUNT_NO_IMPROVE))
		log = self.logger.info('MAX_COUNT_NUM_ROLLBACK: {}'.format(MAX_COUNT_NUM_ROLLBACK))

		torch.cuda.empty_cache()
		if resume:
			# latest_checkpoint_path = Checkpoint.get_latest_checkpoint(self.load_dir)
			latest_checkpoint_path = self.load_dir
			print('resuming {} ...'.format(latest_checkpoint_path))
			resume_checkpoint = Checkpoint.load(latest_checkpoint_path)
			model = resume_checkpoint.model
			print(model)
			self.optimizer = resume_checkpoint.optimizer

			# A walk around to set optimizing parameters properly
			resume_optim = self.optimizer.optimizer
			defaults = resume_optim.param_groups[0]
			defaults.pop('params', None)
			defaults.pop('initial_lr', None)
			self.optimizer.optimizer = resume_optim.__class__(model.parameters(), **defaults)

			# start_epoch = resume_checkpoint.epoch
			# step = resume_checkpoint.step
			model.set_idmap(train_set.src_word2id, train_set.src_id2word)
			model.reset_batch_size(train_set.batch_size)

			for name, param in model.named_parameters():
				log = self.logger.info('{}:{}'.format(name, param.size()))

			# just for the sake of finetuning
			start_epoch = 1
			step = 0

		else:
			start_epoch = 1
			step = 0
			print(model)

			for name, param in model.named_parameters():
				log = self.logger.info('{}:{}'.format(name, param.size()))

			if optimizer is None:
				optimizer = Optimizer(torch.optim.Adam(model.parameters(),
							lr=self.learning_rate), max_grad_norm=self.max_grad_norm) # 5 -> 1
			self.optimizer = optimizer

		self.logger.info("Optimizer: %s, Scheduler: %s"
			% (self.optimizer.optimizer, self.optimizer.scheduler))

		self._train_epoches(train_set, model, num_epochs, start_epoch, step, dev_set=dev_set)

		return model

def main():

	# load config
	parser = argparse.ArgumentParser(description='PyTorch Seq2Seq-DD Training')
	parser = load_arguments(parser)
	args = vars(parser.parse_args())
	config = validate_config(args)

	# record config
	if not os.path.isabs(config['save']):
		config_save_dir = os.path.join(os.getcwd(), config['save'])
	if not os.path.exists(config['save']):
		os.makedirs(config['save'])

	# check device:
	if config['use_gpu'] and torch.cuda.is_available():
		global device
		device = torch.device('cuda')
	else:
		device = torch.device('cpu')
	print('device: {}'.format(device))

	# resume or not
	if config['load']:
		resume = True
		print('resuming {} ...'.format(config['load']))
		config_save_dir = os.path.join(config['save'], 'model-cont.cfg')
	else:
		resume = False
		config_save_dir = os.path.join(config['save'], 'model.cfg')
	save_config(config, config_save_dir)

	# load train set
	train_path_src = config['train_path_src']
	train_path_tgt = config['train_path_tgt']
	path_vocab_src = config['path_vocab_src']
	path_vocab_tgt = config['path_vocab_tgt']
	train_attkey_path = config['train_attkey_path']
	train_set = Dataset(train_path_src, train_path_tgt,
						path_vocab_src, path_vocab_tgt,
						attkey_path=train_attkey_path,seqrev=config['seqrev'],
						max_seq_len=config['max_seq_len'], batch_size=config['batch_size'],
						use_gpu=config['use_gpu'])

	vocab_size_enc = len(train_set.vocab_src)
	vocab_size_dec = len(train_set.vocab_tgt)

	# load dev set
	if config['dev_path_src'] and config['dev_path_tgt']:
		dev_path_src = config['dev_path_src']
		dev_path_tgt = config['dev_path_tgt']
		dev_attkey_path = config['dev_attkey_path']
		dev_set = Dataset(dev_path_src, dev_path_tgt,
						path_vocab_src, path_vocab_tgt,
						attkey_path=dev_attkey_path,seqrev=config['seqrev'],
						max_seq_len=config['max_seq_len'], batch_size=config['batch_size'],
						use_gpu=config['use_gpu'])
	else:
		dev_set = None

	# construct model
	seq2seq = Seq2Seq(vocab_size_enc, vocab_size_dec,
							embedding_size_enc=config['embedding_size_enc'],
							embedding_size_dec=config['embedding_size_dec'],
							embedding_dropout=config['embedding_dropout'],
							hidden_size_enc=config['hidden_size_enc'],
							num_bilstm_enc=config['num_bilstm_enc'],
							num_unilstm_enc=config['num_unilstm_enc'],
							hidden_size_dec=config['hidden_size_dec'],
							num_unilstm_dec=config['num_unilstm_dec'],
							hidden_size_att=config['hidden_size_att'],
							hidden_size_shared=config['hidden_size_shared'],
							dropout=config['dropout'],
							residual=config['residual'],
							batch_first=config['batch_first'],
							max_seq_len=config['max_seq_len'],
							batch_size=config['batch_size'],
							load_embedding_src=config['load_embedding_src'],
							load_embedding_tgt=config['load_embedding_tgt'],
							src_word2id=train_set.src_word2id,
							tgt_word2id=train_set.tgt_word2id,
							src_id2word=train_set.src_id2word,
							att_mode=config['att_mode'],
							hard_att=config['hard_att'],
							use_gpu=config['use_gpu'],
							additional_key_size=config['additional_key_size'],
							ptr_net=config['ptr_net'],
							use_bpe=config['use_bpe'])

	if config['use_gpu']:
		seq2seq = seq2seq.cuda()

	# contruct trainer
	t = Trainer(expt_dir=config['save'],
					load_dir=config['load'],
					batch_size=config['batch_size'],
					random_seed=config['random_seed'],
					checkpoint_every=config['checkpoint_every'],
					print_every=config['print_every'],
					learning_rate=config['learning_rate'],
					eval_with_mask=config['eval_with_mask'],
					scheduled_sampling=config['scheduled_sampling'],
					teacher_forcing_ratio=config['teacher_forcing_ratio'],
					use_gpu=config['use_gpu'],
					max_grad_norm=config['max_grad_norm'],
					ddatt_loss_weight=config['ddatt_loss_weight'],
					ddattcls_loss_weight=config['ddattcls_loss_weight'],
					att_scale_up=config['att_scale_up'])

	# run training
	seq2seq = t.train(train_set, seq2seq,
		num_epochs=config['num_epochs'], resume=resume, dev_set=dev_set)

if __name__ == '__main__':
	main()


from collections import defaultdict
from functools import lru_cache
from typing import Dict, DefaultDict, Tuple, Generator

from covid19_il.logger.logger import Logger
from covid19_il.data_handler.data_handlers.data_handler import DataHandler

class AgeGender(DataHandler):
    """ Covid19_IL Age Gender Data Handler.

    Attributes:
        None.

    Methods:
        statistics_by_gender(self): Yields statistic data via first week day with results which grouped
            by gender..
        _get_statistics_by_columns_names(self, columns_names: Tuple): private method - Yields statistic data as
            generator via first week day with results which grouped by gender.
        statistics_by_given_first_week_day(self, week_day: str): Yields statistic data by given first week day grouped
            by gender.
        statistics_by_age_group(self): Yields statistic of calculated fields by ordered by age group.

    """

    calculated_fields = ('weekly_tests_num', 'weekly_newly_tested', 'weekly_cases', 'weekly_deceased')

    def __init__(self, logger: Logger.logger, json_data: Dict) -> None:
        """ Initialize Base Class & Instance Attributes """
        super().__init__(logger, json_data)

    @lru_cache
    def statistics_by_gender(self) -> \
            Generator[DefaultDict[str, DefaultDict[str, Dict[str, Dict[str, int or str]]]], None, None] or \
            Generator[str, None, None]:
        """ Yields statistic data via first week day with results which grouped by gender.

        Args:
            None.

        Yields:
            Tuple[str, DefaultDict[str, Dict[str, Dict[str, int or str]]] or str: desired data or "No Data" as
                bad result.

        """

        return self._get_statistics_by_columns_names(AgeGender.calculated_fields)

    def _get_statistics_by_columns_names(self, columns_names: Tuple) ->\
            Generator[DefaultDict[str, DefaultDict[str, Dict[str, Dict[str, int or str]]]], None, None] or \
            Generator[str, None, None]:
        """ Yields statistic data via first week day with results which grouped by gender.

        Note:
            private method which get called by other method for data statistic calculation.

        Args:
            columns_names(Tuple): required columns name of data frames for data manipulations.

        Yields:
            Tuple[str, DefaultDict[str, Dict[str, Dict[str, int or str]]]] or str: desired data or "No Data" as
                bad result.

        """

        data_dict = None
        try:
            df = self._get_clean_copy_df_data()
            ser = df.groupby([*df.columns])['gender']
            data = ser.unique()
            data_dict = defaultdict(lambda: defaultdict(lambda: dict()))
            for key in data.keys():
                # key[0]: first week day, key[2]: age_group, key[3]: gender, key[4:]: the actual values
                data_dict[key[3]][key[0]][key[2]] = {key: value for key, value in zip(columns_names, key[4:])}
                # data_dict[key[0]][key[3]][key[2]] = {key: value for key, value in zip(columns_names[1:], key[4:])}
        except KeyError as ke:
            self._logger.exception(ke, "No DataFrame's key exists according to the api client's query results")
        finally:
            if bool(data_dict):
                yield from data_dict.items()
            else:
                yield "No Data"

    @lru_cache
    def statistics_by_given_first_week_day(self, week_day: str) ->\
            Generator[DefaultDict[str, Dict[str, Dict[str, str]]], None, None] or Generator[str, None, None]:
        """ Yields statistic data by given first week day grouped by gender.

        Args:
            week_day(str): given week day for data manipulation.

        Yields:
            Tuple[str, Dict[str, Dict[str, str]] or str: desired data or "No Data" for bad result.

        """

        data_dict = None
        try:
            df = self._get_clean_copy_df_data()
            df = df[df['first_week_day'] == week_day]
            ser = df.groupby([*df.columns])['gender']
            data = ser.unique()
            data_dict = defaultdict(lambda: dict())
            for key in data.keys():
                # key[0]: first week day, key[2]: age_group, key[3]: gender, key[4:]: the actual values
                data_dict[key[3]][key[2]] = {key: value for key, value in zip(AgeGender.calculated_fields, key[4:])}
        except KeyError as ke:
            self._logger.exception(ke, "No DataFrame's key exists according to the api client's query results")
        finally:
            if bool(data_dict):
                yield from data_dict.items()
            else:
                yield "No Data"

    @lru_cache
    def statistics_by_age_group(self) ->\
            Generator[DefaultDict[str, DefaultDict[str, Dict[str, int or float]]], None, None] or\
            Generator[str, None, None]:
        """ Yields statistic of calculated fields by ordered by age group.

        Args:
            None.

        Yields:
            Tuple[str, [DefaultDict[str, Dict[str, int or float]] or str: desired data or "No Data" for bad result.
        """

        data_dict = None
        try:
            df = self._get_clean_copy_df_data()
            for column_name in AgeGender.calculated_fields:
                df[column_name] = [self._convert_string_to_int(item) for item in df[column_name]]
            age_groups = df['age_group'].unique()
            data_dict = defaultdict(lambda: dict())

            for group in age_groups:
                temp_df = df[df['age_group'] == group]
                for field in AgeGender.calculated_fields:
                    df_by_field = temp_df[field]
                    data_dict[group][field] = {"min": df_by_field.min(),
                                               "max": df_by_field.max(),
                                               "mean": df_by_field.mean(),
                                               "total": df_by_field.sum()}
        except KeyError as ke:
            self._logger.exception(ke, "No DataFrame's key exists according to the api client's query results")
        finally:
            if bool(data_dict):
                yield from data_dict.items()
            else:
                yield "No Data"


from playwright.sync_api import expect

class CommonTools:
    def __init__(self, page):
        self.page = page

    def check_page_is_loaded(self, page_title):
        title = self.page.get_by_text(page_title)
        try:
            expect(title).to_be_visible()
        except:
            self.page.reload()
            expect(title).to_be_visible()


import sys
sys.path.append(r'C:\Users\Alireza\PycharmProjects\fitting\io')
import slater_basic as sb
import numpy as np
import scipy.misc
import scipy

elementFile = "/Users/Alireza/Desktop/neutral/be"

a = sb.load_slater_basis(elementFile)

def getSlatorTypeOrbital(exponent, quantumNum, r : 'distance'):
    return ((2 * exponent)**quantumNum)   *    np.sqrt(((2 * exponent) / scipy.misc.factorial(2 * quantumNum)))    *      (r ** (quantumNum - 1)) * (np.exp(-exponent * r))

b = (getSlatorTypeOrbital(a['orbitals_exp']['S'], sb.getQuantumNumber(elementFile, 'S'), [2,2]))

def getLCAO(slatorFunction, coeffMatrix):
    """
    Calculates phi/linear combination of atomic orbitals
    by the dot product of slator and coeffmatrix
    :param slatorFunction:
    :param coeffMatrix:
    :return: a new matrix
    """
    return np.dot(np.transpose(slatorFunction) , coeffMatrix)

c = (getLCAO(b, np.concatenate((a['orbitals_coeff']['1S'], a['orbitals_coeff']['2S']), axis = 1)))

def getOccupationNumber(elementFile):
    """
    Gets the Occupation Number for all orbitals
    of an element
    :param elementFile:
    :return: a dict containing the number and orbital
    """
    file = open(elementFile, 'r')
    electronConfigList = file.readline().split()[1]

    shells = ["K", "L", "M", "N"]

    myDic = {}
    listOrbitals = [str(x) + "S" for x in range(1,8)] + [str(x) + "P" for x in range(2,8)] + [str(x) + "D" for x in range(3,8)] + [str(x) + "F" for x in range(4,8)]
    for list in listOrbitals:
        myDic[list] = 0 #initilize all atomic orbitals to zero electrons

    for x in shells:
        if x in electronConfigList :
            if x == "K":
                myDic["1S"] = 2
            elif x == "L":
                myDic["2S"] = 2; myDic["2P"] = 6
            elif x == "M":
                myDic["3S"] = 2; myDic["3P"] = 6; myDic["3D"] = 10
            elif x== "N":
                myDic["4S"] = 2; myDic["4P"] = 6; myDic["4D"] = 10; myDic["4F"] = 14

    for x in listOrbitals:
        if x in electronConfigList :
            index = electronConfigList .index(x)
            orbital = (electronConfigList [index: index + 2])

            if orbital[1] == "D" or orbital[1] == "F":
                numElectrons = (re.sub('[(){}<>,]', "", electronConfigList.split(orbital)[1]))
                myDic[orbital] = int(numElectrons)
            else:
                myDic[ orbital] = int(  electronConfigList [index + 3: index + 4]  )

    return myDic

def rho(dict :"Occupation Numbers", LCAO : "Matrix rows = points, column =  phi"):
    """
    By Taking the occupation numbers and multiplying it
    to the corresponding phi to obtain rho
    :param dict:
    :param LCAO:
    :return:
    """
    listofAllOrbitals = [str(x) + "S" for x in range(1,6)] + [str(x) + "P" for x in range(2,6)] + [str(x) + "D" for x in range(3,6)] + [str(x) + "F" for x in range(4,6)]

    row, col = np.shape(LCAO)

    column = 0
    for orbital in listofAllOrbitals:
        if dict[orbital] != 0 and column < col:
            LCAO[:,column] = np.absolute(LCAO[:,column] * LCAO[:,column]) * dict[orbital]
            column += 1
    return LCAO


import requests
from bs4 import BeautifulSoup
url1 = "https://cathay-ds-test.s3-ap-northeast-1.amazonaws.com/user_info?userid=A123456789"
url2="https://cathay-ds-test.s3-ap-northeast-1.amazonaws.com/company_info?companyid=1"
response =requests.get (url1)
soup=BeautifulSoup(response.text, "html.parser")
print("url1",soup.prettify())
response =requests.get (url2)
soup=BeautifulSoup(response.text, "html.parser")
print("url2",soup.prettify())
status1=requests.head(url1)
status2=requests.head(url2)

class TestHtml1:

    def test_case2(self):
        assert status1.status_code==200
class TestHtml2:

    def test_case3(self):
        assert status2.status_code==403


import uuid
from datetime import datetime
from flask import request
from flask_socketio import emit
from pony.orm import db_session
from src.entities.User import User
from src.middlewares.jwt import jwt_decode_user
from src.sockets.notifications import get_user_notifications

def page(socketio):
	@socketio.on_error('')
	def error_handler(e):
		emit('TOKEN_EXPIRED')

	@socketio.on('JOIN_PAGE')
	@jwt_decode_user
	@db_session
	def join_page():
		try:
			print(request.user['name'], 'has joined to page')
			user_id = request.user['id']
			user = User.get(id=uuid.UUID(user_id))
			user.last_logged = datetime.now()
			emit(f'NOTIFICATIONS_{user_id}', get_user_notifications(user_id))
		except Exception as e:
			print(e)

	@socketio.on('disconnect')
	@jwt_decode_user
	@db_session
	def disconnect():
		user_id = request.user['id']
		user = User.get(id=uuid.UUID(user_id))
		user.last_logged = datetime.now()
		print(request.user['name'], 'left the page')


import json
import glob

csvFileName = 'finalData.csv'

### some have missing header columns => causes shifts
header = ['id', 'title', 'link', 'Oferit de', 'Categorie', 'Marca', 'Model', 'Versiune',\
		'Anul fabricatiei', 'Kilometraj', 'Capacitate cilindrica', 'VIN', \
		'Combustibil', 'Putere', 'Cutie de viteze', 'Transmisie', 'Norma de poluare', \
		'Filtru de particule', 'Caroserie', 'Culoare', 'Primul proprietar', \
		'Fara accident in istoric', 'Carte de service', 'Stare']

headerBool = 0

if __name__ == '__main__':
	fileNames = glob.glob('carData*')

	with open(csvFileName, 'w') as f:

		###Header
		headerString = ''
		for h in header:
			headerString += h + ','
		f.write(headerString[:-1] + '\n')

		###Data
		for fileName in fileNames:
			with open(fileName, 'r') as jsonFile:
				res = json.loads(jsonFile.read())
				for entry in res:
					dataString = ''

					keys = entry.keys()
					for head in header:
						if head in keys:
							dataString += str(entry[head]) + ','
						else:
							dataString += 'NODATA,'

					f.write(dataString[:-1] + '\n')


import requests
from bs4 import BeautifulSoup

def get_news():
    url = 'https://www.globo.com/'
    page = requests.get(url)
    soup = BeautifulSoup(page.text, 'html.parser')

    noticias = soup.find_all('a')
    tgt_class1 = 'post__title'
    tgt_class2 = 'post-multicontent__link--title__text'

    news_dict = {}
    for noticia in noticias:
        if (noticia.h2!= None) and (noticia.h2.get('class')!=None):
            if tgt_class1 in noticia.h2.get('class'):
                news_dict[noticia.h2.text]=noticia.get('href')
            if tgt_class2 in noticia.h2.get('class'):
                news_dict[noticia.h2.text]=noticia.get('href')
    with open('noticias.txt','w') as f:
        for noticia in noticias:
            f.write('{}\n'.format(noticia))
    return news_dict

news = get_news()


# -*- coding: utf-8 -*-
# __author__ = 'qiuping'
from biocluster.agent import Agent
from biocluster.tool import Tool
from biocluster.core.exceptions import OptionError
from collections import defaultdict
import pandas as pd
import os
from bson.objectid import ObjectId

class KeggRichstatAgent(Agent):
    """
    Kegg富集分析
    version v1.0.1
    author: qiuping
    last_modify: 2016.11.23
    """
    def __init__(self, parent):
        super(KeggRichstatAgent, self).__init__(parent)
        options = [
            {"name": "geneset_kegg_enrich_info", "type": "string"},
            # 筛选过后的基因集kegg详情
            {"name": "geneset_kegg_enrich_stat_id", "type": "string"},
            # 客户输入的gene名字文件
            #{"name": "geneset_name", "type": "string"}  # 基因集名称

        ]
        self.add_option(options)
        self.step.add_steps("kegg_enrich_stat")
        self.on('start', self.stepstart)
        self.on('end', self.stepfinish)

    def stepstart(self):
        self.step.kegg_enrich_stat.start()
        self.step.update()

    def stepfinish(self):
        self.step.kegg_enrich_stat.finish()
        self.step.update()

    def check_options(self):
        """
        重写参数检测函数
        :return:
        """
        return True

    def set_resource(self):
        """
        设置所需资源，需在之类中重写此方法 self._cpu ,self._memory
        :return:
        """
        self._cpu = 10
        self._memory = '4G'

    def end(self):
        result_dir = self.add_upload_dir(self.output_dir)
        result_dir.add_relpath_rules([
            [".", "", "结果输出目录"]
        ])
        result_dir.add_regexp_rules([
            [r"kegg_enrichment.xls$", "xls", "kegg富集分析结果"]
        ])
        super(KeggRichstatAgent, self).end()

class KeggRichstatTool(Tool):
    def __init__(self, config):
        super(KeggRichstatTool, self).__init__(config)

    def run(self):
        """
        运行
        :return:
        """
        super(KeggRichstatTool, self).run()
        list_custom = ['Metabolism', 'Genetic Information Processing', 'Environmental Information Processing',
                       'Cellular Processes', 'Organismal Systems', 'Human Diseases',
                       'Drug Development']
        new_data = pd.read_table(self.option("geneset_kegg_enrich_info"), sep='\t', header=0)
        new_data.groupby("second_category")
        group_obj = new_data.groupby("second_category")
        groups = group_obj.groups.keys()
        genesets = ["seq_list"]
        result = defaultdict(dict)
        for each in groups:
            first = new_data.loc[new_data["second_category"] == each]['first_category']
            first = first.to_dict().values()[0]
            for geneset in genesets:
                group_detail = group_obj.get_group(each)
                genes = list()
                for g in group_detail[geneset]:
                    if not pd.isnull(g):
                        tmp = g.split('|')
                        genes += [x for x in tmp]
                        genes = list(set(genes))
                    else:
                        genes = []
                result[geneset][each] = [len(genes), first]
        a = pd.DataFrame(result)
        a.reset_index(inplace=True)
        a.rename(columns={a.columns[0]: "second_category"}, inplace=True)
        a.to_csv(self.work_dir + "/" + "k", sep='\t', index=False)
        with open(self.work_dir + "/" + "k") as f1, open(self.work_dir + "/" + "kegg_statistic", "w") as fw:
            header = f1.readline()
            fw.write("first_category" + "\t" + header.strip().split("\t")[0] + "\t" + "num" + "\n")
            for line in f1:
                line_split = line.strip().split("\t")
                sec = line_split[0]
                num1 = line_split[1].strip("[]").split(",")[0]
                first_cate = line_split[1].strip("[]").split(",")[1].strip().strip("'")
                fw.write(first_cate + "\t" + sec + "\t" + num1 + "\n")
        df_a = pd.read_table(self.work_dir + "/" + "kegg_statistic", header=0, sep="\t")
        appended_data_new_1 = []
        for i in list_custom:
            if i in list(df_a.first_category):
                data = df_a.loc[df_a['first_category'] == i]
                appended_data_new_1.append(data)

        appended_data_new_1 = pd.concat(appended_data_new_1)
        data_new = appended_data_new_1.to_dict('records')
        appended_data_new_1.to_csv(self.work_dir + "/" + "kegg_statistic", sep='\t', index=False)
        self.end()


import os
import random
import torch
from torch import nn, optim
from torch.utils import data
from torch.utils.tensorboard import SummaryWriter
import numpy as np
from scipy.io import wavfile
from .model import CatsDogsModel
from .dataset import BarkMeowDB

# Define some constants used throughout the training
AUDIO_DIR = './barkmeow'  # directory that contains 'cat' and 'dog' directories
DATA_OUT_ROOT = './data_out'
# directory where tensorboard logs are stored
LOG_DIR = os.path.join(DATA_OUT_ROOT, 'logs')
MODEL_PATH = os.path.join(DATA_OUT_ROOT, 'model.pt')
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
LR = 3e-4  # learning rate
EPOCH = 100
SAMPLE_RATE = 16000  # sample rate - used to save audio of mispredicted examples

# fix random seed to make the training deterministic
torch.manual_seed(8912)
np.random.seed(9912)

writer = SummaryWriter(log_dir=LOG_DIR)

model = nn.DataParallel(CatsDogsModel()).to(DEVICE)

# read dataset and create dataloaders
dataset = BarkMeowDB(AUDIO_DIR)
num_data = len(dataset)
num_train = int(num_data * 0.9)

all_indices = list(range(num_data))
random.shuffle(all_indices)

train_dataloader = data.DataLoader(
    dataset,
    batch_size=5,
    sampler=data.sampler.SubsetRandomSampler(all_indices[:num_train]),
    pin_memory=True,
    drop_last=True)
test_dataloader = data.DataLoader(
    dataset,
    batch_size=5,
    sampler=data.sampler.SubsetRandomSampler(all_indices[num_train:]),
    pin_memory=True,
    drop_last=True)

optimizer = optim.Adam(model.parameters(), lr=LR, weight_decay=0.01)

# binary cross entropy loss
loss_func = nn.BCEWithLogitsLoss()

for i in range(EPOCH):
    losses = []
    accs = []
    for targets, samples in train_dataloader:
        targets = targets.type(torch.FloatTensor).to(DEVICE)
        samples = samples.to(DEVICE)

        out = torch.squeeze(model(samples))
        loss = loss_func(out, targets)

        out_bool = torch.sigmoid(out) > 0.5
        target_bool = targets > 0.5
        pred = (out_bool == target_bool).type(torch.FloatTensor)
        accuracy = pred.sum() / pred.numel()

        # accumulate accuracy and loss values for later calculating mean values of each
        accs.append(accuracy.data)
        losses.append(loss.data)

        # back-prop
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    # log values
    avg_loss = sum(losses) / len(losses)
    avg_acc = sum(accs) / len(accs)
    writer.add_scalar('train/loss', avg_loss, global_step=i)
    writer.add_scalar('train/accuracy', avg_acc, global_step=i)
    print(f'epoch: {i}, loss: {avg_loss}, accuracy: {avg_acc}')

    test_losses = []
    test_accs = []
    with torch.no_grad():
        for targets, samples in test_dataloader:
            targets = targets.type(torch.FloatTensor).to(DEVICE)
            samples = samples.to(DEVICE)
            out = torch.squeeze(model(samples))
            loss = loss_func(out, targets)

            out_bool = torch.sigmoid(out) > 0.5
            target_bool = targets > 0.5
            pred = (out_bool == target_bool).type(torch.FloatTensor)
            accuracy = pred.sum() / pred.numel()

            test_accs.append(accuracy.data)
            test_losses.append(loss.data)

            # save any that are mispredicted at the end of training
            if i == EPOCH - 1:
                for target_class, samp, pred_score in zip(targets, samples, out):
                    pred_class = 1 if pred_score > 0 else 0
                    if target_class != pred_class:
                        # misclassified class name
                        as_ = dataset.class_label_name[pred_class]

                        fname = f'mispred_as_{as_}_{pred_score:04f}.wav'
                        mispred_path = os.path.join(DATA_OUT_ROOT, fname)
                        audio_data = samp.cpu().detach().numpy().T

                        # save misclassified audio
                        wavfile.write(mispred_path, SAMPLE_RATE, audio_data)
                        print(f'mispredicted (as: {as_}): {pred_score:04f}')

    avg_test_acc = sum(test_accs) / len(test_accs)
    avg_test_loss = sum(test_losses) / len(test_losses)
    print(f'TEST: {i}, loss:, {avg_test_loss}, accuracy: {avg_test_acc}')
    writer.add_scalar('test/loss', avg_test_loss, global_step=i)
    writer.add_scalar('test/accuracy', avg_test_acc, global_step=i)

# save module
print(f'Saving model: {MODEL_PATH}')
torch.save(model.module.state_dict(), MODEL_PATH)


import responses
from responses import matchers

@responses.activate
def test_create_token(zpa, session):
    responses.add(
        responses.POST,
        url="https://config.private.zscaler.com/signin",
        json=session,
        status=200,
        match=[
            matchers.urlencoded_params_matcher(
                {
                    "client_id": "1",
                    "client_secret": "yyy",
                }
            ),
            matchers.header_matcher({"Content-Type": "application/x-www-form-urlencoded"}),
        ],
    )

    resp = zpa.session.create_token(client_id="1", client_secret="yyy")

    assert isinstance(resp, str)
    assert resp == "xyz"


import time
import json
import logging
from pymisp import ExpandedPyMISP, MISPAttribute, MISPEvent, MISPSighting
from resilient_lib import IntegrationError

log = logging.getLogger(__name__)

def get_misp_client(URL, API_KEY, VERIFY_CERT, proxies):
    misp_client = ExpandedPyMISP(URL, API_KEY, ssl=VERIFY_CERT, proxies=proxies)
    return misp_client

def create_misp_event(misp_client, misp_distribution, misp_threat_level, misp_analysis_level, misp_event_name):
    misp_event = MISPEvent()
    misp_event.distribution = misp_distribution
    misp_event.threat_level_id = misp_threat_level
    misp_event.analysis = misp_analysis_level
    misp_event.info = misp_event_name
    event_response = misp_client.add_event(misp_event)
    return event_response

def create_misp_attribute(misp_client, misp_event_id, misp_attribute_type, misp_attribute_value):
    misp_event = MISPEvent()
    misp_event.id = int(misp_event_id)
    misp_event.uuid = get_event_uuid(misp_client, misp_event_id)
    misp_attribute = MISPAttribute()
    misp_attribute.type = misp_attribute_type
    misp_attribute.value = misp_attribute_value
    attribute_response = misp_client.add_attribute(misp_event, misp_attribute)
    return attribute_response

def create_misp_sighting(misp_client, my_misp_sighting):
    misp_sighting = MISPSighting()
    misp_sighting.value = my_misp_sighting
    misp_sighting.timestamp = int(time.time())
    misp_sighting.source = "IBM Resilient SOAR"
    sighting_response = misp_client.add_sighting(misp_sighting)
    return sighting_response

def search_misp_attribute(misp_client, search_attribute):
    search_results = misp_client.search(value=search_attribute)
    if not isinstance(search_results, list):
        raise IntegrationError("Received an unexpected response type from the MISP API. Expected a list but received: {}".format(type(search_results)))
    search_results_len = len(search_results)
    if search_results_len == 0:
        success_status = False
    elif search_results_len > 0:
        success_status = True
    else:
        success_status = False
    search_results_response = {
                                "search_status": success_status,
                                "search_results" : search_results
                            }
    return search_results_response

def get_event_tags(event):
    search_tags = []
    if "Tag" in event["Event"]:
        tags = event["Event"]["Tag"]
        for tag in tags:
            log.info("found tag {}".format(tag["name"]))
            search_tags.append(tag["name"])
    return search_tags

def get_attribute_tags(attribute):
    search_tags = []
    if "Tag" in attribute:
        for tag in attribute["Tag"]:
            log.info("found tag {}".format(tag["name"]))
            search_tags.append(tag["name"])
    return search_tags

def get_misp_attribute_tags(misp_client, search_results):
    search_tags = []
    log.debug(json.dumps(search_results, indent=4))
    for event in search_results:
        # Grab Event Tags
        search_tags += get_event_tags(event)
        # Grab Attribute Tags
        for attribute in event["Event"]["Attribute"]:
            search_tags += get_attribute_tags(attribute)
    search_tags = list(set(search_tags))
    return search_tags

def get_misp_sighting_list(misp_client, misp_event_id):
    misp_event = MISPEvent()
    misp_event.id = int(misp_event_id)
    sighting_result = misp_client.sightings(misp_event)
    return sighting_result

def get_event_uuid(misp_client, misp_event_id):
    # returns list with a single element: an event dict
    result = misp_client.search(eventid=misp_event_id)
    for event in result:
        event_uuid = event['Event']['uuid']
    return event_uuid

def get_attribute_uuid(misp_client, misp_attribute_value, misp_event_id):
    misp_event = MISPEvent()
    misp_event.id = int(misp_event_id)
    event_response = misp_client.get_event(misp_event)
    attribute_uuid = None
    if not event_response['Event']['Attribute']:
        log.error("Could not get a uuid for event = {} and attribute = {}. Does it exist?".format(misp_event_id, misp_attribute_value))
        raise IntegrationError("Failed to find any attributes on event {}".format(misp_event_id))

    else:
        for attribute in event_response['Event']['Attribute']:
            if attribute['value'] == misp_attribute_value:
                attribute_uuid = attribute['uuid']
        if attribute_uuid:
            return attribute_uuid
        else:
            raise IntegrationError("Failed to match attribute value = {} for any attributes associated with event = {}".format(misp_attribute_value, misp_event_id))

def create_tag(misp_client, misp_attribute_value, misp_tag_type, misp_tag_name, misp_event_id):
    if misp_tag_type == "Event":
        object_uuid = get_event_uuid(misp_client, misp_event_id)
    elif misp_tag_type == "Attribute":
        object_uuid = get_attribute_uuid(misp_client, misp_attribute_value, misp_event_id)
    tag_result = misp_client.tag(object_uuid, misp_tag_name)
    return tag_result


import pyglet
from pyglet import shapes
import random
from cell import Cell

width = 2500
height = 1300
window = pyglet.window.Window(width, height)
scale = 10
cols = width // scale
rows = height // scale
batch = pyglet.graphics.Batch()
width = width - scale
height = height - scale
DEAD = [0, 0, 0]
ALIVE = [255, 255, 255]
PAUSED = False
LIVELIHOOD = 0.45
cells = []
alreadyvisited = []
lastshownplace = None
mousepos = 0, 0
figureToPlace = None
figures = [
    [(0, -1), (1, -1), (1, 0), (0, 0)],
    [(-4, 0), (-4, -1), (-4, 1), (-3, -2), (-3, 2), (-2, -3), (-2, 3), (-1, -3), (-1, 3), (0,0), (1, -2), (1, 2), (2, -1), (2, 0), (2, 1), (3, 0)],
    [(-7, -8), (-7, 7), (-7, -7), (-7, 6), (-7, -6), (-7, 5),
    (-6, -5), (-6, 4), (-6, -8), (-6, 7),
    (-5, -8), (-5, 7),
    (-4, -4), (-4, 3), (-4, -8), (-4, 7),
    (-3, -4), (-3, 3), (-3, -8), (-3, 7),
    (-2, -3), (-2, 2), (-2, 6), (-2, -7),
    (-1, -2), (-1, 1),
    (0, 0), (0, -1),
    (1, 1), (1, -2), (1, -4), (1, 3),
    (2, -4), (2, 3),
    (3, -3), (3, 2), (3, -2), (3, 1),
    (4, -5), (4, 4), (4, -4), (4, 3),
    (5, -6), (5, 5), (5, -5), (5, 4), (5, -4), (5, 3), (5, -2), (5, 1),
    (6, -7), (6, 6), (6, -6), (6, 5), (6, -3), (6, 2),
    (7, -7), (7, 6), (7, -5), (7, 4),
    (8, -6), (8, 5), (8, -5), (8, 4), (8, -4), (8, 3),
    (9, -6), (9, 5), (9, -5), (9, 4), (9, -4), (9, 3),
    (10, -6), (10, 5), (10, -5), (10, 4)
    ]

]

for j in range(0, rows):
    for i in range(0, cols):
        state = DEAD
        if random.random() > LIVELIHOOD:
            state = ALIVE
        gameobject = shapes.Rectangle(0+i*scale, height-j*scale,
                                        scale-1, scale-1, color=state,
                                        batch=batch)
        cells.append(Cell(gameobject, i, j, state))

for cell in cells:
    cell.getNeighbours(cells, cols)

def calculateNextBoard():
    global cells
    for cell in cells:
        cell.countAliveNeighbours()
    for cell in cells:
        cell.setState()
        cell.setColor()

def cellCountAliveNeighbours(cells):
    for cell in cells:
        cell.countAliveNeighbours()

def cellSetState(cells):
    for cell in cells:
        cell.setState()

def cleanEverything():
    global cells
    for cell in cells:
        cell.die()

@window.event
def on_draw():
    window.clear()
    batch.draw()

@window.event
def on_key_press(symbol, modifiers):
    global PAUSED, figures
    if chr(symbol) == 'p':
        PAUSED = not PAUSED
    if chr(symbol) == 'c':
        PAUSED = True
        cleanEverything()
    if chr(symbol) == '1':
        PAUSED = True
        # figureToPlace = figures[1]
        showFigure(figures[0])
    if chr(symbol) == '2':
        PAUSED = True
        # figureToPlace = figures[1]
        showFigure(figures[1])
    if chr(symbol) == '3':
        PAUSED = True
        # figureToPlace = figures[1]
        showFigure(figures[2])

def showFigure(figure):
    global cells, mousepos, height, cols, lastshownplace
    i, j = mousepos
    x = (i // scale)
    y = (height - j + scale//2)//scale
    lookup = cols * y + x
    origin = cells[lookup]
    offsets = figure
    if lastshownplace != (x, y) and lastshownplace is not None:
        for offset in offsets:
            xoffset = offset[0]
            yoffset = offset[1]
            lastshownplacex = lastshownplace[0]
            lastshownplacey = lastshownplace[1]
            neighbourlookup = (cols * (lastshownplacey + yoffset) + (lastshownplacex + xoffset)) % len(cells) - 1
            neighbour = cells[neighbourlookup]
            neighbour.state = DEAD
            neighbour.gameobject.color = DEAD
    for offset in offsets:
        xoffset = offset[0]
        yoffset = offset[1]
        neighbourlookup = (cols * (y + yoffset) + (x + xoffset)) % len(cells) - 1
        neighbour = cells[neighbourlookup]
        neighbour.state = ALIVE
        neighbour.gameobject.color = ALIVE
    lastshownplace = (x, y)
    pass

@window.event
def on_mouse_press(x, y, button, modifiers):
    print('mousepress')
    global height
    x = x//scale
    y = (height - y + scale//2)//scale
    global cells, cols, ALIVE, DEAD
    lookup = cols * y + x
    print(cells[lookup])
    state = cells[lookup].state
    if state != DEAD:
        state = DEAD
    elif state == DEAD:
        state = ALIVE
    print(state)
    cells[lookup].state = state
    cells[lookup].gameobject.state = state
    print(cells[lookup].state, cells[lookup].gameobject.state)

@window.event
def on_mouse_drag(x, y, dx, dy, buttons, modifiers):
    print('mousedrag')
    global height, cells, cols, ALIVE, DEAD, alreadyvisited
    x = x//scale
    y = (height - y + scale//2)//scale
    lookup = cols * y + x
    if lookup not in alreadyvisited:
        state = cells[lookup].state
        if state != DEAD:
            state = DEAD
        elif state == DEAD:
            state = ALIVE
        cells[lookup].state = state
        cells[lookup].gameobject.state = state
    alreadyvisited.append(lookup)

@window.event
def on_mouse_release(x, y, button, modifiers):
    global alreadyvisited
    alreadyvisited = []

@window.event
def on_mouse_motion(x, y, dx, dy):
    global mousepos
    mousepos = x, y

def update(dt):
    window.clear()
    batch.draw()
    global PAUSED
    if not PAUSED:
        calculateNextBoard()

pyglet.clock.schedule_interval(update, 1/20)
pyglet.app.run()


import os
import re
import scrapy
import string

class DatpiffSpider(scrapy.Spider):
    name = "datpiff"

    custom_settings = {
        'CONCURRENT_REQUESTS': 1,
        'USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:44.0) Gecko/20100101 Firefox/44.0',
        'TELNETCONSOLE_ENABLED': False,
        'ROBOTSTXT_OBEY': False
        }

    page_count = 0
    base_url = 'http://www.datpiff.com'
    cont_crawling = True

    # change this to where you would like to save all the mixtapes. make sure you have enough room!! it will be a lot.
    save_directory = '../../../../../../../Volumes/UNTITLED/datpiff_saves'

    def start_requests(self):
        while self.cont_crawling:
            yield scrapy.Request(
                    url=self.base_url + '/mixtapes.php?filter=all&p=' + str(self.page_count),
                    callback=self.get_mixtape_page)
            self.page_count += 1

    def get_mixtape_page(self, response):
        mixtapes = response.css('div.contentListing > div.contentItem')
        if len(mixtapes) == 0:
            self.cont_crawling = False
        else:
            for m in mixtapes:
                mixtape_url = m.css('div.contentThumb > a[href$=".html"]::attr(href)').extract()
                if not len(mixtape_url) == 0:
                    yield scrapy.Request(
                        url=self.base_url + mixtape_url[0],
                        callback=self.get_player,
                        dont_filter=True)

    def get_player(self, response):
        html_token = response.css('div[onclick*="openMixtape"]::attr(onclick)').extract()
        if len(html_token) > 0:
            token = re.findall(r'\s*\'(.*)\'', html_token[0])[0]

            yield scrapy.Request(
                url=self.base_url + '/player/' + token,
                callback=self.get_embed,
                dont_filter=True)

    def get_embed(self, response):
        embed_url = response.css('iframe::attr(src)').extract()

        yield scrapy.Request(
            url=embed_url[0],
            callback=self.get_mp3_links,
            dont_filter=True)

    def get_mp3_links(self, response):
        script = response.css('script[src="/js/player.js"]:first-of-type + script').extract()

        url_track_prefix = re.findall(r'\s*var trackPrefix = \'(.*?)\';', script[0])
        track_objs = re.findall(r'\s*playerData\.tracks\.push\((.*?)\);', script[0])
        mixtape_title = re.findall(r'\s*\"title\":\"(.*?)\"', script[0])[0].replace('\'', '')
        artist_name = re.findall(r'\s*\"artist\":\"(.*?)\"', script[0])[0].replace('\'', '')

        for t in track_objs:
            url_track =  re.findall(r'\s*concat\((.*?)\),', t)[0].strip().replace('\'', '')

            yield scrapy.Request(
                url=url_track_prefix[0] + url_track,
                callback=self.download_track,
                dont_filter=True,
                meta={
                    'mixtape_title': mixtape_title,
                    'artist_name': artist_name,
                    'url_track': url_track
                    })

    def download_track(self, response):
        for i in string.punctuation:
            clean_mix_title = response.meta['mixtape_title'].replace(i, '')

        mixtape_dir = self.save_directory + '/' + response.meta['artist_name'] + ' - ' + clean_mix_title
        file_path = mixtape_dir + '/' + response.meta['url_track']

        if not os.path.isdir(mixtape_dir):
            os.mkdir(mixtape_dir)

        with open(file_path, 'wb') as f:
            f.write(response.body)
            f.flush()
            os.fsync(f.fileno())


from mainapp.models import Subject, Teacher, Class, Lesson
import xml.etree.ElementTree as ET
import datetime

daysLookup = {
    'monday': 0,
    'tuesday': 1,
    'wednesday': 2,
    'thursday': 3,
    'friday': 4,
}

classNumberHourLookup = {
    '1': datetime.time(8),
    '2': datetime.time(9),
    '3': datetime.time(10),
    '4': datetime.time(11),
    '5': datetime.time(12),
    '6': datetime.time(13),
    '7': datetime.time(14),
    '8': datetime.time(15)
}

def manage():
    tree = ET.parse('.\mainapp\classSchedule.xml')
    for cls in tree.getroot():
        className = cls.find('classname').text
        for day in cls.find('timetable'):
            dayName = day.get('value')
            for subject in day:
                subjectName = subject.find('name').text
                teacher = subject.find('teacher').text
                lessonTime = subject.find('lesson').text

                clz = Class.objects.get(name=className)
                t = Teacher.objects.get(tempFullName=teacher)
                try:
                    sub = Subject.objects.get(name=subjectName, clazz=clz, teacher=t)
                except:
                    sub = Subject(name=subjectName, clazz=clz, teacher=t)
                    sub.save()

                lessonDateTime = datetime.datetime.combine(
                        next_weekday(datetime.date.today() + datetime.timedelta(days=4), daysLookup[dayName]),
                        classNumberHourLookup[lessonTime])

                try:
                    les = Lesson.objects.get(subject=sub, date=lessonDateTime)
                except:
                    les = Lesson(subject=sub, date=lessonDateTime)
                    les.save()

def next_weekday(d, weekday):
    days_ahead = weekday - d.weekday()
    if days_ahead <= 0:  # Target day already happened this week
        days_ahead += 7
    return d + datetime.timedelta(days_ahead)

def getTeacherTimeTable(name):
    tree = ET.parse('D:\django\mainapp\classSchedule.xml')
    root = tree.getroot()

    parent_map = {c: p for p in tree.iter() for c in p}
    teacherSubjects = [parent_map[thc] for thc in root.findall("class/timetable/day/subject/teacher") if
                       thc.text == name]
    result = {}
    result['monday'] = []
    result['tuesday'] = []
    result['wednesday'] = []
    result['thursday'] = []
    result['friday'] = []

    for subject in teacherSubjects:
        day = parent_map[subject].get('value')
        lessonInfo = [subject.find('name').text, subject.find('lesson').text]
        result[day].append(lessonInfo)

    return result

def getStudentTimeTable(className):
    tree = ET.parse('D:\django\mainapp\classSchedule.xml')
    root = tree.getroot()

    parent_map = {c: p for p in tree.iter() for c in p}
    clazz = [clazz for clazz in root.findall("class") if clazz.find('classname').text == className]
    days = clazz[0].findall('timetable/day')
    result = {}
    for day in days:
        dayValue = day.get('value')
        result[dayValue] = []
        for subject in day:
            lessonInfo = [subject.find('name').text, subject.find('lesson').text]
            result[dayValue].append(lessonInfo)

    return result


from datetime import datetime
from manager_rest import storage_manager, models
import base_test

class StorageManagerTests(base_test.BaseServerTestCase):

    def test_store_load_delete_blueprint(self):
        now = str(datetime.now())
        blueprint = models.BlueprintState(id='blueprint-id',
                                          created_at=now,
                                          updated_at=now,
                                          plan={'name': 'my-bp'},
                                          source='bp-source')
        storage_manager.instance().put_blueprint('blueprint-id', blueprint)
        blueprint_from_list = storage_manager.instance().blueprints_list()[0]
        blueprint_restored = \
            storage_manager.instance().get_blueprint('blueprint-id')
        bp_from_delete = storage_manager.instance().delete_blueprint(
            'blueprint-id')
        self.assertEquals(blueprint.__dict__, blueprint_from_list.__dict__)
        self.assertEquals(blueprint.__dict__, blueprint_restored.__dict__)
        # in bp returned from delete operation only 'id' is guaranteed to
        # return
        self.assertEquals(blueprint.id, bp_from_delete.id)
        self.assertEquals(0,
                          len(storage_manager.instance().blueprints_list()))

    def test_get_blueprint_deployments(self):
        now = str(datetime.now())
        blueprint = models.BlueprintState(id='blueprint-id',
                                          created_at=now,
                                          updated_at=now,
                                          plan={'name': 'my-bp'},
                                          source='bp-source')
        storage_manager.instance().put_blueprint('blueprint-id', blueprint)

        deployment1 = models.Deployment(id='dep-1',
                                        created_at=now,
                                        updated_at=now,
                                        blueprint_id='blueprint-id',
                                        plan={'name': 'my-bp'},
                                        permalink=None,
                                        workflows={},
                                        inputs={},
                                        policy_types={},
                                        policy_triggers={},
                                        groups={},
                                        outputs={})
        storage_manager.instance().put_deployment('dep-1', deployment1)

        deployment2 = models.Deployment(id='dep-2',
                                        created_at=now,
                                        updated_at=now,
                                        blueprint_id='blueprint-id',
                                        plan={'name': 'my-bp'},
                                        permalink=None,
                                        workflows={},
                                        inputs={},
                                        policy_types={},
                                        policy_triggers={},
                                        groups={},
                                        outputs={})
        storage_manager.instance().put_deployment('dep-2', deployment2)

        deployment3 = models.Deployment(id='dep-3',
                                        created_at=now,
                                        updated_at=now,
                                        blueprint_id='another-blueprint-id',
                                        plan={'name': 'my-bp'},
                                        permalink=None,
                                        workflows={},
                                        inputs={},
                                        policy_types={},
                                        policy_triggers={},
                                        groups={},
                                        outputs={})
        storage_manager.instance().put_deployment('dep-3', deployment3)

        blueprint_deployments = storage_manager.instance()\
            .get_blueprint_deployments(
                'blueprint-id')

        self.assertEquals(2, len(blueprint_deployments))
        self.assertEquals(deployment1.__dict__,
                          blueprint_deployments[0].__dict__)
        self.assertEquals(deployment2.__dict__,
                          blueprint_deployments[1].__dict__)

    def test_model_serialization(self):
        dep = models.Deployment(id='dep-id',
                                created_at='some-time1',
                                updated_at='some-time2',
                                blueprint_id='bp-id',
                                plan={'field': 'value'},
                                permalink=None,
                                workflows={},
                                inputs={},
                                policy_types={},
                                policy_triggers={},
                                groups={},
                                outputs={})

        serialized_dep = dep.to_dict()
        self.assertEquals(11, len(serialized_dep))
        self.assertEquals(dep.id, serialized_dep['id'])
        self.assertEquals(dep.created_at, serialized_dep['created_at'])
        self.assertEquals(dep.updated_at, serialized_dep['updated_at'])
        self.assertEquals(dep.blueprint_id, serialized_dep['blueprint_id'])
        self.assertEquals(dep.permalink, serialized_dep['permalink'])

        deserialized_dep = models.Deployment(**serialized_dep)
        self.assertEquals(dep.id, deserialized_dep.id)
        self.assertEquals(dep.created_at, deserialized_dep.created_at)
        self.assertEquals(dep.updated_at, deserialized_dep.updated_at)
        self.assertEquals(dep.blueprint_id, deserialized_dep.blueprint_id)
        self.assertEquals(dep.permalink, deserialized_dep.permalink)

    def test_fields_query(self):
        now = str(datetime.now())
        blueprint = models.BlueprintState(id='blueprint-id',
                                          created_at=now,
                                          updated_at=now,
                                          plan={'name': 'my-bp'},
                                          source='bp-source')
        storage_manager.instance().put_blueprint('blueprint-id', blueprint)

        blueprint_restored = \
            storage_manager.instance().get_blueprint('blueprint-id',
                                                     {'id', 'created_at'})
        self.assertEquals('blueprint-id', blueprint_restored.id)
        self.assertEquals(now, blueprint_restored.created_at)
        self.assertEquals(None, blueprint_restored.updated_at)
        self.assertEquals(None, blueprint_restored.plan)


import cv2

img = cv2.imread('./imgs/semC.jpg', 0)

# funcao para calcular o valor do pixel
def calcVarA(pixel, pixelEntrada, pixelSaida):
    return (255 / (pixelSaida - pixelEntrada)) * (pixel - pixelEntrada)

line, collunm = img.shape

# Fazendo a stretching linear
for i in range(0, line):
    for j in range(0, collunm):

        pixelVal = calcVarA(img[i][j], 2, 200)

        img[i][j] = pixelVal if pixelVal <= 255 else 255

# salvando a imagem
cv2.imwrite('./imgs/outputImgs/histoImageStretchingLinear.png', img)


#!/anaconda3/envs/FEALPy/bin python3.8
# -*- coding: utf-8 -*-
# ---
# @Software: PyCharm
# @File: HHOSolver.py
# @Author: Yongchao Zhang, Northwest University, Xi'an
# @E-mail: yoczhang@nwu.edu.cn
# @Site:
# @Time: Sep 29, 2020
# ---

__doc__ = """
The solver hybrid high-order (HHO) method.
"""

import numpy as np
from numpy.linalg import solve, inv
from scipy.sparse.linalg import spsolve
from scipy.sparse import coo_matrix, csc_matrix, csr_matrix, spdiags, eye, bmat
from timeit import default_timer as timer
from multiprocessing.dummy import Pool as ThreadPool

class HHOSolver:
    def __init__(self, M, R, space, pde):
        self.M = M
        self.R = R
        self.space = space
        self.pde = pde
        self.p = self.space.p
        self.mesh = self.space.mesh
        self.NC = self.mesh.number_of_cells()
        self.NE = self.mesh.number_of_edges()

    def StokesSolver(self):
        vSpace = self.space.vSpace
        nu = self.pde.nu
        M = self.M
        R = self.R
        p = self.p
        mesh = self.mesh
        NC = self.NC
        NE = self.NE

        # # 'l' denotes 'local';
        # # 'g' denotes 'global'.
        TlNdof = (p + 1) * (p + 2) // 2
        uTlNdof = TlNdof
        uFlNdof = p + 1
        pTlNdof = TlNdof - 1
        pFlNdof = 1

        uTgNdof = NC * uTlNdof
        uFgNdof = NE * uFlNdof
        ugNdof = uTgNdof + uFgNdof
        pTgNdof = NC * pTlNdof
        pFgNdof = NC * pFlNdof
        pgNdof = pTgNdof + pFgNdof

        # --- matrix setting --- #
        MA1_00 = M[:uTgNdof, :uTgNdof]
        MA1_0b = M[:uTgNdof, uTgNdof:ugNdof]
        MA1_b0 = M[uTgNdof:ugNdof, :uTgNdof]
        MA1_bb = M[uTgNdof:ugNdof, uTgNdof:ugNdof]

        MA2_00 = MA1_00.copy()
        MA2_0b = MA1_0b.copy()
        MA2_b0 = MA1_b0.copy()
        MA2_bb = MA1_bb.copy()

        MB1_00 = M[2*ugNdof:(2*ugNdof + pgNdof), :uTgNdof]
        MB1_0b = M[2*ugNdof:(2*ugNdof + pgNdof), uTgNdof:ugNdof]
        MB2_00 = M[2*ugNdof:(2*ugNdof + pgNdof), ugNdof:(ugNdof + uTgNdof)]
        MB2_0b = M[2*ugNdof:(2*ugNdof + pgNdof), (ugNdof + uTgNdof):2*ugNdof]

        MB1_00t = M[:uTgNdof, 2*ugNdof:(2*ugNdof + pgNdof)]
        MB1_b0 = M[uTgNdof:ugNdof, 2*ugNdof:(2*ugNdof + pgNdof)]
        MB2_00t = M[ugNdof:(ugNdof + uTgNdof), 2*ugNdof:(2*ugNdof + pgNdof)]
        MB2_b0 = M[(ugNdof + uTgNdof):2*ugNdof, 2*ugNdof:(2*ugNdof + pgNdof)]

        # # the right hand side
        r0_0 = R[:uTgNdof, :]
        r0_1 = R[ugNdof:(ugNdof+uTgNdof), :]
        R0 = np.concatenate([r0_0, r0_1, np.zeros((pTgNdof, 1), dtype=np.float)])  # (2*uTgNdof+pTgNdof, 1)
        Rb = np.zeros((2*uFgNdof+pFgNdof+1, 1), dtype=np.float)

        # # the Lagrange multiplier vector (for the pressure condition: \int p = 0)
        # # In HHO-Stokes equation, the velocity and pressure have the same order, here we just use the velocity space.
        pphi = vSpace.basis
        intp = vSpace.integralalg.integral(pphi, celltype=True)  # (NC,pldof)

        LF = intp[:, 0][np.newaxis, :]  # (1,NC)
        LT = intp[:, 1:].reshape(1, -1)  # (1,...)

        # # to reconstruct the global matrix
        pgIdx = np.arange(0, pgNdof, TlNdof)  # (NC,)
        MB1_bb = MB1_0b[pgIdx, :]  # (NC,uFgNdof)
        MB2_bb = MB2_0b[pgIdx, :]  # (NC,uFgNdof)
        MB1_bbt = MB1_b0[:, pgIdx]  # (uFgNdof,NC)
        MB2_bbt = MB2_b0[:, pgIdx]  # (uFgNdof,NC)

        MB1_00 = np.delete(MB1_00.todense(), pgIdx, axis=0)
        MB1_0b = np.delete(MB1_0b.todense(), pgIdx, axis=0)
        MB2_00 = np.delete(MB2_00.todense(), pgIdx, axis=0)
        MB2_0b = np.delete(MB2_0b.todense(), pgIdx, axis=0)

        MB1_00t = np.delete(MB1_00t.todense(), pgIdx, axis=1)
        MB1_b0 = np.delete(MB1_b0.todense(), pgIdx, axis=1)
        MB2_00t = np.delete(MB2_00t.todense(), pgIdx, axis=1)
        MB2_b0 = np.delete(MB2_b0.todense(), pgIdx, axis=1)

        # --- get global matrix --- #
        VP_F0 = np.zeros((pFgNdof, uTgNdof))
        LF0 = np.zeros((1, uFgNdof))
        LT0 = np.zeros((1, uTgNdof))
        A = bmat([[MA1_00, None, MB1_00t], [None, MA2_00, MB2_00t], [MB1_00, MB2_00, None]], format='csr')
        B = bmat([[MA1_0b, None, VP_F0.T, LT0.T], [None, MA2_0b, VP_F0.T, LT0.T], [MB1_0b, MB2_0b, None, LT.T]], format='csr')
        C = bmat([[MA1_b0, None, MB1_b0], [None, MA2_b0, MB2_b0], [VP_F0, VP_F0, None], [LT0, LT0, LT]], format='csr')
        D = bmat([[MA1_bb, None, MB1_bbt, LF0.T], [None, MA2_bb, MB2_bbt, LF0.T], [MB1_bb, MB2_bb, None, LF.T], [LF0, LF0, LF, None]], format='csr')

        # # to get the inv A
        cell2dof, doflocation = vSpace.dof.cell_to_dof()
        StiffM = vSpace.reconstruction_stiff_matrix()  # list, (NC,), each-term.shape (Cldof,Cldof)
        StabM = vSpace.reconstruction_stabilizer_matrix()  # list, (NC,), each-term.shape (Cldof,Cldof)
        divM0, divM1 = self.space.cell_divergence_matrix()  # e.g., divM0, list, (NC,); divM0: (TlNdof,\sum_C{Cldof})
        divM0_split = np.hsplit(divM0, doflocation[1:-1])
        divM1_split = np.hsplit(divM1, doflocation[1:-1])

        AlNdof = 2*uTlNdof + pTlNdof
        invA = np.zeros((NC * AlNdof, NC * AlNdof), dtype=np.float)

        def func_invA(x):
            r = None
            stiffM_TT = nu * x[0][:uTlNdof, :uTlNdof]
            stabM_TT = nu * x[1][:uTlNdof, :uTlNdof]
            divM0_TT = x[2][1:, :uTlNdof]
            divM1_TT = x[3][1:, :uTlNdof]
            CIdx = x[4]

            Al = np.zeros((AlNdof, AlNdof), dtype=np.float)

            Al[:uTlNdof, :uTlNdof] = stiffM_TT + stabM_TT
            Al[uTlNdof:2*uTlNdof, uTlNdof:2*uTlNdof] = stiffM_TT + stabM_TT
            Al[2*uTlNdof:, :uTlNdof] = divM0_TT
            Al[2*uTlNdof:, uTlNdof:2*uTlNdof] = divM1_TT
            Al[:uTlNdof, 2*uTlNdof:] = divM0_TT.T
            Al[uTlNdof:2*uTlNdof, 2*uTlNdof:] = divM1_TT.T

            u0Idx = CIdx * uTlNdof + np.arange(uTlNdof)
            u1Idx = uTgNdof + u0Idx
            pIdx = 2 * uTgNdof + CIdx * pTlNdof + np.arange(pTlNdof)
            Idx = np.concatenate([u0Idx, u1Idx, pIdx])

            # # one way assembling, this way is more quickly
            i, j = np.ix_(Idx, Idx)
            invA[i, j] = inv(Al)

            # # other way assembling
            # rowIdx = np.einsum('i, k->ik', Idx, np.ones((AlNdof, ), dtype=np.int))
            # colIdx = rowIdx.T
            # r = csr_matrix((inv(Al).flat, (rowIdx.flat, colIdx.flat)), shape=(NC * AlNdof, NC * AlNdof), dtype=np.float)
            return r

        start = timer()
        # invAt = sum(list(map(func_invA, zip(StiffM, StabM, divM0_split, divM1_split, range(NC)))))
        invAt = list(map(func_invA, zip(StiffM, StabM, divM0_split, divM1_split, range(NC))))
        # pool = ThreadPool()
        # invAt = sum(pool.map(func_invA, zip(StiffM, StabM, divM0_split, divM1_split, range(NC))))
        # pool.close()
        # pool.join()
        end = timer()
        print("  |___ TIME: get inv matrix (cell-by-cell):", end - start)

        # start = timer()
        # invAtt = inv(A)
        # invAtt = np.linalg.inv(A.todense())
        # end = timer()
        # print("get inv matrix (directly):", end - start)
        # print("max( abs(invA - inv(A)) ) = ", np.max(abs(invAtt-invA)))

        # --- solve the Static Condensation system --- #
        # print("solve system:")
        invA = csr_matrix(invA)  # To makesure invA is sparse matrix is important!!!
        stacM = D - C@invA@B
        stacR = Rb - C@invA@R0

        # # deal the Dirichlet boundary edges
        uD = self.pde.dirichlet  # uD(bcs): (NQ,NC,ldof,2)
        idxDirEdge = self.pde.idxDirEdge
        MD, RD = self.space.applyDirichletBC(stacM, stacR, uD, idxDirEdge=idxDirEdge, StaticCondensation=True)

        # # solve the system
        print("  |___ Begin solve 'sparse' algebraic equations:")
        start = timer()
        Xb = spsolve(MD, RD)
        end = timer()
        print("  |___ TIME: in static solver, solve 'sparse' algebraic equations:", end - start)

        X0 = invA@(np.squeeze(R0) - B@Xb)

        # # rearrange the dofs
        X = np.zeros((2*ugNdof + pgNdof + 1,))
        X[:uTgNdof] = X0[:uTgNdof]
        X[uTgNdof:ugNdof] = Xb[:uFgNdof]
        X[ugNdof:ugNdof+uTgNdof] = X0[uTgNdof:2*uTgNdof]
        X[ugNdof+uTgNdof:2*ugNdof] = Xb[uFgNdof:2*uFgNdof]

        if pTlNdof == 0:
            P = Xb[2*uFgNdof:-1].reshape(-1, 1)  # (PFgNdof,1)
        else:
            P0 = X0[2 * uTgNdof:].reshape(-1, pTlNdof)  # (pTgNdof,pTlNdof)
            Pb = Xb[2 * uFgNdof:-1].reshape(-1, 1)  # (PFgNdof,1)
            P = np.concatenate([Pb, P0], axis=1)

        X[2*ugNdof:-1] = P.flatten()
        return X

    def StokesSolver_1(self):
        vSpace = self.space

        # --- the Poisson (scalar) HHO space
        gdof_scal = self.space.dof.number_of_global_dofs()
        StiffM = self.space.reconstruction_stiff_matrix()  # list, its len is NC, each-term.shape (Cldof,Cldof)
        StabM = self.space.reconstruction_stabilizer_matrix()  # list, its len is NC, each-term.shape (Cldof,Cldof)

        # --- the divergence term
        divM0, divM1 = self.space.cell_divergence_matrix()  # divM0, list, (NC,); divM1, list, (NC,)