#!/usr/bin/env python
__author__ = 'Sergei F. Kliver'
import os
import argparse
from RouToolPa.Parsers.LAST import CollectionLast
from RouToolPa.Collections.General import IdList, SynDict
from RouToolPa.Routines import DrawingRoutines

parser = argparse.ArgumentParser()

parser.add_argument("-i", "--input_last_tab", action="store", dest="input_last_tab", required=True,
                    help="File with LAST output in tab format")
parser.add_argument("-o", "--output_prefix", action="store", dest="output_prefix", required=True,
                    help="Prefix of output files")
parser.add_argument("--format", action="store", dest="format", default="tab",
                    help="Format of LAST alignments. Allowed: tab(default), tab_mismap")
parser.add_argument("-w", "--white_id_file", action="store", dest="white_id_file",
                    help="File with scaffold ids from white list or corresponding comma-separated list."
                         "NOTE: filtering is done BEFORE renaming by synonyms!")
parser.add_argument("-b", "--black_id_file", action="store", dest="black_id_file",
                    help="File with scaffold ids from black list or corresponding comma-separated list."
                         "NOTE: filtering is done BEFORE renaming by synonyms!")

parser.add_argument("-s", "--syn_file", action="store", dest="syn_file",
                    help="File with scaffold id synonyms")
parser.add_argument("--syn_file_key_column", action="store", dest="syn_file_key_column",
                    default=0, type=int,
                    help="Column(0-based) with key(current id) for scaffolds in synonym file")
parser.add_argument("--syn_file_value_column", action="store", dest="syn_file_value_column",
                    default=1, type=int,
                    help="Column(0-based) with value(synonym id) for scaffolds in synonym file synonym")

parser.add_argument("-l", "--label", action="store", dest="label",
                    help="Label for genome(both X and Y axis)")

parser.add_argument("-t", "--title", action="store", dest="title",
                    help="Title of dot plot")
parser.add_argument("-e", "--extensions", action="store", dest="extensions", type=lambda x: x.split(","),
                    default=["png", ],
                    help="Comma-separated list of extensions for histogram files. Default: png")

parser.add_argument("-d", "--dpi", action="store", dest="dpi", type=int, default=400,
                    help="DPI of figure. Default: 400")
parser.add_argument("-f", "--figsize", action="store", dest="figsize", type=lambda s: map(int, s.split(",")),
                    default=(12, 12),
                    help="Size of figure in inches(two comma-separated ints). Default: 12,12")
parser.add_argument("-a", "--antialiasing", action="store_true", dest="antialiasing", default=False,
                    help="Enable antialiasing. Use this option only for small sequences, i.e segments of chromosomes ")

parser.add_argument("--linewidth", action="store", dest="linewidth", type=float,
                    default=0.01,
                    help="Width of alignment lines. Default: 0.01")
parser.add_argument("--gridwidth", action="store", dest="gridwidth", type=float,
                    default=1,
                    help="Width of grid lines. Default: 1")
parser.add_argument("--scaffold_label_fontsize", action="store", dest="scaffold_label_fontsize", type=float,
                    default=13,
                    help="Fontsize for scaffold labels. Default: 13")
parser.add_argument("--grid_color", action="store", dest="grid_color", default='black',
                    help="Color of grid lines. Default: 'black'")
parser.add_argument("--hide_grid", action="store_true", dest="hide_grid", default=False,
                    help="Hide grid. Default: False")
parser.add_argument("--bar_color", action="store", dest="bar_color", default='grey',
                    help="Color of bars. Default: 'grey'")
parser.add_argument("--same_strand_color", action="store", dest="same_strand_color", default='blue',
                    help="Color of alignment line if query and target are in same strand. Default: 'blue'")
parser.add_argument("--diff_strand_color", action="store", dest="diff_strand_color", default='red',
                    help="Color of alignment line if query and target are in different strand. Default: 'red'")

parser.add_argument("--target_scaffold_labels_angle", action="store", dest="target_scaffold_labels_angle",
                    default=45, type=int,
                    help="Angle for labels of target scaffolds. Default: 45")
parser.add_argument("--query_scaffold_labels_angle", action="store", dest="query_scaffold_labels_angle",
                    default=0, type=int,
                    help="Angle for labels of query scaffolds. Default: 0")
parser.add_argument("--hide_labels", action="store_true", dest="hide_labels", default=False,
                    help="Hide labels of scaffolds. Default: False")

parser.add_argument("--bottom_offset", action="store", dest="bottom_offset",
                    default=0.1, type=float,
                    help="Bottom offset for subplot. Default: 0.1")
parser.add_argument("--top_offset", action="store", dest="top_offset",
                    default=0.9, type=float,
                    help="Top offset for subplot. Default: 0.9")
parser.add_argument("--left_offset", action="store", dest="left_offset",
                    default=0.1, type=float,
                    help="Left offset for subplot. Default: 0.1")
parser.add_argument("--right_offset", action="store", dest="right_offset",
                    default=0.9, type=float,
                    help="Right offset for subplot. Default: 0.9")
parser.add_argument("--x_axis_visible", action="store_true", dest="x_axis_visible",
                    default=False,
                    help="Make X axis visible. Default: False")
parser.add_argument("--y_axis_visible", action="store_true", dest="y_axis_visible",
                    default=False,
                    help="Make Y axis visible. Default: False")

args = parser.parse_args()

if args.white_id_file:
    white_list = IdList(filename=args.white_id_file) if os.path.isfile(args.white_id_file) else IdList(args.white_id_file.split(","))
else:
    white_list = IdList()

if args.black_id_file:
    black_list = IdList(filename=args.black_id_file) if os.path.isfile(args.black_id_file) else IdList(args.black_id_file.split(","))
else:
    black_list = IdList()

syn_dict = SynDict(filename=args.syn_file,
                   key_index=args.syn_file_key_column,
                   value_index=args.syn_file_value_column)

if white_list and syn_dict:
    syn_white_list = IdList([syn_dict[entry] for entry in white_list])
elif white_list:
    syn_white_list = white_list
else:
    syn_white_list = IdList()

last_collection = CollectionLast(args.input_last_tab,
                                 target_white_list=white_list,
                                 target_black_list=black_list,
                                 query_white_list=white_list,
                                 query_black_list=black_list,
                                 query_syn_dict=syn_dict,
                                 target_syn_dict=syn_dict,
                                 format=args.format
                                 )

last_collection.write("%s.syn.tab" % args.output_prefix)

DrawingRoutines.draw_dot_plot_per_scaffold_from_last_self_alignment(last_collection,
                                                                    output_prefix=args.output_prefix,
                                                                    extension_list=args.extensions,
                                                                    scaffold_black_list=(),
                                                                    scaffold_white_list=syn_white_list,
                                                                    scaffold_reverse_list=(),
                                                                    figsize=args.figsize, dpi=args.dpi,
                                                                    grid_color=args.grid_color,
                                                                    bar_color=args.bar_color,
                                                                    same_strand_color=args.same_strand_color,
                                                                    diff_strand_color=args.diff_strand_color,
                                                                    title=args.title,
                                                                    label=args.label,
                                                                    linewidth=args.linewidth,
                                                                    gridwidth=args.gridwidth,
                                                                    antialiased_lines=args.antialiasing,
                                                                    scaffold_label_fontsize=args.scaffold_label_fontsize,
                                                                    target_scaffold_labels_angle=args.target_scaffold_labels_angle,
                                                                    query_scaffold_labels_angle=args.query_scaffold_labels_angle,
                                                                    show_grid=not args.hide_grid,
                                                                    show_labels=not args.hide_labels,
                                                                    top_offset=args.top_offset,
                                                                    bottom_offset=args.bottom_offset,
                                                                    left_offset=args.left_offset,
                                                                    right_offset=args.right_offset,
                                                                    x_axis_visible=args.x_axis_visible,
                                                                    y_axis_visible=args.y_axis_visible
                                                                    )


#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# File   : test.py
# Author : Bowen Pan
# Email  : panbowen0607@gmail.com
# Date   : 09/25/2018
#
# Distributed under terms of the MIT license.

"""

"""
from utils import Foo
from models import VPNModel
from datasets import OVMDataset
from opts import parser
from transform import *
import torchvision
import torch
from torch import nn
import os
import time
import cv2
import dominate
from dominate.tags import *

mean_rgb = [0.485, 0.456, 0.406]
std_rgb = [0.229, 0.224, 0.225]

def main():
    global args, web_path, best_prec1
    best_prec1 = 0
    args = parser.parse_args()
    network_config = Foo(
        encoder=args.encoder,
        decoder=args.decoder,
        fc_dim=args.fc_dim,
        num_views=args.n_views,
        num_class=94,
        transform_type=args.transform_type,
        output_size=args.label_resolution,
    )

    val_dataset = OVMDataset(args.data_root, args.eval_list,
                         transform=torchvision.transforms.Compose([
                             Stack(roll=True),
                             ToTorchFormatTensor(div=True),
                             GroupNormalize(mean_rgb, std_rgb)
                         ]),
                         n_views=network_config.num_views, input_size=args.input_resolution,
                         label_size=args.segSize, use_mask=args.use_mask, use_depth=args.use_depth, is_train=False)

    val_loader = torch.utils.data.DataLoader(
        val_dataset, batch_size=args.batch_size,
        num_workers=args.num_workers, shuffle=False,
        pin_memory=True
    )

    mapper = VPNModel(network_config)
    mapper = nn.DataParallel(mapper.cuda())

    if args.weights:
        if os.path.isfile(args.weights):
            print(("=> loading checkpoint '{}'".format(args.weights)))
            checkpoint = torch.load(args.weights)
            args.start_epoch = checkpoint['epoch']
            mapper.load_state_dict(checkpoint['state_dict'])
            print(("=> loaded checkpoint '{}' (epoch {})"
                  .format(args.evaluate, checkpoint['epoch'])))
        else:
            print(("=> no checkpoint found at '{}'".format(args.weights)))

    criterion = nn.NLLLoss(weight=None, size_average=True)
    eval(val_loader, mapper, criterion)

    web_path = os.path.join(args.visualize, args.store_name)
    if os.path.isdir(web_path):
        pass
    else:
        os.makedirs(web_path)

    with dominate.document(title=web_path) as web:
        for step in range(len(val_loader)):
            if step % args.print_freq == 0:
                h2('Step {}'.format(step*args.batch_size))
                with table(border = 1, style = 'table-layout: fixed;'):
                    with tr():
                        for i in range(args.test_views):
                            path = 'Step-{}-{}.png'.format(step * args.batch_size, i)
                            with td(style='word-wrap: break-word;', halign='center', valign='top'):
                                img(style='width:128px', src=path)
                        path = 'Step-{}-pred.png'.format(step * args.batch_size)
                        with td(style='word-wrap: break-word;', halign='center', valign='top'):
                            img(style='width:128px', src=path)
                        path = 'Step-{}-gt.png'.format(step * args.batch_size)
                        with td(style='word-wrap: break-word;', halign='center', valign='top'):
                            img(style='width:128px', src=path)

    with open(os.path.join(web_path, 'index.html'), 'w') as fp:
        fp.write(web.render())

def eval(val_loader, mapper, criterion):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    mapper.eval()

    end = time.time()

    web_path = os.path.join(args.visualize, args.store_name)
    if os.path.isdir(web_path):
        pass
    else:
        os.makedirs(web_path)

    prec_stat = {}
    for i in range(args.num_class):
        prec_stat[str(i)] = {'intersec': 0, 'union': 0, 'all': 0}

    with open('./metadata/colormap_coarse.csv') as f:
        lines = f.readlines()
    cat = []
    for line in lines:
        line = line.rstrip()
        cat.append(line)
    cat = cat[1:]
    label_dic = {}
    for i, value in enumerate(cat):
        key = str(i)
        label_dic[key] = [int(x) for x in value.split(',')[1:]]

    for step, (rgb_stack, target, rgb_origin, OverMaskOrigin) in enumerate(val_loader):
        data_time.update(time.time() - end)
        with torch.no_grad():
            input_rgb_var = torch.autograd.variable(rgb_stack).cuda()
            _, output = mapper(x=input_rgb_var, return_feat=True)
        target_var = target.cuda()
        target_var = target_var.view(-1)
        upsample = output.view(-1, args.label_resolution, args.label_resolution, args.num_class).transpose(3,2).transpose(2,1).contiguous()
        upsample = nn.functional.upsample(upsample, size=args.segSize, mode='bilinear', align_corners=False)
        upsample = nn.functional.softmax(upsample, dim=1)
        output = torch.log(upsample.transpose(1,2).transpose(2,3).contiguous().view(-1, args.num_class))
        _, pred = upsample.data.topk(1, 1, True, True)
        pred = pred.squeeze(1)
        loss = criterion(output, target_var)
        losses.update(loss.data[0], input_rgb_var.size(0))
        prec_stat = count_mean_accuracy(output.data, target_var.data, prec_stat)
        prec1 = accuracy(output.data, target_var.data, topk=(1,))[0]

        batch_time.update(time.time() - end)
        end = time.time()

        if step % args.print_freq == 0:
            output = ('Test: [{0}/{1}]\t'
                    'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                    'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                    'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                    'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                step + 1, len(val_loader), batch_time=batch_time,
                data_time=data_time, loss=losses, top1=top1))
            print(output)

            pred = np.uint8(pred.cpu()[0])
            predMask = np.uint8(np.zeros((args.segSize, args.segSize, 3)))
            for i, _ in enumerate(pred):
                for j, _ in enumerate(pred[0]):
                    key = str(pred[i][j])
                    predMask[i,j] = label_dic[key]
            predMask = cv2.resize(predMask[:, :, ::-1], (256, 256), interpolation=cv2.INTER_NEAREST)
            cv2.imwrite(os.path.join(web_path, 'Step-{}-pred.png'.format(step * args.batch_size, i)), predMask)

            gtMask = OverMaskOrigin[0].cpu().numpy()
            gtMask = cv2.resize(gtMask, (256, 256), interpolation=cv2.INTER_NEAREST)
            cv2.imwrite(os.path.join(web_path, 'Step-{}-gt.png'.format(step * args.batch_size, i)), gtMask)

            rgb = rgb_origin.cpu().numpy()[0]
            for i in range(args.test_views):
                cv2.imwrite(os.path.join(web_path, 'Step-{}-{}.png'.format(step * args.batch_size, i)), cv2.resize(rgb[(i + args.view_bias) % 8], (256, 256), interpolation=cv2.INTER_NEAREST))

    sum_acc = 0
    counted_cat = 0
    sum_iou = 0
    for key in prec_stat:
        if int(prec_stat[key]['all']) != 0:
            acc = prec_stat[key]['intersec'] / (prec_stat[key]['all'] + 1e-10)
            iou = prec_stat[key]['intersec'] / (prec_stat[key]['union'] + 1e-10)
            sum_acc += acc
            sum_iou += iou
            counted_cat += 1
    mean_acc = sum_acc / counted_cat
    mean_iou = sum_iou / counted_cat
    output = ('Testing Results: Prec@1 {top1.avg:.3f} Mean Prec@1 {meantop:.3f} Mean IoU {meaniou:.3f} Loss {loss.avg:.5f}'
              .format(top1=top1, loss=losses, meantop=mean_acc, meaniou=mean_iou))
    print(output)
    output_best = '\nBest Prec@1 of: %.3f' % (best_prec1)
    print(output_best)

    return top1.avg

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def count_mean_accuracy(output, target, prec_stat):
    _, pred = output.topk(1, 1, True, True)
    pred = pred.squeeze(1)
    for key in prec_stat.keys():
        label = int(key)
        pred_map = np.uint8(pred.cpu().numpy() == label)
        target_map = np.uint8(target.cpu().numpy() == label)
        intersection_t = pred_map * (pred_map == target_map)
        union_t = pred_map + target_map - intersection_t
        prec_stat[key]['intersec'] += np.sum(intersection_t)
        prec_stat[key]['union'] += np.sum(union_t)
        prec_stat[key]['all'] += np.sum(target_map)
    return prec_stat

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

if __name__=='__main__':
    main()


#!/usr/bin/env python3
import time, sys, os
from scripts.pscreen import Screen
from scripts.game import Game

################################
# CONSTANTS
WIDTH = 80
HEIGHT = 24
LOG_MAX = 8
STEP = 1/10

################################
# Main app
class App:
	def __init__(self):
		# Set working directory
		os.chdir(sys.path[0])

		# Create screen
		self.screen = Screen(WIDTH, HEIGHT)

		# Debug text log
		self.log_text = []

	def main(self):
		# Create game
		self.speed = 1.0
		self.game = Game(self, WIDTH, HEIGHT-2)

		# Main loop
		while True:
			self.update()
			self.render()
			time.sleep(STEP*self.speed)

	def update(self):
		# Update keyboard events
		self.screen.update()

		# ESC = quit
		if self.screen.down("esc"): self.quit()

		# Update game
		self.game.update()

	def render(self):
		# Clear screen
		self.screen.clear()

		# Render game
		self.game.render()

		# Render screen
		self.screen.render()

	def quit(self, exit_py=True):
		# Destructor
		self.screen.destroy()
		if exit_py: sys.exit()

	def log(self, *args):
		# Add text to debug logs
		self.log_text.append(" ".join([str(x) for x in args]))
		if len(self.log_text) > LOG_MAX:
			self.log_text.pop(0)

################################
# Application start
if __name__ == "__main__":
	try:
		app = App()
		app.main()
	except KeyboardInterrupt:
		pass
	finally:
		if app: app.quit(False)


# -*- coding: utf-8 -*-
import pygame
from pygame.locals import *
from sys import exit
from Roslina import *
import random
class Guarana(Roslina):
	def __init__(self, x, y, swiat):
		self.foto = pygame.image.load('guarana.jpg')
		self.X = x
		self.Y = y
		self.sw = swiat
		self.sila = 0
		self.nazwa = 'guarana'
		self.znak = 'G'
		self.usuniety = 0

	def ruch(self):
		self.akcja()

	def kolizja(self, o):
		pass

	def akcja(self):
		random.seed()
		szansa = random.randint(0, 4)
		tmpX = random.randint(-1,1) + self.X
		tmpY = random.randint(-1,1) + self.Y
		if self.sw.organizmy[tmpX][tmpY] == None and tmpX < 21 and tmpX > 0 and tmpY < 21 and tmpY > 0 and szansa == 0:
			guarana = Guarana(tmpX, tmpY, self.sw)
			self.sw.organizmy[guarana.X][guarana.Y] = guarana
			self.sw.nowy[self.sw.zliczNowy()] = guarana


from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords

from pymystem3 import Mystem
mystem = Mystem()

class TextProcessor:
    def __init__(self):
        self.tokenizer = RegexpTokenizer(r'[а-яА-ЯёЁ]+')
        self.stop_words = stopwords.words('russian')

    def process(self, text):
        tokenized = self.tokenizer.tokenize(text)
        lemmatized = [mystem.lemmatize(token)[0] for token in tokenized]
        without_stopwords = [word for word in lemmatized if word not in self.stop_words]

        return " ".join(without_stopwords)


import datetime

class Denarnica:

    def __init__(self, stanje = 0):
        self.stanje = stanje
        self.slovar = {}

    def __repr__(self):
        return 'Denarnica(stanje = {})'.format(self.stanje)

    def __str__(self):
        return 'Denarnica z {}€ '.format(self.stanje)

    def dodaj_v_slovar(self, ime_izdelka, cena, kolicina):
        if self.slovar.get(ime_izdelka) == None :
            self.slovar[ime_izdelka] = [kolicina * cena, kolicina]
        else:
            self.slovar[ime_izdelka][0] += kolicina * cena
            self.slovar[ime_izdelka][1] += kolicina

    def kupi_izdelek(self, ime_izdelka, cena, kolicina):
        if ime_izdelka == '':
            return 'Ups, prišlo je do napake, ime izdelka mora vsebovati vsaj en znak.'
        elif cena <= 0 or kolicina <= 0:
            return 'Ups, prišlo je do napake, cena in količina morata biti večji od 0.'
        elif kolicina * cena > self.stanje:
            return 'Žal nimate dovolj denarja za ta nakup!'
        else:
            self.stanje -= kolicina * cena
            x = datetime.datetime.now().strftime("%d.%m.%Y %H:%M:%S")
            self.dodaj_v_slovar(ime_izdelka, cena, kolicina)
            with open('denarnica.txt', 'a') as dat:
                print(ime_izdelka, '-' +  str(kolicina * cena) + '€', x, sep = ', ', file = dat)
            return 'Izdelek ste uspešno kupili.'

    def placaj_poloznico(self, namen_poloznice, znesek):
        if namen_poloznice == '':
            return 'Ups, prišlo je do napake, namen položnice mora vsebovati vsaj en znak.'
        elif znesek <= 0:
            return 'Ups, prišlo je do napake, znesek položnice mora biti večji od 0 €.'
        elif znesek > self.stanje:
            return 'Ups, te položnice pa ne morete plačati!'
        else:
            self.stanje -= znesek
            x = datetime.datetime.now().strftime("%d.%m.%Y %H:%M:%S")
            self.dodaj_v_slovar(namen_poloznice, znesek, 1)
            with open('denarnica.txt', 'a') as dat:
                print(namen_poloznice, '-' +  str(znesek) + '€', x, sep = ', ', file = dat)
            return 'Položnico ste uspešno plačali.'

    def prihodek(self, opis_prihodka, velikost_prihodka):
        if opis_prihodka == '':
            return 'Ups, prišlo je do napake, opis prihodka mora vsebovati vsaj en znak.'
        elif velikost_prihodka <= 0:
            return 'Ups, prišlo je do napake, velikost prihodka mora biti večja od 0 €.'
        self.stanje += velikost_prihodka
        x = datetime.datetime.now().strftime("%d.%m.%Y %H:%M:%S")
        with open('denarnica.txt', 'a') as dat:
            print(opis_prihodka,'+' + str(velikost_prihodka) + '€', x, sep = ', ', file = dat)
        return 'Vaše stanje se je povišalo na {}€'.format(self.stanje)

    def poraba_na_izdelek(self, ime_izdelka):
        if self.slovar.get(ime_izdelka) == None:
            return 'Tega izdelka še niste kupili.'
        else:
            return 'Za ta izdelek ste porabili {} €.'.format(self.slovar[ime_izdelka][0])

    def kolicina_na_izdelek(self, ime_izdelka):
        if self.slovar.get(ime_izdelka) == None:
            return 'Tega izdelka še niste kupili.'
        else:
            return 'Kupili ste {} izdelkov.'. format(self.slovar[ime_izdelka][1])

def izpis_prihodkov_in_dohodkov():
    izpis = ''
    with open('denarnica.txt') as dat:
        for vrstica in dat:
            izpis += vrstica
    return izpis

import pickle

def shrani_denarnico(denarnica):
    '''shrani podatke denarnice za prihodno uporabo'''
    with open('denarnice_podatki.pkl', 'wb') as output:
        pickle.dump(denarnica,output)

def nalozi_denarnico():
    '''nalozi podatke denarnice od predhodnje uporabe'''
    with open('denarnice_podatki.pkl', 'rb') as input:
        denarnica = pickle.load(input)
    return denarnica


"""sina.com bounces"""

import re

from email.iterators import body_line_iterator
from flufl.bounce.interfaces import (
    IBounceDetector, NoFailures, NoTemporaryFailures)
from public import public
from zope.interface import implementer

acre = re.compile(r'<(?P<addr>[^>]*)>')

@public
@implementer(IBounceDetector)
class Sina:
    """sina.com bounces"""

    def process(self, msg):
        """See `IBounceDetector`."""
        if msg.get('from', '').lower() != 'mailer-daemon@sina.com':
            return NoFailures
        if not msg.is_multipart():
            return NoFailures
        # The interesting bits are in the first text/plain multipart.
        part = None
        try:
            part = msg.get_payload(0)
        except IndexError:
            pass
        if not part:
            return NoFailures
        addresses = set()
        for line in body_line_iterator(part):
            mo = acre.match(line)
            if mo:
                addresses.add(mo.group('addr').encode('us-ascii'))
        return NoTemporaryFailures, addresses


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymongo
import os

from scrapy import Item

from airticket import constant
from airticket import control

class AirticketMongoPipeline(object):
    """
    存储在mongo
    """

    def process_item(self, item, spider):
        """
        每个item pipeline组件都需要调用该方法
        这个方法必须返回一个Item对象，或是抛出DropItem异常
        :param item: 被爬取的item
        :param spider: 爬取该item的spider
        :return:
        """
        self.insert_db(item)
        return item

    def open_spider(self, spider):
        """
        在spider开启的时候自动被调用，在此可做一些初始化的操作
        :return:
        """
        # 数据库配置
        my_db = control.MyMongo()
        self.db_client = my_db.db_client
        self.db = my_db.db
        # db_uri = spider.settings.get('MONGODB_URI', 'mongodb://localhost:27017')
        # db_name = spider.settings.get('MONGODB_DB_NAME', 'airticketdb')

        # self.db_client = pymongo.MongoClient(db_uri)
        # self.db = self.db_client[db_name]

    def close_spider(self, spider):
        """
        当spider被关闭时，这个方法被调用
        :return:
        """
        self.db_client.close()

    def insert_db(self, item):
        """
        插入数据
        :param item:
        :return:
        """
        if isinstance(item, Item):
            item = dict(item)
            item['search_time'] = constant.NOW_STR_TIME_SS
        self.db.ticket.insert(item)


from unittest.mock import Mock, patch

from django.conf import settings
from django.test import TestCase
from rest_framework import status

from Chat.utils.error_messages import CHAT_ACCESS_MESSAGE
from Chat.utils.helpers import preprocess_count, return_error, check_access_to_chat

class HelpersTestCase(TestCase):

    def test_preprocess_count_none(self):
        output_count = preprocess_count(None)
        self.assertEqual(
            output_count,
            int(settings.CHAT_CONFIGURATION['max_messages_count']))

    def test_preprocess_count_positive_int(self):
        output_count = preprocess_count('10')
        self.assertEqual(output_count, 10)

    def test_return_error(self):
        expected_message = '{"error": {"status": 200, "message": "My message"}}'
        output_message = return_error('My message', 200).content.decode("utf-8")
        self.assertEqual(output_message, expected_message)

    @patch('django.db.models.query.QuerySet.exists')
    def test_check_access_to_chat_accessible(self, mock_exists):
        user_id = 3
        chat_uuid = 'd65d0970-1933-11eb-adc1-0242ac120002'
        mock_func = Mock()
        mock_request = Mock()
        mock_request.user.id = user_id
        mock_exists.return_value = True
        check_access_to_chat(mock_func)(mock_request, chat_uuid)
        mock_func.assert_called_once_with(mock_request, chat_uuid)

    @patch('Chat.utils.helpers.return_error')
    def test_check_access_to_chat_inaccessible(self, mock_return_error):
        user_id = 1
        chat_uuid = 'd65d0970-1933-11eb-adc1-0242ac120002'
        mock_func = Mock()
        mock_request = Mock()
        mock_request.user.id = user_id
        check_access_to_chat(mock_func)(mock_request, chat_uuid)
        mock_return_error.assert_called_once_with(
            CHAT_ACCESS_MESSAGE, status.HTTP_403_FORBIDDEN)


import scrapy
from pixelimg.items import PixelimgItem

class pixelimgSpider(scrapy.Spider):
    name = "pixelimg"
    start_urls = [
        "http://pixeljoint.com/pixels/new_icons.asp?ob=rating",
    ]
    web_url = 'http://pixeljoint.com'
    oldpages = start_urls
    star = 0
    def parse(self, response):
        url = response.xpath('//*[@class="tooltip"]/strong/a/@href').extract()
        for x in url:
            yield scrapy.Request(self.web_url+x, callback=self.parse_content)

        next_pages = response.xpath('//*[@id="leftblockspan"]/div[1]/div/div[50]/a[3]/@href').extract()

        # if next_pages[0] != self.oldpages[self.star]:
        #     self.oldpages[self.star] = next_pages[0]
        # #     ## 将 「下一页」的链接传递给自身，并重新分析
        #     yield scrapy.Request(self.web_url + self.oldpages[self.star], callback=self.parse)

    def parse_content(self, response):
        item = PixelimgItem()
        # //*[@id="leftblockspan"]/div[1]/div/div[50]/a[3]
        item['Title'] = response.xpath('//*[@id="leftblockspan"]/div[1]/div/table/tr[1]/td/table/tr[1]/td[2]/text()').extract()[0]
        item['Artist'] = response.xpath('//*[@id="leftblockspan"]/div[1]/div/table/tr[1]/td/table/tr[2]/td[2]/a/text()').extract()[0]
        item['Posted'] = response.xpath('//*[@id="leftblockspan"]/div[1]/div/table/tr[1]/td/table/tr[3]/td[2]/text()').extract()[0]
        item['Artist_url'] = self.web_url+response.xpath('//*[@id="leftblockspan"]/div[1]/div/table/tr[1]/td/table/tr[2]/td[2]/a/@href').extract()[0]
        item['Pic_url'] = response.xpath('//*[@id="mainimg"]/@src').extract()[0]
        # i = 0
        # for x in lists:
        #     lists[i] = self.web_url + x
        #     i +=1
        # item['image_urls'] = lists

        yield item


from keras.callbacks import TensorBoard, CSVLogger, ModelCheckpoint
from keras.layers import Dense, Embedding, CuDNNLSTM, Bidirectional
from keras.models import Sequential
from sklearn.model_selection import StratifiedKFold
from pandas import get_dummies

from Embedding_Utilities import load_data

util = load_data('TL_BI_cuDNNLSTM_L2_N196_GloVe_5f_final')

lstm_out = 196

model = Sequential()
model.add(Embedding(util.NB_WORDS, util.EMBEDDING_DIM,input_length = util.TWEET_LENGTH,
                    dropout=0.2,weights=[util.embedding_matrix], trainable=False))
model.add(Bidirectional(CuDNNLSTM(lstm_out, return_sequences=True)))
model.add(Bidirectional(CuDNNLSTM(lstm_out, go_backwards=True)))
model.add(Dense(util.NUM_CLASSES,activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='nadam',metrics = ['accuracy'])
print(model.summary())
kfold = StratifiedKFold(n_splits=5,shuffle=True,random_state=util.RANDOM_SEED)
cvscores = []
X = util.X
Y = util.Y
Data_X = X
Data_Y = Y
batch_size = 256
i=0
for train,test in kfold.split(X,Y):
    fold = str(i+1)
    csv_logger = CSVLogger(util.LOG_DIR, append=True)
    foldY_train = get_dummies(Y[train]).values
    foldY_test = get_dummies(Y[test]).values
    checkpoint_name = util.MODEL_DIR + '/' + fold + '.{epoch:02d}-{loss:.2f}-{val_loss:.2f}.hdf5'
    checkpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss',
                                 verbose=0, save_best_only=False,
                                 save_weights_only=False, mode='auto', period=1)
    tb = TensorBoard(log_dir=util.SUMMARY_DIR+fold, histogram_freq=1, batch_size=batch_size, write_graph=True,
                     write_grads=False,write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)
    hist = model.fit(X[train],foldY_train,batch_size=batch_size,
                     epochs=2,verbose=1,callbacks=[csv_logger,tb,checkpoint],validation_data=(X[test],foldY_test))
    i += 1
Data_Y = get_dummies(Data_Y).values
score,acc = model.evaluate(Data_X, Data_Y, verbose = 1, batch_size = batch_size)
print("score: %.2f" % (score))
print("acc: %.2f" % (acc))

Sentiment Analysis POC/BI-cuDnnLSTM 5 Fold.py


Import("env")
import os
import requests
import zipfile
import tempfile
import shutil

url = "https://github.com/makerbase-mks/Mks-Robin-Nano-Marlin2.0-Firmware/archive/master.zip"
zip_path = os.path.join(env.Dictionary("PROJECT_LIBDEPS_DIR"), "mks-assets.zip")
assets_path = os.path.join(env.Dictionary("PROJECT_BUILD_DIR"), env.Dictionary("PIOENV"), "assets")

def download_mks_assets():
	print("Downloading MKS Assets")
	r = requests.get(url, stream=True)
	# the user may have a very clean workspace,
	# so create the PROJECT_LIBDEPS_DIR directory if not exits
	if os.path.exists(env.Dictionary("PROJECT_LIBDEPS_DIR")) == False:
		os.mkdir(env.Dictionary("PROJECT_LIBDEPS_DIR"))
	with open(zip_path, 'wb') as fd:
		for chunk in r.iter_content(chunk_size=128):
			fd.write(chunk)

def copy_mks_assets():
	print("Copying MKS Assets")
	output_path = tempfile.mkdtemp()
	zip_obj = zipfile.ZipFile(zip_path, 'r')
	zip_obj.extractall(output_path)
	zip_obj.close()
	if os.path.exists(assets_path) == True and os.path.isdir(assets_path) == False:
		os.unlink(assets_path)
	if os.path.exists(assets_path) == False:
		os.mkdir(assets_path)
	base_path = ''
	for filename in os.listdir(output_path):
		base_path = filename
	for filename in os.listdir(os.path.join(output_path, base_path, 'Firmware', 'mks_font')):
		shutil.copy(os.path.join(output_path, base_path, 'Firmware', 'mks_font', filename), assets_path)
	for filename in os.listdir(os.path.join(output_path, base_path, 'Firmware', 'mks_pic')):
		shutil.copy(os.path.join(output_path, base_path, 'Firmware', 'mks_pic', filename), assets_path)
	shutil.rmtree(output_path, ignore_errors=True)

if os.path.exists(zip_path) == False:
	download_mks_assets()

if os.path.exists(assets_path) == False:
	copy_mks_assets()


"""
Author: Kaushik Bhimraj
Desc:   The program looks at the point dictionary and will change the suffix in the point database to reflect Desigo
        standards.
"""

import xlrd
import json

def spcheck():

    with open("C:/Users/Z003VVKY/Desktop/44OP-245233_Gainsville_Panel_Upgrade/PointName.json") as ptd:
        data = json.load(ptd)

    file = "C:/Users/Z003VVKY/Desktop/44OP-245233_Gainsville_Panel_Upgrade/PointsCompare1.xlsx"
    wb = xlrd.open_workbook(file)
    ss = wb.sheet_by_index(0)
    rw = ss.nrows

    for val in range(1, rw):
        x = ss.cell(val, 0).value
        bits = x.split('.')
        for n, bit in enumerate(bits):
            try:
                if type(data[bit]) is list:
                    bits[n] = data[bit][0]
                else:
                    bits[n] = data[bit]
                    print(bits)
            except KeyError:
                pass
        if 'BLDG' in bits:
            num1 = bits.index('BLDG')  # Checked whether 'BLDG' is present in spl
            bits.pop(num1)

        if '11' in bits:
            num1 = bits.index('11')  # Checked whether '11' is present in spl
            bits.pop(num1)

        if 'VA' in bits:
            num1 = bits.index('VA')  # Checked whether 'VA' is present in spl
            bits.pop(num1)

        point = "_".join(bits)
        x = point.replace('AC9', 'AC9_')
        x = x.replace('A01', 'A01_')
        x = x.replace('EF1', 'EF1_')
        x = x.replace('EF3', 'EF3_')
        x = x.replace('EF4', 'EF4_')
        x = x.replace('FFS', 'FFS_')
        x = x.replace('A03', 'A03_')
        x = x.replace('A04', 'A04_')
        x = x.replace('A06', 'A06_')
        x = x.replace('A07', 'A07_')
        x = x.replace('P09', 'P09_')
        x = x.replace('P10', 'P10_')
        x = x.replace('RF1', 'RF1_')
        x = x.replace('RF3', 'RF3_')
        x = x.replace('RF6', 'RF6_')
        x = x.replace('P12', 'P12_')
        x = x.replace('SF7', 'SF7_')
        x = x.replace('SF8', 'SF8_')
        x = x.replace('MBC', 'MBC_')
        x = x.replace(' ', '_')
        x = x.replace('/', '_')
        x = x.replace('__', '_')
        if bits[0] != 'GVL':
            print('GVL_' + x)
        else:
            print(x)

spcheck()


from django.http import HttpResponse, Http404
from apis import sf_backends, nb_backends
from . import models
from django.views.decorators.csrf import csrf_exempt
from django.utils import timezone
import json
import time
from django.conf import settings

@csrf_exempt
def send_to_sf(contact):
    person = contact['payload']['person']

    ### for testing ONLY and temporarily saving data to db
    # test_obj = models.TestHook(content=request.body)
    # test_obj.save()

    ### set default language if none is selected
    if person['user_language'] is None:
        person['user_language'] = 'EN'

    try:
        if person['primary_address']['country_code'] == 'Other' or person['primary_address']['country_code'] == 'other':
            country_code = ''
        else:
            country_code = person['primary_address']['country_code']
    except:
        country_code = ''

    contact_obj = {
        'FirstName': person['first_name'],
        'LastName': person['last_name'],
        'Email': person['email'],
        'MailingCountryCode': country_code,
        'Subscriber__c': person['email_opt_in'],
        'Sub_Mozilla_Foundation__c': person['email_opt_in'],
        'Email_Language__c': person['user_language'],
        'RecordTypeId': settings.ADVOCACY_RECORD_TYPE_ID  # advocacy record type
    }
    if person['salesforce_id']:  # not used because we are not saving salesforce ID back
        sf_backends.update_user(person['salesforce_id'], contact_obj)
        sf_contact_id = {'id': person['salesforce_id']}
    else:
        sf_contact_id = sf_backends.insert_user(contact_obj)
        # nb_backends.nb_update_salesforce_id(person['id'], sf_contact_id['id'])  #disable to prevent record duplication

    # print sf_contact_id

    for campaign in person['tags']:
        try:
            campaign_tag = models.Campaign.objects.get(nationbuilder_tag=campaign)
        except models.Campaign.DoesNotExist:
            campaign_tag = None
        if campaign_tag:
            dj_sf_campaign_id = campaign_tag.salesforce_id

            ### add CampaignMember Obj
            sf_backends.upsert_contact_to_campaign({
                'ContactId': sf_contact_id['id'],
                'CampaignId': dj_sf_campaign_id,
                'Campaign_Language__c': person['user_language'],
                'Campaign_Email_Opt_In__c': person['email_opt_in'],
            })
        else:
            ### add campaign obj to SalesForce
            sf_campaign_id = sf_backends.insert_campaign({
                'Name': campaign,
                'Type': 'Petition'
            })

            ### save it in DJ NB -> SF model
            dj_campaign_obj = models.Campaign(
                nationbuilder_tag=campaign,
                salesforce_id=sf_campaign_id['id'],
            )
            dj_campaign_obj.save()

            ### add CampaignMember Obj
            sf_backends.upsert_contact_to_campaign({
                'ContactId': sf_contact_id['id'],
                'CampaignId': sf_campaign_id['id'],
                'Campaign_Language__c': person['user_language'],
                'Campaign_Email_Opt_In__c': person['email_opt_in'],
            })
    return True

@csrf_exempt
def hook(request):
    if request.method == "POST":
        content = request.body
        content = json.loads(content)

        ### save all data to a log table
        log_contact = models.Log(
            email=content['payload']['person']['email'],
            contact=request.body,
            type='created',
        )
        log_contact.save()

        ### add this to prevent NB sending duplicated user_created with user_language as null
        if not content['payload']['person']['user_language']:
            return HttpResponse('not saved, no user_language')

        ### check NB token
        if 'token' not in content:
            raise Http404("Not found")
        if content['token'] != settings.NB_TOKEN:
            raise Http404("Not found")

        try:
            matching_contacts = models.ContactSync.objects.filter(email=content['payload']['person']['email'])
        except models.ContactSync.DoesNotExist:
            matching_contacts = None

        if matching_contacts:
            return HttpResponse('contact exist')
        else:
            db_contact = models.ContactSync(email=content['payload']['person']['email'],
                                            contact=request.body,
                                            type='created')
            db_contact.save()

        saved = send_to_sf(content)
        # saved =  True #For testing
        if saved:
            db_contact.synced = True
            db_contact.save()
            return HttpResponse('saved')
        else:
            return HttpResponse('not saved')
    raise Http404("Not found")

@csrf_exempt
def update(request):
    ### check if there is any duplicate entries due to NB
    # try:
    # contacts = models.ContactSync.objects

    try:
        max_time = timezone.now() - timezone.timedelta(
            minutes=1)  # the newest selected field must be at least 1 minutes old which is the time that salesforce need
        # print timezone.now()
        # print max_time
        contacts = models.ContactSync.objects.filter(synced=False, created_at__lte=max_time).order_by('-created_at')
    except models.ContactSync.DoesNotExist:
        contacts = None
    if contacts:
        for contact_item in contacts:
            # print contact_item.contact
            content = contact_item.contact
            content = json.loads(content)
            # print content
            try:
                sent_to_sf = send_to_sf(content)
            except:
                sent_to_sf = None
            # sent_to_sf = True #For debug without send to salesforce
            if sent_to_sf:
                contact_item.synced = True
                contact_item.save()
        return HttpResponse('All contacts updated')
    else:
        return HttpResponse('no contacts found to be updated')

@csrf_exempt
def save_update(request):
    if request.method == "POST":
        content = request.body
        content = json.loads(content)

        ### save all data to a log table
        log_contact = models.Log(
            email=content['payload']['person']['email'],
            contact=request.body,
            type='updated',
        )
        log_contact.save()

        ### add this to prevent NB sending duplicated user_created with user_language as null
        if not content['payload']['person']['user_language']:
            return HttpResponse('not saved, no user_language')

        ### check NB token
        if 'token' not in content:
            raise Http404("Not found")
        if content['token'] != settings.NB_TOKEN:
            raise Http404("Not found")

        try:
            matching_contacts = models.ContactSync.objects.filter(email=content['payload']['person']['email']).update(
                contact=request.body, synced=False)
        except models.ContactSync.DoesNotExist:
            matching_contacts = None

        if matching_contacts:
            return HttpResponse('contact exist')
        else:
            update_obj = models.ContactSync(email=content['payload']['person']['email'],
                                            contact=request.body,
                                            type='updated')
            update_obj.save()
            return HttpResponse('saved')

    raise Http404("Not found")


import os
import time
import sys

import random
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
plt.switch_backend('agg')

from mil import MIL
from tensorflow.examples.tutorials.mnist import input_data
from utils import np_sigmoid
from utils import print_metrics

os.environ["CUDA_VISIBLE_DEVICES"] = "3"
flags = tf.app.flags
FLAGS = flags.FLAGS
def get_data():
    path = "/home/wykgroup/seongok/ml_study/mnist_mlp/mnist/"
    mnist = input_data.read_data_sets(path, one_hot=False)
    train_set = mnist.train
    valid_set = mnist.validation
    test_set = mnist.test
    return train_set, valid_set, test_set

def instances_to_bag(ds, n_inst, target, n_bags=None):
    x = np.asarray([img.reshape(28,28) for img in ds.images])
    y = ds.labels

    n_total = x.shape[0]
    indices = np.random.randint(n_total, size=n_total)
    if n_bags is None:
        n_bags = n_total//n_inst

    x = x[indices]
    y = y[indices]

    x_bag = []
    y_bag = []
    for i in range(n_bags):
        xi = x[i*n_inst:(i+1)*n_inst]
        yi = y[i*n_inst:(i+1)*n_inst]

        label = 0.0
        if target in yi:
            label = 1.0
        x_bag.append(xi)
        y_bag.append(label)
    return np.asarray(x_bag), np.asarray(y_bag)

def train(opt_dict):
    train_set, valid_set, test_set = get_data()
    x_train, y_train = instances_to_bag(
        train_set, FLAGS.n_inst, FLAGS.target, n_bags=FLAGS.n_train_bags)
    x_valid, y_valid = instances_to_bag(valid_set, FLAGS.n_inst, FLAGS.target)
    x_test, y_test = instances_to_bag(valid_set, FLAGS.n_inst, FLAGS.target)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        for epoch in range(FLAGS.n_epoches):
            # Train
            true_train = np.empty([0,])
            pred_train = np.empty([0,])
            for i in range(x_train.shape[0]):
                st_i = time.time()
                xi = x_train[i]
                yi = np.asarray([y_train[i]])

                feed_dict = {opt_dict.x:xi, opt_dict.y:yi}
                ops = [opt_dict.train_op, opt_dict.loss, opt_dict.logits]
                _, loss, logits = sess.run(ops, feed_dict=feed_dict)
                et_i = time.time()
                print ("Training", epoch, "-th epoch\t", \
                       i, "-th bag\t Loss=", loss,
                       "\t Time:", round(et_i-st_i,3), "(s)")

                true_train = np.concatenate([true_train, yi], axis=0)
                pred_train = np.concatenate([pred_train, np_sigmoid(logits)], axis=0)

            # Validation
            true_valid = np.empty([0,])
            pred_valid = np.empty([0,])
            for i in range(x_train.shape[0]):
                st_i = time.time()
                xi = x_valid[i]
                yi = np.asarray([y_valid[i]])

                feed_dict = {opt_dict.x:xi, opt_dict.y:yi}
                ops = [opt_dict.loss, opt_dict.logits]
                loss, logits = sess.run(ops, feed_dict=feed_dict)
                et_i = time.time()
                print ("Validation", epoch, "-th epoch\t", \
                       i, "-th bag\t Loss=", loss,
                       "\t Time:", round(et_i-st_i,3), "(s)")

                true_valid = np.concatenate([true_valid, yi], axis=0)
                pred_valid = np.concatenate([pred_valid, np_sigmoid(logits)], axis=0)
            print_metrics(true_train, pred_train)
            print_metrics(true_valid, pred_valid)

        # Test
        true_test = np.empty([0,])
        pred_test = np.empty([0,])
        for i in range(x_train.shape[0]):
            st_i = time.time()
            xi = x_test[i]
            yi = np.asarray([y_test[i]])

            feed_dict = {opt_dict.x:xi, opt_dict.y:yi}
            ops = [opt_dict.loss, opt_dict.logits]
            loss, logits = sess.run(ops, feed_dict=feed_dict)
            et_i = time.time()
            print ("Validation", epoch, "-th epoch\t", \
                   i, "-th bag\t Loss=", loss,
                   "\t Time:", round(et_i-st_i,3), "(s)")

            true_test = np.concatenate([true_test, yi], axis=0)
            pred_test = np.concatenate([pred_test, np_sigmoid(logits)], axis=0)
        print_metrics(true_test, pred_test)
        print ("Finish training and test")
    return

def main():
    model = MIL(hw=FLAGS.hw,
                ch=FLAGS.ch,
                attn_dim=FLAGS.attn_dim,
                use_attn=FLAGS.use_attn,
                use_gate=FLAGS.use_gate,
                wd=FLAGS.wd,
                lr=FLAGS.lr)
    opt_dict = model.get_opt_dict()
    train(opt_dict)
    return

if __name__ == '__main__':
    flags.DEFINE_integer('hw', 28, 'height and width')
    flags.DEFINE_integer('ch', 1, 'channel')
    flags.DEFINE_integer('attn_dim', 128, 'hidden dimension of attention')
    flags.DEFINE_integer('n_inst', 50, 'number of instances per bag')
    flags.DEFINE_integer('n_train_bags', 100, 'number of instances per bag')
    flags.DEFINE_integer('target', 9, 'target integer')
    flags.DEFINE_bool('use_attn', True, 'Whether to use the attn mechanism')
    flags.DEFINE_bool('use_gate', False, 'Whether to use the gate mechanism')
    flags.DEFINE_float('wd', 1e-4, 'weight_decay')
    flags.DEFINE_float('lr', 5e-4, 'Initial learning rate')
    flags.DEFINE_integer('n_epoches', 100, 'Epoch size')
    flags.DEFINE_bool('save_model', False, '')
    main()


import numpy as np
import math
import argparse
import os
import gc
from PIL import Image
import time
import tensorflow as tf
from keras import backend as K
from keras.layers import Input
from keras.models import Model
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import Conv3D
from keras.layers.convolutional import MaxPooling2D
from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, Dropout, Activation, Average
from keras.layers.core import Activation
from keras.layers.core import Dense
from keras.layers.core import Flatten
from keras.layers.core import Reshape
from keras.layers import Embedding
from keras.layers import concatenate
import itertools as it
import random
import scipy.io as sio
gc.enable()

def create_cnn(width, height, depth): #CNN design modelled after googlenet inception based modules
	# initialize the input shape and channel dimension, assuming
    # The width and height of the input images in pixels
    # define the model input
    inputShape = (width, height, depth)
    input_image = Input(shape=inputShape, name='input_image')
    #each block represents a single inception module
    conv1 = Conv2D(16, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(input_image)
    conv1 = Conv2D(32, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(conv1)
    # Start of first inception module
    intconv1_1 = Conv2D(64, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(conv1)
    intconv1_1 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv1_1)

    intconv1_2 = Conv2D(64, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(conv1)
    intconv1_2 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv1_2)

    intconv1_3 = Conv2D(64, kernel_size=(5, 5), strides=1, data_format="channels_last", padding='same', activation='relu')(conv1)
    intconv1_3 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv1_3)

    intconv1_4 = MaxPooling2D(pool_size=(3, 3), strides=1, data_format="channels_last", padding='same')(conv1)

    intconv1 = concatenate([intconv1_1, intconv1_2, intconv1_3, intconv1_4])  #end of first inception module

    conv2 = Conv2D(32, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv1)
    conv2 = MaxPooling2D(pool_size=(3, 3), strides=2, data_format="channels_last", padding='same')(conv2)
    # Start of Second inception module
    intconv2_1 = Conv2D(64, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(conv2)
    intconv2_1 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv2_1)

    intconv2_2 = Conv2D(64, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(conv2)
    intconv2_2 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv2_2)

    intconv2_3 = Conv2D(64, kernel_size=(5, 5), strides=1, data_format="channels_last", padding='same', activation='relu')(conv2)
    intconv2_3 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv2_3)

    intconv2_4 = MaxPooling2D(pool_size=(3, 3), strides=1, data_format="channels_last", padding='same')(conv2)

    intconv2 = concatenate([intconv2_1, intconv2_2, intconv2_3, intconv2_4])     # End of second inception module

    conv3 = Conv2D(32, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv2)
    conv3 = MaxPooling2D(pool_size=(3, 3), strides=2, data_format="channels_last", padding='same')(conv3)
    # Start of third inception module
    intconv3_1 = Conv2D(64, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(conv3)
    intconv3_1 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv3_1)

    intconv3_2 = Conv2D(64, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(conv3)
    intconv3_2 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv3_2)

    intconv3_3 = Conv2D(64, kernel_size=(5, 5), strides=1, data_format="channels_last", padding='same', activation='relu')(conv3)
    intconv3_3 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv3_3)

    intconv3_4 = MaxPooling2D(pool_size=(3, 3), strides=1, data_format="channels_last", padding='same')(conv3)

    intconv3 = concatenate([intconv3_1, intconv3_2, intconv3_3, intconv3_4])     # End of Third inception module

    conv4 = Conv2D(32, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv3)
    conv4 = MaxPooling2D(pool_size=(3, 3), strides=2, data_format="channels_last", padding='same')(conv4)
    # Start of Fourth inception module
    intconv4_1 = Conv2D(64, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(conv4)
    intconv4_1 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv4_1)

    intconv4_2 = Conv2D(64, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(conv4)
    intconv4_2 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv4_2)

    intconv4_3 = Conv2D(64, kernel_size=(5, 5), strides=1, data_format="channels_last", padding='same', activation='relu')(conv4)
    intconv4_3 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv4_3)

    intconv4_4 = MaxPooling2D(pool_size=(3, 3), strides=1, data_format="channels_last", padding='same')(conv4)

    intconv4 = concatenate([intconv4_1, intconv4_2, intconv4_3, intconv4_4])     # Start of fourth inception module

    conv5 = Conv2D(32, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv4)
    conv5 = MaxPooling2D(pool_size=(3, 3), strides=2, data_format="channels_last", padding='same')(conv5)
    # Start of fifth inception module
    intconv5_1 = Conv2D(64, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(conv5)
    intconv5_1 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv5_1)

    intconv5_2 = Conv2D(64, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(conv5)
    intconv5_2 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv5_2)

    intconv5_3 = Conv2D(64, kernel_size=(5, 5), strides=1, data_format="channels_last", padding='same', activation='relu')(conv5)
    intconv5_3 = Conv2D(16, kernel_size=(1, 1), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv5_3)

    intconv5_4 = MaxPooling2D(pool_size=(3, 3), strides=1, data_format="channels_last", padding='same')(conv5)

    intconv5 = concatenate([intconv5_1, intconv5_2, intconv5_3, intconv5_4])    # End of fifth inception module

    conv6 = Conv2D(32, kernel_size=(3, 3), strides=1, data_format="channels_last", padding='same', activation='relu')(intconv5)

    flat = Flatten()(conv6) # flatten conv layers into dense layer

    dense = Dense(512, activation='relu')(flat)
    main_output = Dense(196, activation='softmax')(dense) #196 outputs as dataset has 196 classes
    model = Model(inputs=[input_image], outputs=[main_output])
    return model

def save_image(filename, norm_arr): # function for saving image, used for debugging and testing only
    img = (norm_arr.copy()).astype(np.uint8)

    res = Image.fromarray(img).convert('L')
    res.save(filename)

def load_data(folder_path, mat_name): #function for loading test and training data
    os.chdir(folder_path)
    mat = sio.loadmat(str(mat_name) +".mat", squeeze_me=True)
    os.chdir('..')
    idx_name = mat['annotations']
    return idx_name

def load_meta(folder_path, mat_name): #function for loading test and metadata
    os.chdir(folder_path)
    mat = sio.loadmat(str(mat_name) +".mat", squeeze_me=True)
    os.chdir('..')
    idx_name = mat['class_names']
    return idx_name

def load_imgs(folder_path, img_name, width, height): #function for loading and resizing images
    os.chdir(folder_path)
    img = Image.open(str(img_name))
    os.chdir('..')
    return img

def filter_dataset( dataset): #filtering dataset for testing data
    filtering = np.zeros( (len(dataset) ), dtype=np.float)
    for idx in range(len(dataset)):
        test_flag  = dataset[idx][6]
        if test_flag == 1 :
            filtering[idx] = True
        else:
            filtering[idx] = False
    return filtering

def jpg_crop_grey_arr(image_name, width, height): #function for loading and resizing images
    with Image.open(str(image_name)) as img:
        lower_x = train_info[0][0]
        upper_x = train_info[0][2]
        lower_y = train_info[0][1]
        upper_y = train_info[0][3]
        bound_x = upper_x - lower_x
        bound_y = upper_y - lower_y
        img = np.array(img)
        if img.ndim < 3: #for normalizing images that only have one channel
            img_new = np.zeros( (np.size(img,0),np.size(img,1),3), dtype=np.float)
            img_new[:,:,0] = img [:,:]
            img_new[:,:,1] = img [:,:]
            img_new[:,:,2] = img [:,:]
            img = img_new
        else :
            pass

        if upper_y >= np.size(img, 0) : #for filtering images with incorrect boundary conditions
            img_crop = img
        elif upper_x >= np.size(img, 1) :
            img_crop = img
        else :
            img_crop = np.zeros( (bound_y,bound_x,3), dtype=np.float)
            for y_pos in range (bound_y):
                for x_pos in range (bound_x):
                    img_crop[y_pos, x_pos, 0:2] = img[ lower_y + y_pos , lower_x + x_pos, 0:2]
        img_crop = (img_crop.copy()).astype(np.uint8)
        img_crop = Image.fromarray(img_crop)
        img_crop = img_crop.resize((width,height), Image.ANTIALIAS) #resize to standard resolution
        img_crop = np.array(img_crop)
    return img_crop

def aug_data(raw_arr): # augmentation of training data to improve training performance
    res_LR = np.flip(raw_arr, 1) #flips left right

    res_90 = np.rot90(raw_arr, 1) #90 rotation
    res_90LR = np.flip(res_90, 1) #90 + left right flip

    res_180 = np.rot90(raw_arr, 2) #180 rotation
    res_LR180 = np.flip(res_180, 1) #180 + left right flip

    res_270 = np.rot90(raw_arr, 3) #90 rotation
    res_270LR = np.flip(res_270, 1) #90 + left right flip
    res = np.stack((raw_arr, res_90, res_90LR, res_180,res_LR,res_LR180, res_270, res_270LR), axis = 0 )
    return res

def data_load(folder_path, width, height): # function for loading images
    os.chdir(folder_path)
    img_arr = jpg_crop_grey_arr(train_info[0][5], width, height)      # apply greyscale and cropping
    img_arr_stack = np.reshape(img_arr, (1,128,128,3))
    for file_idx in range (2,len(train_info)):
        img_arr = jpg_crop_grey_arr(train_info[file_idx][5], width, height)
        img_arr = np.reshape(img_arr, (1,128,128,3))
        img_arr_stack = np.concatenate((img_arr_stack,img_arr), axis = 0 )
    os.chdir('..')
    return img_arr

def class_load(dataset): # function for loading class dataset
    train_class = np.zeros((len(dataset)), dtype=np.float)
    for file_idx in range (len(dataset)):
        train_class[file_idx] = dataset[file_idx][4]
    return train_class

def test_data_process(folder_path, batch_size, width, height, filtering): # function to load test and train images and classes
    os.chdir(folder_path)
    test_class = np.zeros((8041), dtype=np.int) #number of testing images specified
    train_class = np.zeros((8144), dtype=np.int) #number of training images specified
    test_class_count = 0
    train_class_count = 0
    for file_idx in range (len(test_info)):
        if filtering[file_idx] == True:
            test_class[test_class_count] = test_info[file_idx][5]
            test_class_count = test_class_count +1
        else:
            train_class[train_class_count] = test_info[file_idx][5]
            train_class_count = train_class_count +1
    test_img_stack = np.zeros( (8041,128,128,3) , dtype=np.float) # creating and sorting test and training class datasets
    train_img_stack = np.zeros((8144,128,128,3), dtype=np.float)
    train_count = 0
    test_count = 0
    for file_idx in range (len(test_info)):
        if filtering[file_idx] == True:
            text = test_info[file_idx][0][:0] + test_info[file_idx][0][8:]
            test_img_stack[test_count, :, :, :] = jpg_crop_grey_arr(text, width, height)
            test_count = test_count + 1
        else :
            text = test_info[file_idx][0][:0] + test_info[file_idx][0][8:]
            train_img_stack[train_count,:,:,:] = jpg_crop_grey_arr(text, width, height)
            train_count = train_count + 1
    os.chdir('..')
    return train_img_stack, test_img_stack, train_class, test_class

def hot_encode(dataset, num_classes): #function for hot encoding the dataset the sparse categorical format for softmax activation
    data_classes = np.zeros( (len(dataset), num_classes) , dtype=np.int)
    for idx in range(len(dataset)):
        data_classes[idx, dataset[idx]-1 ] = 1
    return data_classes

def data_gen(dataset, classes, num_classes): #generator for network input #generator improves efficency and RAM management since GPU training is used
    while True:
        class_set = np.zeros((8, num_classes), dtype=np.int)
        for idx in range(len(dataset)):
            img_arr = dataset[idx, : , : , :]
            img_arr = aug_data(img_arr)
            for class_idx in range(8) :
                class_set[class_idx, :] = classes[idx, :]
            yield img_arr, class_set

def data_reduction(dataset): #reduces images to one channel
    reduc_dataset = np.zeros((len(dataset),128,128,1), dtype=np.float)
    reduc_dataset[:,:,:,0] = dataset [:,:,:, 0]
    return reduc_dataset

if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument('-f', '--filepath', required=True, help="specify filepath")
    args = ap.parse_args()
    os.chdir(args.filepath)
    #create network
    model = create_cnn(128, 128, 1)
    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
    model.summary()
    #loading training data
    train_info = load_data('car_devkit', 'cars_train_annos')
    print(np.array(train_info).shape)
    print(len(train_info[1] ))

    #loading testing data
    test_info = load_data('car_devkit', 'cars_annos')
    filtering = filter_dataset(test_info)
    print(len(test_info))
    print(len(filtering))

    #loading meta data
    meta_info = load_meta('car_devkit', 'cars_meta')
    print(len(meta_info))

    #loading and processing training and testing image dataset and classes
    print('dataset 1 processing')
    (train_dataset, test_dataset, train_classes, test_classes)= test_data_process('car_ims', 4, 128, 128, filtering)
    print(np.array(train_dataset).shape)
    print(np.array(test_dataset).shape)
    print(np.array(train_classes).shape)
    print(np.array(test_classes).shape)

    train_classes = hot_encode(train_classes, len(meta_info)) #hot encode class dataset for network training
    print(np.array(train_classes).shape)
    test_classes = hot_encode(test_classes,len(meta_info))
    print(np.array(test_classes).shape)

    test_dataset = data_reduction(test_dataset)
    train_dataset = data_reduction(train_dataset)
    print('final')
    print(np.array(train_dataset).shape)
    print(np.array(test_dataset).shape)

    # save_image(str('test')+".jpg", new_img)

    gc.collect()
    #training network
    model.fit_generator(data_gen(train_dataset,train_classes,len(meta_info)), steps_per_epoch=len(train_classes), epochs=100, validation_data=(test_dataset[:1000], test_classes[:1000]), max_queue_size=100, workers=1, use_multiprocessing=False, verbose = 1)
    gc.collect()
    #testing network on evaluation dataset, img number 1001 to 8041 of the testing dataset
    score = model.evaluate(test_dataset[1001:], test_classes[1001:], verbose=1)
    print('\n', 'Test accuracy:', score[1])


import pygame
import random
import time
import pickle

pygame.init()

sound = pygame.mixer.Sound('eaten.mp3')

width = 800
height = 600
size = width, height

screen = pygame.display.set_mode(size)
pygame.display.set_caption("Змейка")
clock = pygame.time.Clock()

snake_head = (0, 255, 0)
snake_tail = (0, 153, 0)
fruit = (255, 0, 0)

clock = pygame.time.Clock()

snake_speed = 10
snake_size = 10

def snake(snake_size, snake_lst):
    for i in snake_lst:
        pygame.draw.rect(screen, snake_tail, [i[0], i[1], snake_size, snake_size])

def score(score):
    font = pygame.font.Font(None, 50)
    text = font.render('Счёт:' + str(score), True, (0, 153, 0))
    screen.blit(text, [0, 0])

def loss(message):
    font = pygame.font.Font(None, 50)
    text = font.render(message, True, (0, 100, 100))
    text_x = width // 2 - text.get_width() // 2
    text_y = height // 2 - text.get_height() // 2
    text_w = text.get_width()
    text_h = text.get_height()
    screen.blit(text, (text_x, text_y))
    pygame.draw.rect(screen, (0, 255, 0), (text_x - 10, text_y - 10,
                                           text_w + 20, text_h + 20), 1)

def main():
    game_over = False
    game_close = False

    x = width / 2
    y = height / 2
    x_shift = 0
    y_shift = 0

    snake_lst = []
    snake_length = 1

    food_x = round(random.randrange(0, width - snake_size) / 10.0) * 10
    food_y = round(random.randrange(0, width - snake_size) / 10.0) * 10

    while not game_over:
        while game_close == True:
            screen.fill((0, 0, 0))
            loss("Вы проиграли <3")
            score(snake_length - 1)
            pygame.display.update()

            for event in pygame.event.get():
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_q:
                        game_over = True
                        game_close = False
                    if event.key == pygame.K_e:
                        main()

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                game_over = True
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT and x_shift == 0:
                    x_shift = -snake_size
                    y_shift = 0
                elif event.key == pygame.K_RIGHT and x_shift == 0:
                    x_shift = snake_size
                    y_shift = 0
                elif event.key == pygame.K_UP and y_shift == 0:
                    y_shift = -snake_size
                    x_shift = 0
                elif event.key == pygame.K_DOWN and y_shift == 0:
                    y_shift = snake_size
                    x_shift = 0

        if x >= width or y >= height or x < 0 or y < 0:
            game_close = True

        x += x_shift
        y += y_shift
        screen.fill((0, 0, 0))
        pygame.draw.rect(screen, snake_head, [x, y, snake_size, snake_size])
        pygame.draw.rect(screen, fruit, [food_x, food_y, snake_size, snake_size])
        pygame.display.update()
        snake_head1 = []
        snake_head1.append(x)
        snake_head1.append(y)
        snake_lst.append(snake_head1)
        if len(snake_lst) > snake_length:
            del snake_lst[0]

        for i in snake_lst[:-1]:
            if i == snake_head1:
                game_close = True

        snake(snake_size, snake_lst)
        score(snake_length - 1)
        pygame.display.update()

        if x == food_x and y == food_y:
            food_x = round(random.randrange(1, width - snake_size) / 10.0) * 10
            food_y = round(random.randrange(1, height - snake_size) / 10.0) * 10
            snake_length += 1
            sound.play()
            print('<3')
        clock.tick(30)

    pygame.quit()
    quit()

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
@License: Copyright(C) 2019-2020, South China Normal University.
@File   : MySymbolTable.py
@Version: 1.0
@Author : 罗佳海
@Date   : 2020/4/7 12:05
@Desc   : 符号表的实现（最近嵌套作用域）
"""
from collections import OrderedDict

scope_map = {}  # 作用域集合，便于快速获取作用域对象

class Symbol(object):
    """符号类

    主要用于创建符号对象

    Attributes:
        name: 变量名（符号名）字符串
        lines: 保存列号的列表
        mem_loc: 内存位置整型数
        id_kind: NodeKind类的枚举类型（func\var）
        basic_type: BasicType类的枚举类型（arr指针，其引用的memloc与原指针memloc相同）
        size: 数组大小/函数参数个数
        included_scope: 函数声明下的作用域引用
    """
    name = None
    lines = None
    mem_loc = None
    id_kind = None
    basic_type = None
    size = None
    included_scope = None

    def __init__(self, symbol_name, mem_loc, id_kind, basic_type, size, lineno):
        self.name = symbol_name
        self.mem_loc = mem_loc
        self.id_kind = id_kind
        self.basic_type = basic_type
        self.size = size
        self.lines = []
        self.lines.append(lineno)

class SymbolTable(object):
    """符号表类

    主要用于管理符号对象

    Attributes:
        symbols: 符号字典
    """
    symbols = None

    def __init__(self):
        self.symbols = OrderedDict()  # 顺序字典帮助定位函数声明中的参数

    def insert_symbol(self, symbol):
        """向符号表插入符号

        :param symbol: 符号对象引用
        :return:
        """
        st_symbol = self.symbols.get(symbol.name)
        if st_symbol is None:
            self.symbols[symbol.name] = symbol
        else:
            st_symbol.lines.extend(symbol.lines)
            self.symbols[st_symbol.name] = st_symbol

    def lookup_symbol(self, symbol_name):
        """在符号表查找符号并返回其引用

        :param symbol_name: 符号名
        :return: 符号对象引用
        """
        return self.symbols.get(symbol_name)

    def lookup_symbol_by_index(self, index):
        """在符号表查找符号并返回其引用

        :param index: 索引
        :return: 符号对象引用
        """
        return list(self.symbols.values())[index]

class Scope(object):
    """作用域类

    主要用于管理符号表对象

    Attributes:
        id: 作用域id
        level: 作用域级别（最外层到最内层递增）
        symbol_table: 符号表对象的引用（每个作用域维护一个符号表）
        enclosing_scope: 包围当前作用域的外层作用域对象的引用（即最近嵌套）
    """
    id = None
    level = None
    symbol_table = None
    enclosing_scope = None

    def __init__(self, scope_id, scope_level, enclosing_scope=None):
        self.id = scope_id
        self.level = scope_level
        self.symbol_table = SymbolTable()
        self.enclosing_scope = enclosing_scope
        scope_map[self.id] = self  # 维护scope_map

    def insert_symbol(self, symbol_name, mem_loc, id_kind, basic_type, size, lineno):
        """向符号表插入符号"""
        symbol = Symbol(symbol_name, mem_loc, id_kind, basic_type, size, lineno)
        self.symbol_table.insert_symbol(symbol)
        return symbol

    def lookup_symbol(self, symbol_name):
        """在符号表查找符号并返回其引用"""
        return self.symbol_table.lookup_symbol(symbol_name)

    def lookup_symbol_by_index(self, index):
        """在符号表查找符号并返回其引用"""
        return self.symbol_table.lookup_symbol_by_index(index)

    def print_symbol_table(self):
        """打印作用域信息和符号表信息"""
        print("##################################################################")
        print("# Scope")
        print("# Scope Id  Scope Level  Enclosing Scope Id  Enclosing Scope Level")
        print("# --------  -----------  ------------------  ---------------------")
        print("# %8s" % self.id, end="")
        print("%13s" % self.level, end="")
        enclosing_scope_id = None if self.enclosing_scope is None else \
            str(self.enclosing_scope.id)
        enclosing_scope_level = None if self.enclosing_scope is None else \
            str(self.enclosing_scope.level)
        print("%20s" % enclosing_scope_id, end="")
        print("%23s" % enclosing_scope_level)
        print("# \n# Symbol Table")
        print("# Variable Name  Location  Id Kind  Basic Type  Size  Line Numbers")
        print("# -------------  --------  -------  ----------  ----  ------------")
        for symbol in self.symbol_table.symbols.values():
            print("# %13s" % symbol.name, end="")
            print("%10d" % symbol.mem_loc, end="")
            print("%9s" % symbol.id_kind.value, end="")
            print("%12s" % symbol.basic_type.value, end="")
            print("%6d  " % symbol.size, end="")
            for lineno in symbol.lines:
                print("%4d " % lineno, end="")
            print("")
        print("##################################################################\n")

def init_scope_map():
    """初始化作用域集合"""
    global scope_map
    scope_map = {}

def get_scope(scope_id):
    """获取作用域

    :param scope_id: 作用域id
    :return: 作用域对象的引用或None
    """
    return scope_map.get(scope_id)

def update_scope(scope_id, scope_level, enclosing_scope=None):
    """更新作用域

    当作用域在作用域集合里已创建时，仅更新相关属性并返回作用域的引用
    未创建时就创建新作用域对象，并添加到作用域集合中

    :param scope_id: 作用域id
    :param scope_level: 作用域级别
    :param enclosing_scope: 外层作用域对象的引用
    :return: 作用域对象的引用
    """
    if scope_map.__contains__(scope_id):  # not None
        scope_map[scope_id].level = scope_level
        if enclosing_scope is not None:  # 仅在非空时更新
            scope_map[scope_id].enclosing_scope = enclosing_scope
        return scope_map[scope_id]
    scope_map[scope_id] = Scope(scope_id, scope_level, enclosing_scope)  # 维护scope_map
    return scope_map[scope_id]

def st_insert(symbol_name, mem_loc, id_kind, basic_type, size, lineno, scope_id, scope_level):
    """向指定作用域的符号表插入符号

    :param symbol_name: 变量名
    :param mem_loc: 内存位置
    :param id_kind: NodeKind枚举类型
    :param basic_type: BasicType枚举类型
    :param size: 数组大小/函数参数个数
    :param lineno: 行号
    :param scope_id: 作用域id
    :param scope_level: 作用域级别
    :return: 由symbol对象引用和scope对象引用组成的元组
    """
    scope = update_scope(scope_id, scope_level)
    symbol = scope.insert_symbol(symbol_name, mem_loc, id_kind, basic_type, size, lineno)
    return symbol, scope

def st_lookup(symbol_name, scope_id):
    """查找符号

    根据scope_name和scope_level确定scope对象，在scope对象中的符号表进行搜索，找不到再向外层搜索

    :param symbol_name:
    :param scope_id:
    :return: 由symbol对象引用和scope对象引用组成的元组
    """
    scope = scope_map.get(scope_id)
    symbol = None
    while scope is not None:
        symbol = scope.lookup_symbol(symbol_name)
        if symbol is not None:
            break
        scope = scope.enclosing_scope
    return symbol, scope

def print_scope():
    """打印作用域信息和符号表信息"""
    # 按作用域级别排序
    ordered_map = OrderedDict(sorted(scope_map.items(), key=lambda t: t[1].level))
    for scope in ordered_map.values():
        # if len(scope.symbol_table.symbols) > 0:
            scope.print_symbol_table()

# 测试
if __name__ == '__main__':
    from MyTreeNode import NodeKind, BasicType

    scope1 = update_scope(10000, 0)

    scope2 = update_scope(10001, 1)
    scope2.enclosing_scope = scope1

    scope3 = update_scope(10002, 0)

    # Variable Name  Location  Id Kind  Basic Type  Size  Line Numbers
    st_insert("input", 0, NodeKind.FUNC_K, BasicType.INT, 1, 3, 10000, 0)
    st_insert("x", 1, NodeKind.VAR_K, BasicType.INT, 1, 4, 10000, 1)

    st_insert("output", 2, NodeKind.FUNC_K, BasicType.VOID, 1, 6, 10001, 0)
    st_insert("y", 3, NodeKind.VAR_K, BasicType.INT, 1, 7, 10002, 1)

    # print(scope_map)
    print_scope()
    symbol, scope = st_lookup("x", 10000)
    print(scope.lookup_symbol_by_index(0))


# _author: tao wang
# _date: 2019/5/20
from Android_tapai.businessView.PostView import EnterPost
from Android_tapai.common.Android_Main import appium_desired
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
import logging
import random
import time

class VideoView(EnterPost):
    __recordMBtn = (By.ID, "com.maibo.android.tapai:id/recordIMBtn")  # 录制button
    __tabTV = (By.ID, "com.maibo.android.tapai:id/tabTV")  # 切换相册
    __draft = (By.ID, "com.maibo.android.tapai:id/draft_thumbnail")  # 选择视频
    __musicIMG = (By.ID, "com.maibo.android.tapai:id/selectMusicIMG")  # 拍摄页面音乐
    __propsIMG = (By.ID, "com.maibo.android.tapai:id/ivSelectSticker")  # 拍摄页面道具
    __next = (By.ID, "com.maibo.android.tapai:id/nextTv")  # 下一步按钮
    __recordBtn = (By.ID, "com.maibo.android.tapai:id/recordBtn")  # 结束录制
    __propsBtn = (By.ID, "com.maibo.android.tapai:id/videoPublish_ivSelectSticker")  # 道具
    __describeBtn = (By.ID, "com.maibo.android.tapai:id/video_deseTv")  # 描述
    __musicBtn = (By.ID, "com.maibo.android.tapai:id/video_musicIm")  # 音乐
    __coverBtn = (By.ID, "com.maibo.android.tapai:id/video_coverTv")  # 封面
    __saveBtn = (By.ID, "com.maibo.android.tapai:id/video_save")  # 保存
    __publishBtn = (By.ID, "com.maibo.android.tapai:id/publishBtn")  # 发布
    __anonymousBtn = (By.ID, "com.maibo.android.tapai:id/anonymous_tv")  # 匿名
    __coverIMG = (By.ID, "com.maibo.android.tapai:id/coverIMG")  # 选择道具
    __useBtn = (By.ID, "com.maibo.android.tapai:id/useTV")  # 使用
    __mood_video = (By.ID, "com.maibo.android.tapai:id/video_cover_item_tv")  # 选择文字
    __nextBtn = (By.ID, "com.maibo.android.tapai:id/nextTv")  # 完成
    __ediTextBtn = (By.ID, "com.maibo.android.tapai:id/default_editext")  # 文本区域
    __thumbnailBtn = (By.ID, "com.maibo.android.tapai:id/iv_thumbnail")  # 选择封面
    __text_style = (By.ID, "com.maibo.android.tapai:id/ci_text_style")  # 选择文本
    __add_text = (By.ID, "com.maibo.android.tapai:id/add_text")  # 封面文字
    __useIMG = (By.ID, "com.maibo.android.tapai:id/tv_use")  # 封面的使用

    def video_picture(self, video_type, props_type, video_time=3, line_number=1):
        self.choose_video()

        if video_type == 0:
            logging.info('选择拍摄普通视频')
            if props_type == 0:
                logging.info('准备选择道具')
                self.driver.find_element(*self.__propsIMG).click()
                for i in range(2):
                    self.driver.find_elements(*self.__coverIMG)[1].click()
                    time.sleep(2)
                self.get_swipe(0.5, 0.5, 0.5, 0.5, 50)

                logging.info('准备选择音乐')
                self.driver.find_element(*self.__musicIMG).click()
                for i in range(2):
                    self.driver.find_elements(*self.__coverIMG)[2].click()
                    time.sleep(2)
                self.driver.find_element(*self.__useBtn).click()
            else:
                logging.info('不选择音乐和道具')

        elif video_type == 1:
            logging.info('选择拍摄心情视频')
            self.driver.find_elements(*self.__tabTV)[1].click()

        elif video_type == 2:
            logging.info('选择拍摄本地视频')
            self.driver.find_elements(*self.__tabTV)[0].click()
            draft_random = random.choice(range(8))
            self.driver.find_elements(*self.__draft)[draft_random].click()
            self.driver.find_element(*self.__next).click()

        else:
            logging.info('请输入正确的值')

        if video_type == 0 or video_type == 1:
            logging.info('开始录制%s秒' % video_time)
            self.driver.find_element(*self.__recordMBtn).click()
            time.sleep(video_time)
            try:
                self.driver.find_element(*self.__recordBtn)
            except NoSuchElementException:
                logging.info('这个页面又超时了')
            else:
                self.driver.find_element(*self.__recordBtn).click()

        if props_type == 0:

            logging.info('选择道具')
            self.driver.find_element(*self.__propsBtn).click()
            for i in range(2):
                self.driver.find_elements(*self.__coverIMG)[3].click()
                time.sleep(2)
            self.get_swipe(0.5, 0.5, 0.5, 0.5, 50)

            logging.info('选择音乐')
            self.driver.find_element(*self.__musicBtn).click()
            for i in range(2):
                self.driver.find_elements(*self.__coverIMG)[4].click()
                time.sleep(2)
            self.driver.find_element(*self.__useBtn).click()

            logging.info('点击保存')
            self.driver.find_element(*self.__saveBtn).click()

            try:
                self.driver.find_element(*self.__describeBtn)
            except NoSuchElementException:
                logging.info('选择封面')
                self.driver.find_element(*self.__coverBtn).click()
                self.driver.find_element(*self.__mood_video).click()
                self.driver.find_element(*self.__nextBtn).click()
            else:
                logging.info('选择描述')
                self.driver.find_element(*self.__describeBtn).click()
                self.driver.find_element(*self.__ediTextBtn).click()
                self.driver.find_element(*self.__ediTextBtn).clear()
                text_path = self.get_path('data', 'video_number.csv')
                text_file = self.get_csv_data(text_path, line_number)[0]
                self.driver.find_element(*self.__ediTextBtn).send_keys(text_file)
                self.driver.find_element(*self.__nextBtn).click()

                logging.info('选择封面')
                self.driver.find_element(*self.__coverBtn).click()
                random_file = random.choice(range(9))
                self.driver.find_elements(*self.__thumbnailBtn)[random_file].click()

                text_random = random.choice(range(6))
                self.driver.find_elements(*self.__text_style)[text_random].click()

                self.driver.find_element(*self.__add_text).click()
                self.driver.find_element(*self.__add_text).clear()
                self.driver.find_element(*self.__add_text).send_keys(text_file)

                self.get_swipe(0.5, 0.5, 0.2, 0.2, 100)
                self.driver.find_element(*self.__useIMG).click()

        logging.info('点击发布')
        self.driver.find_element(*self.__publishBtn).click()

if __name__ == '__main__':
    obj = VideoView(appium_desired())
    obj.video_picture(2, 0, 10)


# -*- coding: utf-8 -*-
"""
Created on Tue Dec 22 15:25:34 2020

@author: 0100904
不能读取加密文件

"""

import xlrd
import xlwt
from datetime import date,datetime

def read_excel():
# 打开文件
    workbook = xlrd.open_workbook(r'd:/RIC组装表2020.xlsx')
# 获取所有sheet
    sheet_name = workbook.sheet_names()[0]
    sheet = workbook.sheet_by_name(sheet_name)

#显示表内容
    for i in range(6,sheet.nrows):
            for j in range(0,sheet.ncols):
                print(sheet.cell(i,j).value)

if __name__ == '__main__':
    read_excel()


from . import exceptions
from . import jose
from .jws import JWS
import uuid

__all__ = ['Token']

class Token(JWS):
    def __init__(self, payload=None, header=None, signature=None,
            generate_jti=False, **kwargs):
        header = self._validate_header(header)
        header = self._validate_and_set_typ(header)

        payload = self._validate_payload(payload)
        payload = self._validate_and_set_jti(payload, generate_jti)
        # expire time
        # cty - content type
        # ...

        super(Token, self).__init__(payload, header, signature, **kwargs)

    def get_claim_from_namespace(self, namespace, name, uuid_version=5):
        return self.get_claim(_create_uuid_name(namespace, name, uuid_version))

    def get_claim(self, name):
        return self.payload.get(name)

    def _validate_payload(self, payload):
        if isinstance(payload, jose.JOSEDictionary):
            return payload

        elif isinstance(payload, dict):
            return jose.JOSEDictionary(payload)

        else:
            raise TypeError('"payload" must be a dict or JOSEDictionary')

    def _validate_and_set_jti(self, payload, generate_jti):
        if generate_jti:
            if 'jti' in payload:
                raise exceptions.InvalidClaim(
                    'Cannot specify "generate_jti" with existing "jti" claim.')

            else:
                payload['jti'] = _generate_jti()

        return payload

    def _validate_and_set_typ(self, header):
        if 'typ' in header and header['typ'].upper() != 'JWT':
            raise exceptions.InvalidHeader('"typ" parameter must be "JWT"')

        header['typ'] = 'JWT'

        return header

def _generate_jti():
    return uuid.uuid4().hex

_UUID_VERSIONS = {
    3: uuid.uuid3,
    5: uuid.uuid5,
}
def _create_uuid_name(namespace, name, uuid_version):
    name_creator = _UUID_VERSIONS.get(uuid_version)
    if name_creator is None:
        raise exceptions.InvalidUUIDType(
                'Invalid uuid_version specified (%s).  Valid values: %s'
                % (uuid_version, _UUID_VERSIONS.keys()))
    return str(name_creator(namespace, name))


#!/usr/bin/env python

from test_policy import run_policy
from safe_rl.utils.load_utils import load_policy
import xml.etree.ElementTree as ET
import joblib
import time

def shift_dynamics(dynamic, xml, new_val):
    '''
    Shifts some value in a specified mujoco xml file, and then it writes the
    new value to the xml fileself.

    Args:
        dynamic: A string representing a the certain field in the xml that
                 represents a dynamic that will be shifted
        xml: A string that is a file path to an XML that will be edited
        new_val: A float that will be the new value for that dynamic
    '''
    tree = ET.parse(xml)
    root = tree.getroot()
    if dynamic == 'motor':
        x = [i.attrib for i in root.iter('motor')]
        x[0]['forcerange'] = '-' + str(new_val) + ' ' + str(new_val)
        i = 0
        for motor in root.iter('motor'):
            motor.attrib = x[i]
            i += 1
        tree.write(xml)
    elif dynamic == 'joint':
        # TODO: implement joint
        raise NotImplementedError('Have not implemented switch for joint')
    elif dynamic == 'velocity':
        x = [i.attrib for i in root.iter('velocity')]
        x[0]['forcerange'] = '-' + str(new_val) + ' ' + str(new_val)
        i = 0
        for veloc in root.iter('velocity'):
            veloc.attrib = x[i]
            i += 1
        tree.write(xml)
        print("Updated dynamics for", dynamic)

def get_range_vals(dynamic):
    '''
    Returns the list of new values that will be written to an XML file

    Args:
        dynamics: A string representing the dynamic to shift

    Returns:
        The list of floats of the new values
    '''
    if dynamic == 'motor' or dynamic == 'velocity':
        return [0.0025, 0.002, 0.001, 0.0005, 0.0001, 0.005, 0.01, 0.02, 0.03,
                0.04, 0.05, 0.06, 0.07, 0.08, 0.09,
                0.1, 0.125, 0.15, 0.175, 0.2, 0.225, 0.25, 0.275, 0.3, 0.35,
                0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9,
                0.95, 1.0, 0.035, 0.0325, 0.0375, 0.041, 0.042, 0.043, 0.044,
                0.045, 0.046, 0.047, 0.048, 0.049, 0.051, 0.052, 0.053, 0.054,
                0.055, 0.056, 0.057, 0.058, 0.059, 0.055, 0.0575, 0.0625, 0.065,
                0.0675]
        # return [0.005]
    elif dynamic == 'joint':
        return [0.001]
    elif dynamic == 'size':
        return [0.001, 0.0005, 0.0001, 0.0025, 0.005, 0.0075, 0.01, 0.0125,
                0.015, 0.0175, 0.02, 0.021, 0.022, 0.0225, 0.023, 0.024,
                0.025, 0.026, 0.027, 0.028, 0.029, 0.03, 0.031, 0.033, 0.034,
                0.036, 0.037, 0.038, 0.039, 0.0325, 0.032, 0.035, 0.0375, 0.04,
                0.0425, 0.045, 0.0475, 0.041, 0.042, 0.043, 0.044, 0.046,
                0.047, 0.048, 0.049, 0.05, 0.0525, 0.055, 0.0575, 0.06,
                0.0625, 0.065, 0.0675, 0.07, 0.0725, 0.075, 0.0775, 0.08, 0.825,
                0.085, 0.0875, 0.09, 0.0925, 0.095, 0.0975, 1.0]

def test_dynamics(env, get_action, dynamic, xml_file, episodes=20):
    '''
    It shifts a certain dynamic value to a new one, and it then tests a policy
    against these new values for a specified number of episodes. It then
    collects the average episodic returns of each episode for each policy, and
    then dumps the collected data to a pickle file that can be used later for
    visualizations or other analysis.

    Args:
        env: The openAI gym environment
        get_action: An openAI policy function that consumes obseravtions and returns actions
        dynamic: A string representing the dynamic to shift
        xml_file: A string of a path to a XML file to edit
        episodes: The number of episodes to run a policy for

    Returns:
        Nothing, but dumps a pickle file with the collected data
    '''
    assert dynamic != "", "You did not specify a dynamic"
    points = {}
    forcerange_vals = get_range_vals(dynamic)
    for x_val in forcerange_vals:
        shift_dynamics(dynamic, xml_file, x_val)
        y_vals = run_policy(env, get_action, 0, episodes, False)
        points[x_val] = y_vals
    pkl_name = str("shift_data_" + str(dynamic) + '_point_' + str(time.strftime("%Y-%m-%d")) + ".pkl")
    joblib.dump(points, pkl_name)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('fpath', type=str)
    parser.add_argument('--len', '-l', type=int, default=0)
    parser.add_argument('--episodes', '-n', type=int, default=20)
    parser.add_argument('--itr', '-i', type=int, default=-1)
    parser.add_argument('--deterministic', '-d', action='store_true')
    parser.add_argument('--dynamic', '-dn', type=str, default="")
    parser.add_argument('--xml_file', '-xf', type=str, default="")
    args = parser.parse_args()
    env, get_action, _ = load_policy(args.fpath,
                                     args.itr if args.itr >= 0 else 'last',
                                     args.deterministic)
    test_dynamics(env, get_action, args.dynamic, args.xml_file, args.episodes)


from bs4 import BeautifulSoup

class Transforms:
    @classmethod
    def data_types(cls):
        return cls.convert_rules.keys()

    # If there is a rule defined here, the corresponding CamelCase model
    # will be indexed
    convert_rules = {
        "committee": {
            "id": "id",
            "title": "name",
            "description": ["info", "about"]
        },
        "committee_meeting": {
            "id": "id",
            "title": "title",
            "description": "summary",
            "fulltext": "body",
            "date": "date",
            "committee_name": ["organisation", "name"],
            "committee_id": ["organisation", "id"],
        },
        "member": {
            "id": "id",
            "title": "name",
            "description": "bio",
            "date": "start_date",
        },
        "bill": {
            "id": "id",
            "title": "title",
            "description": "bill_code"
        },
        "hansard": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
        },
        "briefing": {
            "id": "id",
            "title": "title",
            "description": "summary",
            "fulltext": "minutes",
            "date": "start_date",
            "committee_name": ["committee", "name"],
            "committee_id": ["committee", "id"],
        },
        "question_reply": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
            "committee_name": ["committee", "name"],
            "committee_id": ["committee", "id"],
        },
        "tabled_committee_report": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
            "committee_name": ["committee", "name"],
            "committee_id": ["committee", "id"],
        },
        "call_for_comment": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
            "committee_name": ["committee", "name"],
            "committee_id": ["committee", "id"],
        },
        "policy_document": {
            "id": "id",
            "title": "title",
            "date": "start_date",
        },
        "gazette": {
            "id": "id",
            "title": "title",
            "date": "start_date",
        },
        "daily_schedule": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
        },
    }

    @classmethod
    def serialise(cls, data_type, obj):
        item = {}
        for key, field in Transforms.convert_rules[data_type].iteritems():
            val = cls.get_val(obj, field)
            if isinstance(val, unicode):
                val = BeautifulSoup(val).get_text().strip()
            item[key] = val
        return item

    @classmethod
    def get_val(cls, obj, field):
        if isinstance(field, list):
            if (len(field) == 1):
                if hasattr(obj, field[0]):
                    return getattr(obj, field[0])
            key = field[:1][0]
            if isinstance(key, int):
                if len(obj) > key:
                    return cls.get_val(obj[key], field[1:])
                return None
            if hasattr(obj, key):
                newobj = getattr(obj, key)
                return cls.get_val(newobj, field[1:])
            else:
                return None
        else:
            return getattr(obj, field)