from flask import Flask, render_template, Response
from datetime import datetime
import io
import cbpro
# import matplotlib.pyplot as plt
# import matplotlib
import pandas as pd
import yaml
import math
import os
import time
# from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
# from matplotlib.backends.backend_svg import FigureCanvasSVG
# from matplotlib.figure import Figure
from apscheduler.schedulers.background import BackgroundScheduler

import plotly
import plotly.graph_objs as go
import plotly.express as px
import json

BUY_LOCK = False
PLT_LOCK = True

product = []
STATE_P = []

def main():

    with open('app/state.yaml') as f:
        STATE = yaml.load(f,Loader=yaml.FullLoader)

    # STATE['bought'] = not STATE['bought']

    #print(auth_client.get_accounts())
    balance = float(auth_client.get_account("c7e286f1-e04f-46fb-ace5-34a27245c95f")['available'])
    balance = math.floor(balance*100)/100.0

    bitcoin = float(auth_client.get_account("e02112f4-5daf-4970-9ce1-563e5460e523")['available'])

    #print(balance)

    ##print(auth_client.get_product_order_book('BTC-USD'))

    #print(auth_client.get_product_ticker(product_id='BTC-USD'))

    ##print(auth_client.get_product_historic_rates('BTC-USD',granularity=60))

    #print(auth_client.get_product_24hr_stats('BTC-USD'))

    #this should be done with web socket instead if polling frequently?
    items = auth_client.get_product_historic_rates('BTC-USD',granularity=300)

    x = []
    y = []
    for item in items[::-1]:
        #plt.plot(item[0],item[4],'.')
        x.append(item[0])
        y.append(item[4])
    # plt.plot(x,y,'-g')
    price = pd.Series(y)

    #sma1 = ta.volatility.bollinger_mavg(price, n=n1, fillna=False)
    # for i in range(len(x)):
    #     plt.plot(x[i],sma1[i],'.')

    global product
    product = pd.DataFrame({'time' : x, 'price': y})

    # n1 = 8
    # n2 = 20
    # n1 = 20
    # n2 = 45
    n1 = 25
    n2 = 50
    product['SMA1'] = product['price'].rolling(n1).mean()
    product['SMA2'] = product['price'].rolling(n2).mean()

    stance = STATE['bought']
    money = balance
    btc = bitcoin
    numbuy = STATE['numbuy']
    numsell = STATE['numsell']
    #print(money)
    #.5% == *.005
    # for t,s1,s2,p in zip(product['time'][n2:],product['SMA1'][n2:],product['SMA2'][n2:],product['price'][n2:]):
    #     if not stance and (s1>s2):
    #         stance = True
    #         plt.plot(t,p,'.g')
    #         btc += (money*.995)/p #only 99.5% goes in
    #         money -= money ##fix this
    #         print(str(money) + '    ' + str(btc))
    #     elif stance and (s2>s1):
    #         stance = False
    #         plt.plot(t,p,'.r')
    #         money += btc*p*.995 #only 99.5% comes back out
    #         btc -= btc
    #         print(str(money) + '    ' + str(btc))

    s1 = product['SMA1'].iloc[-1]
    s2 = product['SMA2'].iloc[-1]
    #t = product['time'].iloc[-1]
    # print(stance)
    # print(s1)
    # print(s2)

    if not stance and (s1 > s2):

        print('Buying BTC with funds: ' + str(money))
        if not BUY_LOCK:
            auth_client.place_market_order(product_id='BTC-USD',side='buy',funds = money)
            stance = True
            numbuy+=1
            STATE['buylocs']['locs'].append(float(product['time'].iloc[-1]))
            STATE['buylocs']['vals'].append(float(product['price'].iloc[-1]))
    elif stance and (s2>s1):

        print('Selling BTC with size: ' + str(btc))
        if not BUY_LOCK:
            auth_client.place_market_order(product_id='BTC-USD',side='sell',size = btc)
            stance = False
            numsell+=1
            STATE['selllocs']['locs'].append(float(product['time'].iloc[-1]))
            STATE['selllocs']['vals'].append(float(product['price'].iloc[-1]))

    #update STATE

    # print('---')
    # print(money)
    # print(btc)
    # print(stance)
    if not PLT_LOCK:
        plt.show()

    #time.sleep(5)
    date = datetime.now()
    mpldate = matplotlib.dates.date2num(date)
    #print(date.isoformat())

    balance = float(auth_client.get_account("c7e286f1-e04f-46fb-ace5-34a27245c95f")['available'])
    bitcoin = float(auth_client.get_account("e02112f4-5daf-4970-9ce1-563e5460e523")['available'])

    STATE['bought'] = stance
    STATE['money'] = balance
    STATE['BTC'] = bitcoin
    STATE['last'] = date.isoformat()
    STATE['numbuy'] = numbuy
    STATE['numsell'] = numsell
    STATE['moneyOT'].append(balance)
    STATE['BTCOT'].append(bitcoin)
    STATE['timeOT'].append(mpldate)
    with open('app/state.yaml', 'w') as f:
        yaml.dump(STATE, f)
        print('Updated YAML')

    global STATE_P
    STATE_P = STATE
    # return STATE

def create_plot_p():
    # fig = px.line(product,x='time',y=['price','SMA1','SMA2'])
    #
    # color_discrete_map = {'vals': 'rgb(255,0,0)'}
    # fig2 = go.Scatter(x=STATE_P['buylocs']['locs'],y=STATE_P['buylocs']['vals'],mode='markers',marker_color='rgba(0, 255, 0, .8)')
    # fig3 = go.Scatter(x=STATE_P['selllocs']['locs'],y=STATE_P['selllocs']['vals'],mode='markers',marker_color='rgba(255, 0, 0, .8)')
    # data = [
    #     fig.data[0],fig.data[1],fig.data[2],fig2,fig3
    # ]

    fig = go.Figure()
    strstamps = []
    for elem in product['time']:
        strstamps.append(datetime.fromtimestamp(elem).strftime('%Y-%m-%d %H:%M:%S'))
    fig.add_trace(go.Scatter(x=strstamps,y=product['price'],mode='lines',name='price',marker_color='rgba(0,135,0,.6)'))
    fig.add_trace(go.Scatter(x=strstamps,y=product['SMA1'],mode='lines',name='SMA1',marker_color='rgba(255,0,0,.6)'))
    fig.add_trace(go.Scatter(x=strstamps,y=product['SMA2'],mode='lines',name='SMA2',marker_color='rgba(0,0,255,.6)'))

    strstamps = []
    for elem in STATE_P['buylocs']['locs']:
        strstamps.append(datetime.fromtimestamp(elem).strftime('%Y-%m-%d %H:%M:%S'))
    fig.add_trace(go.Scatter(x=strstamps,y=STATE_P['buylocs']['vals'],name='buy',mode='markers',marker_color='rgba(0, 245, 95, 1)'))
    strstamps = []
    for elem in STATE_P['selllocs']['locs']:
        strstamps.append(datetime.fromtimestamp(elem).strftime('%Y-%m-%d %H:%M:%S'))
    fig.add_trace(go.Scatter(x=strstamps,y=STATE_P['selllocs']['vals'],name='sell',mode='markers',marker_color='rgba(255, 0, 0, .9)'))
    fig.update_layout(autosize=True,hovermode='x unified',title='Overview')
    data = fig
    graphJSON = json.dumps(data, cls=plotly.utils.PlotlyJSONEncoder)

    return graphJSON

def create_plot_m():
    # fig = px.line(product,x='time',y=['price','SMA1','SMA2'])
    #
    # color_discrete_map = {'vals': 'rgb(255,0,0)'}
    # fig2 = go.Scatter(x=STATE_P['buylocs']['locs'],y=STATE_P['buylocs']['vals'],mode='markers',marker_color='rgba(0, 255, 0, .8)')
    # fig3 = go.Scatter(x=STATE_P['selllocs']['locs'],y=STATE_P['selllocs']['vals'],mode='markers',marker_color='rgba(255, 0, 0, .8)')
    # data = [
    #     fig.data[0],fig.data[1],fig.data[2],fig2,fig3
    # ]

    fig = go.Figure()

    newdict = {'timeOT' : [], 'moneyOT': []}
    for time,money in zip(STATE_P['timeOT'],STATE_P['moneyOT']):
        if money > .1:
            newdict['timeOT'].append(time)
            newdict['moneyOT'].append(money)

    strstamps = []
    for elem in newdict['timeOT']:
        strstamps.append(matplotlib.dates.num2date(elem))

    fig.add_trace(go.Scatter(x=strstamps,y=newdict['moneyOT'],mode='lines',name='money',marker_color='rgba(0,155,0,.6)'))
    fig.update_layout(hovermode='x unified',title='Money')

    data = fig
    graphJSON = json.dumps(data, cls=plotly.utils.PlotlyJSONEncoder)

    return graphJSON

def create_plot_b():
    # fig = px.line(product,x='time',y=['price','SMA1','SMA2'])
    #
    # color_discrete_map = {'vals': 'rgb(255,0,0)'}
    # fig2 = go.Scatter(x=STATE_P['buylocs']['locs'],y=STATE_P['buylocs']['vals'],mode='markers',marker_color='rgba(0, 255, 0, .8)')
    # fig3 = go.Scatter(x=STATE_P['selllocs']['locs'],y=STATE_P['selllocs']['vals'],mode='markers',marker_color='rgba(255, 0, 0, .8)')
    # data = [
    #     fig.data[0],fig.data[1],fig.data[2],fig2,fig3
    # ]

    fig = go.Figure()
    newdict = {'timeOT' : [], 'BTCOT': []}
    for time,money in zip(STATE_P['timeOT'],STATE_P['BTCOT']):
        if money != 0.0:
            newdict['timeOT'].append(time)
            newdict['BTCOT'].append(money)

    strstamps = []
    for elem in newdict['timeOT']:
        strstamps.append(matplotlib.dates.num2date(elem))
    fig.add_trace(go.Scatter(x=strstamps,y=newdict['BTCOT'],mode='lines',name='money',marker_color='rgba(0,155,0,.6)'))
    fig.update_layout(hovermode='x unified',title='BTC')

    data = fig
    graphJSON = json.dumps(data, cls=plotly.utils.PlotlyJSONEncoder)

    return graphJSON

sched = BackgroundScheduler(daemon=True)
sched.add_job(main,'interval',minutes=5)
sched.start()

app = Flask(__name__,template_folder='templates')

@app.route("/")
def home_view():
    # global STATE_P
    # STATE_P = main()
    # return "
Welcome to " + str(stance) + "Geeks for Geeks
"
    main()
    plot_p = create_plot_p()
    plot_m = create_plot_m()
    plot_b = create_plot_b()
    return render_template('index.html',plot_p=plot_p, plot_m=plot_m, plot_b=plot_b)

@app.route('/prices.png')
def plot_prices():
    fig = Figure()
    axis = fig.add_subplot(1,1,1)
    axis.plot(product['time'],product['price'],'-g')
    axis.plot(product['time'],product['SMA1'],'-r')
    axis.plot(product['time'],product['SMA2'],'-b')
    axis.plot(STATE_P['buylocs']['locs'],STATE_P['buylocs']['vals'],'.y')
    axis.plot(STATE_P['selllocs']['locs'],STATE_P['selllocs']['vals'],'.r')
    output = io.BytesIO()
    FigureCanvasSVG(fig).print_svg(output)
    return Response(output.getvalue(), mimetype='image/svg+xml')

@app.route('/money.png')
def plot_money():
    fig = Figure()
    axis = fig.add_subplot(1,1,1)
    axis.plot_date(STATE_P['timeOT'],STATE_P['moneyOT'],'-')
    output = io.BytesIO()
    FigureCanvasSVG(fig).print_svg(output)
    return Response(output.getvalue(), mimetype='image/svg+xml')

@app.route('/btc.png')
def plot_btc():
    fig = Figure()
    axis = fig.add_subplot(1,1,1)
    axis.plot_date(STATE_P['timeOT'],STATE_P['BTCOT'],'-')
    output = io.BytesIO()
    FigureCanvasSVG(fig).print_svg(output)
    return Response(output.getvalue(), mimetype='image/svg+xml')


# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""test ogb
"""
import argparse

import pgl
import numpy as np
import paddle.fluid as fluid
from pgl.contrib.ogb.nodeproppred.dataset_pgl import PglNodePropPredDataset
from pgl.utils import paddle_helper
from ogb.nodeproppred import Evaluator

def train():
    pass

def send_func(src_feat, dst_feat, edge_feat):
    return (src_feat["h"] + edge_feat["h"]) * src_feat["norm"]

class GNNModel(object):
    def __init__(self, name, emb_dim, num_task, num_layers):
        self.num_task = num_task
        self.emb_dim = emb_dim
        self.num_layers = num_layers
        self.name = name

    def forward(self, graph):
        h = fluid.layers.embedding(
            graph.node_feat["x"],
            size=(2, self.emb_dim))  # name=self.name + "_embedding")
        edge_attr = fluid.layers.fc(graph.edge_feat["feat"], size=self.emb_dim)
        for layer in range(self.num_layers):
            msg = graph.send(
                send_func,
                nfeat_list=[("h", h), ("norm", graph.node_feat["norm"])],
                efeat_list=[("h", edge_attr)])
            h = graph.recv(msg, "sum")
            h = fluid.layers.fc(
                h,
                size=self.emb_dim,
                bias_attr=False,
                param_attr=fluid.ParamAttr(name=self.name + '_%s' % layer))
            h = h * graph.node_feat["norm"]
            bias = fluid.layers.create_parameter(
                shape=[self.emb_dim],
                dtype='float32',
                is_bias=True,
                name=self.name + '_bias_%s' % layer)
            h = fluid.layers.elementwise_add(h, bias, act="relu")
        pred = fluid.layers.fc(h,
                               self.num_task,
                               act=None,
                               name=self.name + "_pred_output")
        return pred

def main():
    """main
    """
    # Training settings
    parser = argparse.ArgumentParser(description='Graph Dataset')
    parser.add_argument(
        '--epochs',
        type=int,
        default=100,
        help='number of epochs to train (default: 100)')
    parser.add_argument(
        '--dataset',
        type=str,
        default="ogbn-proteins",
        help='dataset name (default: proteinfunc)')
    args = parser.parse_args()

    #device = torch.device("cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
    #place = fluid.CUDAPlace(0)
    place = fluid.CPUPlace()  # Dataset too big to use GPU

    ### automatic dataloading and splitting
    dataset = PglNodePropPredDataset(name=args.dataset)
    splitted_idx = dataset.get_idx_split()

    ### automatic evaluator. takes dataset name as input
    evaluator = Evaluator(args.dataset)

    graph_data, label = dataset[0]

    train_program = fluid.Program()
    startup_program = fluid.Program()
    test_program = fluid.Program()
    # degree normalize
    indegree = graph_data.indegree()
    norm = np.zeros_like(indegree, dtype="float32")
    norm[indegree > 0] = np.power(indegree[indegree > 0], -0.5)
    graph_data.node_feat["norm"] = np.expand_dims(norm, -1).astype("float32")
    graph_data.node_feat["x"] = np.zeros((len(indegree), 1), dtype="int64")
    graph_data.edge_feat["feat"] = graph_data.edge_feat["feat"].astype(
        "float32")
    model = GNNModel(
        name="gnn", num_task=dataset.num_tasks, emb_dim=64, num_layers=2)

    with fluid.program_guard(train_program, startup_program):
        gw = pgl.graph_wrapper.StaticGraphWrapper("graph", graph_data, place)
        pred = model.forward(gw)
        sigmoid_pred = fluid.layers.sigmoid(pred)

    val_program = train_program.clone(for_test=True)

    initializer = []
    with fluid.program_guard(train_program, startup_program):
        train_node_index, init = paddle_helper.constant(
            "train_node_index", dtype="int64", value=splitted_idx["train"])
        initializer.append(init)

        train_node_label, init = paddle_helper.constant(
            "train_node_label",
            dtype="float32",
            value=label[splitted_idx["train"]].astype("float32"))
        initializer.append(init)
        train_pred_t = fluid.layers.gather(pred, train_node_index)
        train_loss_t = fluid.layers.sigmoid_cross_entropy_with_logits(
            x=train_pred_t, label=train_node_label)
        train_loss_t = fluid.layers.reduce_sum(train_loss_t)
        train_pred_t = fluid.layers.sigmoid(train_pred_t)

        adam = fluid.optimizer.Adam(
            learning_rate=1e-2,
            regularization=fluid.regularizer.L2DecayRegularizer(
                regularization_coeff=0.0005))
        adam.minimize(train_loss_t)

    exe = fluid.Executor(place)
    exe.run(startup_program)
    gw.initialize(place)
    for init in initializer:
        init(place)

    for epoch in range(1, args.epochs + 1):
        loss = exe.run(train_program, feed={}, fetch_list=[train_loss_t])
        print("Loss %s" % loss[0])
        print("Evaluating...")
        y_pred = exe.run(val_program, feed={}, fetch_list=[sigmoid_pred])[0]
        result = {}
        input_dict = {
            "y_true": label[splitted_idx["train"]],
            "y_pred": y_pred[splitted_idx["train"]]
        }
        result["train"] = evaluator.eval(input_dict)
        input_dict = {
            "y_true": label[splitted_idx["valid"]],
            "y_pred": y_pred[splitted_idx["valid"]]
        }
        result["valid"] = evaluator.eval(input_dict)
        input_dict = {
            "y_true": label[splitted_idx["test"]],
            "y_pred": y_pred[splitted_idx["test"]]
        }
        result["test"] = evaluator.eval(input_dict)
        print(result)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import copy
import numpy as np
import open3d as o3
import transformations as trans
from probreg import filterreg
from probreg import callbacks
import utils
import os
import glob
import sys

##################################
# FilterRegに用いるパラメータ
##################################
sig = None
ter = 1
ol = 0.00010
##################################

def read_rotation(model_num):

    rot_in = trans.identity_matrix()
    rote_path = '/mnt/container-data/rotation/' + model_num + ".txt"

    with open(rote_path, mode='r') as f:

        l_strip = [s.strip() for s in f.readlines()]

        k=0
        for i in range(4):
            for j in range(4):
                rot_in[j][i] = np.array(l_strip[k])
                k=k+1

    return rot_in

def write_rotation(tf, scene_num):

    #
    # rot =
    #     | rot[00] rot[01] rot[02] t[0] |
    #     | rot[10] rot[11] rot[12] t[1] |
    #     | rot[20] rot[21] rot[22] t[2] |
    #     |    0       0       0     1   |
    #

    rot = trans.identity_matrix()
    rot[:3, :3] = tf.rot

    rot[0][3] = tf.t[0]
    rot[1][3] = tf.t[1]
    rot[2][3] = tf.t[2]

    print(rot)

    print("result: ", np.rad2deg(trans.euler_from_matrix(rot)),tf.scale, tf.t)

    path = '/mnt/container-data/rotation/' + scene_num + ".txt"

    with open(path, mode='w') as f:

        for i in range(4):
            for j in range(4):
                f.write(np.array2string(rot[j][i]) + "\n")

def main():

    argvs = sys.argv
    model_num = str(argvs[1])
    scene_num = str(argvs[2])
    cloud_dir = "/mnt/container-data/remove_plane/"

    model_path = cloud_dir + model_num + ".ply"
    model_cloud = o3.read_point_cloud(model_path)

    scene_path = cloud_dir + scene_num + ".ply"
    scene_cloud = o3.read_point_cloud(scene_path)

    rot_in = read_rotation(model_num)
    model_cloud.transform(rot_in)

    model_cloud = o3.voxel_down_sample(model_cloud, voxel_size=0.020)
    scene_cloud = o3.voxel_down_sample(scene_cloud, voxel_size=0.020)

    # 基準とするpointcloud, 回転させたいpointcloud の順番
    cbs = [callbacks.Open3dVisualizerCallback(model_cloud, scene_cloud)]
    objective_type = 'pt2pt'

    # 基準とするpointcloud, 回転させたいpointcloud の順番
    tf_param, _, _ = filterreg.registration_filterreg(model_cloud, scene_cloud,
                                                      scene_cloud.normals,
                                                      objective_type=objective_type,
                                                      sigma2=sig,
                                                      callbacks=cbs,
                                                      maxiter=ter,
                                                      tol=ol)

    write_rotation(tf_param, scene_num)

if __name__ == "__main__":

    main()


from django.contrib import admin
from . import models

class LitterAdmin(admin.ModelAdmin):
    search_fields = ['name']
    list_display = ['name', 'created', 'modified']

class MedicationAdmin(admin.ModelAdmin):
    search_fields = ['cat__name', 'notes']
    list_display = ['duration', 'notes', 'frequency', 'dosage_guidelines', 'dosage_unit',
                    'created', 'modified']

class CareLogInline(admin.TabularInline):
    model = models.CareLog
    extra = 1
    readonly_fields = ['created']

class CareLogAdmin(admin.ModelAdmin):
    search_fields = ['cat__name', 'amount_of_food_taken', 'notes']
    list_display = ['cat', 'weight_unit_measure', 'weight_before_food', 'food_unit_measure', 'amount_of_food_taken',
                    'food_type', 'weight_after_food', 'stimulated', 'stimulation_type', 'notes', 'photo',
                    'created']

class CatAdmin(admin.ModelAdmin):
    search_fields = ['name', 'reference_id', 'short_name', 'gender', 'notes']
    list_display = ['name', 'slug', 'gender', 'weight', 'weight_unit',
                    'birthday', 'first_weight_loss', 'second_weight_loss',
                    'third_weight_loss', 'many_weight_losses', 'created', 'modified']
    inlines = [
        CareLogInline
    ]

__custom_admins__ = {
    'Cat': CatAdmin,
    'CareLog': CareLogAdmin,
    'Medication': MedicationAdmin,
    'Litter': LitterAdmin,
}

for model in models.__admin__:
    params = [getattr(models, model)]
    if model in __custom_admins__:
        params.append(__custom_admins__[model])
    else:
        _dyn_class = type('%sAdmin' % ( model,), (admin.ModelAdmin,), {})
        params.append(_dyn_class)
    admin.site.register(*params)

admin.site.site_title = 'KittyTracker Admin'
admin.site.site_header = 'KittyTracker Admin'


#!/usr/bin/env python3

import os
import datetime as dt
import reports
import emails
#import json
x_dat=[]
now = dt.datetime.now()
#files = os.listdir('./supplier-data/description')
for i in os.listdir('./supplier-data/descriptions/'):
	buff={}

	f=open('./supplier-data/descriptions/'+i)
	lines=f.readlines()

	buff["name"]=lines[0].strip()
	buff["weight"]=lines[1].strip()
	'''buff["description"]=lines[2].strip()
	buff["image_name"]=(i.strip('.txt'))+'.jpeg' '''
	f.close()
	x=('name: '+buff["name"]+'
'+'weight: '+buff['weight'])
	x_dat.append(x)
x_dat='

'.join(x_dat)
reports.generate("processed.pdf", "Processed Update on "+now.strftime('%B %d,%Y'), x_dat)

sender = "automation@example.com"
receiver = "{}@example.com".format(os.environ.get('USER'))
subject = "Upload Completed - Online Fruit Store"
body = "All fruits are uploaded to our website successfully. A detailed list is attached to this email."

message = emails.generate(sender, receiver, subject, body, "processed.pdf")
emails.send(message)


import pyaudio
import time
import wave

CHUNK = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 2
RATE = 44100
RECORD_SECONDS = 5
WAVE_OUTPUT_FILENAME = "a.wav"
frames = []

p = pyaudio.PyAudio()
def callback(in_data, frame_count, time_info, status):
	frames.append(in_data)
	return (in_data, pyaudio.paContinue)

stream = p.open(format=FORMAT,
                channels=CHANNELS,
                rate=RATE,
                input=True,
                frames_per_buffer=CHUNK,
		stream_callback=callback)

print("* recording")

#while stream.is_active():
#    time.sleep(1)
for i in range(0,5):
	time.sleep(1)

print("* finished")

stream.stop_stream()
stream.close()
p.terminate()

wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')
wf.setnchannels(CHANNELS)
wf.setsampwidth(p.get_sample_size(FORMAT))
wf.setframerate(RATE)
wf.writeframes(b''.join(frames))
wf.close()


import base64
import json
import os
import random
import time
from functools import wraps
from queue import Queue
from threading import Thread

import cats

fernet = None

COMMON_WORDS_SET = set(cats.lines_from_file("data/common_words.txt"))
CAPTCHA_QUEUE_LEN = 200
CAPTCHA_LENGTH = 10
CAPTCHA_WORD_LEN = 10

captcha_queue = Queue()

def require_fernet(f):
    @wraps(f)
    def wrapped(*args, **kwargs):
        global fernet
        if not fernet:
            from cryptography.fernet import Fernet

            fernet = Fernet(os.environ.get("FERNET_KEY", Fernet.generate_key()))
        return f(*args, **kwargs)

    return wrapped

def token_writer(f):
    @wraps(f)
    @require_fernet
    def wrapped(*args, **kwargs):
        data = f(*args, **kwargs)
        decoded = json.dumps(data).encode("utf-8")
        return fernet.encrypt(decoded).decode("utf-8")

    return wrapped

def token_reader(fail):
    def decorator(f):
        @wraps(f)
        @require_fernet
        def wrapped(*, token, **kwargs):
            from cryptography.fernet import InvalidToken

            if not token:
                return fail
            try:
                return f(
                    token=json.loads(fernet.decrypt(token.encode("utf-8"))), **kwargs
                )
            except (TypeError, InvalidToken):
                return fail

        return wrapped

    return decorator

@token_writer
def create_wpm_authorization(user, wpm):
    return {
        "user": user,
        "wpm": wpm,
    }

@token_reader(fail=0)
def get_authorized_limit(user, token):
    if token["user"] != user:
        return 0
    return token["wpm"]

@token_writer
def encode_challenge(user, words):
    return {
        "user": user,
        "words": words,
        "startTime": time.time(),
    }

@token_reader(fail=(False, False))
def decode_challenge(token):
    return token["user"], token["words"], token["startTime"]

def populate_captcha_queue():
    while captcha_queue.qsize() < CAPTCHA_QUEUE_LEN:
        captcha_queue.put(generate_captcha())

def generate_captcha():
    from claptcha import Claptcha

    word = random.choice([x for x in COMMON_WORDS_SET if len(x) < CAPTCHA_WORD_LEN])
    c = Claptcha(word, "gui_files/FreeMono.ttf", margin=(20, 10))
    image_b64 = base64.b64encode(c.bytes[1].getvalue()).decode("utf-8")
    return "data:image/png;base64," + image_b64, word

def get_captcha_urls(num_words=CAPTCHA_LENGTH):
    Thread(target=populate_captcha_queue).start()

    images, words = [], []
    for _ in range(num_words):
        image, word = captcha_queue.get()
        images.append(image)
        words.append(word)

    return images, words


import numpy as np
import tensorflow as tf
from tqdm import trange
from . import dl_ops as ops

def param_dict_to_flag_grid(param_dict):
    flag_grid = dict()
    for k, v in param_dict.items():
        if k in ['EMB_DATASETS', 'DATASETS_INFO', 'NEW_GAME', 'DATA_FOLDER_PATH', 'CONFIG_FOLDER_PATH', 'RESULT_FOLDER_PATH',
                 'LEFT_CONTEXT_SIZE', 'RIGHT_CONTEXT_SIZE', 'DIRECTED', 'SKIP_EDGE_EXTRACTING',
                 'PROJECT_NAME', 'EMB_PARAMS_FILE_NAME', 'CDM_DB_NAME', ]:
            continue
        if k=='DUMPING_PATH':
            flag_grid[k] = [v]
        else:
            flag_grid[k] = v
    return flag_grid

def get_flag_list(flag_grid, FEATURE_SIZE):
    #update flag_grid
    flag_grid['FEATURE_SIZE'] = [FEATURE_SIZE]

    #flag_grid to flag_list
    from itertools import product
    flag_list = [dict(list(zip(list(flag_grid.keys()), values)))
                 for values in list(product(*flag_grid.values()))]

    #remove duplicated flag
    flag_list_new_unique = []
    for d_item in flag_list:
        if d_item not in flag_list_new_unique:
            flag_list_new_unique.append(d_item)
    return flag_list_new_unique

def get_model_list(param_dict):
    ##make flag_list
    flag_grid = param_dict_to_flag_grid(param_dict)
    flag_list = get_flag_list(flag_grid, param_dict['EMB_DATASETS'].info['FEATURE_SIZE'])

    ##get model_list
    MODEL_DICT = {'LINE_MODEL': LINE_MODEL}

    model_list = []
    for m_idx, flag in enumerate(flag_list):
        flag['MODEL_NAME'] = 'MODEL_{}'.format(m_idx+1)
        model_list.append(MODEL_DICT[flag['MODEL_ARCH']](flag))
    return model_list

class LINE_MODEL():
    def __init__(self, flag):
        self.flag = flag
        self.tensorDict = dict()
        self.resultDict = dict()
        self.g = tf.Graph()
        self.g_vis = tf.Graph()
        self.Building_graph()

    def _get_logger(self):
        from .utils import get_logger_instance
        self.logger = get_logger_instance(logger_name=self.flag['MODEL_NAME'],
                                          DUMPING_PATH=self.flag['DUMPING_PATH'])

    def _basic_tensors(self):
        with tf.name_scope('Learning_Rate'):
            self.tensorDict['global_step'] = tf.Variable(0, name="Global_step", trainable=False, dtype=tf.int32)
            if ('DECAY_STEPS' in self.flag.keys())&('DECAY_RATE' in self.flag.keys()):
                self.tensorDict['lr_p1'] = tf.train.exponential_decay(self.flag['LR_p1'], self.tensorDict['global_step'],
                                                               self.flag['DECAY_STEPS'], self.flag['DECAY_RATE'],
                                                               staircase=True, name='ExpDecay_lr_p1')
                self.tensorDict['lr_p2'] = tf.train.exponential_decay(self.flag['LR_p2'], self.tensorDict['global_step'],
                                                               self.flag['DECAY_STEPS'], self.flag['DECAY_RATE'],
                                                               staircase=True, name='ExpDecay_lr_p2')
            else:
                self.tensorDict['lr_p1'] = tf.constant(self.flag['LR_p1'], name='Constant_lr_p1')
                self.tensorDict['lr_p2'] = tf.constant(self.flag['LR_p2'], name='Constant_lr_p2')

    def _input_layer_tensors(self):
        with tf.name_scope('Input_Layer'):
            self.tensorDict['focus_w'] = tf.placeholder(tf.int32, shape=[self.flag['BATCH_SIZE']], name='focus_w')
            self.tensorDict['context_w'] = tf.placeholder(tf.int32, shape=[self.flag['BATCH_SIZE']], name='context_w')
            self.tensorDict['score'] = tf.placeholder(tf.float32, shape=[self.flag['BATCH_SIZE']], name='score')

    def _embedding_layer_tensors(self):
        with tf.name_scope('EMB_Layer'):
            ## p1
            self.tensorDict['p1_emb_matrix'] = tf.Variable(tf.random_uniform([self.flag['FEATURE_SIZE'], self.flag['EMB_SIZE']],
                                                                             -0.01, 0.01),
                                                           name='p1_emb_matrix')
            self.tensorDict['p1_focus_emb_vec'] = tf.nn.embedding_lookup(self.tensorDict['p1_emb_matrix'],
                                                                         self.tensorDict['focus_w'],
                                                                         name='p1_focus_emb_vec')
            self.tensorDict['p1_context_emb_vec'] = tf.nn.embedding_lookup(self.tensorDict['p1_emb_matrix'],
                                                                           self.tensorDict['context_w'],
                                                                           name='p1_context_emb_vec')

            ## p2
            self.tensorDict['p2_emb_focus_matrix'] = tf.Variable(tf.random_uniform([self.flag['FEATURE_SIZE'],
                                                                                self.flag['EMB_SIZE']], -0.01, 0.01),
                                                                 name='p2_emb_focus_matrix')
            self.tensorDict['p2_focus_emb_vec'] = tf.nn.embedding_lookup(self.tensorDict['p2_emb_focus_matrix'],
                                                                         self.tensorDict['focus_w'],
                                                                         name='p2_focus_emb_vec')

            self.tensorDict['p2_emb_context_matrix'] = tf.Variable(tf.random_uniform([self.flag['FEATURE_SIZE'],
                                                                                self.flag['EMB_SIZE']],
                                                                             -0.01, 0.01),
                                                                   name='p2_emb_context_matrix')
            self.tensorDict['p2_context_emb_vec'] = tf.nn.embedding_lookup(self.tensorDict['p2_emb_context_matrix'],
                                                                           self.tensorDict['context_w'],
                                                                           name='p2_context_emb_vec')

    def _Inference(self):
        with tf.name_scope('Inference'):
            self._embedding_layer_tensors()

    def _Loss(self):
        with tf.variable_scope('Loss'):
            with tf.name_scope('First-order_proximity'):
                p1_inner_product = tf.reduce_sum((self.tensorDict['p1_focus_emb_vec'] * self.tensorDict['p1_context_emb_vec']),
                                                 axis=1, name='p1_inner_product')
                p1_logits = tf.nn.sigmoid(p1_inner_product, name='p1_logits')
                self.tensorDict['loss_1st_prox'] = tf.reduce_mean(-self.tensorDict['score']*tf.log(p1_logits+1e-10),
                                                                  name='loss_1st_prox')

            with tf.name_scope('Second-order_proximity'):
                p2_inner_product = tf.reduce_sum((self.tensorDict['p2_focus_emb_vec'] * self.tensorDict['p2_context_emb_vec']),
                                                 axis=1, name='p2_inner_product')
                #issue; add negative sampling
                p2_numerator = tf.exp(p2_inner_product, name='p2_numerator')
                p2_denominator = tf.reduce_sum(tf.exp(tf.matmul(self.tensorDict['p2_focus_emb_vec'],
                                                                tf.transpose(self.tensorDict['p2_emb_context_matrix']))) + 1e-10,
                                               axis=1, name='p2_denominator')
                p2_logits = tf.divide(p2_numerator, p2_denominator, name='p2_logits')
                self.tensorDict['loss_2nd_prox'] = tf.reduce_mean(-self.tensorDict['score']*tf.log(p2_logits+1e-10),
                                                                  name='loss_2nd_prox')

    def _Optimizer(self):
        with tf.name_scope('Optimizer_p1'):
            optimizer_p1 = tf.train.AdamOptimizer(self.tensorDict['lr_p1'], name='optimizer_p1')
            self.tensorDict['trainOp_p1'] = optimizer_p1.minimize(self.tensorDict['loss_1st_prox'],
                                                                  global_step=self.tensorDict['global_step'])
        with tf.name_scope('Optimizer_p2'):
            optimizer_p2= tf.train.AdamOptimizer(self.tensorDict['lr_p2'], name='optimizer_p2')
            self.tensorDict['trainOp_p2'] = optimizer_p2.minimize(self.tensorDict['loss_2nd_prox'],
                                                                  global_step=self.tensorDict['global_step'])

    def _Summary(self):
        ## logging
        self.logger.info("\n[FLAG]")
        for k, v in self.flag.items():
            self.logger.info("\t{}:  {}".format(k, v))

        self.logger.info("\n[INPUT_LAYERS]")
        self.logger.info("\tfocus_w: {}".format(self.tensorDict['focus_w']))
        self.logger.info("\tcontext_w: {}".format(self.tensorDict['context_w']))
        self.logger.info("\tscore: {}".format(self.tensorDict['score']))

        self.logger.info("\n[EMB_LAYERS]")
        self.logger.info("\tp1_emb_matrix: {}".format(self.tensorDict['p1_emb_matrix']))
        self.logger.info("\tp1_focus_emb_vec: {}".format(self.tensorDict['p1_focus_emb_vec']))
        self.logger.info("\tp1_context_emb_vec: {}".format(self.tensorDict['p1_context_emb_vec']))
        self.logger.info("\tp2_emb_focus_matrix: {}".format(self.tensorDict['p2_emb_focus_matrix']))
        self.logger.info("\tp2_focus_emb_vec: {}".format(self.tensorDict['p2_focus_emb_vec']))
        self.logger.info("\tp2_emb_context_matrix: {}".format(self.tensorDict['p2_emb_context_matrix']))
        self.logger.info("\tp2_context_emb_vec: {}".format(self.tensorDict['p2_context_emb_vec']))

        self.logger.info("\n[LOSS]")
        self.logger.info("\tloss_1st_prox: {}".format(self.tensorDict['loss_1st_prox']))
        self.logger.info("\tloss_2nd_prox: {}".format(self.tensorDict['loss_2nd_prox']))

        ## summary
        tf.summary.scalar('lr_p1', self.tensorDict['lr_p1'])
        tf.summary.scalar('lr_p2', self.tensorDict['lr_p2'])
        tf.summary.scalar('loss_1st_prox', self.tensorDict['loss_1st_prox'])
        tf.summary.scalar('loss_2nd_prox', self.tensorDict['loss_2nd_prox'])

    def Building_graph(self):
        with self.g.as_default():
            self._basic_tensors()
            self._input_layer_tensors()
            self._Inference()
            self._Loss()
            self._Optimizer()
            self._get_logger()
            self._Summary()


# -*- coding:utf-8 -*-

from sklearn.model_selection import train_test_split

from deeptables.models import deeptable, deepnets
from deeptables.datasets import dsutils
from deeptables.models.layers import register_custom_objects
from tensorflow.keras import layers

from tensorflow.keras import backend as K
import tensorflow as tf

import tempfile, os

class Test_DeepTable:

    def run_nets(self, nets):
        df_train = dsutils.load_adult().head(100)
        y = df_train.pop(14).values
        X = df_train

        conf = deeptable.ModelConfig(nets=nets,
                                     metrics=['AUC'],
                                     fixed_embedding_dim=True,
                                     embeddings_output_dim=2,
                                     apply_gbm_features=False,
                                     apply_class_weight=True)
        dt = deeptable.DeepTable(config=conf)

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
        model, history = dt.fit(X_train, y_train, epochs=1)
        result = dt.evaluate(X_test, y_test)
        assert result['AUC'] >= 0.0
        return dt, result

    def test_DeepFM(self):
        self.run_nets(nets=deepnets.DeepFM)

    def test_xDeepFM(self):
        self.run_nets(nets=deepnets.xDeepFM)

    def test_WideDeep(self):
        self.run_nets(nets=deepnets.WideDeep)

    def test_callable(self):
        self.run_nets(nets=[deepnets.linear])

    def test_custom_nets(self):
        def custom_net(embeddings, flatten_emb_layer, dense_layer, concat_emb_dense, config, model_desc):
            out = layers.Dense(10)(flatten_emb_layer)
            return out

        self.run_nets(nets=[deepnets.linear, custom_net, 'dnn_nets'])

    def test_linear(self):
        self.run_nets(nets=['linear'])

    def test_afm(self):
        self.run_nets(nets=['afm_nets'])

    def test_cin(self):
        self.run_nets(nets=['cin_nets'])

    def test_pnn_nets(self):
        self.run_nets(nets=['opnn_nets', 'ipnn_nets', 'pnn_nets'])

    def test_autoint_nets(self):
        self.run_nets(nets=['autoint_nets'])

    def test_fgcnn_nets(self):
        self.run_nets(nets=['fg_nets'])
        self.run_nets(nets=['fgcnn_cin_nets'])
        self.run_nets(nets=['fgcnn_ipnn_nets'])
        self.run_nets(nets=['fgcnn_dnn_nets'])
        self.run_nets(nets=['fgcnn_fm_nets'])
        self.run_nets(nets=['fgcnn_afm_nets'])

    def test_fibi_nets(self):
        self.run_nets(nets=['fibi_nets'])
        self.run_nets(nets=['fibi_dnn_nets'])

    def test_custom_dnn(self):
        df_train = dsutils.load_adult().head(100)
        y = df_train.pop(14).values
        X = df_train

        conf = deeptable.ModelConfig(nets=['dnn_nets'],
                                     dnn_params={
                                         'custom_dnn_fn': deepnets.custom_dnn_D_A_D_B,
                                         'hidden_units': ((128, 0.2, True), (64, 0, False)),
                                     },
                                     metrics=['AUC'],
                                     fixed_embedding_dim=True,
                                     embeddings_output_dim=2,
                                     apply_gbm_features=False,
                                     apply_class_weight=True)
        dt = deeptable.DeepTable(config=conf)
        model, history = dt.fit(X, y, epochs=1)
        l1 = model.model.get_layer('dnn_custom_dense_1')
        l2 = model.model.get_layer('dnn_custom_dropout_1')
        l3 = model.model.get_layer('dnn_custom_bn_1')
        l4 = model.model.get_layer('dnn_custom_dense_2')

        assert l1
        assert l2
        assert l3
        assert l4

    def test_save_load_custom_nets(self):
        def custom_net(embeddings, flatten_emb_layer, dense_layer, concat_emb_dense, config, model_desc):
            if embeddings is None or len(embeddings) <= 0:
                model_desc.add_net('fm', (None), (None))
                return None
            fm_concat = layers.Concatenate(axis=1, name='concat_fm_embedding')(embeddings)
            fm_output = CustomFM(name='fm_layer')(fm_concat)  # use custom layer
            model_desc.add_net('fm', fm_concat.shape, fm_output.shape)
            return fm_output

        dt, result = self.run_nets(nets=[custom_net, 'dnn_nets'])

        filepath = tempfile.mkdtemp()
        dt.save(filepath)
        #assert os.path.exists(f'{filepath}/dt.pkl')
        #assert os.path.exists(f'{filepath}/custom_net+dnn_nets.h5')

        register_custom_objects(
            {
                'CustomFM': CustomFM,
            })

        newdt = deeptable.DeepTable.load(filepath)
        assert newdt.best_model

class CustomFM(layers.Layer):

    def __init__(self, **kwargs):
        super(CustomFM, self).__init__(**kwargs)

    def call(self, x, **kwargs):
        if K.ndim(x) != 3:
            raise ValueError(f'Wrong dimensions of inputs, expected 3 but input {K.ndim(x)}.')
        square_of_sum = tf.square(tf.reduce_sum(
            x, axis=1, keepdims=True))
        sum_of_square = tf.reduce_sum(
            x * x, axis=1, keepdims=True)
        cross = square_of_sum - sum_of_square
        cross = 0.5 * tf.reduce_sum(cross, axis=2, keepdims=False)
        return cross


import requests
import json
from flask import render_template, url_for, jsonify, request
from app import app
from .transitivity_rankings import TransRank

app.trans = None

@app.route('/')
def home_page():
    if app.trans is None:
        get_trans()
    top_ten = app.trans.rank_df.query('comb_rank <= 10')
    top_ten_teams = list(top_ten.index)
    top_ten_ranks = top_ten['comb_rank'].to_dict()
    top_ten_win_ranks = top_ten['win_rank'].to_dict()
    top_ten_loss_ranks = top_ten['loss_rank'].to_dict()
    top_ten_avg_ranks = top_ten['avg_rank'].to_dict()
    return render_template('index.html', all_teams=app.trans.teams, all_team_urls=app.trans.all_team_urls(),
                           top_ten_teams=top_ten_teams, top_ten_ranks=top_ten_ranks,
                           top_ten_win_ranks=top_ten_win_ranks, top_ten_loss_ranks=top_ten_loss_ranks,
                           top_ten_avg_ranks=top_ten_avg_ranks, logos=app.logos, urls=app.trans.all_team_urls())

@app.route('/rankings')
def rank_page():
    if app.trans is None:
        get_trans()
    html_table = app.trans.get_html_table(images=app.logos)
    return render_template('ranks.html', table=html_table,
                           all_teams=app.trans.teams, all_team_urls=app.trans.all_team_urls())

@app.route('/<team_with_underscores>')
def team_page_router(team_with_underscores):
    team = team_with_underscores.replace('_', ' ')
    return team_page(team)

def team_page(team):
    if app.trans is None:
        get_trans()
    if team not in app.trans.teams:
        return 'Team not found'
        # todo: make this nicer
    win_paths = app.trans.win_paths[team]
    win_levels, no_wins = get_levels(win_paths, app.trans.teams, team)
    loss_paths = app.trans.loss_paths[team]
    loss_levels, no_losses = get_levels(loss_paths, app.trans.teams, team)
    paths = {'victories': win_paths, 'defeats': loss_paths}
    levels = {'victories': win_levels, 'defeats': loss_levels}
    no_paths = {'victories': no_wins, 'defeats': no_losses}
    rank_info = dict(app.trans.rank_df.loc[team])
    return render_template('team.html', team=team, paths=paths, no_paths=no_paths, levels=levels, rank_info=rank_info,
                           logos=app.logos, all_teams=app.trans.teams, all_team_urls=app.trans.all_team_urls())

@app.route('/rankings/about')
def about_rank_page():
    return render_template('ranks-about.html')

@app.route('/_get_path', methods=['POST'])
def get_path():
    my_team = request.form['my_team']
    other_team = request.form['other_team']
    if other_team in app.trans.win_paths[my_team]:
        return jsonify(app.trans.win_paths[my_team][other_team])
    else:
        return jsonify([])

def get_levels(paths, all_teams, team):
    included_teams = []
    if len(paths) == 0:
        max_length = 0
    else:
        max_length = max(len(paths[t]) - 1 for t in paths)
    levels = {i + 1: [] for i in range(max_length)}
    for t in paths:
        levels[len(paths[t]) - 1] += [t]
        included_teams += [t]
    for l in levels:
        levels[l] = sorted(levels[l])
    excluded_teams = sorted(set(all_teams) - set(included_teams) -{team})
    return levels, excluded_teams

def get_trans():
    conferences = ['ACC', 'American Athletic', 'Big 12', 'Big Ten', 'Conference USA', 'FBS Independents',
                   'Mid-American', 'Mountain West', 'Pac-12', 'SEC', 'Sun Belt']
    game_data_url = 'https://api.collegefootballdata.com/games?year=2018'
    game_data = json.loads(requests.get(game_data_url).text)
    game_data = [g for g in game_data if g['home_conference'] in conferences and g['away_conference'] in conferences]
    winners = [g['away_team'] if g['away_points'] > g['home_points'] else g['home_team'] for g in game_data]
    losers = [g['home_team'] if g['away_points'] > g['home_points'] else g['away_team'] for g in game_data]
    app.trans = TransRank(winners, losers)
    teams_url = 'https://api.collegefootballdata.com/teams'
    team_info = json.loads(requests.get(teams_url).text)
    app.logos = {x['school']: x['logos'][0] for x in team_info if x['school'] in app.trans.teams}


from tqdm import tqdm
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
import unicodedata2
import string

class Cleaner:

    def __init__(self):
        with open("r_words.txt", 'r') as r_w_files:
            common_words=[line.replace('\n', '') for line in r_w_files.readlines()]
        with open("r_sentences.txt", 'r') as r_s_files:
            common_sentences=[line.replace('\n', '') for line in r_s_files.readlines()]
        self.common_words = common_words
        self.common_sentences = common_sentences
        self.stemming_dict_ = {}

    def strip_accents(self, text):
        text = unicodedata2.normalize('NFD', text)
        text = text.encode('ascii', 'ignore')
        text = text.decode("utf-8")
        return str(text)

    def stemmer(self, filtered_words):
        stemmer = SnowballStemmer('spanish')
        stemmed_words = []
        for i in filtered_words:
            stemmed = stemmer.stem(i)
            stemmed_words.append(stemmed)
            if stemmed in self.stemming_dict_:
                dict_stem = self.stemming_dict_[stemmed]
                if i in dict_stem:
                    dict_stem[i] =  dict_stem[i] + 1
                else:
                    dict_stem[i] = 1
            else:
                 self.stemming_dict_[stemmed] = {i:1}
        doc = (' ').join(stemmed_words)
        return  doc

    def de_stemmer(self, data):
        #busco el mas frecuente
        replacement = {}
        for key in tqdm(self.stemming_dict_.keys(), desc="select most representative word of stem"):
            stem_dict = self.stemming_dict_[key]
            replacement[key] = max(stem_dict, key=stem_dict.get)
        self.replacement = replacement
        #remplazo en los documentos
        de_stemmed = []
        for doc in tqdm(data, desc = "de-stemming"):
            word_list = doc.split(' ')
            new_list = []
            for word in word_list:
                if word in self.stemming_dict_:
                    new_list.append(replacement.get(word))
                else:
                    new_list.append(word)
            de_stemmed.append(' '.join(new_list))

        return de_stemmed

    def preprocess(self, doc,stem, stopwords_lang = "spanish"):
        for frase in self.common_sentences:
            doc=doc.replace(frase, '')
        stop_words = stopwords.words(stopwords_lang)
        remove_words  = set(stop_words).union(set(self.common_words))
        remove_punctuation_map = dict((ord(char), ' ') for char in string.punctuation)
        doc = self.strip_accents(doc).lower()
        doc = doc.translate(remove_punctuation_map)
        querywords = doc.split()
        #Agrego replacement de numeros por NUM
        querywords = ["NUM" if c.isdigit() else c for c in querywords]
        filtered_words = [palabra for palabra in querywords if palabra not in remove_words]
        if stem:
            doc = self.stemmer(filtered_words)
        else:
            doc = (' ').join(filtered_words)
        return doc

    def data_clean(self,data, stemming):
        data_clean = [self.preprocess(doc, stem = stemming) for doc in tqdm(data, desc = "preprocess")]
        if stemming:
            data_clean = self.de_stemmer(data_clean)
        return data_clean


# coding=utf-8
"""Outlet Group Call feature tests."""
# python standard library
from functools import partial

# from pypi
from expects import (
    be_false,
    be_true,
    expect,
    raise_error,
)
from pytest_bdd import (
    given,
    then,
    when,
)

import pytest_bdd

# testing
from ..fixtures import katamari  # NOQA: F401

# software under test
from thestrip.api.outlets import OutletGroup
from thestrip.parts import ConfigurationError

scenario = partial(pytest_bdd.scenario,
                   '../../features/api/outlet_group_call.feature')

# ******************** Call ******************** #
# ********** off ********** #

@scenario('The outlets are called off')
def test_the_outlets_are_turned_off():
    return

@given('an outlet group with the off state')  # NOQA: F811
def an_outlet_group_with_the_off_group(katamari, faker, mocker):
    katamari.outlet_group = OutletGroup(faker.ipv4(), OutletGroup.ALL,
                                        OutletGroup.OFF)
    katamari.outlet_group.turn_off = mocker.MagicMock()
    return

@when('the outlet group is called')  # NOQA: F811
def the_outlet_group_is_called(katamari):
    katamari.outlet_group()
    return

@then('turn off is called')  # NOQA: F811
def the_outlets_are_turned_off(katamari):
    expect(katamari.outlet_group.turn_off.called).to(be_true)
    return

# ******************** Turn me on, big boy ******************** #

@scenario("The outlets are called on")
def test_all_on():
    return

@given("an outlet group with the on state")  # NOQA: F811
def setup_on_group(katamari, faker, mocker):
    katamari.outlet_group = OutletGroup(faker.ipv4(), OutletGroup.ALL,
                                        set_state="on")
    katamari.outlet_group.turn_off = mocker.MagicMock()
    katamari.outlet_group.turn_on = mocker.MagicMock()
    return

#  When the outlet group is called

@then("the outlets are turned on")  # NOQA: F811
def check_all_on(katamari):
    expect(katamari.outlet_group.turn_on.called).to(be_true)
    expect(katamari.outlet_group.turn_off.called).to(be_false)
    return

# ********** What? ********** #

@scenario("The outlets are called to an unknown state")
def test_unknown_state():
    return

@given("an outlet group with an unknown state")  # NOQA: F811
def setup_bad_state(katamari, faker, mocker):
    katamari.outlet_group = OutletGroup(faker.ipv4(),
                                        OutletGroup.ALL,
                                        set_state=faker.word())
    return

@when("the bad outlet group is called")  # NOQA: F811
def call_bad_outlet_group(katamari):
    def bad_call():
        katamari.outlet_group()
        return

    katamari.bad_call = bad_call
    return

@then("it raises a ConfigurationError")  # NOQA: F811
def check_error(katamari):
    expect(katamari.bad_call).to(raise_error(ConfigurationError))
    return


import pygame
from pygame.sprite import Group

from settings import Settings   #导入设置
from game_stats import GameStats
from bush import Bush
from button import Botton
from scoreboard import Scoreboard

import game_functions as gf

def run_game():
	'''初始化游戏并创建一个屏幕对象'''
	pygame.init()   #初始化背景设置,使pygame能正常工作
	ai_settings = Settings()
	# pygame.display.set_mode()方法,创建screen窗口,第一个参数为分辨率
	screen = pygame.display.set_mode((ai_settings.screen_width,
	                                  ai_settings.screen_height), 0, 32) # pygame.FULLSCREEN
	pygame.display.set_caption("Pengpeng Invasion") #caption方法设置当前窗口的标题

	# 设置背景颜色:设置bg_color,并在循环中使用方法scree.fill(bg_color)即可
	# bg_color = ai_settings.bg_color

	# 创建Play按钮
	play_button = Botton(ai_settings, screen, msg= 'Play')

	# 创建一个游戏信息统计实例,创建一个计分板
	stats = GameStats(ai_settings)
	sb = Scoreboard(ai_settings, screen, stats)

	# 创建一个灌木,一个芃芃编组,一个子弹编组
	bush = Bush(ai_settings, screen)
	bullets = Group()
	pengs = Group()

	# 创建芃芃们
	gf.create_tribe(ai_settings, screen, bush, pengs)

	#开始游戏主循环
	while True:
		gf.check_events(ai_settings, screen, stats, sb, play_button, bush, pengs, bullets)

		if stats.game_active:
			bush.update()
			gf.update_bullets(ai_settings, screen, stats, sb, bush, pengs, bullets)
			gf.update_pengs(ai_settings, stats, sb, screen, bush, pengs, bullets)

		gf.update_screen(ai_settings, screen, stats, sb, bush, pengs, bullets, play_button)

run_game()


from django.db import models
from django.contrib.auth.models import User
from django.core.validators import MaxValueValidator, MinValueValidator
from PIL import Image
from books.models import Book

# Create your models here.

class Profile(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE)
    image = models.ImageField(default='default.jpg', upload_to='profile_pics')

    def __str__(self):
        return f'{self.user.username} Profile'

    def save(self, *args, **kwargs):
        super(Profile, self).save(*args, **kwargs)

        img = Image.open(self.image.path)

        if img.height > 300 or img.width > 300:
            output_size = (300, 300)
            img.thumbnail(output_size)
            img.save(self.image.path)

class WishListNames(models.Model):
    class Meta:
        unique_together = (('user', 'wishlist_num'),)

    user = models.ForeignKey(User, on_delete=models.CASCADE)
    wishlist_num = models.IntegerField(
        default=0,
        validators=[MaxValueValidator(0), MinValueValidator(2)]
    )
    name = models.CharField(max_length=120)

class WishList(models.Model):
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    book = models.ForeignKey(Book, on_delete=models.CASCADE)
    wishlist_num = models.IntegerField(
        default=0,
        validators=[MaxValueValidator(0), MinValueValidator(2)]
    )

    def __str__(self):
        return f'{self.user.username}, {self.book.title}'

class GuestEmail(models.Model):
    email = models.EmailField()
    active = models.BooleanField(default=True)
    update = models.DateTimeField(auto_now=True)
    timestamp = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.email


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup # BeautifulSoup is in bs4 package
import requests
import json
import sys
import csv

verb = sys.argv[1]
URL = "http://conjugador.reverso.net/conjugacion-aleman-verbo- " + verb +".html"
content = requests.get(URL)
soup = BeautifulSoup(content.text, 'html.parser')
row = soup.find('div', { "class" : "blue-box-wrap"})
txt = row.get_text(separator=" ").split()

json_data =  {
    "Ich": txt[2],
    "Du": txt[4],
    "Er_Sie_Es":txt[6],
    "Wir": txt[8],
    "Ihr": txt[10],
    "Sie_sie": txt[12]
    }

data = ''
for i in range(2,13):
    if i%2 == 0:
        if i == 12:
            data += txt[i]
        else:
            data += txt[i] + ","

print(json.dumps(json_data, ensure_ascii=True).encode('utf8'))
sys.stdout.flush()


import pandas as pd
import re
import json

def format_name(s):
    exceptions = ['and', 'or', 'an']
    tokens = str(s).split(' ')
    tokens = [str(x).lower().capitalize() if x not in exceptions else x for x in tokens]
    return str(' '.join(tokens))

def parse_ingredients(ingredients):
    ingredients = re.sub(r'\([^)]*\)', '', ingredients)
    ingredients = str(ingredients).replace('(', '')
    ingredients = str(ingredients).replace(')', '')
    return list(set([x.strip() for x in str(ingredients).split(',')]))

print('Started Loading')
df_nutr = pd.read_excel('data/Nutrient.xlsx', converters={'NDB_No': lambda x: str(x)})
df_prod = pd.read_excel('data/Products.xlsx', converters={'NDB_Number': lambda x: str(x)})
df_serv = pd.read_excel('data/Serving_Size.xlsx', converters={'NDB_No': lambda x: str(x)})

df_prod['long_name'] = [format_name(x) for x in df_prod['long_name']]
df_prod['manufacturer'] = [format_name(x) for x in df_prod['manufacturer']]
df_prod['ingredients_english'] = [format_name(x) for x in df_prod['ingredients_english']]

nutr_ndb = list(df_nutr['NDB_No'])
serv_ndb = list(df_serv['NDB_No'])

ABBREV = []

i = 1
print('Finished Loading')
print('Started Parsing')
for index, row in df_prod.iterrows():
    if row['NDB_Number'] in nutr_ndb and row['NDB_Number'] in serv_ndb:
        nutr = df_nutr.loc[df_nutr['NDB_No'] == row['NDB_Number']]
        serv = df_serv.loc[df_serv['NDB_No'] == row['NDB_Number']]

        kingredients = row['ingredients_english']
        kingredients = parse_ingredients(kingredients)

        serving_size = float(serv['Serving_Size'].values[0])
        serving_measement = str(serv['Serving_Size_UOM'].values[0])

        knutrients = []
        knutrientsNames = []
        for index2, row2 in nutr.iterrows():
            value = None
            if serving_measement == 'g':
                value = float(row2['Output_value']) * (serving_size / 100.0)
            elif serving_measement == 'mg':
                value = float(row2['Output_value']) * (serving_size / 1000.0)
            knutrientsNames.append(str(row2['Nutrient_name']))
            knutrients.append({
                'name': str(row2['Nutrient_name']),
                'value': str(value),
                'measurement': str(row2['Output_uom'])
            })

        print(knutrients)

        kdict = {
            'ndb': str(row['NDB_Number']),
            'description': str(row['long_name']),
            'manufacturer': str(row['manufacturer']),
            'ingredients': list(kingredients),
            'nutrients': list(knutrients),
            'serving_size': str(serv['Serving_Size'].values[0]),
            'serving_measurement': str(serv['Serving_Size_UOM'].values[0]),
            'serving_size_household': str(serv['Household_Serving_Size'].values[0]),
            'serving_measurement_household': str(serv['Household_Serving_Size_UOM'].values[0])
        }
        print(kdict['description'], i)
        i += 1
        ABBREV.append(kdict)

# print(ABBREV)
out = {'payload': ABBREV}
with open('data/ABBREV.json', 'w') as outfile:
    json.dump(out, outfile)

print('Finished Parsing')


'''
Created on 2018年4月25日
@author: 杨少华
经过周密的特征筛选得到最后需要的特征以及，
多值有序变量的map以及多值无序变量的映射，得到最优的训练特征集合的确定
特征工程完备下的最优预测结果
'''
import pandas as pd
import numpy as np
import time
import matplotlib.pyplot as plt
import seaborn as sns

####################################    读取数据+填充缺失值        ########################################
start_time=time.time()
features_list = ['acc_now_delinq','loan_amnt','funded_amnt_inv','term','int_rate','installment','grade', \
'emp_length','annual_inc','verification_status','dti','total_rec_int', \
'revol_util','total_acc','revol_bal','out_prncp','out_prncp_inv','total_pymnt_inv', \
'total_rec_prncp', 'mths_since_last_record','mths_since_last_major_derog','tot_coll_amt', \
'tot_cur_bal','total_rev_hi_lim','home_ownership','loan_status','total_pymnt']
df = pd.read_csv('data/train.csv')

#train.shape  (709903, 26)
train = df[features_list]
#'mths_since_last_record','mths_since_last_major_derog' 两个特征确实严重，填充0作为常识
train = train.fillna(0)

###########################       多值无序变量，多值有序变量处理：       #################################
train['emp_length']=train['emp_length'].map({'< 1 year':0,'1 year':1,'2 years':2,'3 years':3,'4 years':4,'5 years':5, \
                                                                  '6 years':6,'7 years':7,'8 years':8,'9 years':9,'10+ years':10,'n/a':0}).astype(float)
train['grade'] = train['grade'].map( {'D':4, 'A':1, 'E':5, 'B':2, 'C':3, 'F':6, 'G':7} ).astype(float)
train['loan_status']=train['loan_status'].map({'Charged Off':1,'Fully Paid':0,'Current':0,'In Grace Period':1,\
                                                         'Late (31-120 days)':1,'Late (16-30 days)':1,'Default':1, \
                                                         'Does not meet the credit policy. Status:Fully Paid':0, \
                                                         'Does not meet the credit policy. Status:Charged Off':1,'Issued':1}).astype(int)

#print("缺失值处理多值无序变量处理：\n")
#check_null = train.isnull().sum(axis=0).sort_values(ascending=False)/float(len(train)) #查看缺失值比例
#print(check_null[check_null > 0.1]) # 查看缺失比例大于20%的属性。
#objectColumns = train.select_dtypes(include=["object"]).columns
#print(train[objectColumns].isnull().sum().sort_values(ascending=False))

#"多值无序变量处理
n_columns = ["home_ownership", "verification_status", "term"]
dummy_df = pd.get_dummies(train[n_columns])# 用get_dummies进行one hot编码
loans = pd.concat([train, dummy_df], axis=1)#当axis = 1的时候，concat就是行对齐，然后将不同列名称的两张表合并
#筛选包含home_ownership的所有变量
#print(loans.loc[:,loans.columns.str.contains("home_ownership")].head())
loans = loans.drop(n_columns, axis=1)  #清除原来的分类变量
#用pandas的info( )的方法作最后检查，发现已经将所有类型为object的变量作转化，所有数据类型均满足下一步算法的要求。
#print(loans.info())
#我们采用的是标准化的方法，与归一化相比，标准化的方法保持了异常值所包含的有用信息，并且使得算法受到异常值的影响较小。
col = loans.select_dtypes(include=['int64','float64']).columns
loans_ml_df = loans # 复制数据至变量loans_ml_df

##############################             标准化                                           ##########################
from sklearn.preprocessing import StandardScaler # 导入模块
sc =StandardScaler() # 初始化缩放器
loans_ml_df[col] =sc.fit_transform(loans_ml_df[col]) #对数据进行标准化
#print(loans_ml_df.head()) #查看经标准化后的数据
print(type(loans_ml_df))

############################       过采样处理数据并运用逻辑斯谛回归模型           ##############################################
# 构建自变量和因变量
X = loans_ml_df.drop(['acc_now_delinq'],axis=1)
y = loans_ml_df['acc_now_delinq']
n_sample = y.shape[0]
n_pos_sample = y[y == 0].shape[0] #正样本
n_neg_sample = y[y == 1].shape[0] #负样本
#样本个数：709903; 正样本占99.54%; 负样本占0.46%

from imblearn.over_sampling import SMOTE # 导入SMOTE算法模块
# 处理不平衡数据
sm = SMOTE(random_state=42)    # 处理过采样的方法
X, y = sm.fit_sample(X, y)
n_sample = y.shape[0]
n_pos_sample = y[y == 0].shape[0]
n_neg_sample = y[y == 1].shape[0]
#样本个数：1413220; 正样本占50.00%; 负样本占50.00%
########################   XGB    #################
import xgboost as xgb
from sklearn.cross_validation import train_test_split
train_x, test_x, train_y, test_y=train_test_split(X,y,random_state=0)

dtrain=xgb.DMatrix(train_x,label=train_y)
dtest=xgb.DMatrix(test_x)

params={'booster':'gbtree',
    #'objective': 'reg:linear',
    'objective': 'binary:logistic',
    'eval_metric': 'auc',
    'max_depth':4,
    'lambda':10,
    'subsample':0.75,
    'colsample_bytree':0.75,
    'min_child_weight':2,
    'eta': 0.025,
    'seed':0,
    'nthread':8,
     'silent':1}

watchlist = [(dtrain,'train')]

bst=xgb.train(params,dtrain,num_boost_round=100,evals=watchlist)

ypred=bst.predict(dtest)

# 设置阈值, 输出一些评价指标
list3 = list(ypred)
for i in range(len(list3)):
    if list3[i] > 0.5:
        list3[i] = 1
    else:
        list3[i] = 0
#y_pred = (ypred >= 0.5)*1

#模型校验
from sklearn import metrics
#print ('AUC: %.4f' % metrics.roc_auc_score(test_y,list3))
#print ('ACC: %.4f' % metrics.accuracy_score(test_y,list3))
print ('Recall: %.4f' % metrics.recall_score(test_y,list3))
#print ('F1-score: %.4f' %metrics.f1_score(test_y,list3))
print ('Precesion: %.4f' %metrics.precision_score(test_y,list3))
metrics.confusion_matrix(test_y,list3)

print("xgboost:")
print("accuracy on the training subset:{:.3f}".format(bst.get_score(train_x,train_y)))
print("accuracy on the test subset:{:.3f}".format(bst.get_score(test_x,test_y)))
print('Feature importances:{}'.format(bst.get_fscore()))

'''
AUC: 0.8135
ACC: 0.7640
Recall: 0.9641
F1-score: 0.8451
Precesion: 0.7523

#特征重要性和随机森林差不多
Feature importances:{'Account Balance': 80, 'Duration of Credit (month)': 119,
 'Most valuable available asset': 54, 'Payment Status of Previous Credit': 84,
 'Value Savings/Stocks': 66, 'Age (years)': 94, 'Credit Amount': 149,
 'Type of apartment': 20, 'Instalment per cent': 37,
 'Length of current employment': 70, 'Sex & Marital Status': 29,
 'Purpose': 67, 'Occupation': 13, 'Duration in Current address': 25,
 'Telephone': 15, 'Concurrent Credits': 23, 'No of Credits at this Bank': 7,
 'Guarantors': 28, 'No of dependents': 6}
'''

'''
##################################    01构建逻辑回归分类器             #################################
from sklearn.linear_model import LogisticRegression
clf1 = LogisticRegression()
clf1.fit(X, y)
predicted1 = clf1.predict(X) # 通过分类器产生预测结果
from sklearn.metrics import accuracy_score
#print("Test set accuracy score: {:.5f}".format(accuracy_score(predicted1, y,)))
#采用classification_report快速查看混淆矩阵precision、recall、f1-score的计算值。
from sklearn.metrics import classification_report
#print(classification_report(y, predicted1))
from sklearn.metrics import roc_auc_score
roc_auc1 = roc_auc_score(y, predicted1)
#print("Area under the ROC curve : %f" % roc_auc1)

####  模型优化    cross-validation+grid search    ####
from sklearn.model_selection import GridSearchCV
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0) # random_state = 0 每次切分的数据都一样
# 构建参数组合
param_grid = {'C': [0.01,0.1, 1, 10, 100, 1000,],
                            'penalty': [ 'l1', 'l2']}
grid_search = GridSearchCV(LogisticRegression(),  param_grid, cv=10) # 确定模型LogisticRegression，和参数组合param_grid ，cv指定5折
grid_search.fit(X_train, y_train) # 使用训练集学习算法

results = pd.DataFrame(grid_search.cv_results_)
best = np.argmax(results.mean_test_score.values)
print("Best parameters: {}".format(grid_search.best_params_))
print("Best cross-validation score: {:.5f}".format(grid_search.best_score_))
scores = np.array(results.mean_test_score).reshape(2, 6)
sns.heatmap(scores, ylabel='penalty', yticklabels=param_grid['penalty'],
                      xlabel='C', xticklabels=param_grid['C'], cmap="viridis")

print("Best estimator:\n{}".format(grid_search.best_estimator_))#grid_search.best_estimator_ 返回模型以及他的所有参数（包含最优参数）

y_pred = grid_search.predict(X_test)
print("Test set accuracy score: {:.5f}".format(accuracy_score(y_test, y_pred,)))

print(classification_report(y_test, y_pred))

roc_auc2 = roc_auc_score(y_test, y_pred)
print("Area under the ROC curve : %f" % roc_auc2)

from sklearn.externals import joblib
#lr是一个LogisticRegression模型
joblib.dump(grid_search, 'lr.model')
lr = joblib.load('lr.model')
'''
print("{}{}{}".format("一共消耗时间为",int(time.time()-start_time),"秒"))


from rest_framework import serializers

from .models import UserCollect, UserRead, UserForward, ExportTask, UserMark
from django_server.apps.user.models import Branch
from django_server.apps.manage.models import MedOrgPrincipal, MedOrgType
from django_server.utils.translate import MEDICAL_CATEGORY_LST, MEDICAL_LST
from django_server.utils.utils import list_item_same
# from django_server.apps.account.models import Module

class CollectListSerializer(serializers.ModelSerializer):
    name = serializers.SerializerMethodField()
    marks = serializers.SerializerMethodField()
    related_orgs = serializers.SerializerMethodField()
    last_status = serializers.SerializerMethodField()
    product_tags = serializers.SerializerMethodField()
    is_self_product = serializers.SerializerMethodField()

    class Meta:
        model = UserCollect
        fields = ('id', 'uuid', 'title', 'branch', 'released_at', 'created_at', 'name', 'marks',
                  'state', 'extra', 'last_status', 'update_at', 'info_type', 'related_orgs',
                  'is_self_product', 'product_tags')

    def get_name(self, obj):
        try:
            return Branch.objects.get(ename=obj.branch).name
            # return Module.objects.get(ename=obj.module).name
        # except Module.DoesNotExist:
        except Branch.DoesNotExist:
            return ''

    def get_marks(self, obj):
        return {'collect': obj.uuid}
        # marks = {'collect': obj.uuid}
        # try:
        #     user = self.context['request'].user.user_extra
        # except:
        #     return marks
        # if UserRead.objects.filter(user=user, uuid=obj.uuid).exists():
        #     marks['read'] = obj.uuid
        # return marks

    def get_crm_code(self, obj):
        try:
            user = self.context['request'].user.user_extra
        except:
            return ''
        try:
            return UserMark.objects.filter(user=user, uuid=obj.uuid, module=obj.module)[0].crm_code_number
        except:
            return ''

    # todo
    def get_related_orgs(self, obj):
        payload = []
        if obj.extra.get('related_orgs'):
            related_orgs = obj.extra.get('related_orgs')
            user_id = self.context['request'].user.id
            for item in related_orgs:
                if item['org_role']['value'] in ('ER0001', 'ER0002'):
                    org_uuid = item['org_uuid']
                    if MedOrgPrincipal.objects.filter(user_id=user_id, org_uuid=org_uuid).exists():
                        item.update({'state':  '我负责的'})
                    elif MedOrgType.objects.filter(user_id=user_id, org_uuid=org_uuid).exists():
                        item.update({'state': '我关注的'})
                    else:
                        item.update({'state': None})
                    payload.append(item)
        return payload

    def get_last_status(self, obj):
        if obj.last_status == '0':
            return '我：收藏成功'
        elif obj.last_status == '1':
            status = '我：更新状态为' + ('新收藏' if obj.state == '0' else ('跟进中' if obj.state == '1' else ('已签约' if obj.state == '2' else ('储备库' if obj.state == '3' else ()))))
            return status
        elif obj.last_status == '2':
            try:
                contents = UserMark.objects.get(uuid=obj.uuid, user=self.context['request'].user.user_extra).contents
                status = '我：编辑标注为' + contents[0]['content']
                return status
            except:
                return '我：编辑标注'
        else:
            return '中标公告'

    def get_product_tags(self, obj):
        product_tags = []
        product_tags2 = []
        # 获取departments
        departments = self.context['request'].user.user_extra.departments.values('base_range_product',
                                                                                'info_range_product', )
        products_lst = []
        for department in departments:
            products = department['info_range_product']
            products = department['base_range_product'] if not products else products
            if list_item_same(products, MEDICAL_LST):
                products_lst = []
            else:
                products_lst.extend(products)

        for tag in obj.extra['custom_tags']:
            if tag not in MEDICAL_CATEGORY_LST + MEDICAL_LST:
                continue
            elif tag in products_lst or tag[:4] in products_lst:
                product_tags.append({"product_tag": tag, "related": True})
            else:
                product_tags2.append({"product_tag": tag, "related": False})
        # 排序
        product_tags.extend(product_tags2)
        return product_tags

    def get_is_self_product(self, obj):
        try:
            self_products = self.context['request'].user.tenant.products_categories[0]['keywords']
            departments = self.context['request'].user.user_extra.departments.values('base_range_product',
                                                                                'info_range_product', )
            products_lst = []
            for department in departments:
                products = department['info_range_product']
                products = department['base_range_product'] if not products else products
                if list_item_same(products, MEDICAL_LST):
                    products_lst = []
                else:
                    products_lst.extend(products)
            self_products = list(set(self_products).intersection(set(products_lst)))
            custom_tags = obj.extra['custom_tags']
            union = set(self_products).intersection(set(custom_tags))
            if union:
                return True
            return False
        except Exception as e:
            return False

class MarkInfoSerializer(serializers.Serializer):
    contacts = serializers.JSONField()
    contents = serializers.JSONField()

    class Meta:
        fields = ('contacts', 'contents')

class ExportTaskListSerializer(serializers.ModelSerializer):
    class Meta:
        model = ExportTask
        fields = ('id', 'name', 'download_url', 'state', 'completed', 'created')

class ForwardListSerializer(serializers.ModelSerializer):
    name = serializers.SerializerMethodField()
    readed_count = serializers.SerializerMethodField()

    # location = serializers.SerializerMethodField()
    # custom_tags = serializers.SerializerMethodField()

    class Meta:
        model = UserForward
        fields = ('branch', 'module', 'uuid', 'title', 'released_at',
                  'created_at', 'name', 'readed_count', 'id', 'industry_tags', 'extra')

    def get_name(self, obj):
        try:
            return Branch.objects.get(ename=obj.branch).name
        except:
            return ''

    def get_readed_count(self, obj):
        readed_count = 0
        for user in obj.to_users:
            if UserRead.objects.filter(user_id=user, uuid=obj.uuid).exists():
                readed_count += 1
        return readed_count

        # def get_location(self, obj):
        #     resource = BulletinResource()
        #     fetch = resource.fetch()
        #     fetch.params.update({
        #         'uuid__in': obj.uuid
        #     })
        #     for result in fetch:
        #         return getattr(result, 'location')

        # def get_custom_tags(self, obj):
        #     resource = BulletinResource()
        #     fetch = resource.fetch()
        #     fetch.params.update({
        #         'uuid__in': obj.uuid
        #     })
        #     for result in fetch:
        #         return getattr(result, 'custom_tags')


# !/usr/bin/env python3
# -*- coding: utf-8 -*-

# @Time    : 2019/11/4 22:43
# @File    : 尼泊尔hamropatro文学博客.py
# @Date    : 2019/11/4
# @Author  : Yuwenjun
# @Desc    : https://www.hamropatro.com/posts

import requests
from lxml import etree
from queue import Queue
from multiprocessing.dummy import Pool
import re
from retrying import retry
import time

letter_regex = re.compile(r'[a-zA-z]')
special_symbol = re.compile(r"[#',\\/<=>，。★、【】《》？“”‘’！[\]_`{|\u4e00-\u9fa5}~]+")
proxies = {
    'https': '183.154.52.85:9999',
    'https': '36.25.40.241:9999'
}

class NewsSpider:
    def __init__(self):
        self.url_temp = "http://www.hamropatro.com/posts/"
        self.url_temp_header = "https://www.hamropatro.com/posts/"
        self.host_header = "https://www.hamropatro.com/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, "
                          "like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
            'Referer': 'https://www.hamropatro.com/',
            # 'Connection': 'close'
        }
        self.queue = Queue()
        self.pool = Pool(5)
        self.cookies = {"security_session_mid_verify": "e63dce6af1e6dee46a3a145b79f7557d"}
        self.is_running = True
        self.total_requests_num = 0
        self.total_response_num = 0

    def parse_url_list(self, html):
        html = etree.HTML(html)
        url_list = html.xpath("//div[@class='BlogcategoriesMobile']/select/option/@value")
        return url_list

    def parse_url_detail_list(self, html):
        html = etree.HTML(html)
        url_list = html.xpath("//div[@class='column8']/div/div/div/div[@class='blogTitle']/a/@href")
        return url_list

    def get_url_list(self):  # 获取url列表
        try:
            category_html = self.parse_url(self.url_temp)
            category_url_list = self.parse_url_list(category_html)
            print(category_url_list)
            for url in category_url_list:
                detail_html = self.parse_url(self.url_temp_header + url)
                detail_url_list = self.parse_url_detail_list(detail_html)
                print(detail_url_list)
                for url2 in detail_url_list:
                    print(url2)
                    self.queue.put(self.host_header + url2)
                    self.total_requests_num += 1
        except Exception as e:
            print(e)

    @retry(stop_max_attempt_number=3)
    def parse_url(self, url):  # 发送请求，获取响应
        response = requests.get(url, headers=self.headers, timeout=0.5)
        # session = requests.session()
        # response = session.get(url, headers=self.headers, timeout=2, proxies=proxies)
        return response.content.decode()

    def split_content(self, content):
        content_list = list()
        e = 0
        for k, v in enumerate(content):
            if k == 0:
                if v == ".":
                    e = 1
            elif k < len(content) - 1:
                if v == ".":
                    if content[k - 1] != "." and content[k + 1] != ".":
                        if content[k - 1].isdecimal() and content[k + 1].isdecimal():
                            continue
                        if content[k - 1].isalpha() and content[k + 1].isalpha():
                            continue
                        content_list.append(content[e:k].strip())
                        e = k + 1
                if v == "?":
                    if content[k - 1] != "?" and content[k + 1] != "?" and content[k + 1] != ")":
                        content_list.append(content[e:k + 1].strip())
                        e = k + 1
                if v == "|":
                    content_list.append(content[e:k + 1].strip())
                    e = k + 1
                if v == "!":
                    content_list.append(content[e:k + 1].strip())
                    e = k + 1
            else:
                content_list.append(content[e:].strip())
        return content_list

    def get_content_list(self, html_str):  # 提取段子
        html = etree.HTML(html_str)
        contents = html.xpath("//p/text()")
        content_list = []
        print(contents)
        for content in contents:
            if letter_regex.findall(content):
                continue
            if content.strip():
                content_merge_list = self.split_content(content.replace(u'\xa0', u' ').replace("<br>", ""))
                content_list.append(content_merge_list)
        return content_list

    def save_content_list(self, content_list):  # 保存数据
        with open('尼泊尔语hamropatro文学博客.txt', 'a', encoding='utf-8') as f:
            for contents in content_list:
                for content in contents:
                    if len(content) < 5:
                        continue
                    f.write(content + '\n')

    def exetute_requests_item_save(self):
        url = self.queue.get()
        print(url)
        html_str = self.parse_url(url)
        content_list = self.get_content_list(html_str)
        self.save_content_list(content_list)
        self.total_response_num += 1

    def _callback(self, temp):
        if self.is_running:
            self.pool.apply_async(self.exetute_requests_item_save, callback=self._callback)

    def run(self):
        self.get_url_list()

        for i in range(2):  # 控制并发
            self.pool.apply_async(self.exetute_requests_item_save, callback=self._callback)

        while True:  # 防止主线程结束
            time.sleep(0.0001)  # 避免cpu空转，浪费资源
            # print("总任务数%s个,已完成任务%s个" % (self.total_requests_num, self.total_response_num))
            if self.total_response_num >= self.total_requests_num:
                self.is_running = False
                break

        self.pool.close()  # 关闭线程池，防止新的线程开启
        # self.pool.join()  # 等待所有的子线程结束

if __name__ == '__main__':
    news = NewsSpider()
    news.run()


#!/usr/bin/env python

# Ahsen Uppal
#
# CS 6421 - Simple Python based discovery server, including path find.
# Uses a dictionary dispatch table to process simple get
# and set commands.
#
# For path-finding, it builds an internal graph representation of the
# conversion server network, and processes conversion requests by
# doing a shortest-path traversal through the conversion network.
#

import socket, sys
import traceback
import numpy as np
from collections import defaultdict

BUFFER_SIZE = 1024

# Key-Value stores
# Tricky data structure for unit_to_server, a dict of dicts
unit_to_server = defaultdict(dict)
server_to_unit = {}

# Process an bidirectional add conversion server command
def cmd_add(tokens):
    if len(tokens) != 5:
        return 'Failure invalid command. Expected: add u1 u2 host port.\n'
    u1,u2,host,port = tokens[1:]
    if (host,port) in unit_to_server[(u1,u2)]:
        return 'Failure entry exists.\n'
    if (host,port) in unit_to_server[(u2,u1)]:
        return 'Failure entry exists.\n'
    unit_to_server[(u1,u2)][(host,port)] = 1
    unit_to_server[(u2,u1)][(host,port)] = 1
    server_to_unit[(host,port)] = (u1,u2)
    return 'Success\n'

# Process an unidirectional add conversion server command
def cmd_adduni(tokens):
    if len(tokens) != 5:
        return 'Failure invalid command. Expected: add u1 u2 host port.\n'
    u1,u2,host,port = tokens[1:]
    if (host,port) in unit_to_server[(u1,u2)]:
        return 'Failure entry exists.\n'
    unit_to_server[(u1,u2)][(host,port)] = 1
    server_to_unit[(host,port)] = (u1,u2)
    return 'Success\n'

# Process a remove conversion server command
def cmd_remove(tokens):
    if len(tokens) != 3:
        return 'Failure invalid command. Expected: remove host port.\n'
    host,port = tokens[1:]
    if not (host,port) in server_to_unit:
        return 'Failure entry does not exist.\n'

    u1,u2 = server_to_unit[(host,port)]
    del server_to_unit[(host,port)]

    if (u1,u2) in unit_to_server:
        del unit_to_server[(u1,u2)][(host,port)]
        if len(unit_to_server[(u1,u2)]) == 0:
            del unit_to_server[(u1,u2)]

    if (u2,u1) in unit_to_server:
        del unit_to_server[(u2,u1)][(host,port)]
        if len(unit_to_server[(u2,u1)]) == 0:
            del unit_to_server[(u2,u1)]

    return 'Success\n'

# Process a lookup a a single conversion server from u1 to u2
def cmd_lookup(tokens):
    if len(tokens) != 3:
        return 'Failure invalid command. Expected: lookup u1 u2.\n'
    u1,u2 = tokens[1:]
    if not (u1,u2) in unit_to_server:
        return 'None.\n'
    L = [i for i in unit_to_server[(u1,u2)].items()]
    host,port = L[0][0]
    msg = '%s %s\n' % (host,port)
    return msg

# A list of all available units, and conversions to and from unit ids
units = ('ft', 'in', 'cm', 'm', 'kg', 'g', 'lbs', 'b', '$', 'y')
n_units = len(units)
unit_to_id = {}
id_to_unit = {}
for i,e in enumerate(units):
    unit_to_id[e] = i
    id_to_unit[i] = e

MAX_DIST = 99

# Find paths using the Floyd-Warshall
# algorithm for all-pairs shortest distances.
def compute_distances(edges, n_units):
    dist = np.zeros((n_units, n_units), dtype=int)
    next = np.empty((n_units, n_units), dtype=int)
    next.fill(-1)

    for i in range(n_units):
        for j in range(n_units):
            dist[i][j] = MAX_DIST

    for i in range(n_units):
        dist[i][i] = 0

    for i in range(n_units):
        for j in range(n_units):
            if edges[i][j] > 0:
                dist[i][j] = edges[i][j]
                next[i][j] = j

    for k in range(n_units):
        for i in range(n_units):
            for j in range(n_units):
                if dist[i][j] > dist[i][k] + dist[k][j]:
                    dist[i][j] = dist[i][k] + dist[k][j]
                    next[i][j] = next[i][k]

    return (next, dist)

# Path reconstruction from next-neighbor matrix.
def get_path(src, dst):
    print("Get path %s %s\n" % (src, dst))

    edges = np.zeros((n_units, n_units), dtype=int)
    for i in unit_to_server.items():
        u0 = i[0][0]
        u1 = i[0][1]
        edges[unit_to_id[u0]][unit_to_id[u1]] = 1

    next,dist = compute_distances(edges, n_units)

    path = []
    u = unit_to_id[src]
    v = unit_to_id[dst]

    if next[u, v] == -1:
        return path

    while u != v:
        u = next[u, v]
        path.append(u)

    return path

# Process a lookup path from u1 to u2 command
def cmd_path(tokens):
    if len(tokens) != 3:
        return 'Failure invalid command. Expected: path u1 u2.\n'
    u1,u2 = tokens[1:]

    if u1 not in unit_to_id:
        return 'Failure invalid unit1\n'

    if u2 not in unit_to_id:
        return 'Failure invalid unit2\n'

    p = get_path(u1, u2)

    if not p:
        msg = "Failure could not find a suitable input conversion server :(\n"
    else:
        msg = ""
        u = unit_to_id[u1]
        for i in p:
            v = i
            s,d = id_to_unit[u],id_to_unit[v]
            # Tricky dereference: lookup in a dictionary, get a dictionary, get the first entry, and its key.
            host,port = unit_to_server[(s,d)].items()[0][0]
            msg += ('Query %s %s to server at %s %s\n' % (s, d, host, port))
            u = v
    return msg

# Dict for command dispatch.
commands = { "add" : cmd_add,
             "adduni" : cmd_adduni,
             "remove" : cmd_remove,
             "lookup" : cmd_lookup,
             "path" : cmd_path,
             }

# The standard recv function is not neccessarily line-delimited.
def recv_lines(conn):
    if not conn in recv_lines.keep_line:
        recv_lines.keep_line[conn] = ''
    lines_recieved = 0
    while lines_recieved == 0:
        buf = conn.recv(BUFFER_SIZE).decode('UTF-8')
        print("conn.recv buf: %s" % buf)
        if not buf:
            yield None
        lines = buf.splitlines(True)
        for l in lines:
            if l.endswith('\n'):
                yield recv_lines.keep_line[conn] + l
                recv_lines.keep_line[conn] = ''
                lines_recieved = 1
            else:
                recv_lines.keep_line[conn] += lines[-1]
recv_lines.keep_line = {}

# Main processing function for a single connection
def process(conn):
    greeting = "Welcome, you are connected to a Python-based simple command server.\n"
    conn.sendall(greeting.encode('UTF-8'))

    for userInput in recv_lines(conn):
        if not userInput:
            print("Error reading message")
            return

        sys.stdout.write("Received message: %s\n" % userInput)
        tokens = userInput.lower().split()

        if not tokens:
            # Empty line received
            continue

        try:
            if tokens[0] in commands:
                print("Processing command.")
                response = commands[tokens[0]](tokens)
            else:
                response = ("Invalid command: %s\n" % tokens[0])

            sys.stdout.write("Sending response: %s" % response)
            conn.sendall(response.encode('UTF-8'))
        except:
            traceback.print_exc(file=sys.stdout)
            msg = ("Exception occurred: %s\n" % (userInput))
            conn.sendall(msg.encode('UTF-8'))

    return

if __name__ == '__main__':
    interface = ""

    if len(sys.argv) < 2:
        sys.stderr.write("usage: python {0} portnum\n".format(sys.argv[0]))
        sys.exit(1)

    portnum = int(sys.argv[1])
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind((interface, portnum))
    s.listen(5)
    exit_flag = False
    try:
        print("Started Python-based discovery command server on port %s" % (portnum))
        while not exit_flag:
            conn, addr = s.accept()
            print ('Accepted connection from client ', addr)
            try:
                process(conn)
            except:
                traceback.print_exc(file=sys.stdout)
                print ('Failed to process request from client. Continuing.')
            print('Closing connection.')
            conn.close()
    except KeyboardInterrupt:
        exit_flag = True

    print("Exiting...")
    s.close()
    sys.exit(0)


from tkinter import *
from tkinter import filedialog
from PIL import Image
from time import time

def ouvrir(nominage):
    """
        Ouvre l'image que l'on veut traiter.

        :param nominage: Chemin de l'image.
        :type nominage: str.
        :returns: img.
    """
    img = Image.open(nominage)
    print(img.format,img.size,img.mode)
    return img

def hexa2(n):
    L=[]
    lettres = ['A','B','C','D','E','F']
    hexarep = ""
    if n==0:
        hexarep += "0"
    while n!=0:
        if n%16 < 10:
            hexarep += str(n%16)
        else:
            hexarep += lettres[n%16-16]
        n=n//16
    return(hexarep[::-1])

def dessinerGrille(ims):
    ''' dessine la grille en traçant chaque cercle un à un '''
    cv.delete(ALL)
    pixs = ims.load()
    for j in range(height):
        for i in range(width):
            n = pixs[i,j]
            hexaval = hexa2(n)
            if len(hexaval) == 1:
                hexaval = '0' + hexaval
            couleur = '#' + str(3*(hexaval))
            dessinerCase(1,j,i,couleur)
        cv.update()

def dessinerGrilleCouleur(ims):
    cv.delete(ALL)
    pixs = ims.load()
    for j in range(height):
        for i in range(width):
            hexaval = ""
            for k in range(3):
                hexa = hexa2(pixs[i,j][k])
                if len(hexa) == 1:
                    hexaval += '0' + hexa
                else:
                    hexaval += hexa
            couleur = '#' + hexaval
            dessinerCase(1,j,i,couleur)
        cv.update()

def dessinerCase(cote,j,i,couleur):
    x0,y0 = (cote)*i,(cote)*j
    x1,y1 = x0 + cote, y0 + cote
    cv.create_rectangle(x0,y0,x1,y1,fill=couleur,outline="")

imopen = filedialog.askopenfilename(title="Ouvrir une image")

ims = ouvrir(imopen)

fen = Tk()

(width,height) = ims.size

cv = Canvas(fen,height=height,width=width,bg="white")
cv.pack()

t1 = time()
if ims.mode == 'L':
    imd = Image.new('F', ims.size)
    pixs = ims.load()
    dessinerGrille(ims)
elif ims.mode == 'RGB' or ims.mode == 'RGBA':
    imd = Image.new('RGB', ims.size)
    pixs = ims.load()
    dessinerGrilleCouleur(ims)
t2 = time()

print(round(t2-t1,3))

fen.mainloop()