import pandas as pd
import matplotlib.pyplot as plt

# Load data from your three tables: Original, Developers, and Management
original_data = pd.read_excel('Data/output/readability_metrics3Original.xlsx')  # Replace with the actual file path
developers_data = pd.read_excel('Data/output/readability_metrics3Devs.xlsx')  # Replace with the actual file path
management_data = pd.read_excel('Data/output/readability_metrics3Manager.xlsx')  # Replace with the actual file path

# Ensure that the three tables have the same structure (columns)
# You can rename columns if needed to match
original_data = original_data.rename(columns={'Project Name': 'Project', 'Type of test': 'Test', 'Average': 'Original'})
developers_data = developers_data.rename(columns={'Project Name': 'Project', 'Type of test': 'Test', 'Average': 'Developers'})
management_data = management_data.rename(columns={'Project Name': 'Project', 'Type of test': 'Test', 'Average': 'Management'})

# Merge the data from the three tables using 'Project' and 'Test' columns as keys
merged_data = pd.merge(original_data, developers_data, on=['Project', 'Test'])
merged_data = pd.merge(merged_data, management_data, on=['Project', 'Test'])

# Create a comparison plot
plt.figure(figsize=(10, 6))

# Customize plot style for a research paper
plt.style.use('seaborn-whitegrid')

# Sort the data for better visualization
merged_data = merged_data.sort_values(by=['Original'])

# Plot the data
plt.plot(merged_data['Project'], merged_data['Original'], marker='o', label='Original', linestyle='-', color='b', linewidth=2)
plt.plot(merged_data['Project'], merged_data['Developers'], marker='s', label='Developers', linestyle='--', color='g', linewidth=2)
plt.plot(merged_data['Project'], merged_data['Management'], marker='^', label='Management', linestyle='-.', color='r', linewidth=2)

# Add labels and title
plt.xlabel('Project')
plt.ylabel('Average Value')
plt.title('Comparison of Developers and Management to Original Data')
plt.xticks(rotation=45)
plt.legend()

# Save the plot as a high-resolution image suitable for a research paper
plt.savefig('comparison_plot.png', dpi=300, bbox_inches='tight')

# Show the plot
plt.show()


import streamlit as st
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn import preprocessing
import tezhenggx as tz

class Act_fun(nn.Module):
    def __init__(self):
        super(Act_fun, self).__init__()

    def forward(self, x):
        x = torch.sigmoid(x)
        x = x * 100
        return x

st.title('demo')
add_selectbox = st.sidebar.selectbox(
    "How would you like to be contacted?",
    ("Email", "Home phone", "Mobile phone")
)

# 载入数据
@st.cache
def load():
    X_train = pd.read_csv(r'C:\Users\zhao\Desktop\streamlit_webapp\X_train_sc_st_pca.csv')
    X_test = pd.read_csv(r'C:\Users\zhao\Desktop\streamlit_webapp\X_test_sc_st_pca.csv')
    y_train = pd.read_csv(r'C:\Users\zhao\Desktop\streamlit_webapp\train_y.csv')
    y_test = pd.read_csv(r'C:\Users\zhao\Desktop\streamlit_webapp\test_y.csv')
    test_data = pd.read_csv(r'D:\jupyter_project\大创\test_data.csv')
    train_data = pd.read_csv(r'D:\jupyter_project\大创\train_data.csv')
    X_train = np.array(X_train)
    X_test=np.array(X_test)
    y_train=np.array(y_train)
    y_test=np.array(y_test)
    return X_train,X_test,y_train,y_test,train_data,test_data

data_load_state = st.text('Loading data...')
X_train,X_test,y_train,y_test,train_data,test_data = load()
data_load_state.text('Loading data...done!')
if st.checkbox('Show raw data'):
    st.subheader('Raw data')
    st.write(train_data)
lr_to_filter = st.slider('学习率X10^5', 1,50,10)
num_to_filter = st.slider('学习次数', 1,5000,1000)
#模型
input_size = X_train.shape[1]
hidden_size1 = 100
output_size = 1
batch_size = 10
# 框架

my_nn = torch.nn.Sequential(
    torch.nn.Linear(input_size, hidden_size1),
    torch.nn.ReLU(),
    torch.nn.Linear(hidden_size1, output_size),
    Act_fun(),
)
cost = torch.nn.MSELoss(reduction='mean')
optimizer = torch.optim.Adam(my_nn.parameters(), lr=lr_to_filter*10**-4)
for i in range(num_to_filter):
    batch_loss = []
    # MINI-Batch方法来进行训练
    for start in range(0, len(X_train), batch_size):
        end = start + batch_size if start + batch_size < len(X_train) else len(X_train)
        xx = torch.tensor(X_train[start:end], dtype=torch.float, requires_grad=True)
        yy = torch.tensor(y_train[start:end], dtype=torch.float, requires_grad=True)
        prediction = my_nn(xx)
        # 降为一维，否则loss会告警
        prediction = prediction.squeeze(-1)
        loss = cost(prediction, yy)
        optimizer.zero_grad()
        loss.backward(retain_graph=True)
        optimizer.step()
        batch_loss.append(loss.data.numpy())

    # 打印损失
    if i % 100 == 0:
        #losses.append(np.mean(batch_loss))
        st.write('第',i,'次训练', '均方误差为:',np.mean(batch_loss))


# -*- coding: utf-8 -*-
import numpy as np
import cupy as cp
#from numba import jit
import time
import pandas as pd
import random
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
import copy
import os
os.chdir('D:/R.data/DSL_paper') #设定路径

#----时间纪录(开始)----
startTime = time.time()

#----资料初始化(本地端)----
sourceData = pd.read_csv('assets_cupy/StoreData.csv') #读取原始资料
preferenceTable = pd.read_csv('assets_cupy/preferenceTable.csv') #读取商品偏好表
sourceData.drop(['产品代号', '品名','输入格式(序,产品代码+品名,单价,体积,种类)'], axis = 1, inplace=True) #移除不必要的资料字段
sourceData.rename(index=str, columns={"重量(g)": "重量"}, inplace=True) #重新命名字段名称
goodData = copy.copy(sourceData) #将原始资料复制一份
sourceData = sourceData.values

goodData['Selected'], goodData['Preference'] = 0, 1 #新增被选择字段
#goodData = goodData.values #转换成ndarry型态
#preferenceTable = preferenceTable.values

#----环境参数设定----
maxVolume = 47*32*39 #最大箱子体积
maxWeight = 16000 #最大重量(g)
popAmount = 30 #人口数量
crossRate = 1 #交配率
mutationRate = 0.001#突变率
eliteValues = round(popAmount*0.1) #菁英数量
maxGen = 100 #世代次数

#----使用者需输入的参数(假设)----
dietHabit = '荤食' #荤食与素食的选择
userItemValues = 22 #使用者需要的数量
maxPrice = 1500 #使用者金额
exceptBrandList = np.array(['大同']) #将要剔除的品牌

#----Function----
#饮食习惯(荤或素食):
#在开始算法前先将该饮食习惯给加入
#若为荤食则包含素食和荤食, 反之只有素食
def diet_select(good_data, diet_habit_list):
    #good_data: 原始商品资料集
    #diet_habit_list: 荤食或素食的选择
    if diet_habit_list=='素食':
        good_data = good_data[good_data.iloc[:, 8] == diet_habit_list]
    return(good_data)

#剔除品牌的方法:
#在开始算法前先将该品牌给移除
def except_brand(good_data, except_brand_list):
    #good_data: 原始商品资料集
    #except_brand_list: 剔除品牌的名称
    for i in range(len(except_brand_list)):
        good_data = good_data[good_data.iloc[:,3] != exceptBrandList[i]] #将要剔除的厂牌移除
    return(good_data)

#偏好值与类别合并:
#将使用者对商品种类的偏好与原始商品资料进行合并成一个Data Frame, 使原始资料有使用者对每个商品的品项偏好
def preference_match(good_data, preference_table):
    #gene_list: 被选择出的基因清单
    #require_goods: 必要性的商品清单
    #non_require_goods: 不必要性的商品清单
    #user_preference: 使用者对商品种类的偏好
    for i in range(len(preference_table)):
        temp = good_data[good_data.iloc[:, 7] == preference_table.iloc[:,0][i]]
        temp.loc[:, 'Preference'] = preference_table.iloc[:,1][i]
        good_data[good_data.iloc[:, 7] == preference_table.iloc[:,0][i]] = temp
    return(good_data)

#初始族群
def initial_pop(good_data, require_goods, non_require_goods, non_require_values, limit_weight):
    #good_data: 原始商品资料集
    #require_goods: 必要性的商品清单
    #non_require_goods: 不必要性的商品清单
    #non_require_values: 不必要性的商品数量
    #limit_weight: 最大重量限制
    while True:
        temp_good = copy.copy(good_data) #先将原始资料暂时给另外一个变数使用
        for i in range(len(require_goods)):
            get_index = temp_good[(temp_good[:,6] == require_goods[i]) & (temp_good[:, 8] !=1)] #取得符合条件的资料
            temp_index = random.randint(0, (len(get_index)-1)) #随机取得品项
            get_index = get_index[temp_index] #取得商品名称
            get_index[8] = 1
            temp_good[temp_good[:,0] == get_index[0]] = get_index

        selected_require = np.random.choice(non_require_goods, non_require_values, replace=False)

        for i in range(len(selected_require)):
            get_index = temp_good[(temp_good[:,6] == selected_require[i]) & (temp_good[:, 8] !=1)] #取得符合条件的资料
            temp_index = random.randint(0, (len(get_index)-1)) #随机取得品项
            get_index = get_index[temp_index] #取得商品名称
            get_index[8] = 1
            temp_good[temp_good[:,0] == get_index[0]] = get_index

        selected_good = temp_good[temp_good[:, 8] == 1]
        sum_weight = np.sum(selected_good[:,7])
        if sum_weight <= limit_weight:
            break
    return(selected_good) #回传结果

#编码染色体:
#必要性商品必定方在最前段, 选择性商品必定放在后段
def create_chromosome(gene_list):
    #gene_list: 被选择出的基因清单
    for i in range(len(gene_list)):
        chromosome = (gene_list[i]['data.frame'][:,0])
        gene_list[i]['chromosome'] = chromosome
    return(gene_list)

#计算总重量
def total_weight(gene_list):
    #gene_list: 被选择出的基因清单
    for i in range(len(gene_list)):
        sum_weight = cp.sum(gene_list[i]['data.frame'][:,8])
        gene_list[i]['totalWeight'] = cp.array([sum_weight])
    return(gene_list)

#偏好的适应度方法(算式分母为偏好值1~偏好的最大值)
def fitness_preference(gene_list, require_goods, non_require_values, preference_table):
    #gene_list: 被选择出的基因清单
    #require_goods: 必要性的商品清单
    #non_require_goods: 不必要性的商品清单
    #user_preference: 使用者对商品种类的偏好

    max_preference = np.max(preference_table[:,1])
    total_preference = 0
    for i in range(0, max_preference+1, 1):
        total_preference = total_preference + i**2

    for i in range(len(gene_list)):
        reuslt = 1
        for j in range((len(require_goods) + non_require_values)):
            temp_preferenced = 1+(((gene_list[i]['data.frame'][:,9][j]**2)-1) / total_preference)
            reuslt *=temp_preferenced
            #temp.append(reuslt)

        gene_list[i]['fitPreference'] = reuslt
        sum_preferenced = (gene_list[i]['data.frame'][:,9]).sum()
        gene_list[i]['totalPreference'] = sum_preferenced
    return(gene_list)

#----执行----
goodData = diet_select(good_data = goodData, diet_habit_list = dietHabit) #荤素的方法
goodData = except_brand(good_data = goodData, except_brand_list = exceptBrandList) #剔除掉不想要的品牌

level = preferenceTable.iloc[:,0] #取得所有种类
requiredList = level[0:6] #必要性商品种类
nonRequiredList = (level[len(requiredList):len(level)]) #选择性商品种类
nonRequiredValues = userItemValues-len(requiredList) #选择性商品的数量

goodData = preference_match(good_data = goodData, preference_table = preferenceTable) #必要性商品种类
goodData.drop(['厂牌', '荤素'], axis = 1, inplace=True) #移除不必要的资料字段
goodData = goodData.values #转换成ndarry型态
preferenceTable = preferenceTable .values

##基因算法开始
#产生初始口(遵照popAmount数量)
geneList = []
for i in range(popAmount):
    geneDict = {'data.frame': initial_pop(good_data = goodData, require_goods = requiredList, non_require_goods = nonRequiredList, non_require_values = nonRequiredValues, limit_weight = maxWeight)}
    geneList.append(geneDict)

#编码染色体
geneList = create_chromosome(gene_list = geneList)

#计算每个基因总重量
geneList = total_weight(gene_list = geneList)

#计算偏好适应度(目前仅计算总偏好值)
fitnessPreference = fitness_preference(gene_list = geneList, require_goods = requiredList, non_require_values =  nonRequiredValues, preference_table = preferenceTable)


# -*- coding:utf-8
import os
import stat
import magic
import codecs
import re
import logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

def is_binary_file_1(file_path):
    '''
    根据text文件数据类型判断是否是二进制文件
    :param file_path: 文件名（含路径）
    :return: True或False，返回是否是二进制文件
    '''
    TEXT_BOMS = (
        codecs.BOM_UTF16_BE,
        codecs.BOM_UTF16_LE,
        codecs.BOM_UTF32_BE,
        codecs.BOM_UTF32_LE,
        codecs.BOM_UTF8,
    )
    with open(file_path, 'rb') as file:
        CHUNKSIZE = 8192
        initial_bytes = file.read(CHUNKSIZE)
        file.close()
    #: BOMs to indicate that a file is a text file even if it contains zero bytes.
    return not any(initial_bytes.startswith(bom) for bom in TEXT_BOMS) and b'\0' in initial_bytes

def is_binary_file_2(ff):
    '''
    根据magic文件的魔术判断是否是二进制文件
    :param ff: 文件名（含路径）
    :return: True或False，返回是否是二进制文件
    '''
    mime_kw = 'x-executable|x-sharedlib|octet-stream|x-object'  ###可执行文件、链接库、动态流、对象
    try:
        magic_mime = magic.from_file(ff, mime=True)
        magic_hit = re.search(mime_kw, magic_mime, re.I)
        if magic_hit:
            return True
        else:
            return False
    except Exception as e:
        return False

# 判断文件是否是elf文件
def is_ELFfile(filepath):
    if not os.path.exists(filepath):
        # logger.info('file path {} doesnot exits'.format(filepath))
        return False
    # 文件可能被损坏，捕捉异常
    try:
        FileStates = os.stat(filepath)
        FileMode = FileStates[stat.ST_MODE]
        if not stat.S_ISREG(FileMode) or stat.S_ISLNK(FileMode):  # 如果文件既不是普通文件也不是链接文件
            return False
        with open(filepath, 'rb') as f:
            header = (bytearray(f.read(4))[1:4]).decode(encoding="utf-8")
            # logger.info("header is {}".format(header))
            if header in ["ELF"]:
                # print header
                return True
    except UnicodeDecodeError as e:
        # logger.info("is_ELFfile UnicodeDecodeError {}".format(filepath))
        # logger.info(str(e))
        pass

    return False

def is_binary_file(path):
    return any((is_ELFfile(path), is_binary_file_1(path), is_binary_file_2(path)))

def count_line(path):
    sum = 0
    for root, dirs, files in os.walk(path):
        for name in files:
            tmp_file = os.path.join(root, name)
            if tmp_file.find('venv') != -1:
                continue
            if tmp_file.find('idea') != -1:
                continue
            if tmp_file.find('__pycache__') != -1:
                continue
            if tmp_file.find('.git') != -1:
                continue
            tmp = 0
            if is_binary_file(tmp_file):
                continue
            for index, line in enumerate(open(tmp_file, 'r', encoding='utf8')):
                tmp += 1
            # print(tmp_file, ":", tmp)
            sum += tmp
    return sum

if __name__ == '__main__':
    print(count_line("."))


import sys
from collections import deque

M,N,H = map(int, sys.stdin.readline().split())

li = []
que = deque()
zcnt = 0

for i in range(H):
    tmp = []
    for j in range(N):
        tmp.append(list(map(int, sys.stdin.readline().split())))
    li.append(tmp)

for k in range(H):
    for i in range(N):
        for j in range(M):
            if li[k][i][j] == 1:
                que.append([k,i,j])
            if li[k][i][j] == 0:
                zcnt += 1

def BFS():
    cnt = 0
    global zcnt
    if zcnt == 0:
        return 0
    while que:
        cnt += 1
        for i in range(len(que)):
            xyz = que.popleft()
            k = xyz[0]
            i = xyz[1]
            j = xyz[2]

            if k+1 < H and li[k+1][i][j] == 0:
                li[k+1][i][j] = 1
                zcnt -= 1
                que.append([k+1,i,j])

            if k-1 >= 0 and li[k-1][i][j] == 0:
                li[k-1][i][j] = 1
                zcnt -= 1
                que.append([k-1,i,j])

            if i+1 < N and li[k][i+1][j] == 0:
                li[k][i+1][j] = 1
                zcnt -= 1
                que.append([k,i+1,j])

            if i-1 >= 0 and li[k][i-1][j] == 0:
                li[k][i-1][j] = 1
                zcnt -= 1
                que.append([k,i-1,j])

            if j+1 < M and li[k][i][j+1] == 0:
                li[k][i][j+1] = 1
                zcnt -= 1
                que.append([k,i,j+1])

            if j-1 >= 0 and li[k][i][j-1] == 0:
                li[k][i][j-1] = 1
                zcnt -= 1
                que.append([k,i,j-1])

            if zcnt == 0:
                return cnt
    return -1
print(BFS())


from os import urandom
from typing import List, Optional
from secp256k1 import Secp256k1, Message
from secp256k1.key import SecretKey, PublicKey, Signature
from ._libsecp256k1 import ffi, lib

def sign_single(secp: Secp256k1, message: Message, secret_key: SecretKey, secret_nonce: Optional[SecretKey],
                public_nonce: Optional[PublicKey], public_key_sum: Optional[PublicKey],
                public_nonce_sum: Optional[PublicKey], extra_secret_key: Optional[SecretKey]) -> Signature:
    signature_out = ffi.new("char [64]")
    res = lib.secp256k1_aggsig_sign_single(
        secp.ctx, signature_out, bytes(message.message), bytes(secret_key.key), ffi.NULL if secret_nonce is None
        else bytes(secret_nonce.key), ffi.NULL if extra_secret_key is None else bytes(extra_secret_key.key),
        ffi.NULL if public_nonce is None else public_nonce.key, ffi.NULL if public_nonce_sum is None
        else public_nonce_sum.key, ffi.NULL if public_key_sum is None else public_key_sum.key, urandom(32)
    )
    assert res, "Unable to sign message"
    return Signature.from_bytearray(secp, bytearray(ffi.buffer(signature_out, 64)))

def verify_single(secp: Secp256k1, signature: Signature, message: Message, public_key: PublicKey,
                  public_key_sum: Optional[PublicKey], public_nonce_sum: Optional[PublicKey],
                  extra_public_key: Optional[PublicKey], partial=True) -> bool:
    res = lib.secp256k1_aggsig_verify_single(
        secp.ctx, bytes(signature.signature), bytes(message.message), ffi.NULL if public_nonce_sum is None
        else public_nonce_sum.key, public_key.key, ffi.NULL if public_key_sum is None else public_key_sum.key,
        ffi.NULL if extra_public_key is None else extra_public_key.key, 1 if partial else 0
    )
    return True if res else False

def add_single(secp: Secp256k1, signatures: List[Signature], public_nonce_sum: PublicKey) -> Signature:
    signature_out = ffi.new("char [64]")
    signature_ptrs = []
    for signature in signatures:
        signature_ptr = ffi.new("char []", bytes(signature.signature))
        signature_ptrs.append(signature_ptr)

    res = lib.secp256k1_aggsig_add_signatures_single(
        secp.ctx, signature_out, signature_ptrs, len(signatures), public_nonce_sum.key
    )
    assert res, "Unable to add signatures"
    return Signature.from_bytearray(secp, bytearray(ffi.buffer(signature_out, 64)))


import torch as torch
import copy
import os
import yaml
import numpy as np
from tqdm.autonotebook import tqdm
from .layers.coupling_cells import  AffineCoupling, PWLin, PWQuad
from .layers.layers import *
from .misc import tqdm_recycled
from statistics import mean
import time
import gc
import datetime
import math

TF_CPP_MIN_LOG_LEVEL="2"

def get_bin(x, n=0):
    """
    Get the binary representation of x.

    Parameters
    ----------
    x : int
    n : int
        Minimum number of digits. If x needs less digits in binary, the rest
        is filled with zeros.

    Returns
    -------
    list of binary digits
    """
    y=format(x, 'b').zfill(n)
    return [int(i) for i in str(y)]

def normal(x,mu, sigma, n_flow):
    return (torch.exp(-torch.sum((x-mu)**2/(2*sigma**2),-1)))/(sigma*np.sqrt((2*np.pi)**n_flow))

class ModelAPI():

    @property
    def model(self):
        if self._model is not None:
            return self._model
        else:
            raise AttributeError("No model was instantiated")

class BasicManager(ModelAPI):
    """Basic training methods"""

    def __init__(self,
                  n_flow=2,*args):

        self.n_flow = n_flow
        self._model = None
        self._inverse_model = None

        self.optimizer_object = None

    def _train_variance_forward_seq(self, f, optimizer_object, log=True, logdir=None, batch_size = 10000, epochs=10,
                                    epoch_start=0,
                               pretty_progressbar=True,
                                    save_best=True, run=None,dev=0,mini_batch_size=2000, integrate=False,
                                    preburn_time=75, kill_counter=7, impr_ratio=1e-2, loss_mode="var"):
        """Train the model using the integrand variance as loss and compute the Jacobian in the forward pass
        (fixed latent space sample mapped to a phase space sample)

        """

        dev = torch.device("cuda:"+str(dev)) if torch.cuda.is_available() else torch.device("cpu")

        if preburn_time>10:
            check_time=preburn_time
        else:
            check_time=50

        if(mini_batch_size>batch_size):
            mini_batch_size=batch_size

        n_minibatches = int(batch_size/mini_batch_size)
        batch_size=batch_size-(batch_size%mini_batch_size)

        if(run!=None and log):
            filename=logdir+"/"+str(run._id)+"/torch"
            filename2=filename+"_int"
            if not os.path.exists(logdir+"/"+str(run._id)):
                os.mkdir(logdir+"/"+str(run._id))

        else:
            filename=logdir+"/torch"
            filename2=filename+"_int"
            if not os.path.exists(logdir):
                os.makedirs(logdir)

        try:
            if(log):
                file= open(filename,"w+")
                file2=open(filename2,"w+")
                file2.close()
                file.close()
                torch.save({
                    'model_state_dict': self.best_model.state_dict()
                    },filename2)

        except:
            print("Torch save not possible")

        integ=torch.zeros((epochs+1,),device=dev)
        err=torch.zeros((epochs+1,),device=dev)

        # Instantiate a pretty progress bar if needed
        if pretty_progressbar:
            epoch_progress = tqdm(range(epoch_start,epoch_start+epochs), leave=False, desc="Loss: {0:.3e} | Epoch".format(0.))
            # Instantiate a pretty progress bar for the minibatch loop if it is not trivial
            if n_minibatches>1:
                minibatch_progress = tqdm_recycled(range(n_minibatches), leave=False, desc="Step")
            else:
                minibatch_progress = range(n_minibatches)

        else:
            epoch_progress = range(epoch_start, epoch_start+epochs)
            minibatch_progress = range(n_minibatches)

        i=0
        self.model.to(dev)
        self.best_loss=0

        self.best_var=0
        maxf=0

        while (i<self.n_flow):
            w = torch.empty(2*mini_batch_size, self.n_flow).to(torch.double)

            torch.nn.init.uniform_(w).to(dev)  # Generate a batch of points in latent space

            dkl=torch.nn.KLDivLoss(reduction='batchmean')
            fres=f(w)
            integ[0]+=torch.sum(fres)/(self.n_flow*2*mini_batch_size)
            err[0]+=torch.var(fres)/(self.n_flow)
            if torch.max(fres)>maxf:
                maxf=torch.max(fres)

            if(loss_mode=="var"):
                self.best_loss+=torch.var(fres/maxf).detach()/self.n_flow
            elif(loss_mode=="est" ):
                self.best_loss+=torch.mean((fres)**2).detach()/self.n_flow
            else:
                print("Unknown loss function")
                return

            i=i+1

            self.best_var+=float((torch.var((fres/maxf)**2)/2*mini_batch_size).detach())

        if save_best or log:

            XJ = self.model(  # Pass through the model
                self.format_input(  # Append a unit Jacobian to each point
                    w,dev
                )
              )
            X=XJ[:,:-1]
            J = XJ[:,-1]
            self.varJ=torch.mean(J**2).detach()

            self.DKL=dkl(torch.log(X+torch.ones_like(X).fill_(1e-45)).to(dev),w.to(dev)).detach()

            self.best_model=copy.deepcopy(self.model)

            self.best_epoch=0
            self.best_time=0
            self.best_loss_rel=torch.ones_like(self.best_loss)
            self.best_func_count=2*mini_batch_size*self.n_flow/(batch_size/mini_batch_size)
            self.best_func_count=2*batch_size*self.n_flow
            self.history=[]

            del XJ,X,J
        if(run!=None and log):
            run.log_scalar("training.int_loss", self.best_loss.tolist(), 0)

        self.int_loss=self.best_loss

        stale_save=1000

        preburner=(preburn_time>0)

        counter=0
        last_loss=1000

        for i in epoch_progress:

            loss = 0

            var = 0
            optimizer_object.zero_grad()

            for j in minibatch_progress:

                w = torch.empty(mini_batch_size, self.n_flow).to(dev).to(torch.double)
                torch.nn.init.uniform_(w)

                XJ = self.model(                                            # Pass through the model
                    self.format_input(                                      # Append a unit Jacobian to each point
                        w,dev
                    )
                )

                # Separate the points and their Jacobians:
                # This sample is fixed, we optimize the Jacobian
                X = (XJ[:, :-1]).detach()

                if(preburner):
                    fres=f(w)
                    fXJ=torch.mul(fres, XJ[:, -1])/maxf

                    integ[i+1]+=torch.mean(fres)/n_minibatches
                    err[i+1]+=torch.var(fres)/n_minibatches

                else:
                    fres=torch.mul(f(X), XJ[:, -1])
                    fXJ = fres/maxf

                    integ[i+1]+=torch.mean(fres.detach())/n_minibatches
                    err[i+1]+=torch.var(fres.detach())/n_minibatches

                if(loss_mode=="var"):
                    loss+=torch.var(fXJ)

                elif(loss_mode=="est"):
                    loss+=torch.mean((fXJ*maxf)**2)

                # The Monte Carlo integrand is fXJ: we minimize its variance up to the constant term
                #in newer versions, the variance estimator is used in order to increase
                #stability for flat distributions

                var+=float(torch.var(fXJ**2)/mini_batch_size) # variance of the mean is variance/N
                del X, fXJ, XJ
                gc.collect()

            loss=loss/n_minibatches

            loss.backward()

            optimizer_object.step()

            self.history.append(loss.detach())
            # Update the progress bar
            if pretty_progressbar:
                epoch_progress.set_description("Loss: {0:.3e} | Epoch".format(loss))
            if(run!=None and log):
                run.log_scalar("training.loss", loss.tolist(), i)
                run.log_scalar("training.loss_rel",(loss/self.int_loss).tolist(),i)

            if (save_best or log):
                self.best_func_count=self.best_func_count+batch_size
            if (save_best or log) and (loss < self.best_loss) and not preburner:
                self.best_loss = loss
                self.best_var=var
                self.best_loss_rel=loss/self.int_loss
                self.best_model=copy.deepcopy(self.model)
                self.best_epoch=i

                if(run!=None):
                    self.best_time=(datetime.datetime.utcnow()-run.start_time).total_seconds()
                else:
                    self.best_time=0

            if loss<last_loss:
                counter=0
            else:
                counter+=1
                if counter>kill_counter and preburner:
                    counter=0
                    preburner=False
                elif counter>kill_counter:
                    break
            last_loss=loss
            if ( i%check_time==0) and i>(preburn_time+1) and float(self.best_loss/stale_save)>(1-impr_ratio) and not preburner:
                break
            elif  i%check_time==0 and not preburner and (self.best_loss<self.int_loss or i>300):

                stale_save=self.best_loss

            if preburner and ((loss<0.25*self.best_loss) or i>preburn_time):

                preburner=False

        endpoint=i+1
        with torch.no_grad():
            if(integrate and endpoint<epochs-1):
                model=self.best_model.eval()
                for s in range(endpoint,epochs):
                    for t in range(n_minibatches):
                        w = torch.empty(mini_batch_size, self.n_flow).to(dev).to(torch.double)
                        torch.nn.init.uniform_(w)

                        XJ = model(self.format_input(w,dev)).detach()
                        fres=torch.mul(f(XJ[:,:-1]), XJ[:, -1])
                        integ[s+1]+=torch.mean(fres)/(n_minibatches*math.sqrt(mini_batch_size))
                        err[s+1]+=torch.std(fres)/n_minibatches

                    self.best_func_count=self.best_func_count+batch_size

        self.integ_tot=torch.sum(integ/err)/torch.sum(1/err)
        self.err_tot=torch.sqrt(1/torch.sum(1/err))

        if(run!=None and integrate):
            run.log_scalar("training.integ", self.integ_tot.tolist(), 0)
            run.log_scalar("training.err", self.err_tot.tolist(), 0)

        try:
            if (log):
                torch.save({
                    'best_epoch': self.best_epoch,
                    'best_loss': self.best_loss,
                    'int_loss': self.int_loss,
                    'best_loss_rel' : self.best_loss_rel,
                    'best_func_count' : self.best_func_count,
                    'model_state_dict': self.best_model.state_dict(),
                    'integ' : self.integ_tot,
                    'err' : self.err_tot
                    },filename)
        except:
            print("Torch save not possible")

        del loss, var, w, optimizer_object

        if(integrate):
            return (self.integ_tot.detach().tolist(),self.err_tot.detach().tolist())
        else:
            return (0,0)

    def integrate(self, f, nitn, neval, dev=None):
        if(self.best_model==None):
            print("No model has been trained")
            return (0,0)
        if(dev==None or not torch.cuda.is_available()):
            dev=torch.device("cpu")
        else:
            dev=dev = torch.device("cuda:"+str(dev))
        neval=int(neval)
        nitn=int(nitn)
        w = torch.empty(neval, self.n_flow).to(dev)
        var=torch.zeros((nitn,))
        mean=torch.zeros((nitn,))
        for i in range(nitn):

            torch.nn.init.uniform_(w)
            Y=self.format_input(w,dev)
            X=self.best_model(Y).detach()
            fres=f(X[:,:-1])*X[:,-1]
            var[i]=torch.var(fres)
            mean[i]=torch.mean(fres)

        sig=torch.sum(mean/var)/torch.sum(1/var)
        sig_err=torch.sqrt(1/torch.sum(1/var))/np.sqrt(neval*nitn)

        return (sig, sig_err)

class AffineManager(BasicManager):

    """A manager for normalizing flows with affine coupling cells interleaved with rolling layers that
    apply cyclic permutations on the variables. All cells have the same number of pass through variables and the
    same step size in the cyclic permutation.
    Each coupling cell has a fully connected NN with a fixed number of layers (depth) of fixed size (width)

    Hyperparameters:
    - n_pass_through
    - n_bins
    - nn_width
    - nn_depth
    - roll_step
    """

    format_input = AddJacobian()

    def create_model(self,
                     n_pass_through,
                     n_cells,
                     NN,
                     roll_step
                     ):

        dev = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

        self._model = torch.nn.Sequential()

        for i_cell in range(n_cells):#create coupling cells
            self._model.add_module(str(i_cell),
                AffineCoupling(flow_size=self.n_flow, pass_through_size=n_pass_through,
                                                NN_layers=NN)
            )
            self._model.add_module("roll",RollLayer(roll_step)) #add roll layer

        w = torch.empty(10, self.n_flow).to(dev)
        torch.nn.init.uniform_(w)
        # Do one pass forward:
        self._model(self.format_input(w, dev))

class PWLinManager(BasicManager):

    """A manager for normalizing flows with piecewise-linear coupling cells interleaved with rolling layers that
    apply cyclic permutations on the variables. All cells have the same number of pass through variables and the
    same step size in the cyclic permutation.
    Each coupling cell has a fully connected NN with a fixed number of layers (depth) of fixed size (width)

    Hyperparameters:
    - n_pass_through
    - n_bins
    - nn_width
    - nn_depth
    - roll_step
    """

    format_input = AddJacobian()

    def create_model(self,
                     n_pass_through,
                     n_cells,
                     n_bins,
                     NN,
                     roll_step
                     ):

        dev = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

        self._model = torch.nn.Sequential()

        for i_cell in range(n_cells):#create coupling cells
            self._model.add_module(str(i_cell),
                PWLin(flow_size=self.n_flow, pass_through_size=n_pass_through,n_bins=n_bins,
                                                NN_layers=NN)
            )
            self._model.add_module("roll",RollLayer(roll_step)) #add roll layer
        self._model.to(dev)
        self.best_model=self.model

        w = torch.empty(5, self.n_flow).to(torch.double)
        torch.nn.init.uniform_(w).to(dev)
        # Do one pass forward:
        self._model(self.format_input(w,dev))

class PWQuadManager(BasicManager):

    """A manager for normalizing flows with piecewise-quadratic coupling cells interleaved with masking layers that
    mask subsets of the varialbes. The cells have different numbers of pass through variables.
    Each coupling cell has a fully connected NN with a fixed number of layers (depth) of fixed size (width)

    Hyperparameters:
    - n_bins
    - nn_width
    - nn_depth
    - n_cells
    """

    format_input = AddJacobian()

    def create_model(self,
                     n_cells,
                     n_bins,
                     NN,
                     dev=0
                     ):

        if(n_cells<2*np.ceil(np.log2(self.n_flow)) and n_cells<self.n_flow):

            if(self.n_flow<=6):
                n_cells=self.n_flow
            elif self.n_flow==7:
                n_cells=6
            else:
                n_cells=int(2*np.ceil(np.log2(self.n_flow)))
            print("Adjusted # coupling cells to "+str(n_cells))
        dev = torch.device("cuda:"+str(dev)) if torch.cuda.is_available() else torch.device("cpu")

        self._model = torch.nn.Sequential()
        if(self.n_flow<=7):
            if(self.n_flow<=6):
                roll_step=1
                n_pass_through=1
            elif (self.n_flow==7):
                roll_step=1
                n_pass_through=2

            for i_cell in range(n_cells):#create coupling cells
                self._model.add_module(str(i_cell),
                    PWQuad(flow_size=self.n_flow, pass_through_size=n_pass_through,n_bins=n_bins,
                                                    NN_layers=NN)
                )
                if(i_cell<n_cells-1):
                    self._model.add_module("roll"+str(i_cell),RollLayer(roll_step)) #add roll layer
                else:
                     self._model.add_module("roll"+str(i_cell),RollLayer(self.n_flow-((n_cells-1)%self.n_flow)))

        else:
            roll_step=1
            n_pass_through=int(self.n_flow/2)
            dims=torch.arange(self.n_flow)
            n=len(get_bin(self.n_flow-1,0))
            dims_bin=torch.ones(self.n_flow,n)

            dims_bin=torch.IntTensor(list(map(get_bin, dims,[n]*self.n_flow))).to(dev)

            for i_cells in range(2*n):#create coupling cells
                masker=MaskLayer(dims_bin,i_cells,dev)
                self._model.add_module("mask"+str(i_cells),masker)
                self._model.add_module(str(i_cells),
                    PWQuad(flow_size=self.n_flow, pass_through_size=masker.pass_through,n_bins=n_bins,
                                                    NN_layers=NN)
                )

                self._model.add_module("demask"+str(i_cells),DeMaskLayer(masker.feeder,masker.trafoer))
            for i_cells in range(n_cells-2*n):
                self._model.add_module(str(i_cells+2*n),
                    PWQuad(flow_size=self.n_flow, pass_through_size=n_pass_through,n_bins=n_bins,
                                                    NN_layers=NN)
                )
                if(i_cells<n_cells-2*n-1):
                    self._model.add_module("roll"+str(i_cells+2*n),RollLayer(roll_step)) #add roll layer
                else:
                    self._model.add_module("roll"+str(i_cells+2*n),RollLayer(self.n_flow-((n_cells-2*n-1)%self.n_flow)))

        self._model.to(dev).to(torch.double)
        self.best_model=self.model

        w = torch.empty(5, self.n_flow).to(torch.double)
        torch.nn.init.uniform_(w)

        # Do one pass forward:

        self._model(self.format_input(w,dev))

        return


#import pyperclip
import time
import requests
import subprocess
import json
import requests
import pyperclip
import pyautogui
import calendar
from datetime import date

#Varíavel que armazena o dia atual
todayName = date.today()
#Varíavel que armazena o nome do dia atual
todayNameShow = calendar.day_name[todayName.weekday()]
print(todayNameShow)

#Função de execução do Whatsapp
def openWhatsAppHome():

    # pyautogui.alert("O código vai começar. Não utilize nada do computador até o código finalizar!")
    #Tempo de espera do código
    time.sleep(2)
    #Abre o menu, como se o botão windows tivesse sido clicado
    pyautogui.press('winleft')
    #Tempo de espera do código
    time.sleep(3)
    pyautogui.PAUSE = 1.5
    #Escreve Whatsapp na busca já selecionada
    pyautogui.write('whatsapp')
    #Tempo de espera do código
    time.sleep(3)
    #Pressiona 'enter'
    pyautogui.press('enter')
    #Tempo de espera do código
    time.sleep(3)
    #Move o cursor do mouse para as coordenadas(x,y)
    pyautogui.moveTo(1180,50)
    #Clique
    pyautogui.click()
    #Tempo de espera do código
    time.sleep(3)
    pyautogui.write('@Dúvidas diversas')
    #Move o cursor do mouse para as coordenadas(x,y)
    pyautogui.moveTo(150,178)
    #Clique
    pyautogui.click()
    pyautogui.click()
    #Tempo de espera do código
    time.sleep(4)
    #Se o dia da semana estiver entre as posições abaixo, ele abre uma API
    if todayNameShow == 'Tuesday' or todayNameShow == 'Thursday':
        subprocess.call("C:/Albacete-automation/Automation-of-Spreadsheets/BOLETOS/support_files/translationApiValyrian.py", shell=True)

    #Caso contrário abre outro
    if todayNameShow == 'Monday' or todayNameShow == 'Wednesday' or todayNameShow == 'Friday':
        subprocess.call("C:/Albacete-automation/Automation-of-Spreadsheets/BOLETOS/support_files/translationApiDothrarki.py", shell=True)

    #Pressiona 'enter'
    pyautogui.press('enter')
    #Move o cursor do mouse para as coordenadas(x,y)
    pyautogui.moveTo(1255,15)
    #Clique
    pyautogui.click()

    '''

    pyautogui.press('enter')
    pyautogui.moveTo(150,178)
    pyautogui.click()
    time.sleep(3)
    text_with_special_chars = 'Teste 01 - Garoto de programa\n 17/10/2022'
    pyperclip.copy(text_with_special_chars)
    pyautogui.hotkey('ctrl', 'v')
    pyautogui.press('enter')
    time.sleep(5)
    pyautogui.moveTo(1000,15)
    pyautogui.click()

    #pyautogui.mouseDown()
    #pyautogui.moveTo(756,635)
'''
#Chama a função
openWhatsAppHome()


'''
# Module name:
    wxbot.py

# Prerequisite:
    Based on Python 3.4
    Need python module pillow and wxpy.

# Purpose:
    An independent wechat service based on Wxpy for hass component wechat3.py.

# Author:
    Retroposter retroposter@outlook.com

# Created:
    Nov.22th 2017

# Last Modified:
    Nov.22th 2017
'''

REQUIREMENTS = ['wxpy==0.3.9.8','pillow']

import configparser
import json
import logging
import os
import socket

from wxpy import *

# No attribute config???
#logging.config.fileConfig('logging.conf')
_LOGGER = logging.getLogger('wxbot')

ATTR_TARGET = 'target'
ATTR_MESSAGE = 'message'
ATTR_DATA = 'data'

ATTR_IMAGE = 'image'
ATTR_VIDEO = 'video'
ATTR_FILE = 'file'

GROUP_POSTFIX = '#group#'

SERVICE_PORT = 18001
CUSTOMER_PORT = 18002

is_running = False

def get_config_value(conf, section, option, default=''):
    try:
        value = conf.get(section, option)
    except:
        value = default
    return value

def run_service():
    is_running = True
    conf = configparser.ConfigParser()
    conf.read('config.conf')
    tuling_api_key = get_config_value(conf, 'bot', 'tuling_api_key')
    cmder = get_config_value(conf, 'cmd', 'cmder')
    cmd_prefix = get_config_value(conf, 'cmd', 'cmd_prefix')
    tts_prefix = get_config_value(conf, 'cmd', 'tts_prefix')

    cache_path = os.path.join(os.getcwd(), 'wxpy.pkl')
    bot = Bot(cache_path=cache_path, console_qr=True)

    if cmder != '':
        try:
            cmder = ensure_one(bot.friends().search(cmder))
        except:
            _LOGGER.error('Failed to set commander: no friend named %s.', cmder)
            cmder = None
    def process_special_msg(msg, msg_type, sender, sex, chat):
        msg_data = {'msg_type': msg_type, 'message': msg, 'sender': sender, 'sex': sex}
        msg_to_send = json.dumps(msg_data)
        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect(('localhost', CUSTOMER_PORT))
            sock.send(msg_to_send.encode())
            sock.close()
            replay = '{0} msg: {1} was sent successfully.'.format(msg_type, msg)
        except Exception as ex:
            replay = '{0} msg: {1} failed to be sent.'.format(msg_type, msg)
            _LOGGER.exception(ex)
        chat.send_msg(replay)

    def is_specified_fmt(fmt_prefix, msg_text):
        if fmt_prefix is None or msg_text is None:
            return False
        text = msg_text.lower()
        fmt_pf = fmt_prefix.lower()
        return text.startswith(fmt_pf)

    def is_cmd_fmt(msg_text):
        return is_specified_fmt(cmd_prefix, msg_text)

    def is_tts_fmt(msg_text):
        return is_specified_fmt(tts_prefix, msg_text)

    @bot.register(Friend)
    def on_msg_received(msg):
        if msg.text is not None:
            # If it is sent by specified user, check if it metches cmd fmt. If it does,
            # invoke cmd handler to process
            if cmder is not None and msg.sender == cmder and is_cmd_fmt(msg.text):
                # If it matches cmd fmt
                process_special_msg(msg.text, 'cmd', msg.sender.remark_name, msg.sender.sex, msg.chat)
                return
            # If it matches tts fmt
            if is_tts_fmt(msg.text):
                process_special_msg(msg.text, 'tts', msg.sender.remark_name, msg.sender.sex, msg.chat)
                return
        if tuling_api_key != '':
            tuling = Tuling(api_key=tuling_api_key)
            tuling.do_reply(msg)

    wechat_service = WeChatService(bot)

    def process_buffer(msg_bytes):
        try:
            msg_str = str(msg_bytes, encoding='utf8')
            json_obj = json.loads(msg_str)
            target = json_obj.get(ATTR_TARGET)
            message = json_obj.get(ATTR_MESSAGE)
            data = json_obj.get(ATTR_DATA)
            wechat_service.send_message(message, target=target, data=data)
        except Exception as ex:
            _LOGGER.error('Something wrong when processing msg.')
            _LOGGER.exception(ex)

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind(('localhost', SERVICE_PORT))
    sock.listen(5)
    while True:
        conn, addr = sock.accept()
        try:
            conn.settimeout(50)
            buf = conn.recv(1024)
            process_buffer(buf)
        except socket.timeout:
            _LOGGER.error('Conn timeout.')
        conn.close()

class WeChatService():
    def __init__(self, bot):
        """Initialize the service."""
        self.bot = bot

    def send_message(self, message="", **kwargs):
        targets = kwargs.get(ATTR_TARGET)
        data = kwargs.get(ATTR_DATA)
        # Send to file helper if no target specified.
        if targets is None:
            if data is not None:
                if ATTR_IMAGE in data:
                    image = data.get(ATTR_IMAGE, None)
                    self.bot.file_helper.send_image(image)
                elif ATTR_VIDEO in data:
                    video = data.get(ATTR_VIDEO, None)
                    self.bot.file_helper.send_video(video)
                elif ATTR_FILE in data:
                    file = data.get(ATTR_FILE, None)
                    self.bot.file_helper.send_file(file)
                else:
                    _LOGGER.error('No image, video or file in data.')
            self.bot.file_helper.send_msg(message)
        else:
            for tar in targets:
                if tar == '':
                    _LOGGER.error('Invalid target.')
                    continue
                # Determine the target is a friend or group.
                target = tar.split(GROUP_POSTFIX)
                if len(target) == 1:
                    chats = self.bot.friends().search(target[0])
                else:
                    chats = self.bot.groups().search(target[0])
                chats_count = len(chats)
                if chats is not None and chats_count > 0:
                    # Only send to the first chat matched specified target.
                    chat = chats[0]
                    if data is not None:
                        if ATTR_IMAGE in data:
                            image = data.get(ATTR_IMAGE, None)
                            chat.send_image(image)
                        elif ATTR_VIDEO in data:
                            video = data.get(ATTR_VIDEO, None)
                            chat.send_video(video)
                        elif ATTR_FILE in data:
                            file = data.get(ATTR_FILE, None)
                            chat.send_file(file)
                        else:
                            _LOGGER.error('No image, video or file in data.')
                    chat.send_msg(message)
                else:
                    _LOGGER.error('Incorrect name of friend or group: %s', target[0])

if __name__ == '__main__':
    if not is_running:
        run_service()
    else:
        _LOGGER.warning('It is running.')


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os
import sys
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtNetwork import *
# Download OpenGL dll from here
# http://download.qt.io/development_releases/prebuilt/llvmpipe/windows/
#
from PyQt5.QtWebEngineWidgets import *
from PyQt5.QtXmlPatterns import *
if __package__ is None or __package__ == '':
    from executor import PyExecutor
    import meshandler, dbpool
    import util as cu
else:
    from . import (meshandler, dbpool)
    from .executor import PyExecutor
    from . import util as cu
from datetime import timedelta, datetime

class XmlRestExecutor(PyExecutor):
    def __init__(self, iniFile, parent=None):
        super().__init__(iniFile,parent)

        self.lastReply = None
        self.man = QNetworkAccessManager(self)
        #self.man.finished.connect(self.netFinished)
        self.man.sslErrors.connect(self.netSslErrors)
        self.man.authenticationRequired.connect(self.authenticationRequired)
        self.man.proxyAuthenticationRequired.connect(self.authenticationRequired)
        self.page = QWebEngineView(self)
        self.resultLay.addWidget(self.page)
        #self.loadIni(iniFile)
        self.startTime = datetime.now()
        self.timer = QTimer(self)
        self.timer.setInterval(1000)
        self.timer.timeout.connect(self.showTime)

    def loadIni(self, iniFile):
        super().loadIni(iniFile)
        ini = QSettings(iniFile, QSettings.IniFormat)
        ini.setIniCodec("utf-8")
        print(iniFile,ini.childGroups())
        ini.beginGroup("WebPage")
        self.url = ini.value('Url')
        self.bodyFile = ini.value('Body')
        self.method = ini.value('Method')
        if self.method is None:
            self.method = 'get' if self.bodyFile == None else 'post'
        if self.bodyFile:
            self.bodyFile = cu.nearFile(iniFile, self.bodyFile)

##        print(self.method)
        self.transformTemplate = ini.value('Transform')
        if self.transformTemplate:
            self.transformTemplate = cu.nearFile(iniFile, self.transformTemplate)
        ini.endGroup()

##        self.readInputs(ini)
##
##        self.runBtn = QPushButton("Run")
##        self.runBtn.setDefault(True)
##        self.runBtn.clicked.connect(self.run)
##        self.btnLay = QHBoxLayout()
##        self.btnLay.addStretch()
##        self.btnLay.addWidget(self.runBtn)
##        self.lay.addRow(self.btnLay)

    def readInputs(self, ini):
        self.inputs = {}
        self.params = []
        ini.beginGroup("Input")
        for key in sorted(ini.childKeys()):
            v = ini.value(key)
            print(v)
            if type(v) != type([]):
                v = [v]
            if len(v)>1:
                paramTitle = v[0]
                paramValue = v[1]
            else:
                paramTitle = key
                paramValue = v[0]
            self.params.append([key, paramTitle, paramValue])
            if paramTitle != '':
                le = QLineEdit()
                self.inputs[key] = le
                le.setText(paramValue)
                le.paramTitle = paramTitle
                self.lay.addRow(paramTitle, le)
        for kp in self.params:
            key = kp[0]
            paramTitle = kp[1]
            paramValue = kp[2]
            if paramTitle == '':
                le = self.inputs[paramValue]
                self.inputs[key] = le
        ini.endGroup()

    def startedRun(self):
        self.runBtn.setEnabled(False)
        self.startTime = datetime.now()
        self.timer.start()
        self.showTime()

    def finishedRun(self):
        self.timer.stop()
        self.showTime()
        self.runBtn.setEnabled(True)
        self.lastReply = None

    def run(self):
        try:
            self.startedRun()
            values = { kp[0]:self.inputs[kp[0]].text() for kp in self.params}
##            print(values)
            url = self.url.format(**values)
##            print('url:',url)
            req = QNetworkRequest(QUrl(url))
##            print('method:',self.method)
            if self.method == 'get':
##                print("get", url)
                req.setHeader(QNetworkRequest.ContentTypeHeader,"text/xml;charset=UTF-8")
                req.setHeader(QNetworkRequest.ContentLengthHeader,"0")
##                req.setRawHeader("Accept","application/xhtml+xml")
##                print(req.rawHeaderList())
                reply = self.man.get(req)
            else:
                f = QFile(self.bodyFile)
                if f.open(QIODevice.ReadOnly):
                    body = str(f.readAll(),'utf-8-sig')
                else:
                    body = ''
                body = body.format(**values)
##                print('body:')
##                print(body)
##                print('----')
                body = bytes(body,'utf-8')
                req.setHeader(QNetworkRequest.ContentTypeHeader,"text/xml;charset=UTF-8")
##                print("post", url)
                reply = self.man.post(req,body)
            reply.finished.connect(self.replyFinished)
            reply.error.connect(self.replyError)
            #reply.sslErrors.connect(self.replySslErrors)
            reply.downloadProgress.connect(self.downloadProgress)
            err = reply.error()
            self.lastReply = reply
##            print("Error", err, reply.errorString())
##            print("Reply is running:", reply.isRunning())
##            print("Reply is finished:", reply.isFinished())
##            print("Reply man:", reply.manager())
##            print("Self man:", self.man)
            if err != QNetworkReply.NoError:
                self.finishedRun()
##            r = requests.get(self.url)
##            print(r.content)
        except:
            print(str(sys.exc_info()[1]))
            self.finishedRun()

    def replyFinished(self):
        self.netFinished(self.sender())

    def netFinished(self, reply):
        #reply = self.sender()
        #buf = reply.readAll()
##        print("finished")
        self.finishedRun()
        if reply is None:
##            print("reply is None")
            return
        if reply.isOpen():
            hasTempl = True
            if self.transformTemplate != None:
                filename, ext = os.path.splitext(self.transformTemplate)
    ##            print(filename, ext)
                if ext.lower() == '.xq' or ext.lower() == '.xquery':
                    lang = QXmlQuery.XQuery10
                elif  ext.lower() == '.xsl' or ext.lower() == '.xslt':
                    lang = QXmlQuery.XSLT20
                else:
                    return
                templ = QFile(self.transformTemplate)
                if not templ.open(QIODevice.ReadOnly):
                    print("Can't open template",self.transformTemplate)
                    hasTempl = False
            if hasTempl:
                query = QXmlQuery(lang)
                query.setMessageHandler(XmlQueryMessageHandler())
                query.setFocus(reply)
                query.setQuery(templ)
                if query.isValid():
                    out = query.evaluateToString()
    ##                print("out",out)
                    if out != None:
                        self.page.setHtml(out)
                        #self.finishedRun()
                        return
            b = reply.readAll()
        if reply.error() != QNetworkReply.NoError:
            print("Error:", reply.error())
        else:
            self.page.setContent(b,"text/plain;charset=windows-1251",reply.url())
        #self.finishedRun()

    def showTime(self):
        time = datetime.now()
        delta = time - self.startTime
        s = "In progress" if self.timer.isActive() else "Finished"
        self.bar.showMessage("%s. %s s" % (s, delta.seconds))
        if delta.seconds >= 30 and not (self.lastReply is None):
            self.lastReply.abort()

    def netSslErrors(self, reply, errors):
        print(errors)
        reply.ignoreSslErrors()

    def replySslErrors(self, errors):
        print(errors)
        self.netSslErrors(self.sender(), errors)

    def replyError(self, errorCode):
        reply = self.sender()
        print("Error", errorCode, reply.errorString())

    def authenticationRequired(self, reply, auth):
        print("authenticationRequired")

    def downloadProgress(self, received, total):
        print("progress %s/%s bytes" % (received, total))

class XmlQueryMessageHandler(QAbstractMessageHandler):
    def handleMessage(self, msgType, desc, idUrl, source):
        print("Msg(%s) on (%s,%s): %s" % (msgType, source.line(), source.column(), desc))

if __name__ == '__main__':
    os.environ.putenv('QT_ANGLE_PLATFORM','warp') # d3d11, d3d9 and warp
## warp:
##    ---------------------------
##QtWebEngineProcess.exe - Системная ошибка
##---------------------------
##Запуск программы невозможен, так как на компьютере отсутствует VCRUNTIME140.dll. Попробуйте переустановить программу.
##---------------------------
##ОК
##---------------------------

    os.environ.putenv('QT_OPENGL','software') # desktop, software, angle
#    os.environ.putenv('QT_OPENGL','angle') # desktop, software, angle
## software - нужно положить vcruntime140 в Qt/bin

    #os.environ.putenv('QT_D3DCOMPILER_DLL','C:/Windows/System32/D3DCompiler_47.dll')
    app = QApplication(sys.argv)
    ex = XmlRestExecutor("../data/ЦБ/valcurs.ini")
    ex.show()
    sys.exit(app.exec_())


from __future__ import print_function
import numpy as np
import networkx as nx
import scipy.sparse as sp
import tensorflow as tf
from utilities import run_random_walks_n2v
import dill

flags = tf.app.flags
FLAGS = flags.FLAGS
np.random.seed(123)

def load_graphs(dataset_str):
    """Load graph snapshots given the name of dataset"""
#np.load()专门用来加载.npy文件，.npy文件是一种用来保存数据的文件。
#graphs.npz文件里面存储的是一个graph文件（可以看作一个对象）。.npz文件是一个压缩文件，文件下面会有很多子文件
#graphs是一个列表，列表存储了图的静态快照，列表中的每个元素是一个图对象。
    graphs = np.load("data/{}/{}".format(dataset_str, "graphs.npz"), allow_pickle=True, encoding="latin1")['graph']
    print("Loaded {} graphs ".format(len(graphs)))
#map()会根据提供的函数对指定序列做映射。
#第一个参数 function 以参数序列中的每一个元素调用 function 函数，返回包含每次 function 函数返回值的新列表。
#lambda后面接的是函数的形参
#python3的map函数返回的是一个迭代器，python2的map函数返回的是一个新列表
#adj_matrices可以看作是一个新的列表，列表中的每个元素为图对象转换后的邻接矩阵。
    adj_matrices = list(map(lambda x: nx.adjacency_matrix(x), graphs))
    return graphs, adj_matrices

def load_feats(dataset_str):
    """ Load node attribute snapshots given the name of dataset (not used in experiments)"""
    features = np.load("data/{}/{}".format(dataset_str, "features.npz"), allow_pickle=True)['feats']
    print("Loaded {} X matrices ".format(len(features)))
    return features

def sparse_to_tuple(sparse_mx):
    """Convert scipy sparse matrix to tuple representation (for tf feed dict)."""
    #将邻接矩阵转化成元组表示[[],[],[],...]=[[(),a],[(),a],[(),a]...[(),a]]
    def to_tuple(mx):
#coo矩阵用来压缩系数矩阵，coo矩阵由三个数组组成，一个存放非零元素的值。
#剩下的两个数组，一个用来存放非零元素位置的行，一个用来存放非零元素位置的列。
        if not sp.isspmatrix_coo(mx):
            mx = mx.tocoo()
#[row],[col](2*10)=[[第一非零元素行位置，第一非零元素列位置],[],...,[],[]](10*2)
        coords = np.vstack((mx.row, mx.col)).transpose()
        #values是一个列表[_,_,_,...,_]
        values = mx.data
        shape = mx.shape
        return coords, values, shape

    def to_tuple_list(matrices):
        # Input is a list of matrices.
        coords = []
        values = []
        shape = [len(matrices)]
        for i in range(0, len(matrices)):
            mx = matrices[i]
            if not sp.isspmatrix_coo(mx):
                mx = mx.tocoo()
            # Create proper indices - coords is a numpy array of pairs of indices.
            #np.vstack((a,b))返回的是一个a和b堆叠的数组.
            coords_mx = np.vstack((mx.row, mx.col)).transpose()
            z = np.array([np.ones(coords_mx.shape[0]) * i]).T
#z就变为了[[_,_,_],[_,_,_],...,[_,_,_]]
            z = np.concatenate((z, coords_mx), axis=1)
            #astype(int)将数组z中的所有数据元素类型更换为int类型。
            z = z.astype(int)
#extend()函数用于在列表末尾一次性追加另一个序列中的多个值（用新列表扩展原来的列表）。
#coords = [[_,_,0],[_,_,1],[_,_,2],...,[_,_,n]]最后一个数字是几，就表示是第几个矩阵。
#values = [_,_,_,...,_]
            coords.extend(z)
            values.extend(mx.data)

#shape = [_,_,_] 三维
        shape.extend(matrices[0].shape)
        shape = np.array(shape).astype("int64")
        values = np.array(values).astype("float32")
        coords = np.array(coords)
        return coords, values, shape

#isinstance() 函数来判断一个对象是否是一个已知的类型，类似 type()。
    if isinstance(sparse_mx, list) and isinstance(sparse_mx[0], list):
        # Given a list of lists, convert it into a list of tuples.
        for i in range(0, len(sparse_mx)):
            sparse_mx[i] = to_tuple_list(sparse_mx[i])

    elif isinstance(sparse_mx, list):
        for i in range(len(sparse_mx)):
            sparse_mx[i] = to_tuple(sparse_mx[i])
    else:
        sparse_mx = to_tuple(sparse_mx)

    return sparse_mx

def preprocess_features(features):
    """Row-normalize feature matrix and convert to tuple representation"""
    #sum(1)将一个向量[_,_,...,_]全部元素加和。
    #rowsum = [_]
    rowsum = np.array(features.sum(1))
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = sp.diags(r_inv)
    features = r_mat_inv.dot(features)
    return features.todense(), sparse_to_tuple(features)

def normalize_graph_gcn(adj):
    """GCN-based normalization of adjacency matrix (scipy sparse format). Output is in tuple format"""
#adj就变为了coo类型的存储矩阵
#sp.coo_matrix()用来创建稀疏矩阵，该函数返回的是一个矩阵。
    adj = sp.coo_matrix(adj)
#sp.eye函数返回一个adj.shape[0]*adj.shape[0]形状的对角矩阵，对角元素为1。
    adj_ = adj + sp.eye(adj.shape[0])
#先将adj_的元素进行加和，加和后的结果再加上1。
    rowsum = np.array(adj_.sum(1))
#a.flatten()：a是个数组，a.flatten()就是把a降到一维，默认是按行的方向降。
#           ：a若是个矩阵，a.flatten()之后还是个矩阵[[_,_,...,_]]。
#power(x, y) 函数，计算 x 的 y 次方。
#    当 np.diag(array) 中
#    array是一个1维数组时，结果形成一个以一维数组为对角线元素的矩阵
#    array是一个二维矩阵时，结果输出矩阵的对角线元素
    degree_mat_inv_sqrt = sp.diags(np.power(rowsum, -0.5).flatten())
    #dot()返回的是矩阵之间的乘积。（向量之间就是点积）（两个数之间就是乘法）
    #tocoo([copy])：返回稀疏矩阵的coo_matrix形式
    adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt).tocoo()
    return sparse_to_tuple(adj_normalized)

def get_context_pairs_incremental(graph):
    return run_random_walks_n2v(graph, graph.nodes())

def get_context_pairs(graphs, num_time_steps):
    """ Load/generate context pairs for each snapshot through random walk sampling."""
    load_path = "data/{}/train_pairs_n2v_{}.pkl".format(FLAGS.dataset, str(num_time_steps - 2))
    try:
#        dill 可以用于保存对象等大多数Python的数据格式
        context_pairs_train = dill.load(open(load_path, 'rb'))
        print("Loaded context pairs from pkl file directly")
    except (IOError, EOFError):
        print("Computing training pairs ...")
        context_pairs_train = []
        #num_time_steps表示训练的静态快照图，
        #所以这里的context_pairs_train列表包含num_time_steps个数的字典。
        for i in range(0, num_time_steps):
            #run_random_walks_n2v()返回的是一个字典。
            context_pairs_train.append(run_random_walks_n2v(graphs[i], graphs[i].nodes()))
        #保存不同时刻图的随机游走的固定窗口的节点对。
        dill.dump(context_pairs_train, open(load_path, 'wb'))
        print ("Saved pairs")

    return context_pairs_train

def get_evaluation_data(adjs, num_time_steps, dataset):
    """ Load train/val/test examples to evaluate link prediction performance"""
    eval_idx = num_time_steps - 2
    #.npz文件相当于一个文件夹，文件夹目录下面有许多文件。
    eval_path = "data/{}/eval_{}.npz".format(dataset, str(eval_idx))
    try:
        #空格加反斜杠是续行的意思，一行代码太长后面加上 \表示接下来的一行是跟着这一行的。
        #左斜杠/也叫正斜杠，右斜杠\又叫做反斜杠。
        train_edges, train_edges_false, val_edges, val_edges_false, test_edges, test_edges_false = \
            np.load(eval_path, encoding='bytes', allow_pickle=True)['data']
        print("Loaded eval data")
    except IOError:
#adj是图的邻接矩阵，[[第一个图的邻接矩阵],[第二个图的邻接矩阵],[第三个图的邻接矩阵]]
        #next_adjs表示的是一个邻接矩阵
        next_adjs = adjs[eval_idx + 1]
        print("Generating and saving eval data ....")
        train_edges, train_edges_false, val_edges, val_edges_false, test_edges, test_edges_false = create_data_splits(adjs[eval_idx], next_adjs, val_mask_fraction=0.2, test_mask_fraction=0.6)
#数据是以字典的形式存储在.npz文件中，这里键是data。
        np.savez(eval_path, data=np.array([train_edges, train_edges_false, val_edges, val_edges_false,
                                           test_edges, test_edges_false]))

    return train_edges, train_edges_false, val_edges, val_edges_false, test_edges, test_edges_false

def create_data_splits(adj, next_adj, val_mask_fraction=0.2, test_mask_fraction=0.6):
    """In: (adj, next_adj) along with test and val fractions. For link prediction (on all links), all links in
    next_adj are considered positive examples.
    Out: list of positive and negative pairs for link prediction (train/val/test)"""
#edges_all就是[[第一个非零元素行位置，第一个非零元素列位置],[],...,[],[]]
    edges_all = sparse_to_tuple(next_adj)[0]  # All edges in original adj.
#diagonal()查看矩阵对角线上的元素，返回矩阵对角线元素的列表。
#np.newaxis的作用就是在这一位置增加一个一维。就是shape的形状多一个位置用1填充。
    adj = adj - sp.dia_matrix((adj.diagonal()[np.newaxis, :], [0]), shape=adj.shape)  # Remove diagonal elements
    #将adj矩阵的非零元素给消除掉，adj依旧是一个矩阵。
    adj.eliminate_zeros()
#adj.todense()返回一个矩阵。
#np.diag(a)将a数组元素变化为一个对角矩阵。
#当参数a是一个1维数组时，结果形成一个以一维数组为对角线元素的矩阵。[[],[],...,[]]
#当参数a是一个二维矩阵时，结果输出矩阵的对角线元素，[_,_,_,...,_]。
    #sum()该函数返回一个数。
    assert np.diag(adj.todense()).sum() == 0
    if next_adj is None:
        raise ValueError('Next adjacency matrix is None')

#list函数是替换的意思，将原先的数据类型去除掉，然后替换成list这种数据类型。
    edges_next = np.array(list(set(nx.from_scipy_sparse_matrix(next_adj).edges())))
    edges = []   # Constraint to restrict new links to existing nodes.
    #edges_next = [(_,_),(_,_),...,(_,_)]
    for e in edges_next:
        if e[0] < adj.shape[0] and e[1] < adj.shape[0]:
            edges.append(e)
    #edges = [(_,_),(_,_),...,(_,_)]
    edges = np.array(edges)

    def ismember(a, b, tol=5):
    #np.all是与操作，所有元素为true，则输出为true。
    #np.any是或操作，任意一个元素为true，则输出为true。
#round() 方法返回浮点数x的四舍五入值
        rows_close = np.all(np.round(a - b[:, None], tol) == 0, axis=-1)
        return np.any(rows_close)

#edges.shape[0]表示的是边的个数。
    all_edge_idx = list(range(edges.shape[0]))
    np.random.shuffle(all_edge_idx)
    #用来测试的边数和用来验证的边数。
    num_test = int(np.floor(edges.shape[0] * test_mask_fraction))
    num_val = int(np.floor(edges.shape[0] * val_mask_fraction))
    #将图中的前num_val数量的边用来验证，后num_test数量的边用来测试。
    val_edge_idx = all_edge_idx[:num_val]
    test_edge_idx = all_edge_idx[num_val:(num_val + num_test)]
#test_edges = [(),(),...,()]    val_edges = [(),(),...,()]
    test_edges = edges[test_edge_idx]
    val_edges = edges[val_edge_idx]
#np.delete(_,_,_)表明删除目标向量中的子向量
#第一个参数为目标向量，第二个参数为删除子向量的位置，第三个参数表明沿着哪个维度删除。
#将一个图分为训练集，验证集，测试集。
#先将t3时刻的图的节点向量表示出来，然后用t3时刻更新的节点向量表示t4时刻节点的向量。
#也就是以t3时刻更新的节点向量来预测t4时刻哪些节点之间会有连边。
#将t4时刻的图分为训练集，验证集，测试集。以t3时刻更新的节点向量作为输入特征，用训练集训练一个分类器。
    train_edges = np.delete(edges, np.hstack([test_edge_idx, val_edge_idx]), axis=0)

    # Create train edges.
    train_edges_false = []
    while len(train_edges_false) < len(train_edges):
        #adj.shape[0]表示的是adj这个图有多少个节点。
        idx_i = np.random.randint(0, adj.shape[0])
        idx_j = np.random.randint(0, adj.shape[0])
        if idx_i == idx_j:
            #continue退出整个循环。
            continue
        if ismember([idx_i, idx_j], edges_all):
            continue
        if ismember([idx_j, idx_i], edges_all):
            continue
        if train_edges_false:
            if ismember([idx_j, idx_i], np.array(train_edges_false)):
                continue
            if ismember([idx_i, idx_j], np.array(train_edges_false)):
                continue
        #train_edges_false是[[_,_],[_,_],[_,_],...,[_,_]]
        train_edges_false.append([idx_i, idx_j])

    # Create test edges.
    test_edges_false = []
    while len(test_edges_false) < len(test_edges):
        idx_i = np.random.randint(0, adj.shape[0])
        idx_j = np.random.randint(0, adj.shape[0])
        if idx_i == idx_j:
            continue
        if ismember([idx_i, idx_j], edges_all):
            continue
        if ismember([idx_j, idx_i], edges_all):
            continue
        if test_edges_false:
            if ismember([idx_j, idx_i], np.array(test_edges_false)):
                continue
            if ismember([idx_i, idx_j], np.array(test_edges_false)):
                continue
        test_edges_false.append([idx_i, idx_j])

    # Create val edges.
    val_edges_false = []
    while len(val_edges_false) < len(val_edges):
        idx_i = np.random.randint(0, adj.shape[0])
        idx_j = np.random.randint(0, adj.shape[0])
        if idx_i == idx_j:
            continue
        if ismember([idx_i, idx_j], edges_all):
            continue
        if ismember([idx_j, idx_i], edges_all):
            continue

        if val_edges_false:
            if ismember([idx_j, idx_i], np.array(val_edges_false)):
                continue
            if ismember([idx_i, idx_j], np.array(val_edges_false)):
                continue
        val_edges_false.append([idx_i, idx_j])

    assert ~ismember(test_edges_false, edges_all)
    assert ~ismember(val_edges_false, edges_all)
    assert ~ismember(val_edges, train_edges)
    assert ~ismember(test_edges, train_edges)
    assert ~ismember(val_edges, test_edges)
    #无false的是正样本（正类），有false的是负样本（负类）。
    print("# train examples: ", len(train_edges), len(train_edges_false))
    print("# val examples:", len(val_edges), len(val_edges_false))
    print("# test examples:", len(test_edges), len(test_edges_false))

#为什么会有两个train_edges和train_edges_false,因为一个是正样本，一个是负样本。
    return list(train_edges), train_edges_false, list(val_edges), val_edges_false, list(test_edges), test_edges_false


#encoding=utf-8
import logging
import os
from datetime import datetime
class RecordLog(object):
    def __init__(self,logger):
        self.logger = logging.getLogger(logger)
        self.logger.setLevel(logging.DEBUG)

        # 2.将log信息输出到log文件中
        # 2.1 先定位看将log文件输出到哪里去
        current_dir = os.path.dirname(os.path.abspath(__file__))
        print(current_dir)  # D:\MySpace\Python\WebTesting\util
        log_dir = os.path.join('../Result')
        # 日志名称构建
        log_file_name = datetime.now().strftime("%Y-%m-%d") + '.log'
        log_file_path = log_dir + '/' + log_file_name
        print(log_file_path)

        # 2.2 好的，将日志写进log文件中
        self.file_handle = logging.FileHandler(log_file_path, 'a', encoding='utf-8')
        self.file_handle.setLevel(logging.INFO)
        formatter = logging.Formatter(
            '%(asctime)s %(name)s %(levelname)s ---> %(message)s')
        self.file_handle.setFormatter(formatter)
        self.logger.addHandler(self.file_handle)

    def get_log(self):
        return self.logger

if __name__ == "__main__":
    rl = RecordLog()
    log_info = rl.get_log()
    log_info.debug('输出到文件中去')
    rl.close_handle()


# -*- coding: utf-8 -*-

import chardet,requests,sys,os,time,base64,json

from selenium import webdriver

#头部参数
headers = {
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language': 'zh-CN,zh;q=0.8',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Referer': 'http://pub.alimama.com/myunion.htm',
    'X-Requested-With': 'XMLHttpRequest',
    'Connection': 'keep-alive',
}

#1. 读取cookies文件,如果存在则获取它
def load():
    try:
        with open('cookies.json') as f:
            cookies = json.load(f)
        return cookies
    except FileNotFoundError:
        return None

#2. 验证Cookies值,成功则输出名字，失败输出空值
def validate():
    cookies = load()

    url = 'http://pub.alimama.com/common/getUnionPubContextInfo.json'
    s = requests.session()

    html = s.get(url, headers=headers, cookies=cookies, timeout=10)
    result = html.json()

    try:
        # 成功读取用户名
        name = result['data']['mmNick']
        # print(name)
        return name
    except:
        # 当前Cookies失效
        return None

#3. 不存在Cookies文件或Cookies已失效，则更新 Cookies 文件并输出最新 Cookie 的值
def update():
    #driver = webdriver.Chrome(r"C:\Python35\selenium\webdriver\chrome\chromedriver.exe")
    # driver = webdriver.Chrome(r'/Users/Chao/bin/chromedriver') # Mac版本
    driver = webdriver.Chrome(r'/usr/local/chromedriver-Linux64')
    # loginurl = 'https://login.taobao.com/member/login.jhtml?style=mini&newMini2=true&from=alimama&redirectURL=http://login.taobao.com/member/taobaoke/login.htm?is_login=1&full_redirect=true&disableQuickLogin=true'
    loginurl = 'https://login.taobao.com/member/login.jhtml?style=mini&newMini2=true&from=alimama&redirectURL=http://pub.alimama.com/common/getUnionPubContextInfo.json'
    driver.get(loginurl)
    driver.set_window_size(500,300)
    time.sleep(1)

    # 点击用户名登陆
    driver.find_element_by_xpath('//*[@id="J_Quick2Static"]').click()
    # print('✔ 恭喜，点击用户名登陆按钮成功！')
    time.sleep(1)

    # 输入用户名
    driver.find_element_by_xpath('//*[@id="TPL_username_1"]').send_keys('CPS')
    # print('✔ 恭喜，用户名输入成功！')
    time.sleep(1)

    # 输入密码
    driver.find_element_by_xpath('//*[@id="TPL_password_1"]').send_keys('TaoKe')
    # print('✔ 恭喜，密码输入成功！')
    time.sleep(1)

    # 点击登陆按钮
    driver.find_element_by_xpath('//*[@id="J_SubmitStatic"]').click()
    # print ('✔ 恭喜，点击登陆按钮成功！')
    time.sleep(1)

    #页面标题没有‘验证’字样
    if '验证' not in driver.title:
        time.sleep(1)

    #需要输入手机验证码
    else:
        print('✘ Maybe you need type in the sms code.')
        time.sleep(20)
        # update()

    # 获得cookie2信息
    cookie2 = driver.get_cookie('cookie2')['value']
    #获取token值
    token = driver.get_cookie('_tb_token_')['value']

    #
    # print ('✔ 恭喜，成功获取到最新到Cookies值！')
    # print(cookie2)

    # 组装Cookies
    cookies = {
        'cookie2':cookie2,
        '_tb_token_':token
        }
    with open('cookies.json','w',encoding='utf-8') as f:
        f.write(json.dumps(cookies))

    #关闭窗口
    try:
        driver.quit()
    except:
        print('关闭出错')

    return cookies

# 输出欢迎信息
def login():
    name = validate()
    if name:
        # print('\n♛ 主人,欢迎回来,江湖一直有你的传说.  ┊┊  %s  ┊┊ \n' % name)
        cookies = load()
        return cookies
    else:
        print("\n✎ 主人，需要更新Cookies，即将打开浏览器窗口，请耐心等待...")
        time.sleep(0.5)
        cookies = update()
        return cookies

if __name__ == "__main__":
        login()
        # update()


"""Tests for the random variable implementation."""

import itertools
import unittest

import numpy as np
import pytest
import scipy.stats

import probnum
from probnum import linops, randvars
from tests.testing import NumpyAssertions

def test_rng_not_generator_raises_error():

    # dummy random variable that implements a sample method.
    rv = randvars.Normal(0.0, 1.0)

    # incorrect random number generator (np.random.Generator is expected)
    rng = np.random.RandomState(seed=1)

    with pytest.raises(TypeError):
        rv.sample(rng=rng)

class RandomVariableTestCase(unittest.TestCase, NumpyAssertions):
    """General test case for random variables."""

    def setUp(self) -> None:
        """Scalars, arrays, linear operators and random variables for tests."""
        # Seed
        self.rng = np.random.default_rng(42)

        # Random variable instantiation
        self.scalars = [0, int(1), 0.1, np.nan, np.inf]
        self.arrays = [np.empty(2), np.zeros(4), np.array([]), np.array([1, 2])]

        # Random variable arithmetic
        self.arrays2d = [
            np.empty(2),
            np.zeros(2),
            np.array([np.inf, 1]),
            np.array([1, -2.5]),
        ]
        self.matrices2d = [np.array([[1, 2], [3, 2]]), np.array([[0, 0], [1.0, -4.3]])]
        self.linops2d = [linops.Matrix(A=np.array([[1, 2], [4, 5]]))]
        self.randvars2d = [
            randvars.Normal(mean=np.array([1, 2]), cov=np.array([[2, 0], [0, 5]]))
        ]
        self.randvars2x2 = [
            randvars.Normal(
                mean=np.array([[-2, 0.3], [0, 1]]),
                cov=linops.SymmetricKronecker(A=np.eye(2), B=np.ones((2, 2))),
            ),
        ]

        self.scipyrvs = [
            scipy.stats.bernoulli(0.75),
            scipy.stats.norm(4, 2.4),
            scipy.stats.multivariate_normal(np.random.randn(10), np.eye(10)),
            scipy.stats.gamma(0.74),
            scipy.stats.dirichlet(alpha=np.array([0.1, 0.1, 0.2, 0.3])),
        ]

class InstantiationTestCase(RandomVariableTestCase):
    """Test random variable instantiation."""

    def test_rv_dtype(self):
        """Check the random variable types."""
        pass

    def test_rv_from_number(self):
        """Create a random variable from a number."""
        for x in self.scalars:
            with self.subTest():
                rv = probnum.asrandvar(x)
                self.assertIsInstance(rv, randvars.RandomVariable)

    def test_rv_from_ndarray(self):
        """Create a random variable from an array."""
        for arr in self.arrays2d:
            with self.subTest():
                rv = probnum.asrandvar(arr)
                self.assertIsInstance(rv, randvars.RandomVariable)

    # def test_rv_from_linearoperator(self):
    #     """Create a random variable from a linear operator."""
    #     for linop in linops:
    #       with self.subTest():
    #           probnum.asrandvar(A)

    def test_rv_from_scipy(self):
        """Create a random variable from a scipy random variable."""
        for scipyrv in self.scipyrvs:
            with self.subTest():
                rv = probnum.asrandvar(scipyrv)
                self.assertIsInstance(rv, randvars.RandomVariable)

class ArithmeticTestCase(RandomVariableTestCase):
    """Test random variable arithmetic and broadcasting."""

    def test_rv_addition(self):
        """Addition with random variables."""
        for (x, rv) in list(itertools.product(self.arrays2d, self.randvars2d)):
            with self.subTest():
                z1 = x + rv
                z2 = rv + x
                self.assertEqual(z1.shape, rv.shape)
                self.assertEqual(z2.shape, rv.shape)
                self.assertIsInstance(z1, randvars.RandomVariable)
                self.assertIsInstance(z2, randvars.RandomVariable)

    def test_rv_scalarmult(self):
        """Multiplication of random variables with scalar constants."""
        for (alpha, rv) in list(itertools.product(self.scalars, self.randvars2d)):
            with self.subTest():
                if np.isinf(alpha):
                    with self.assertWarns(RuntimeWarning):
                        z = alpha * rv
                else:
                    z = alpha * rv
                self.assertEqual(z.shape, rv.shape)
                self.assertIsInstance(z, randvars.RandomVariable)

    def test_rv_broadcasting(self):
        """Broadcasting for arrays and random variables."""
        for alpha, rv in list(itertools.product(self.scalars, self.randvars2d)):
            with self.subTest():
                z = alpha + rv
                self.assertEqual(z.shape, rv.shape)

                z = rv - alpha
                self.assertEqual(z.shape, rv.shape)

    def test_rv_dotproduct(self):
        """Dot product of random variables with constant vectors."""
        for x, rv in list(
            itertools.product([np.array([1, 2]), np.array([0, -1.4])], self.randvars2d)
        ):
            with self.subTest():
                z1 = rv @ x
                z2 = x @ rv
                self.assertIsInstance(z1, randvars.RandomVariable)
                self.assertIsInstance(z2, randvars.RandomVariable)
                self.assertEqual(z1.shape, ())
                self.assertEqual(z2.shape, ())

    def test_rv_matmul(self):
        """Multiplication of random variables with constant matrices."""
        for A, rv in list(itertools.product(self.matrices2d, self.randvars2d)):
            with self.subTest():
                y2 = A @ rv
                self.assertEqual(y2.shape[0], A.shape[0])
                self.assertIsInstance(y2, randvars.RandomVariable)

    def test_rv_linop_matmul(self):
        """Linear operator applied to a random variable."""
        for A, rv in list(itertools.product(self.linops2d, self.randvars2d)):
            with self.subTest():
                y = A @ rv + np.array([-1, 1.1])
                self.assertEqual(y.shape[0], A.shape[0])

    def test_rv_vector_product(self):
        """Matrix-variate random variable applied to vector."""
        for rv in self.randvars2x2:
            with self.subTest():
                x = np.array([[1], [-4]])
                y = rv @ x
                X = np.kron(np.eye(rv.shape[0]), x)
                truemean = rv.mean @ x
                truecov = X.T @ rv.cov.todense() @ X
                self.assertIsInstance(
                    y,
                    randvars.RandomVariable,
                    "The variable y does not have the correct type.",
                )
                self.assertEqual(
                    y.shape, (2, 1), "Shape of resulting random variable incorrect."
                )
                self.assertAllClose(
                    y.mean, truemean, msg="Means of random variables do not match."
                )
                self.assertAllClose(
                    y.cov.todense(),
                    truecov,
                    msg="Covariances of random variables do not match.",
                )

class ShapeTestCase(RandomVariableTestCase):
    """Test methods related to the shape of a random variable or its realizations."""

    def test_reshape(self):
        """Reshape a random variable and test for correct output shape."""
        for rv in self.randvars2x2:
            for shape in [(4, 1), (2, 2), (4,), (1, 4)]:
                with self.subTest():
                    try:
                        reshaped_rv = rv.reshape(newshape=shape)

                        self.assertEqual(reshaped_rv.shape, shape)
                        self.assertEqual(
                            reshaped_rv.sample(rng=self.rng, size=1).shape, shape
                        )
                    except NotImplementedError:
                        pass
        for rv in self.randvars2d:
            for shape in [(2, 1), (2,), (1, 2)]:
                with self.subTest():
                    try:
                        reshaped_rv = rv.reshape(newshape=shape)

                        self.assertEqual(reshaped_rv.shape, shape)
                        self.assertEqual(
                            reshaped_rv.sample(rng=self.rng, size=1).shape, shape
                        )
                    except NotImplementedError:
                        pass

    def test_sample_shape(self):
        """Sample from a random variable with different sizes and check sample
        shapes."""
        pass


import logging
import os
import datetime as dt
import json
import azure.functions as func
from azure.storage.blob import BlobClient, ContainerClient, ContentSettings
from .tablestorage import tablestorage
from .event import event

def uploadblob(json_in, blobname, conn_str, lin_container):

    container_client = ContainerClient.from_connection_string(conn_str, container_name=lin_container)
    blob = BlobClient.from_connection_string(conn_str, container_name= lin_container, blob_name=blobname)
    BlobClient
    blob.upload_blob(json_in, overwrite=True)

def main(req: func.HttpRequest) -> func.HttpResponse:

    logging.info("http trigger function kicked off")

    lineageContainerStr = os.environ["LINEAGE_STORAGE_CONN_STR"]
    lineageContainer = os.environ["LINEAGE_CONTAINER"]

    data = req.get_json()

    currenttimestamp = dt.datetime.utcnow().strftime("%Y%m%d%H%M%S")
    eventType = data["eventType"]
    className = data["run"]["facets"]["spark.logicalPlan"]["plan"][0]["class"]
    runId = data["run"]["runId"]
    notebookName = data["job"]["name"]
    notebookName = notebookName[0 : notebookName.index('.')]

    fileName = runId + '_' + notebookName + '_' + currenttimestamp + '.json'
    filePath = lineageContainer + '/' + fileName

    predefined_class_list = ["org.apache.spark.sql.execution.datasources.CreateTable",
                "org.apache.spark.sql.catalyst.plans.logical.CreateViewStatement",
                "org.apache.spark.sql.catalyst.plans.logical.CreateTableAsSelectStatement",
                "org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement",
                "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand"]

    if eventType is not None and eventType == "COMPLETE" and className in predefined_class_list :
        # upload file as json into blob storage
        uploadblob(json.dumps(data), fileName, lineageContainerStr, lineageContainer)

        #code to add new unprocessed row for same uploaded json into azure table
        eventrow = event('HRSI', fileName)
        eventrow.Status = 'Unprocessed'
        eventrow.RetryCount = 3
        eventrow.FilepPath = filePath
        eventrow.isArchived = False
        eventrow.Message = ''

        tableStorage = tablestorage()
        tableStorage.insertEventMetadata(eventrow.__dict__)

        return func.HttpResponse(f"Func App successfully processed http request")

    else:

        return func.HttpResponse(f"Event Type is not COMPLETE Or ClassName Not Matched")


import os, json
from Task4 import capitalize_string
from Task5 import News, Tweet, Ads

DEFAULT_FILENAME = 'Task8_default.json'

class FeedJSON:
    def __init__(self, file_path):
        if not file_path:
            self.__create_default_file()
            self.file_path = DEFAULT_FILENAME
        else:
            self.file_path = file_path
        self.fail_flag = False
        self.read_file(self.file_path)
        if not self.fail_flag:
            os.remove(self.file_path)

    def __create_default_file(self):
        with open(DEFAULT_FILENAME, 'w') as default_file:
            content = {
                "feeds": [
                    {
                        "id": 1,
                        "type": "N",
                        "text": "This IS the eXAMple of task 8! news text",
                        "city": "Prague"
                    },
                    {
                        "id": 2,
                        "type": "A",
                        "text": "This IS the eXAMple of task 8! aDS text",
                        "city": "Brno",
                        "expired": "2024-01-01"
                    },
                    {
                        "id": 3,
                        "type": "T",
                        "text": "This IS the eXAMple of task 8! twEET text",
                        "author": "Unknown"
                    }
                ]
            }
            json_data = json.dumps(content)
            default_file.write(json_data)

    def read_file(self, filepath):
        with open(filepath, 'r') as file:
            feeds = json.load(file)
            for feed in feeds['feeds']:
                if feed['type'] == "N":
                    text = capitalize_string(feed['text'])
                    city = feed['city']
                    News(text, city)
                elif feed['type'] == "A":
                    text = capitalize_string(feed['text'])
                    city = feed['city']
                    expired = feed['expired']
                    Ads(text, city, expired)
                elif feed['type'] == "T":
                    text = capitalize_string(feed['text'])
                    author = feed['author']
                    Tweet(text, author)
                else:
                    self.fail_flag = True
                    print("Unsupported feed content: ", feed)


from selenium import webdriver
import scrapy
from scrapy.selector import Selector
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from shutil import which
from fake_useragent import UserAgent
import time
from selenium.webdriver.common.action_chains import ActionChains
import csv
from pandas import DataFrame

path = which("chromedriver")
options = Options()
options.add_experimental_option("detach", True)
options.add_argument("--headless")
ua = UserAgent()
userAgent = ua.random
options.add_argument(f'user-agent={userAgent}')
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(executable_path=path,options=options)
driver.get("https://www.fashiontiy.com/men-swimwear.html")
time.sleep(2)

def aloo():
    color,siz,images,qity_pri=[],[],[],[]
    try:
        coli_item = driver.find_elements_by_xpath("//div[@class='color_colors']/div")
        for i in range(0,len(coli_item)):
            btn = driver.find_elements_by_xpath("//div[@class='color_colors']/div")[i]
            driver.execute_script("arguments[0].click();", btn)
            time.sleep(2)
            col = driver.find_element_by_xpath("//div[@class='base_key_value size_title_left']/span[@class='notranslate']").text
            color.append(col)
            img = driver.find_elements_by_xpath("//div[@class='detail_left']/div[@class='left_thums']/img")
            for im in range(0,len(img)):
                img = driver.find_elements_by_xpath("//div[@class='detail_left']/div[@class='left_thums']/img")[im].get_attribute("src")
                images.append(img)

            sizes = driver.find_elements_by_xpath("//div[@class='size_list notranslate']/div/span[@class='size_item_name ']")
            for j in range(0,len(sizes)):
                sizes_avaiable = driver.find_elements_by_xpath("//div[@class='size_list notranslate']/div/span[@class='size_item_name ']")[j].text
                siz.append(sizes_avaiable)

        g1 =  ', '.join(images)
        g2 =  '|'.join(siz)
        g3 = ', '.join(color)
    except:
        print("There is nothing to scrape.")

    html = driver.page_source
    resp = Selector(text=html)
    for xlo in resp.xpath("//div[@class='right_price_parent']/div"):
        Quantity_price = xlo.xpath(".//p/text()").getall()
        qity_pri.append(Quantity_price)

    absolute_id = ""
    Id = resp.xpath("//div[@class='right_title notranslate']/text()").get()
    absolute_id = Id
    absolute_id = absolute_id.replace("SKU:","")
    absolute_id = absolute_id.lstrip()
    Name = resp.xpath("//div[@class='right_desc']/span/text()").get()
    Min_order_req = resp.xpath("//div[@class='right_moq base_key_value']/span[@class='notranslate']/text()").get()
    Gross_Weight = resp.xpath("(//div[@class='base_key_value']/span[not(contains(@class,'key'))])[1]/text()").get()
    Material = resp.xpath("(//div[@class='base_key_value']/span[not(contains(@class,'key'))])[2]/text()").get()

    with open("mens_swimwear.csv", 'a',newline='',encoding="utf-8") as file:
        writer = csv.writer(file)
        writer.writerow([absolute_id,Name,Min_order_req,g3,qity_pri,g2,g1,Gross_Weight,Material])
        print("Data Saved in CSV :")
def moli():

    links = driver.find_elements_by_xpath("//div[@class='list_products']/div//a[@class='product_infos_2 notranslate']")
    for i in range(0,len(links)):
        link = driver.find_elements_by_xpath("//div[@class='list_products']/div//a[@class='product_infos_2 notranslate']")[i].get_attribute("href")
        driver.get(link)
        time.sleep(2)
        aloo()

        driver.back()
        time.sleep(5)
    next_page = driver.find_element_by_xpath("//div[contains(text(),'NEXT')]")
    if next_page:
        next_page.click()
        time.sleep(4)
        moli()

with open("mens_swimwear.csv", 'w', newline='') as file:
    writer = csv.writer(file)
    writer.writerow(["ID","Name","Minimum Order requirenment","color","Quantity and price","Size","images_url","Gross Weight","Material"])
moli()