import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.spatial.distance import pdist, squareform
from collections import OrderedDict

#根据反向近邻求噪声
def zaoSheng(n,k1,k_mindist,distMatrix):
    fknn = OrderedDict()
    zaoshenglist=[]
    lenzaosheng=0
    zaosheng = np.zeros(n, dtype=int)
    if k1!=0:
        for i in range(n):
            fknn[i] = []
        k = 0
        while k != -1:
            for j in range(n):
                fknn[k_mindist[j][k]].append(j)
            k = k + 1
            if k == k1:
                break
        lenfknn = np.zeros(n, dtype=int)
        for i in range(n):
            lenfknn[i] = len(fknn[i])
        juli = np.zeros(n, dtype=float)
        for i in range(n):
            for j in range(k1):
                juli[i] = juli[i] + distMatrix[i, k_mindist[i][j]]
        mean_juli = np.mean(juli)
        median_fknn=np.median(lenfknn)
        for i in range(n):
            if juli[i] > mean_juli and lenfknn[i]<median_fknn:
            # if lenfknn[i]<median_fknn:
                zaosheng[i]=1
                zaoshenglist.append(i)
                lenzaosheng = lenzaosheng + 1
    return zaosheng,lenzaosheng,zaoshenglist

def getRho(n,k1,k_mindist):
    fknn = OrderedDict()
    for i in range(n):
        fknn[i] = []
    k = 0
    while k != -1:
        for j in range(n):
            fknn[k_mindist[j][k]].append(j)
        k = k + 1
        if k == k1:
            break
    lenfknn = np.zeros(n, dtype=int)
    for i in range(n):
        lenfknn[i] = len(fknn[i])
    rho = np.zeros(n, dtype=float)
    for i in range(n):
        if len(fknn[i])==0:
            continue
        else:
            for j in range(len(fknn[i])):
                rho[i] = rho[i] + lenfknn[fknn[i][j]]
        rho[i]=lenfknn[i]
    return rho,fknn

#求每个点的最近距离k个点
def k_Mindist(n,distMatrix,k):
    k_mindist=np.zeros((n,k), dtype=int)
    for i in range(n):
        sortdist = np.argsort(distMatrix[i])
        if sortdist[0]!=i:
            for j in range(1,len(sortdist)):
                if sortdist[j]==i:
                    temp=sortdist[0]
                    sortdist[0]=i
                    sortdist[j]=temp
                    break
        k_mindist[i]=sortdist[1:k+1]
    return k_mindist

def hexindianshibie(n,k,k_mindist1,rho,distMatrix):
    sortrho = np.sort(rho)
    print(sortrho)
    maxrhoindex=np.zeros(n, dtype=int)
    median_rho = np.median(rho)
    print(median_rho)
    kdist = np.zeros(n, dtype=float)
    for i in range(0,n):
        for j in range(0,k):
            kdist[i]=kdist[i]+distMatrix[i,k_mindist1[i][j]]
    meankdist=np.mean(kdist)
    maxkdist=np.max(kdist)
    for i in range(0,n):
        maxrho=rho[i]
        maxrhoindex[i]=i
        if rho[i]<=median_rho and kdist[i]>(meankdist+(maxkdist-meankdist)/2):
            continue
        else:
            for j in range(0,k):
                if rho[k_mindist1[i][j]]>maxrho:
                    maxrho=rho[k_mindist1[i][j]]
                    maxrhoindex[i]=k_mindist1[i][j]
    return maxrhoindex

#删除重复核心点
def delete_repeat_hexindian(maxrhoindex):
    finalmaxrhoindex=np.unique(maxrhoindex)
    return finalmaxrhoindex

def prime(gaorho,distMatrix):
    gaorho=gaorho.tolist()
    n=len(gaorho)
    selected_nodes = []  # 存储选中节点的集合
    candidate_nodes = gaorho  # 备选的集合
    result_edges = []
    dist_edges=[]
    for i in range(n):
        if i == 0:  # 这里随机选择第一个顶点作为起始点，这个起始点
            selected_nodes.append(gaorho[i])
            candidate_nodes.remove(gaorho[i])
        else:
            candidate_cost = distMatrix[selected_nodes, :][:, candidate_nodes]
            idx = np.nonzero(candidate_cost == np.min(candidate_cost))  # 选出从选中节点的出度中最短的边
            #         print(candidate_cost)
            start_p = selected_nodes[idx[0][0]]
            end_p = candidate_nodes[idx[1][0]]
            #         print(start_p, end_p)
            selected_nodes.append(end_p)
            candidate_nodes.remove(end_p)
            result_edges.append([start_p, end_p])
            dist_edges.append(distMatrix[start_p, end_p])
    return selected_nodes,result_edges,dist_edges

def qiebian(dist_edges,result_edges,maxrhoindex,k_mindist1,k):
    dist_edges_sort = np.argsort(dist_edges)
    cut_edges = []
    # cut_edges.append(dist_edges_sort[len(dist_edges_sort) - 1])
    # cut_edges.append(dist_edges_sort[len(dist_edges_sort) - 2])
    for i in range(len(dist_edges_sort) - 1, 0, -1):
        dingdian_and_fushudian1 = []
        dingdian_and_fushudian2 = []
        dingdian_and_fushudian1.append(result_edges[dist_edges_sort[i]][0])
        dingdian_and_fushudian2.append(result_edges[dist_edges_sort[i]][1])
        for j in range(0,len(maxrhoindex)):
            if maxrhoindex[j] == result_edges[dist_edges_sort[i]][0]:
                dingdian_and_fushudian1.append(j)
            if maxrhoindex[j] == result_edges[dist_edges_sort[i]][1]:
                dingdian_and_fushudian2.append(j)
        biaoji=0
        for j in range(0,len(dingdian_and_fushudian1)):
            for h in range(0,len(dingdian_and_fushudian2)):
                if dingdian_and_fushudian1[j] in k_mindist1[dingdian_and_fushudian2[h]]:
                    if dingdian_and_fushudian2[h] in k_mindist1[dingdian_and_fushudian1[j]]:
                        biaoji=1
                        break
            if biaoji==1:
                break
        if biaoji==1:
            break
        if biaoji==0:
            cut_edges.append(dist_edges_sort[i])
    return cut_edges

#每个噪声点分配给离他最近的非噪声点所属的簇
def fenpeizaoshengdian(zaoshenglist,distMatrix,n,zaosheng):
    mindistzaosheng=np.zeros(n,dtype=int)
    for i in range(len(zaoshenglist)):
        mindist=np.max(distMatrix[zaoshenglist[i]])
        for j in range(2,n):
            if i==j:
                continue
            if zaosheng[j]==1:
                continue
            if distMatrix[zaoshenglist[i]][j]<mindist:
                mindist=distMatrix[zaoshenglist[i]][j]
                mindistzaosheng[zaoshenglist[i]]=j
    return mindistzaosheng

def selected_nodes_Cluster(selected_nodes,cut_edges,result_edges):
    selected_nodes_cluster = OrderedDict()  # 存储选中节点的集合字典
    selected_nodes_label = np.ones(len(selected_nodes), dtype=int)
    for i in range(len(cut_edges)):
        selected_nodes_label[selected_nodes.index(result_edges[cut_edges[i]][1])] = -1
    count = 1
    selected_nodes_cluster[count] = []
    for i in range(len(selected_nodes)):
        if selected_nodes_label[i] == -1:
            count = count + 1
            selected_nodes_cluster[count] = []
            selected_nodes_cluster[count].append(selected_nodes[i])
        else:
            selected_nodes_cluster[count].append(selected_nodes[i])
    return selected_nodes_cluster,count

if __name__ == '__main__':
    # data = np.array(pd.read_table(r'D:\project\数据集\Aggregation.txt', sep='\t', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\flame有噪声.txt', sep='\t', engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\Spiral.txt', sep='\t', engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\cth3.txt', sep='\t', engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\d6.txt', sep='\t', engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\ls3.txt', sep='\t', engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\Compound.txt', sep='\t', usecols=[0,1],engine='python', header=None))
    data = np.array(pd.read_table(r'D:\project\数据集\jain.txt', sep='\t', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\lineblobs.txt', sep='   ', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\threecircles.txt', sep='   ', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\Twomoons.txt', sep=' ', usecols=[1,2],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\Zigzag.txt', sep=' ', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\pathbased.txt', sep='\t', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\t4.txt', sep='\t', engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\t7.txt', sep='\t', engine='python', header=None))
    # data, y1 = datasets.make_circles(n_samples=2000, factor=0.6, noise=0.04,shuffle=True,random_state=44)
    # data, y2 = datasets.make_moons(n_samples=500, noise=0.1,shuffle=True,random_state=44)
    # data, y2 = datasets.make_blobs(n_samples=1000,n_features=2,cluster_std=0.8, centers=10,shuffle=True,random_state=44)
    # data = np.array(pd.read_table(r'D:\project\数据集\D31.txt', sep=',', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\grid.txt', sep=' ', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\worms_2d.txt', sep=' ', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\2d-20c-no0.txt', sep=',', usecols=[0,1],engine='python', header=None))
    # data = np.array(pd.read_table(r'D:\project\数据集\heartshaped.txt', sep=',', usecols=[0,1],engine='python', header=None))

    x = data[:, 0]
    y = data[:, 1]
    plt.scatter(x,y,s=0.002,c='black')
    plt.scatter(x,y,c='deepskyblue',s=50,marker='o',edgecolors='black')
    plt.xticks([])
    plt.yticks([])
    # plt.savefig('D:\project\论文需要的图\英文论文算法步骤图\英文论文算法步骤图1.png')
    plt.show()
    n = x.shape[0]
    distList = pdist(data, metric='euclidean')
    distMatrix = squareform(distList)
    k1=0
    k2=7
    k_mindist = k_Mindist(n, distMatrix, k1)
    zaosheng, lenzaosheng, zaoshenglist = zaoSheng(n, k1, k_mindist, distMatrix)
    mindistzaosheng = fenpeizaoshengdian(zaoshenglist, distMatrix, n,zaosheng)
    #
    data1 = []
    data1todata = np.zeros(n - lenzaosheng, dtype=int)
    count = -1
    for i in range(n):
        if zaosheng[i]==1:
            plt.scatter(data[i,0],data[i,1],c='black',s=50,marker='s')
            continue
        else:
            data1.append(data[i])
            count = count + 1
            data1todata[count] = i
            plt.scatter(data[i,0],data[i,1],c='deepskyblue',s=50,marker='o',edgecolors='black')
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图\英文论文算法步骤图2.png')
    plt.show()
    distList = pdist(data1, metric='euclidean')
    distMatrix = squareform(distList)
    data1 = np.array(data1)

    xx = data1[:, 0]
    yy = data1[:, 1]
    plt.scatter(xx, yy,c='deepskyblue',s=50,marker='o',edgecolors='black')
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图\英文论文算法步骤图3.png')
    plt.show()
    n = xx.shape[0]
    k_mindist1 = k_Mindist(n, distMatrix, k2)
    rho, fknn = getRho(n, k2, k_mindist1)
    maxrhoindex =hexindianshibie(n, k2, k_mindist1, rho,distMatrix)
    finalmaxrhoindex = delete_repeat_hexindian(maxrhoindex)
    print('数量', len(maxrhoindex), len(finalmaxrhoindex))
    for i in range(n):
        if i in finalmaxrhoindex:
            plt.scatter(data1[i,0],data1[i,1],marker='*',c='red',s=30)
        else:
            plt.scatter(data1[i,0],data1[i,1],c='skyblue',s=30,marker='o')
    for i in range(n):
        start = data1[i]
        end = data1[maxrhoindex[i]]
        plt.arrow(start[0], start[1], end[0]-start[0], end[1]-start[1],head_width=0.3, lw=0.2, facecolor="black", length_includes_head=True,fill=False)  # ⻓度计算包含箭头箭尾
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图2\代表点指向图1.png')
    plt.show()
    for i in range(n):
        if i in finalmaxrhoindex:
            plt.scatter(data1[i,0],data1[i,1],marker='*',c='red',s=30)
        else:
            plt.scatter(data1[i,0],data1[i,1],c='skyblue',s=30,marker='o')
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图2\代表点无指向图1.png')
    plt.show()
    print("构建最小生成树点的数量", len(finalmaxrhoindex))
    print("开始记录时间")

    selected_nodes, result_edges, dist_edges = prime(finalmaxrhoindex, distMatrix)

    for i in range(len(selected_nodes)):
        plt.scatter(data1[selected_nodes[i], 0], data1[selected_nodes[i], 1], marker='*',c='red',s=50)
    for i in range(len(result_edges)):
        plt.plot([data1[result_edges[i][0], 0], data1[result_edges[i][1], 0]],
                 [data1[result_edges[i][0], 1], data1[result_edges[i][1], 1]], color='b')
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图2\英文论文算法步骤图5.png')
    plt.show()

    cut_edges = qiebian(dist_edges, result_edges, maxrhoindex, k_mindist1, k2)
    print('聚类簇数', len(cut_edges) + 1)

    for i in range(len(selected_nodes)):
        plt.scatter(data1[selected_nodes[i], 0], data1[selected_nodes[i], 1], marker='*',c='red',s=30)
    for i in range(len(result_edges)):
        if i not in cut_edges:
            plt.plot([data1[result_edges[i][0], 0], data1[result_edges[i][1], 0]],
                     [data1[result_edges[i][0], 1], data1[result_edges[i][1], 1]], color='b')
        if i in cut_edges:
            plt.plot([data1[result_edges[i][0], 0], data1[result_edges[i][1], 0]],
                     [data1[result_edges[i][0], 1], data1[result_edges[i][1], 1]], color='darkorange')
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图2\英文论文算法步骤图6.png')
    plt.show()

    selected_nodes_cluster,count=selected_nodes_Cluster(selected_nodes,cut_edges,result_edges)
    for k, v in selected_nodes_cluster.items():
        E = data1[v]
        plt.scatter(E[:, 0], E[:, 1],s=50,marker='o',edgecolors='black')
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图\英文论文算法步骤图7.png')
    plt.show()

    b=[]
    for i in range(n):
        b.append(i)
    dirho=list(set(b).difference(set(finalmaxrhoindex)))  # b中有⽽a中没有的

    for i in range(len(dirho)):
        for j in range(len(selected_nodes_cluster)):
            if maxrhoindex[dirho[i]] in selected_nodes_cluster[j+1]:
                selected_nodes_cluster[j+1].append(dirho[i])
                break

    for k, v in selected_nodes_cluster.items():
        E = data1[v]
        plt.scatter(E[:, 0], E[:, 1],s=50,marker='o',edgecolors='black')
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图\英文论文算法步骤图8.png')
    plt.show()

    aset=[]
    for i in range(count):
        aset.append(i+1)
    cluster_data = (-1) * np.ones(x.shape[0], dtype=int)
    for i in range(len(aset)):
        for j in range(len(selected_nodes_cluster[aset[i]])):
            cluster_data[data1todata[selected_nodes_cluster[aset[i]][j]]] = data1todata[aset[i]]
    for i in range(x.shape[0]):
        if cluster_data[i] == -1:
            cluster_data[i] = cluster_data[mindistzaosheng[i]]
    clusterSetFinally = OrderedDict()
    for i in range(len(aset)):
        clusterSetFinally[aset[i]] = []
    for i in range(x.shape[0]):
        for j in range(len(aset)):
            if cluster_data[i] == data1todata[aset[j]]:
                clusterSetFinally[aset[j]].append(i)
                break
    print(clusterSetFinally)

    for k, v in clusterSetFinally.items():
        E = data[v]
        plt.scatter(E[:, 0], E[:, 1],s=50,marker='o',edgecolors='black')
    plt.xticks([])
    plt.yticks([])
    plt.savefig('D:\project\论文需要的图\英文论文算法步骤图\英文论文算法步骤图9.png')
    plt.show()


"""
============================
Author:柠檬班-木森
Time:2019/10/24
E-mail:3247119728@qq.com
Company:湖南零檬信息技术有限公司
============================
"""
import openpyxl

# 第一步：打开excle文件

workbook = openpyxl.load_workbook("cases.xlsx")

# 第二步：选择工作薄中的某个表单（sheet）
sh = workbook["test"]

# 读取表格的内容
# print(sh.cell(row=1, column=3).value)
#
# # 往表格中写入内容
# sh.cell(row=1,column=4).value = 999
# sh.cell(row=1,column=5,value=8888)
#
# # 写入完之后，一定要保存
# workbook.save("cases.xlsx")

# 获取表单中最大的行
# max_row = sh.max_row
# print(max_row)
#
# # 获取表单中最大的列
# max_column = sh.max_column
# print(max_column)

# 按行获取表单中的，所有格子对象，每一行的格子放在一个元组当中/
rows = sh.rows
for r in rows:
    print(r)


import pytest

from lava_dispatcher.actions.boot.fvp import CheckFVPVersionAction
from tests.lava_dispatcher.test_basic import Factory

@pytest.fixture
def factory():
    return Factory()

fvp_version_output = """ARM V8 Foundation Platformr0p0 [1.2.3]
Copyright 2000-2021 ARM Limited.
All Rights Reserved."""

class TestCheckFVPVersionAction:
    @pytest.fixture
    def action(self, factory):
        job = factory.create_job("fvp-01.jinja2", "sample_jobs/fvp_foundation.yaml")
        return job.pipeline.actions[1].pipeline.actions[0].pipeline.actions[0]

    def test_action_class(self, action):
        assert type(action) is CheckFVPVersionAction

    def test_basic(self, action, mocker):
        action.validate()
        action.logger = mocker.MagicMock()
        conn = mocker.MagicMock()
        run_cmd = mocker.patch(
            "lava_dispatcher.actions.boot.fvp.CheckFVPVersionAction.run_cmd",
            return_value=0,
        )
        parsed_command = mocker.patch(
            "lava_dispatcher.actions.boot.fvp.CheckFVPVersionAction.parsed_command",
            return_value=fvp_version_output,
        )
        conn2 = action.run(conn, 60)
        assert conn2 is conn
        action.logger.results.assert_called()
        entry = action.logger.results.call_args[0][0]
        assert entry["extra"]["fvp-version"] == "ARM V8 Foundation Platformr0p0 [1.2.3]"
        run_cmd.assert_called_once_with(
            [
                "docker",
                "image",
                "inspect",
                "--format",
                "Image foundation:11.8 exists locally",
                "foundation:11.8",
            ],
            allow_fail=True,
        )
        parsed_command.assert_called_once_with(
            [
                "sh",
                "-c",
                "docker run --rm foundation:11.8 /opt/model/Foundation_Platformpkg/models/Linux64_GCC-6.4/Foundation_Platform --version",
            ]
        )


# encoding: utf8

import re
from random import *
from collections import Counter
from pprint import pprint
from heapq import nlargest
from time import time

# -> http://obphio.us/pdfs/lda_tutorial.pdf
# -> http://brooksandrew.github.io/simpleblog/articles/latent-dirichlet-allocation-under-the-hood/

K = 4
ITERATIONS = 1000
alpha = 0.1 # document alpha factor
eta = 0.1 # word eta factor
RANDOMIZE = True
MULTIPLICATION = 10

TOP_N = 6

documents = [
"reksio szczeka na koty",
"pies glosno szczeka",
"koty cicho mrucza",
"reksio to madry pies",
"kerbale buduja rakiety",
"rakiety wynosza satelity",
"satelity sa na orbicie",
"rakiety glosno startuja",
"szybowce leca cicho",
"szybowce startuja z wyciagarki",
"samoloty szturmowe leca nisko",
"krowy jedza trawe",
"kury jedza ziarno",
"krowy pija wode"
]
documents *= MULTIPLICATION

def get_tf(text):
	terms = re.findall('(?u)\w+',text.lower())
	#terms = [t for t in terms if t not in set(['sa','na','z'])]
	terms = [t for t in terms if len(t)>=4]
	tf = Counter(terms)
	return dict(tf)

def weighted_choice(w_map):
	items = w_map.items()
	cum_weights = [0]*len(items)
	cum_weights[0] = items[0][1]
	for i,item in enumerate(items):
		if i==0: continue
		w = item[1]
		cum_weights[i] = cum_weights[i-1]+w
	total = cum_weights[-1]
	r = random() * total
	for i,c in enumerate(cum_weights):
		if r<c: return items[i][0]

#print(weighted_choice({11:.1,22:.1,33:.1}))

docs = list(map(get_tf,documents))
corpus = Counter()
for doc in docs:
	corpus.update(doc)
corpus = dict(corpus)

# TODO min_df

topics = list(range(1,K+1))

# initialize
t_of_w_in_d = {w:{} for w in corpus}
total_d_in_t = {d:{t:0 for t in topics} for d in range(len(docs))} # ~phi
total_w_in_t = {w:{t:0 for t in topics} for w in corpus} # ~theta
total_in_t = {t:0 for t in topics}
## pass # gamma

# randomly select new topic for word in document
i=0
for d,doc in enumerate(docs):
	for w in doc:
		t = randint(1,K) if RANDOMIZE else (i%K)+1
		t_of_w_in_d[w][d] = t
		total_in_t[t] += 1
		total_d_in_t[d][t] += 1
		total_w_in_t[w][t] += 1
		i += 1

if 0:
	pprint(total_d_in_t)
	pprint(total_w_in_t)
	pprint(total_in_t)
	#exit()

t0=time()
for _ in range(ITERATIONS):
	# select new topic for word in document
	for d,doc in enumerate(docs):
		for w in doc:

			# gdzie to powinno byc?
			t = t_of_w_in_d[w][d]
			total_in_t[t] -= 1
			total_d_in_t[d][t] -= 1
			total_w_in_t[w][t] -= 1
			old_t = t

			p_of_t = {}
			for t in topics:
				# ??? czu tu nie powinna powstac macierz kwadratowa?
				denom_a = len(doc) + K*alpha
				denom_b = total_in_t[t] + len(corpus)*eta
				p_of_word_in_t = 1.* (total_w_in_t[w][t] + eta)   / denom_b
				p_of_t_in_doc  = 1.* (total_d_in_t[d][t] + alpha) / denom_a
				p_of_t[t] = p_of_word_in_t * p_of_t_in_doc
				#print(d,w,t,p_of_word_in_t,p_of_t_in_doc)
			#print(p_of_t)

			t = weighted_choice(p_of_t)

			t_of_w_in_d[w][d] = t
			total_in_t[t] += 1
			total_d_in_t[d][t] += 1
			total_w_in_t[w][t] += 1

if 1:
	#pprint(total_d_in_t)
	pprint(total_w_in_t)
	pprint(total_in_t)

# theta - topics probabilities per document
p_d_in_t = {d:{} for d in range(len(docs))}
for d,doc in enumerate(docs):
	denom_a = 1.* len(doc) + K*alpha
	for t in topics:
		p_d_in_t[d][t] = 1.* (total_d_in_t[d][t] + alpha) / denom_a

#pprint(p_d_in_t)

# phi - topic probabilities per word
p_w_in_t = {w:{} for w in corpus} # phi
for w in corpus:
	for t in topics:
		denom_b = total_in_t[t] + len(corpus)*eta
		p_w_in_t[w][t] = 1.* (total_w_in_t[w][t] + eta) / denom_b
pprint(p_w_in_t)

print(time()-t0,'s',ITERATIONS/(time()-t0),'iter/s')

# topic best words
by_t = {t:{} for t in topics}
for w in p_w_in_t:
	for t in total_w_in_t[w]:
		p = p_w_in_t[w][t]
		by_t[t][w] = p
for t in by_t:
	total = sum(by_t[t].values())
	for w in by_t[t]:
		by_t[t][w] /= total
	pprint((t,nlargest(TOP_N,by_t[t].items(),key=lambda x:x[1])))
	#print(t,by_t[t])


from django.contrib import admin

# Register your models here.

from .models import Cart, CartItem

class CartAdmin(admin.ModelAdmin):
    list_display = ['cart_id', 'date_added', 'user']
    list_editable = ['user']
    list_per_page = 20

admin.site.register(Cart,CartAdmin)

class CartItemAdmin(admin.ModelAdmin):
    list_display = ['product', 'cart', 'quantity', 'active']
    list_editable = ['quantity', 'active']
    list_per_page = 20

admin.site.register(CartItem,CartItemAdmin)


from __future__ import print_function
import web
import sys
import codecs
import os.path
from catalog.read_rc import read_rc
import psycopg2
from catalog.infostore import get_site
from catalog.merge.merge_marc import attempt_merge, build_marc
import catalog.marc.fast_parse as fast_parse

sys.stdout = codecs.getwriter('utf-8')(sys.stdout)

# need to use multiple databases
# use psycopg2 to until open library is upgraded to web 3.0

rc = read_rc()
threshold = 875

conn = psycopg2.connect("dbname='%s' user='%s' host='%s' password='%s'" \
        % ('ol_merge', rc['user'], rc['host'], rc['pw']));
cur1 = conn.cursor()
cur2 = conn.cursor()

site = get_site()

marc_path = '/2/pharos/marc/'

def get_marc(loc):
    try:
        filename, p, l = loc.split(':')
    except ValueError:
        return None
    if not os.path.exists(marc_path + filename):
        return None
    f = open(marc_path + filename)
    f.seek(int(p))
    buf = f.read(int(l))
    f.close()
    rec = fast_parse.read_edition(buf)
    if rec:
        return build_marc(rec)

for line in open('dups'):
    v, num = eval(line)
    cur2.execute('select key from isbn where value=%(v)s', {'v':v})
    editions = []
    for i in cur2.fetchall():
        key = i[0]
        t = site.withKey(key)
        mc =  site.versions({'key': key})[0].machine_comment
        editions.append({'key': key, 'title': t.title, 'loc': mc})
    if len(editions) != 2:
        continue
    if any(not i['loc'] or i['loc'].startswith('amazon:') for i in editions):
        continue
    e1 = get_marc(editions[0]['loc'])
    if not e1:
        continue
    e2 = get_marc(editions[1]['loc'])
    if not e2:
        continue

#    print v, [i['title'] for i in editions]
#    print e1
#    print e2
    match = attempt_merge(e1, e2, threshold, debug=False)
    if match:
        print(tuple([v] + [i['key'] for i in editions]))

sys.exit(0)
cur1.execute('select value, count(*) as num from isbn group by value having count(*) > 1')
for i in cur1.fetchall():
    print(i)
    cur2.execute('select key from isbn where value=%(v)s', {'v':i[0]})
    print(cur2.fetchall())


"""Synchronise a pipeline TEMPLATE branch with the template.
"""

import json
import logging
import os
import re
import shutil

import git
import questionary
import requests
import requests_cache
import rich
import yaml
from git import GitCommandError, InvalidGitRepositoryError

import nf_core
import nf_core.create
import nf_core.list
import nf_core.utils

log = logging.getLogger(__name__)

class SyncException(Exception):
    """Exception raised when there was an error with TEMPLATE branch synchronisation"""

    pass

class PullRequestException(Exception):
    """Exception raised when there was an error creating a Pull-Request on GitHub.com"""

    pass

class PipelineSync:
    """Object to hold syncing information and results.

    Args:
        pipeline_dir (str): The path to the Nextflow pipeline root directory
        from_branch (str): The branch to use to fetch config vars. If not set, will use current active branch
        make_pr (bool): Set this to `True` to create a GitHub pull-request with the changes
        gh_username (str): GitHub username
        gh_repo (str): GitHub repository name
        template_yaml_path (str): Path to template.yml file for pipeline creation settings. DEPRECATED

    Attributes:
        pipeline_dir (str): Path to target pipeline directory
        from_branch (str): Repo branch to use when collecting workflow variables. Default: active branch.
        original_branch (str): Repo branch that was checked out before we started.
        made_changes (bool): Whether making the new template pipeline introduced any changes
        make_pr (bool): Whether to try to automatically make a PR on GitHub.com
        required_config_vars (list): List of nextflow variables required to make template pipeline
        gh_username (str): GitHub username
        gh_repo (str): GitHub repository name
    """

    def __init__(
        self,
        pipeline_dir,
        from_branch=None,
        make_pr=False,
        gh_repo=None,
        gh_username=None,
        template_yaml_path=None,
    ):
        """Initialise syncing object"""

        self.pipeline_dir = os.path.abspath(pipeline_dir)
        self.from_branch = from_branch
        self.original_branch = None
        self.original_merge_branch = f"nf-core-template-merge-{nf_core.__version__}"
        self.merge_branch = self.original_merge_branch
        self.made_changes = False
        self.make_pr = make_pr
        self.gh_pr_returned_data = {}
        self.required_config_vars = ["manifest.name", "manifest.description", "manifest.version", "manifest.author"]

        self.gh_username = gh_username
        self.gh_repo = gh_repo
        self.pr_url = ""

        self.config_yml_path, self.config_yml = nf_core.utils.load_tools_config(self.pipeline_dir)

        # Throw deprecation warning if template_yaml_path is set
        if template_yaml_path is not None:
            log.warning(
                f"The `template_yaml_path` argument is deprecated. Saving pipeline creation settings in .nf-core.yml instead. Please remove {template_yaml_path} file."
            )
            if "template" in self.config_yml:
                overwrite_template = questionary.confirm(
                    f"A template section already exists in '{self.config_yml_path}'. Do you want to overwrite?",
                    style=nf_core.utils.nfcore_question_style,
                    default=False,
                ).unsafe_ask()
            if overwrite_template or "template" not in self.config_yml:
                with open(template_yaml_path, "r") as f:
                    self.config_yml["template"] = yaml.safe_load(f)
                with open(self.config_yml_path, "w") as fh:
                    yaml.safe_dump(self.config_yml, fh)
                log.info(f"Saved pipeline creation settings to '{self.config_yml_path}'")
                raise SystemExit(
                    f"Please commit your changes and delete the {template_yaml_path} file. Then run the sync command again."
                )

        # Set up the API auth if supplied on the command line
        self.gh_api = nf_core.utils.gh_api
        self.gh_api.lazy_init()
        if self.gh_username and "GITHUB_AUTH_TOKEN" in os.environ:
            log.debug(f"Authenticating sync as {self.gh_username}")
            self.gh_api.setup_github_auth(
                requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"])
            )

    def sync(self):
        """Find workflow attributes, create a new template pipeline on TEMPLATE"""

        # Clear requests_cache so that we don't get stale API responses
        requests_cache.clear()

        log.info(f"Pipeline directory: {self.pipeline_dir}")
        if self.from_branch:
            log.info(f"Using branch '{self.from_branch}' to fetch workflow variables")
        if self.make_pr:
            log.info("Will attempt to automatically create a pull request")

        self.inspect_sync_dir()
        self.get_wf_config()
        self.checkout_template_branch()
        self.delete_template_branch_files()
        self.make_template_pipeline()
        self.commit_template_changes()

        # Push and make a pull request if we've been asked to
        if self.made_changes and self.make_pr:
            try:
                # Check that we have an API auth token
                if os.environ.get("GITHUB_AUTH_TOKEN", "") == "":
                    raise PullRequestException("GITHUB_AUTH_TOKEN not set!")

                # Check that we know the github username and repo name
                if self.gh_username is None and self.gh_repo is None:
                    raise PullRequestException("Could not find GitHub username and repo name")

                self.push_template_branch()
                self.create_merge_base_branch()
                self.push_merge_branch()
                self.make_pull_request()
                self.close_open_template_merge_prs()
            except PullRequestException as e:
                self.reset_target_dir()
                raise PullRequestException(e)

        self.reset_target_dir()

        if not self.made_changes:
            log.info("No changes made to TEMPLATE - sync complete")
        elif not self.make_pr:
            log.info(
                f"Now try to merge the updates in to your pipeline:\n  cd {self.pipeline_dir}\n  git merge TEMPLATE"
            )

    def inspect_sync_dir(self):
        """Takes a look at the target directory for syncing. Checks that it's a git repo
        and makes sure that there are no uncommitted changes.
        """
        # Check that the pipeline_dir is a git repo
        try:
            self.repo = git.Repo(self.pipeline_dir)
        except InvalidGitRepositoryError:
            raise SyncException(f"'{self.pipeline_dir}' does not appear to be a git repository")

        # get current branch so we can switch back later
        self.original_branch = self.repo.active_branch.name
        log.info(f"Original pipeline repository branch is '{self.original_branch}'")

        # Check to see if there are uncommitted changes on current branch
        if self.repo.is_dirty(untracked_files=True):
            raise SyncException(
                "Uncommitted changes found in pipeline directory!\nPlease commit these before running nf-core sync"
            )

    def get_wf_config(self):
        """Check out the target branch if requested and fetch the nextflow config.
        Check that we have the required config variables.
        """
        # Try to check out target branch (eg. `origin/dev`)
        try:
            if self.from_branch and self.repo.active_branch.name != self.from_branch:
                log.info(f"Checking out workflow branch '{self.from_branch}'")
                self.repo.git.checkout(self.from_branch)
        except GitCommandError:
            raise SyncException(f"Branch `{self.from_branch}` not found!")

        # If not specified, get the name of the active branch
        if not self.from_branch:
            try:
                self.from_branch = self.repo.active_branch.name
            except GitCommandError as e:
                log.error(f"Could not find active repo branch: {e}")

        # Fetch workflow variables
        log.debug("Fetching workflow config variables")
        self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir)

        # Check that we have the required variables
        for rvar in self.required_config_vars:
            if rvar not in self.wf_config:
                raise SyncException(f"Workflow config variable `{rvar}` not found!")

    def checkout_template_branch(self):
        """
        Try to check out the origin/TEMPLATE in a new TEMPLATE branch.
        If this fails, try to check out an existing local TEMPLATE branch.
        """
        # Try to check out the `TEMPLATE` branch
        try:
            self.repo.git.checkout("origin/TEMPLATE", b="TEMPLATE")
        except GitCommandError:
            # Try to check out an existing local branch called TEMPLATE
            try:
                self.repo.git.checkout("TEMPLATE")
            except GitCommandError:
                raise SyncException("Could not check out branch 'origin/TEMPLATE' or 'TEMPLATE'")

    def delete_template_branch_files(self):
        """
        Delete all files in the TEMPLATE branch
        """
        # Delete everything
        log.info("Deleting all files in 'TEMPLATE' branch")
        for the_file in os.listdir(self.pipeline_dir):
            if the_file == ".git":
                continue
            file_path = os.path.join(self.pipeline_dir, the_file)
            log.debug(f"Deleting {file_path}")
            try:
                if os.path.isfile(file_path):
                    os.unlink(file_path)
                elif os.path.isdir(file_path):
                    shutil.rmtree(file_path)
            except Exception as e:
                raise SyncException(e)

    def make_template_pipeline(self):
        """
        Delete all files and make a fresh template using the workflow variables
        """
        log.info("Making a new template pipeline using pipeline variables")

        # Only show error messages from pipeline creation
        logging.getLogger("nf_core.create").setLevel(logging.ERROR)

        # Re-write the template yaml info from .nf-core.yml config
        if "template" in self.config_yml:
            with open(self.config_yml_path, "w") as config_path:
                yaml.safe_dump(self.config_yml, config_path)

        try:
            nf_core.create.PipelineCreate(
                name=self.wf_config["manifest.name"].strip('"').strip("'"),
                description=self.wf_config["manifest.description"].strip('"').strip("'"),
                version=self.wf_config["manifest.version"].strip('"').strip("'"),
                no_git=True,
                force=True,
                outdir=self.pipeline_dir,
                author=self.wf_config["manifest.author"].strip('"').strip("'"),
                plain=True,
            ).init_pipeline()
        except Exception as err:
            # Reset to where you were to prevent git getting messed up.
            self.repo.git.reset("--hard")
            raise SyncException(f"Failed to rebuild pipeline from template with error:\n{err}")

    def commit_template_changes(self):
        """If we have any changes with the new template files, make a git commit"""
        # Check that we have something to commit
        if not self.repo.is_dirty(untracked_files=True):
            log.info("Template contains no changes - no new commit created")
            return False
        # Commit changes
        try:
            self.repo.git.add(A=True)
            self.repo.index.commit(f"Template update for nf-core/tools version {nf_core.__version__}")
            self.made_changes = True
            log.info("Committed changes to 'TEMPLATE' branch")
        except Exception as e:
            raise SyncException(f"Could not commit changes to TEMPLATE:\n{e}")
        return True

    def push_template_branch(self):
        """If we made any changes, push the TEMPLATE branch to the default remote
        and try to make a PR. If we don't have the auth token, try to figure out a URL
        for the PR and print this to the console.
        """
        log.info(f"Pushing TEMPLATE branch to remote: '{os.path.basename(self.pipeline_dir)}'")
        try:
            self.repo.git.push()
        except GitCommandError as e:
            raise PullRequestException(f"Could not push TEMPLATE branch:\n  {e}")

    def create_merge_base_branch(self):
        """Create a new branch from the updated TEMPLATE branch
        This branch will then be used to create the PR
        """
        # Check if branch exists already
        branch_list = [b.name for b in self.repo.branches]
        if self.merge_branch in branch_list:
            merge_branch_format = re.compile(rf"{self.original_merge_branch}-(\d+)")
            max_branch = max(
                [1]
                + [
                    int(merge_branch_format.match(branch).groups()[0])
                    for branch in branch_list
                    if merge_branch_format.match(branch)
                ]
            )
            new_branch = f"{self.original_merge_branch}-{max_branch+1}"
            log.info(f"Branch already existed: '{self.merge_branch}', creating branch '{new_branch}' instead.")
            self.merge_branch = new_branch

        # Create new branch and checkout
        log.info(f"Checking out merge base branch '{self.merge_branch}'")
        try:
            self.repo.create_head(self.merge_branch)
        except GitCommandError as e:
            raise SyncException(f"Could not create new branch '{self.merge_branch}'\n{e}")

    def push_merge_branch(self):
        """Push the newly created merge branch to the remote repository"""
        log.info(f"Pushing '{self.merge_branch}' branch to remote")
        try:
            origin = self.repo.remote()
            origin.push(self.merge_branch)
        except GitCommandError as e:
            raise PullRequestException(f"Could not push branch '{self.merge_branch}':\n  {e}")

    def make_pull_request(self):
        """Create a pull request to a base branch (default: dev),
        from a head branch (default: TEMPLATE)

        Returns: An instance of class requests.Response
        """
        log.info("Submitting a pull request via the GitHub API")

        pr_title = f"Important! Template update for nf-core/tools v{nf_core.__version__}"
        pr_body_text = (
            "Version `{tag}` of [nf-core/tools](https://github.com/nf-core/tools) has just been released with updates to the nf-core template. "
            "This automated pull-request attempts to apply the relevant updates to this pipeline.\n\n"
            "Please make sure to merge this pull-request as soon as possible, "
            f"resolving any merge conflicts in the `{self.merge_branch}` branch (or your own fork, if you prefer). "
            "Once complete, make a new minor release of your pipeline.\n\n"
            "For instructions on how to merge this PR, please see "
            "[https://nf-co.re/docs/contributing/sync/](https://nf-co.re/docs/contributing/sync/#merging-automated-prs).\n\n"
            "For more information about this release of [nf-core/tools](https://github.com/nf-core/tools), "
            "please see the `v{tag}` [release page](https://github.com/nf-core/tools/releases/tag/{tag})."
        ).format(tag=nf_core.__version__)

        # Make new pull-request
        stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors())
        with self.gh_api.cache_disabled():
            try:
                r = self.gh_api.request_retry(
                    f"https://api.github.com/repos/{self.gh_repo}/pulls",
                    post_data={
                        "title": pr_title,
                        "body": pr_body_text,
                        "maintainer_can_modify": True,
                        "head": self.merge_branch,
                        "base": self.from_branch,
                    },
                )
            except Exception as e:
                stderr.print_exception()
                raise PullRequestException(f"Something went badly wrong - {e}")
            else:
                self.gh_pr_returned_data = r.json()
                self.pr_url = self.gh_pr_returned_data["html_url"]
                log.debug(f"GitHub API PR worked, return code {r.status_code}")
                log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}")

    def close_open_template_merge_prs(self):
        """Get all template merging branches (starting with 'nf-core-template-merge-')
        and check for any open PRs from these branches to the self.from_branch
        If open PRs are found, add a comment and close them
        """
        log.info("Checking for open PRs from template merge branches")

        # Look for existing pull-requests
        list_prs_url = f"https://api.github.com/repos/{self.gh_repo}/pulls"
        with self.gh_api.cache_disabled():
            list_prs_request = self.gh_api.get(list_prs_url)
        try:
            list_prs_json = json.loads(list_prs_request.content)
            list_prs_pp = json.dumps(list_prs_json, indent=4)
        except:
            list_prs_json = list_prs_request.content
            list_prs_pp = list_prs_request.content

        log.debug(f"GitHub API listing existing PRs:\n{list_prs_url}\n{list_prs_pp}")
        if list_prs_request.status_code != 200:
            log.warning(f"Could not list open PRs ('{list_prs_request.status_code}')\n{list_prs_url}\n{list_prs_pp}")
            return False

        for pr in list_prs_json:
            log.debug(f"Looking at PR from '{pr['head']['ref']}': {pr['html_url']}")
            # Ignore closed PRs
            if pr["state"] != "open":
                log.debug(f"Ignoring PR as state not open ({pr['state']}): {pr['html_url']}")
                continue

            # Don't close the new PR that we just opened
            if pr["head"]["ref"] == self.merge_branch:
                continue

            # PR is from an automated branch and goes to our target base
            if pr["head"]["ref"].startswith("nf-core-template-merge-") and pr["base"]["ref"] == self.from_branch:
                self.close_open_pr(pr)

    def close_open_pr(self, pr):
        """Given a PR API response, add a comment and close."""
        log.debug(f"Attempting to close PR: '{pr['html_url']}'")

        # Make a new comment explaining why the PR is being closed
        comment_text = (
            f"Version `{nf_core.__version__}` of the [nf-core/tools](https://github.com/nf-core/tools) pipeline template has just been released. "
            f"This pull-request is now outdated and has been closed in favour of {self.pr_url}\n\n"
            f"Please use {self.pr_url} to merge in the new changes from the nf-core template as soon as possible."
        )
        with self.gh_api.cache_disabled():
            self.gh_api.post(url=pr["comments_url"], data=json.dumps({"body": comment_text}))

        # Update the PR status to be closed
        with self.gh_api.cache_disabled():
            pr_request = self.gh_api.patch(url=pr["url"], data=json.dumps({"state": "closed"}))
        try:
            pr_request_json = json.loads(pr_request.content)
            pr_request_pp = json.dumps(pr_request_json, indent=4)
        except:
            pr_request_json = pr_request.content
            pr_request_pp = pr_request.content

        # PR update worked
        if pr_request.status_code == 200:
            log.debug(f"GitHub API PR-update worked:\n{pr_request_pp}")
            log.info(
                f"Closed GitHub PR from '{pr['head']['ref']}' to '{pr['base']['ref']}': {pr_request_json['html_url']}"
            )
            return True
        # Something went wrong
        else:
            log.warning(f"Could not close PR ('{pr_request.status_code}'):\n{pr['url']}\n{pr_request_pp}")
            return False

    def reset_target_dir(self):
        """
        Reset the target pipeline directory. Check out the original branch.
        """
        log.info(f"Checking out original branch: '{self.original_branch}'")
        try:
            self.repo.git.checkout(self.original_branch)
        except GitCommandError as e:
            raise SyncException(f"Could not reset to original branch `{self.original_branch}`:\n{e}")


"""
https://github.github.com/gfm/#paragraph
"""
from test.utils import act_and_assert

import pytest

@pytest.mark.gfm
def test_paragraph_series_c_b():
    """
    Test case:  Paragraphs ends with a backslash escape
    was:        test_paragraph_extra_21
    """

    # Arrange
    source_markdown = """a fun day\\\\"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):a fun day\\\b\\:]",
        "[end-para:::True]",
    ]
    expected_gfm = """
a fun day\\
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_bh():
    """
    Test case:  Paragraph ends with a backslash as in a hard line break
    was:        test_paragraph_extra_22
    """

    # Arrange
    source_markdown = """this was \\
"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):this was \\:]",
        "[end-para:::True]",
        "[BLANK(2,1):]",
    ]
    expected_gfm = """
this was \\
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_sh():
    """
    Test case:  Paragraph ends with 2+ spaces as in a hard line break
    was:        test_paragraph_extra_23
    """

    # Arrange
    source_markdown = """no line break?\a\a\a
""".replace(
        "\a", " "
    )
    expected_tokens = [
        "[para(1,1)::   ]",
        "[text(1,1):no line break?:]",
        "[end-para:::True]",
        "[BLANK(2,1):]",
    ]
    expected_gfm = """
no line break?
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_cs():
    """
    Test case:  Paragraph string ending with a code span.
    was:        test_paragraph_extra_24
    """

    # Arrange
    source_markdown = """a fun ``day``"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):a fun :]",
        "[icode-span(1,7):day:``::]",
        "[end-para:::True]",
    ]
    expected_gfm = """
a fun day
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_cr():
    """
    Test case:  Paragraph string ending with a character reference.
    was:        test_paragraph_extra_25
    """

    # Arrange
    source_markdown = """played on &"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):played on \a&\a\a&\a&\a\a:]",
        "[end-para:::True]",
    ]
    expected_gfm = """
played on &
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_rh():
    """
    Test case:  Paragraph string ending with a raw html block.
    was:        test_paragraph_extra_26
    """

    # Arrange
    source_markdown = """really, <there it='is'>"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):really, :]",
        "[raw-html(1,9):there it='is']",
        "[end-para:::True]",
    ]
    expected_gfm = """
really, <there it='is'>
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_ua():
    """
    Test case:  Paragraph string ending with an URI autolink
    was:        test_paragraph_extra_27
    """

    # Arrange
    source_markdown = """at <http://www.google.com>"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):at :]",
        "[uri-autolink(1,4):http://www.google.com]",
        "[end-para:::True]",
    ]
    expected_gfm = (
        """
at http://www.google.com
"""
    )

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_ea():
    """
    Test case:  Paragraph string ending with an email autolink
    was:        test_paragraph_extra_28
    """

    # Arrange
    source_markdown = """contact <foo@bar.com>"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):contact :]",
        "[email-autolink(1,9):foo@bar.com]",
        "[end-para:::True]",
    ]
    expected_gfm = """
contact foo@bar.com
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_e():
    """
    Test case:  Paragraph string ending with an emphasis
    was:        test_paragraph_extra_29
    """

    # Arrange
    source_markdown = """it's *me*"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):it's :]",
        "[emphasis(1,6):1:*]",
        "[text(1,7):me:]",
        "[end-emphasis(1,9)::]",
        "[end-para:::True]",
    ]
    expected_gfm = """
it's me
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_l():
    """
    Test case:  Paragraph string ending with a link.
    was:        test_paragraph_extra_30
    """

    # Arrange
    source_markdown = """like [Foo](/uri "t")"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):like :]",
        '[link(1,6):inline:/uri:t::::Foo:False:":: :]',
        "[text(1,7):Foo:]",
        "[end-link::]",
        "[end-para:::True]",
    ]
    expected_gfm = """
like Foo
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)

@pytest.mark.gfm
def test_paragraph_series_c_i():
    """
    Test case:  Paragraph string ending with an image
    was:        test_paragraph_extra_31
    """

    # Arrange
    source_markdown = """an ![foo](/url "t")"""
    expected_tokens = [
        "[para(1,1):]",
        "[text(1,1):an :]",
        '[image(1,4):inline:/url:t:foo::::foo:False:":: :]',
        "[end-para:::True]",
    ]
    expected_gfm = """
an
"""

    # Act & Assert
    act_and_assert(source_markdown, expected_gfm, expected_tokens)


#!/usr/bin/python3

"""
Create a symbolic link to the latest recorded camera image in
/var/www/html/flight_data/images/
"""

# A watched directory will watch for new files and new directories.
# Any new directory detected will, in turn, be watched is the same
# manor as the the initial directory is.

import logging, logging.handlers
import os, sys, pyinotify, re, time
from pyinotify import WatchManager, Notifier, ProcessEvent #, EventsCodes
import ac_config

mask = pyinotify.IN_CREATE | pyinotify.IN_CLOSE_WRITE

class transferNotifier(pyinotify.ProcessEvent):

#   def process_IN_CREATE(self, event):
#       logging.info("IN_CREATE saw:   %s" % event.pathname)

    def process_IN_CLOSE(self, event):
#        logging.info("IN_CLOSE  saw:   %s" % event.pathname)
        if event.dir: return

        m = reLatest.match(event.pathname)
        if m:
#            logging.info("IN_CLOSE  match: %s" % event.pathname)
            if os.path.islink(sym_link):
                os.remove(sym_link)
            os.symlink(event.pathname, sym_link)

def Monitor(data):

    wm = pyinotify.WatchManager()
    notifier = pyinotify.Notifier(wm, transferNotifier())
    wm.add_watch(data, mask, rec=True, auto_add=True)

    try:
        while 1:
            notifier.process_events()
            if notifier.check_events():
                notifier.read_events()
    except KeyboardInterrupt:
        notifier.stop()
        return

if __name__ == '__main__':

    data     = ac_config.get_config("cameras.path")+"/"
    logfile  = "/tmp/camera_symlink.log"
    sym_link = "/var/www/html/flight_data/images/latest_forward.jpg"

    # This regexp defines a constrained path to prevent falsly linking
    # to any rogue .jpg(s) that are created in the watched directory.
    # regexp hint:       any number of non...   /'s            .'s
    #                                         /----\         /----\
    reLatest = re.compile(data+"forward/[^\.]*\.jpg")

    # setup logging
    logger = logging.getLogger()
    handler = logging.handlers.RotatingFileHandler(logfile, maxBytes=8192, backupCount=10)
    formatter = logging.Formatter("%(asctime)s|%(levelname)s|%(message)s")
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(logging.DEBUG)

    if not os.path.isdir(data):
        logging.critical("Folder '%s' does not exist, holding." % data)

    # hang until 'data' folder is created
    while (not os.path.isdir(data)):
        time.sleep(1)

    # watch the data folder for new files being created
    Monitor(data)


import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d

class SE_Block(nn.Module):                         # Squeeze-and-Excitation block
    def __init__(self, in_planes):
        super(SE_Block, self).__init__()
        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.conv1 = nn.Conv2d(in_planes, in_planes // 16, kernel_size=1)
        self.relu = nn.ReLU()
        self.conv2 = nn.Conv2d(in_planes // 16, in_planes, kernel_size=1)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x = self.avgpool(x)
        x = self.conv1(x)
        x = self.relu(x)
        x = self.conv2(x)
        out = self.sigmoid(x)
        return out

class ECALayer(nn.Module):
    def __init__(self, k_size=3):
        super(ECALayer, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.conv = nn.Conv1d(1, 1, kernel_size=k_size, padding=(k_size - 1) // 2, bias=False)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        # x: input features with shape [b, c, h, w]
        b, c, h, w = x.size()
        # feature descriptor on the global spatial information
        y = self.avg_pool(x)
        # Two different branches of ECA module
        y = self.conv(y.squeeze(-1).transpose(-1, -2)).transpose(-1, -2).unsqueeze(-1)
        # Multi-scale information fusion
        y = self.sigmoid(y) ## y为每个通道的权重值
        return x * y.expand_as(x) ##将y的通道权重一一赋值给x的对应通道

class SELayer(nn.Module):
    def __init__(self, channel, reduction=16):
        super(SELayer, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.fc = nn.Sequential(
            nn.Linear(channel, channel // reduction, bias=False),
            nn.ReLU(inplace=True),
            nn.Linear(channel // reduction, channel, bias=False),
            nn.Sigmoid()
        )

    def forward(self, x):
        b, c, _, _ = x.size()
        y = self.avg_pool(x).view(b, c)
        y = self.fc(y).view(b, c, 1, 1)
        return x * y.expand_as(x)

class _ASPPModule(nn.Module):
    def __init__(self, inplanes, planes, kernel_size, padding, dilation, BatchNorm):
        super(_ASPPModule, self).__init__()
        self.atrous_conv = nn.Conv2d(inplanes, planes, kernel_size=kernel_size,
                                            stride=1, padding=padding, dilation=dilation, bias=False)
        self.bn = BatchNorm(planes)
        self.relu = nn.ReLU()

        self._init_weight()

    def forward(self, x):
        x = self.atrous_conv(x)
        x = self.bn(x)

        return self.relu(x)

    def _init_weight(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                torch.nn.init.kaiming_normal_(m.weight)
            elif isinstance(m, SynchronizedBatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

class ASPP(nn.Module):
    def __init__(self, backbone, output_stride, BatchNorm):
        super(ASPP, self).__init__()
        if backbone == 'drn':
            inplanes = 512
        elif backbone == 'mobilenet':
            inplanes = 320
        else:
            inplanes = 2048
        if output_stride == 16:
            dilations = [1, 6, 12, 18]
        elif output_stride == 8:
            dilations = [1, 12, 24, 36]
        else:
            raise NotImplementedError

        self.aspp1 = _ASPPModule(inplanes, 256, 1, padding=0, dilation=dilations[0], BatchNorm=BatchNorm)
        self.aspp2 = _ASPPModule(inplanes, 256, 3, padding=dilations[1], dilation=dilations[1], BatchNorm=BatchNorm)
        self.aspp3 = _ASPPModule(inplanes, 256, 3, padding=dilations[2], dilation=dilations[2], BatchNorm=BatchNorm)
        self.aspp4 = _ASPPModule(inplanes, 256, 3, padding=dilations[3], dilation=dilations[3], BatchNorm=BatchNorm)

        self.global_avg_pool = nn.Sequential(nn.AdaptiveAvgPool2d((1, 1)),
                                             nn.Conv2d(inplanes, 256, 1, stride=1, bias=False),
                                             BatchNorm(256),
                                             nn.ReLU())
        self.conv1 = nn.Conv2d(1280, 256, 1, bias=False)
        self.bn1 = BatchNorm(256)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.5)
        self._init_weight()
        #
        self.conv_cat = nn.Sequential(
            nn.Conv2d(256 * 5, 256, 1, 1, padding=0, bias=True),
            nn.BatchNorm2d(256, momentum=0.1),
            nn.ReLU(inplace=True),
        )
        self.senet = SE_Block(in_planes=256 * 5)
        #
        k_size = 3
        self.eca = ECALayer(k_size)
    #
        """se"""
        reduction = 16  #
        self.se1 = SELayer(256*5, reduction)  # 我的c1_in_channels为256，所以直接就是10*256=2560，记得修改

    def forward(self, x):
        x1 = self.aspp1(x)
        x2 = self.aspp2(x)
        x3 = self.aspp3(x)
        x4 = self.aspp4(x)
        x5 = self.global_avg_pool(x)
        x5 = F.interpolate(x5, size=x4.size()[2:], mode='bilinear', align_corners=True)
        x = torch.cat((x1, x2, x3, x4, x5), dim=1)
        #
        # seaspp1 = self.senet(x)
        # se_feature_cat = seaspp1 * x
        # result = self.conv_cat(se_feature_cat)
        # return result
        # return self.dropout(result)
        # aspp_outs = self.eca(x)
        # return aspp_outs
        """SE"""
        aspp_outs = self.se1(x)
        # return aspp_outs

        x = self.conv1(aspp_outs)
        x = self.bn1(x)
        x = self.relu(x)

        return self.dropout(x)

    def _init_weight(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                # m.weight.data.normal_(0, math.sqrt(2. / n))
                torch.nn.init.kaiming_normal_(m.weight)
            elif isinstance(m, SynchronizedBatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

def build_aspp(backbone, output_stride, BatchNorm):
    return ASPP(backbone, output_stride, BatchNorm)


import cv2
from keras.models import model_from_json
from keras.preprocessing import image
import numpy as np
model = model_from_json(open("facial_expression_model_structure.json", "r").read())
model.load_weights('facial_expression_model_weights.h5') #load weights

face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0)

while(True):
    ret, img = cap.read()

    #apply same face detection procedures
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)

    for (x,y,w,h) in faces:
        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)

    detected_face = img[int(y):int(y+h), int(x):int(x+w)] #crop detected face
    detected_face = cv2.cvtColor(detected_face, cv2.COLOR_BGR2GRAY) #transform to gray scale
    detected_face = cv2.resize(detected_face, (48, 48)) #resize to 48x48

    img_pixels = image.img_to_array(detected_face)
    img_pixels = np.expand_dims(img_pixels, axis = 0)

    img_pixels /= 255

    predictions = model.predict(img_pixels)

    #find max indexed array
    max_index = np.argmax(predictions[0])

    emotions = ('angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral')
    emotion = emotions[max_index]

    cv2.putText(img, emotion, (int(x), int(y)), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2)
    cv2.imshow('img',img)

    if cv2.waitKey(1) & 0xFF == ord('q'): #press q to quit
        break

cap.release()
cv2.destroyAllWindows()


from goods.models import GoodsCategory,ProductId
from django.conf import settings

htt = settings.HTT
htts = settings.HTTS
def get_category():
    '''
    获取商品分类
    :return: 商品分类信息
    '''
    goods = dict()
    #获取商品类别
    categories = GoodsCategory.objects.filter(parent_id__isnull=True)
    #构造分类信息
    for category in categories:
        #商品一级分类
        obj = category.goodslist_set.get()
        ppid = ProductId.objects.get(id=obj.pid_id)
        pid = ppid.pid
        goods[pid] = {}
        goods[pid]['id'] = obj.category_id
        goods[pid]['name'] = obj.category.name
        goods[pid]['url'] = ''
        goods[pid]['subs'] = []
        #商品二级分类
        for subs in category.subs.all()[0:3]:
            if subs.name == '帽子':#这里只开发出帽子这一类别商品
                category_id = subs.id
                url = htts + 'goodslist/' + str(category_id)#待优化
            else:
                url = "#"
            goods[pid]['subs'].append({
                'id': subs.id,
                'name': subs.name,
                'url':url
            })
    #返回主题分类信息
    return goods
data1,data2,data3,data4,data5,data6,data7,data33,data44,data55,data66,data77 = [],[],[],[],[],[],[],[],[],[],[],[]
def get_data(cc_title,title,price,discountprice,image):
    '''
    #构造首页商品广告楼层信息
    :param cc_title: 二级类别名称
    :param title: 分类内容商品名称
    :param price: 商品价格
    :param discountprice: 商品折后价格
    :param image: 商品图片
    '''
    if cc_title == '今日必抢':
        today_grab = {
            "title": title,
            "price": float(price),
            "discountprice": float(discountprice),
            "image": htt + str(image)
        }
        data1.append(today_grab)
    elif cc_title == "女装":
        woman_cloth = {
            "title": title[0:10] +'...',
            "price": float(price),
            "image": htt + str(image)
        }
        data2.append(woman_cloth)
    elif cc_title.strip() == '女鞋&包包':
        woman_shoe = {
        "title": title[0:10] + '...',
        "price": float(price),
        "image": htt + str(image)
        }
        data3.append(woman_shoe)
    elif cc_title == '女鞋&包包 推荐':
        woman_shoe_recomm = {
            "title": title[0:18] + '...',
            "price": float(price),
            "image": htt + str(image)
        }
        data33.append(woman_shoe_recomm)
    elif cc_title.strip() == '男装&男鞋':
        man_shoe = {
        "title": title[0:10] + '...',
        "price": float(price),
        "image": htt + str(image)
        }
        data4.append(man_shoe)
    elif cc_title == '男装&男鞋 推荐':
        man_shoe_recomm = {
            "title": title[0:18] + '...',
            "price": float(price),
            "image": htt + str(image)
        }
        data44.append(man_shoe_recomm)
    elif cc_title.strip() == '内衣':
        underWare = {
        "title": title[0:10] + '...',
        "price": float(price),
        "image": htt + str(image)
        }
        data5.append(underWare)
    elif cc_title == '内衣 推荐':
        underWare_recomm = {
            "title": title[0:18] + '...',
            "price": float(price),
            "image": htt + str(image)
        }
        data55.append(underWare_recomm)
    elif cc_title.strip() == '家纺&家饰':
        home_textile = {
        "title": title[0:10] + '...',
        "price": float(price),
        "image": htt + str(image)
        }
        data6.append(home_textile)
    elif cc_title == '家纺&家饰 推荐':
        jiaju_recomm = {
            "title": title[0:18] + '...',
            "price": float(price),
            "image": htt + str(image)
        }
        data66.append(jiaju_recomm)
    elif cc_title.strip() == '母装&儿童':
        mvying = {
        "title": title[0:10] + '...',
        "price": float(price),
        "image": htt + str(image)
        }
        data7.append(mvying)
    elif cc_title == '母装&儿童 推荐':
        mvying_recomm = {
            "title": title[0:18] + '...',
            "price": float(price),
            "image": htt + str(image)
        }
        data77.append(mvying_recomm)
def get_contents(contents):
    '''
    获取首页广告楼层信息
    :param contents: 广告内容对象
    :return:
    '''
    contents['today_grab'] = []
    contents['woman_cloth'] = []
    contents['woman_shoe'] = []
    contents['woman_shoe_recomm'] = []
    contents['man_shoe'] = []
    contents['man_shoe_recomm'] = []
    contents['underWare'] = []
    contents['underWare_recomm'] = []
    contents['home_textile'] = []
    contents['jiaju_recomm'] = []
    contents['mvying'] = []
    contents['mvying_recomm'] = []
    for i in range(0, len(data1), 4):
        d = [k for k in data1[i:i + 4]]
        contents['today_grab'].append(d)
    for i in range(0, len(data2), 4):
        d = [k for k in data2[i:i + 4]]
        contents['woman_cloth'].append(d)
    for i in range(0, len(data3), 6):
        d = [k for k in data3[i:i + 6]]
        contents['woman_shoe'].append(d)
    for i in range(0, len(data33), 4):
        d = [k for k in data33[i:i + 4]]
        contents['woman_shoe_recomm'].append(d)
    for i in range(0, len(data4), 6):
        d = [k for k in data4[i:i + 6]]
        contents['man_shoe'].append(d)
    for i in range(0, len(data44), 4):
        d = [k for k in data44[i:i + 4]]
        contents['man_shoe_recomm'].append(d)
    for i in range(0, len(data5), 6):
        d = [k for k in data5[i:i + 6]]
        contents['underWare'].append(d)
    for i in range(0, len(data55), 4):
        d = [k for k in data55[i:i + 4]]
        contents['underWare_recomm'].append(d)
    for i in range(0, len(data6), 6):
        d = [k for k in data6[i:i + 6]]
        contents['home_textile'].append(d)
    for i in range(0, len(data66), 4):
        d = [k for k in data66[i:i + 4]]
        contents['jiaju_recomm'].append(d)
    for i in range(0, len(data7), 6):
        d = [k for k in data7[i:i + 6]]
        contents['mvying'].append(d)
    for i in range(0, len(data77), 4):
        d = [k for k in data77[i:i + 4]]
        contents['mvying_recomm'].append(d)


# by gaoxin
from django.urls import path
from .views import CategoryView, CourseView, CourseDetailView, CourseChapterView, CourseCommentView, QuestionView
from .video_view import PolyvView

urlpatterns = [
    path('category', CategoryView.as_view()),
    path('list', CourseView.as_view()),
    path('detail/<int:pk>', CourseDetailView.as_view()),
    path('chapter/<int:pk>', CourseChapterView.as_view()),
    path('comment/<int:pk>', CourseCommentView.as_view()),
    path('question/<int:pk>', QuestionView.as_view()),
    path('polyv', PolyvView.as_view()),

]


from rest_framework import permissions

from courses.models import Course, HomeworkAnswer
from users.models import User

class IsTeacherOrReadOnly(permissions.BasePermission):
    def has_permission(self, request, view):
        if request.method in permissions.SAFE_METHODS:
            return True
        return request.user.role == User.RoleChoices.TEACHER

class IsCourseTeacher(permissions.BasePermission):
    def has_permission(self, request, view):
        course = Course.objects.get(pk=view.kwargs['course_pk'])
        if course not in Course.objects.user_courses(request.user):
            return False
        if request.method in permissions.SAFE_METHODS:
            return True
        return request.user in course.teachers.all()

class IsCourseTeacherOrReadOnly(permissions.BasePermission):
    def has_permission(self, request, view):
        if request.method in permissions.SAFE_METHODS and \
           view.get_object() in Course.objects.user_courses(request.user):
            return True
        return request.user in view.get_object().teachers.all()

class IsCourseStudent(permissions.BasePermission):
    def has_permission(self, request, view):
        course = Course.objects.get(pk=view.kwargs['course_pk'])
        return request.user in course.students.all()

class IsHomeworkAnswerOwnerOrCourseTeacher(permissions.BasePermission):
    def has_permission(self, request, view):
        course = Course.objects.get(pk=view.kwargs['course_pk'])
        return request.user == view.get_object().user or request.user in course.teachers.all()

class IsHomeworkAnswerOwner(permissions.BasePermission):
    def has_permission(self, request, view):
        return request.user == view.get_object().user

class IsCourseTeacherOrHomeworkAnswerOwner(permissions.BasePermission):
    def has_permission(self, request, view):
        course = Course.objects.get(pk=view.kwargs['course_pk'])
        homework_answer = HomeworkAnswer.objects.get(pk=view.kwargs['homework_answer_pk'])
        return request.user == homework_answer.user or request.user in course.teachers.all()

class IsCommentOwner(permissions.BasePermission):
    def has_object_permission(self, request, view, obj):
        return request.user == obj.user


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from datetime import date

from schemas import Item
from application import TodoApplication

app: FastAPI = FastAPI()

origins = [
    "http://localhost:3001",
    "https://localhost:3001",
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

db: TodoApplication = TodoApplication()

@app.post("/add")
async def add_item(item: Item):
  result = db.add_item(item)
  return {"id": result}

@app.get("/read_all")
async def read_all_items():
  result = db.read_all_items()
  return {"items": result}


from __future__ import (print_function, division)
import os
from obspy import Stream, Trace
import pyadjoint
from .plot_util import plot_adjoint_source
from .process_adjsrc import process_adjoint
from .io import _extract_window_time, _extract_window_id
from .utils import calculate_chan_weight

def calculate_adjsrc_on_trace(obs, syn, windows, config, adj_src_type,
                              figure_mode=False, figure_dir=None,
                              adjoint_src_flag=True):
    """
    Calculate adjoint source on a pair of trace and windows selected

    :param obs: observed trace
    :type obs: obspy.Trace
    :param syn: synthetic trace
    :type syn: obspy.Trace
    :param window_time: window time information, 2-dimension array, like
        [[win_1_left, win_1_right], [win_2_left, win_2_right], ...]
    :type windows: 2-d list or numpy.array
    :param config: config of pyadjoint
    :type config: pyadjoint.Config
    :param adj_src_type: adjoint source type, options include:
        1) "cc_traveltime_misfit"
        2) "multitaper_misfit"
        3) "waveform_misfit"
    :type adj_src_type: str
    :param adjoint_src_flag: whether calcualte adjoint source or not.
        If False, only make measurements
    :type adjoint_src_flag: bool
    :param plot_flag: whether make plots or not. If True, it will lot
        a adjoint source figure right after calculation
    :type plot_flag:  bool
    :return: adjoint source(pyadjoit.AdjointSource)
    """
    if not isinstance(obs, Trace):
        raise ValueError("Input obs should be obspy.Trace")
    if not isinstance(syn, Trace):
        raise ValueError("Input syn should be obspy.Trace")
    # if not isinstance(config, pyadjoint.Config):
    #    raise ValueError("Input config should be pyadjoint.Config")

    window_time = _extract_window_time(windows)
    if len(window_time.shape) != 2 or window_time.shape[1] != 2:
        raise ValueError("Input windows dimension incorrect, dimension"
                         "(*, 2) expected")

    adjsrc = pyadjoint.calculate_adjoint_source(
        adj_src_type=adj_src_type, observed=obs, synthetic=syn,
        config=config, window=window_time, adjoint_src=adjoint_src_flag,
        plot=figure_mode)

    if figure_mode:
        if figure_dir is None:
            figname = None
        else:
            figname = os.path.join(figure_dir, "%s.pdf" % obs.id)
        plot_adjoint_source(adjsrc, win_times=window_time, obs_tr=obs,
                            syn_tr=syn, figname=figname)

    return adjsrc

def calculate_adjsrc_on_stream(observed, synthetic, windows, config,
                               adj_src_type, figure_mode=False,
                               figure_dir=None, adjoint_src_flag=True):
    """
    calculate adjoint source on a pair of stream and windows selected

    :param observed: observed stream
    :type observed: obspy.Stream
    :param synthetic: observed stream
    :type synthetic: obspy.Stream
    :param windows: list of pyflex windows, like:
        [[Windows(), Windows(), Windows()], [Windows(), Windows()], ...]
        For each element, it contains windows for one channel
    :type windows: list
    :param config: config for calculating adjoint source
    :type config: pyadjoit.Config
    :param adj_src_type: adjoint source type
    :type adj_src_type: str
    :param figure_mode: plot flag. Leave it to True if you want to see adjoint
        plots for every trace
    :type figure_mode: bool
    :param adjoint_src_flag: adjoint source flag. Set it to True if you want
        to calculate adjoint sources
    :type adjoint_src_flag: bool
    :return:
    """
    if not isinstance(observed, Stream):
        raise ValueError("Input observed should be obspy.Stream")
    if not isinstance(synthetic, Stream):
        raise ValueError("Input synthetic should be obspy.Stream")
    if windows is None or len(windows) == 0:
        return None
    # if not isinstance(config, pyadjoint.Config):
    #    raise ValueError("Input config should be pyadjoint.Config")

    adjsrcs_list = []

    for chan_win in windows.itervalues():
        if len(chan_win) == 0:
            continue

        obsd_id, synt_id = _extract_window_id(chan_win)

        try:
            obs = observed.select(id=obsd_id)[0]
        except:
            raise ValueError("Missing observed trace for window: %s" % obsd_id)

        if synt_id == "UNKNOWN":
            syn = synthetic.select(channel="*%s" % obs.stats.channel[-1])[0]
        else:
            syn = synthetic.select(id=synt_id)[0]

        adjsrc = calculate_adjsrc_on_trace(
            obs, syn, windows[obsd_id], config, adj_src_type,
            adjoint_src_flag=adjoint_src_flag,
            figure_mode=figure_mode, figure_dir=figure_dir)

        if adjsrc is None:
            continue
        adjsrcs_list.append(adjsrc)

    return adjsrcs_list

def calculate_and_process_adjsrc_on_stream(
        observed, synthetic, windows, inventory, config, event,
        adj_src_type, postproc_param, figure_mode=False,
        figure_dir=None):
    """
    (API for pypaw)
    Calculate based on config, then process adjoint sources
    based on postproc_param
    """
    # check total number of windows. If total number of
    # window is 0, return None
    nwin_total = 0
    for value in windows.itervalues():
        nwin_total += len(value)
    if nwin_total == 0:
        return

    adjsrcs = calculate_adjsrc_on_stream(
        observed, synthetic, windows, config, adj_src_type,
        figure_mode=figure_mode, figure_dir=figure_dir,
        adjoint_src_flag=True)

    if postproc_param["weight_flag"]:
        chan_weight_dict = calculate_chan_weight(adjsrcs, windows)
    else:
        chan_weight_dict = None

    origin = event.preferred_origin() or event.origins[0]
    focal = event.preferred_focal_mechanism()
    hdr = focal.moment_tensor.source_time_function.duration / 2.0
    # according to SPECFEM starttime convention
    time_offset = -1.5 * hdr
    starttime = origin.time + time_offset

    new_adjsrcs = process_adjoint(
        adjsrcs, interp_starttime=starttime,
        inventory=inventory, event=event,
        weight_dict=chan_weight_dict,
        **postproc_param)

    # return new_adjsrcs, time_offset
    return new_adjsrcs

def measure_adjoint_on_stream(
        observed, synthetic, windows, config, adj_src_type,
        figure_mode=False, figure_dir=None):
    """
    (API for pypaw)
    Calculate the measurement of adjoint sources. Only measurments
    are returned(adjoint source is not returned).
    """

    nwin_total = 0
    for value in windows.itervalues():
        nwin_total += len(value)
    if nwin_total == 0:
        return

    adjsrcs = calculate_adjsrc_on_stream(
        observed, synthetic, windows, config, adj_src_type,
        figure_mode=False, figure_dir=None,
        adjoint_src_flag=True)

    results = {}
    for adj in adjsrcs:
        results[adj.id] = adj.measurement
    return results


import math
import time
from datetime import datetime
#from multiprocessing import Pool
#from functools import partial
import json

def pcalc(rge):
    global x
    global xc
    global c

    #print(x)
    #print (rge)

    for y in rge :
        #print(y)
        if (x % y == 0) :
            #print("Failed with " + str(y) + " in " + str(c) + " calcs")
            nfile = open('lastNumber.txt', 'w', encoding='utf-8')
            write = str(x) + "," + str(c)
            nfile.write(write)
            nfile.close()
            break

        if len(str(x)) > 20 :
            if(c % 100000000000 == 0 and c <999999999999):
                print(str(int(c*0.000000001)) + "B calculations processed ")

            elif(c % 10000000000 == 0 and c < 99999999999):
                print(strrge(int(c*0.000000001)) + "B calculations processed")

            elif(c % 1000000000 == 0 and c < 9999999999):
                print(str(int(c*0.000000001)) + "B calculations processed")

            elif(c % 100000000 == 0 and c < 999999999):
                print(str(int(c*0.000001)) + "M calculations processed ")

            #if(c % 10000000 == 0 and c < 99999999):
                #print(str(int(c*0.000001)) + "M calculations processed")
                #print(x)
                #print(y)
        #print("Prime = " + str(x) + " with " + str(c) + " calculations")
        c=c+1

    else:
        nfile = open('lastNumber.txt', 'w', encoding='utf-8')
        write = str(x) + "," + str(c)
        nfile.write(write)
        nfile.close()

        f = open('primes.txt', 'a')

        if len(str(x)) <= 5 :
            if c % 10 == 0 :
        #print(str(c)+" calc "+str(y))
                print("Prime = " + str(x) + " with " + str(c) + " calculations")
        elif 5 < len(str(x)) <= 7  :
            if c % 50 == 0 :
                #print(str(c)+" calc "+str(y))
                print("Prime = " + str(x) + " with " + str(c) + " calculations")
        elif 7 < len(str(x)) <= 14  :
            if c % 250 == 0 :
                #print(str(c)+" calc "+str(y))
                print("Prime = " + str(x) + " with " + str(c) + " calculations")
        elif 14 < len(str(x)) <= 15 :
            if c % 100 == 0 :
                #print(str(c)+" calc "+str(y))
                print("Prime = " + str(x) + " with " + str(c) + " calculations")
        elif 15 < len(str(x)) <= 20 :
            if c % 10 == 0 :
                #print(str(c)+" calc "+st)
                print("Prime = " + str(x) + " with " + str(c) + " calculations")
        elif len(str(x)) > 20 :
            print("Prime = " + str(x) + " with " + str(c) + " calculations")

        if xc%10 == 0:

            lbr = "\n"
            f.write(lbr)

        prime = str(x) + ","
        f.write(prime)
        f.close()
        xc = xc+1

#        return x

try:
    file = open('lastNumber.txt', 'r')
    info = (file.readline())
    splift = info.split(",",1)[0]
    clinfo = info.split(",",1)[1]
    print("previous number checked " + str(splift))
    print("Calculations for last number: " + str(clinfo))

except IOError :
    print()

ui1 = int(input("How many numbers would you like to process?: "))
#start = 3
start = int(splift) + 2
stop = start + int(ui1)

plist = []
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
tstart = time.time()
xc = 0

print(current_time)

x = start
sqr = int(math.sqrt(x))
rge = range(3,sqr,2)

for x in range(start, stop +1):

    s = int(str(x)[-1])
#print(s)
    if s in (1,3,7,9) :
        #print(s)
        if len(str(x)) > 20 :
            print("Testing " + str(x))

        sqr = int(math.sqrt(x))

        c = 1

        pcalc(rge)

tend = time.time()

#print(plist)
print("Completed in " + str(round((tend-tstart),3)/60) + " minutes")
file = open('lastNumber.txt', 'r')
info = (file.readline())
splift = info.split(",",1)[0]
clinfo = info.split(",",1)[1]
print("last number checked " + str(splift))