from flask import (render_template, url_for, flash,
                   redirect, request, abort, Blueprint)
from flask_login import current_user, login_required
from approot import db
from approot.models import Expense
from approot.expenses.forms import ExpenseForm

expenses = Blueprint('expenses', __name__)

@expenses.route("/expense")
@login_required
def expense():
    page = request.args.get('page', 1, type=int)
    expenses = Expense.query.order_by(Expense.expense_date.desc()).paginate(page=page, per_page=5)
    form = ExpenseForm()
        #return redirect(url_for('expenses.expense'))
    return render_template('expense/expense.html', expenses=expenses, form=form)

@expenses.route("/expense/new", methods=['GET', 'POST'])
@login_required
def new_expense():
    page = request.args.get('page', 1, type=int)
    expenses_from_query = Expense.query.order_by(Expense.expense_date.desc()).paginate(page=page, per_page=5)
    form = ExpenseForm()
    if form.validate_on_submit():
        expense = Expense(description=form.description.data, expense_date=form.expense_date.data,
                        amount=form.amount.data,vat_amount=form.vat_amount.data,Transferrable=form.Transferrable.data, author=current_user)
        db.session.add(expense)
        db.session.commit()
        flash('Your expense has been created!', 'success')
        return redirect(url_for('expenses.expense'))
    #return render_template('expense/create_expense.html', title='New Expense',
    #                       form=form, legend='New Expense')
    return render_template('expense/expense.html', expenses=expenses_from_query, form=form)

@expenses.route("/expense/update/<int:expense_id>", methods=['GET', 'POST'])
@login_required
def update_expense(expense_id):
    page = request.args.get('page', 1, type=int)
    expenses_from_query = Expense.query.order_by(Expense.expense_date.desc()).paginate(page=page, per_page=5)
    expense = Expense.query.get_or_404(expense_id)
    if expense.author != current_user:
        abort(403)
    form = ExpenseForm()
    if form.validate_on_submit():
        expense.description = form.description.data
        expense.amount = form.amount.data
        expense.expense_date = form.expense_date.data
        expense.vat_amount = form.vat_amount.data
        expense.Transferrable = form.Transferrable.data
        db.session.commit()
        flash('Your expense has been updated!', 'success')
        return redirect(url_for('expenses.expense'))
        """ Commented to see if this will have any negative effect on the flow - I think it won't
    elif request.method == 'GET':
        form.description.data = expense.description
        form.amount.data = expense.amount
        form.expense_date.data = expense.expense_date
        form.vat_amount.data = expense.vat_amount
        form.Transferrable.data = expense.Transferrable """
    #return render_template('expense/expense.html', title='Update Expense',
    #                       form=form, legend='Update Expense')
    return render_template('expense/expense.html', expenses=expenses_from_query, form=form, expense_id=expense_id)

@expenses.route("/expense/delete/<int:expense_id>", methods=['POST'])
@login_required
def delete_expense(expense_id):
    expense = Expense.query.get_or_404(expense_id)
    if expense.author != current_user:
        abort(403)
    db.session.delete(expense)
    db.session.commit()
    flash('Your expense has been deleted!', 'success')
    return redirect(url_for('expenses.expense'))


import unittest
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

class IpiviTermsOfUse(unittest.TestCase):

    def setUp(self):
        self.driver = webdriver.Firefox()

    def test_terms_of_use(self):
        driver = self.driver
        driver.maximize_window()
        driver.get("http://www.ipivi.com")
        element = WebDriverWait(driver, 10).until(EC.visibility_of_element_located
                                                  ((By.XPATH, '//a[contains(@href, "/page/terms-of-use")]')))
        element.click()
        assert "အသံုးျပဳမႈစည္းကမ္းမ်ား" in driver.title

if __name__ == "__main__":
    unittest.main()


#2020-11-18 (8일차)
#model_load()
#fit 이전에 save: 모델만

import matplotlib.pyplot as plt
import numpy as np
from tensorflow.keras.datasets import mnist #텐서플로우에서 제공해 준다(수치로 변환해서 제공)

#train_test_split 할 필요 없이 알아서 나눠 준다
(x_train, y_train), (x_test, y_test) = mnist.load_data() #괄호 주의

#60000장 * 28pixel * 28pixel
# print(x_train.shape, x_test.shape) #(60000, 28, 28)(10000, 28, 28)
# print(y_train.shape, y_test.shape) #(60000, )      (10000,)        : 스칼라

# print(x_train[0])
# print(y_train[1]) #label

# plt.imshow(x_train[0], 'gray')
# plt.show()

#8은 2보다 4배의 가치? 3은 1보다 3배의 가치? no
#One-Hot Encoder
#y_train: 60000, -> OneHotEncoding : 1 0 0 0 0 0 0 0 0 0 (60000, 10) : 분류가 10개니까 (0~9)

#1. 데이터 전처리: OneHotEncoding 대상은 Y
from tensorflow.keras.utils import to_categorical
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
print(y_train)

# print(y_train.shape, y_test.shape)
# print(y_train[0]) #y_train[0]=5 -> [0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]

#shape 바꿀 줄 알아야 함
#60000, 14, 14, 4도 가능하고 60000, 28, 14, 2도 가능
#LSTM으로도 바꿀 수 있다

x_train = x_train.reshape(60000, 28, 28, 1).astype('float32')/255.
x_test = x_test.reshape(10000, 28, 28, 1).astype('float32')/255.
                        #x_test.shape[0], x_test.shape[1] ...

#predict data, answer data
x_predict = x_train[20:30]
y_answer = y_train[20:30]

#CNN에 넣을 수 있는 4차원 reshape + y도 onehotencoding
#scaler 사용해야: 어떤 게 더 좋을지는 해 봐야 안다
#지금 이 상황에서 M은 255라는 걸 알고 있음. 그러므로 MinMax에서는 255로 나누면 0~1 사이로 수렴 가능

# print(x_train[0])

#2. 모델
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten

# model = Sequential()
# model.add(Conv2D(30, (2, 2), padding='same', input_shape=(28, 28, 1))) #padding 주의!
# model.add(Conv2D(50, (2, 2), padding='valid'))
# model.add(Conv2D(120, (3, 3))) #padding default=valid
# model.add(Conv2D(200, (2, 2), strides=2))
# model.add(Conv2D(30, (2, 2)))
# model.add(MaxPooling2D(pool_size=2)) #pool_size default=2
# model.add(Flatten())
# model.add(Dense(10, activation='relu')) #flatten하면서 곱하고 dense에서 또 100 곱함
#                                         #Conv2d의 activation default='relu'
#                                         #LSTM의 activation default='tanh'
# #MaxPooling2D-Flatten:reshape 개념

# model.add(Dense(10, activation='softmax')) #softmax** : 2 이상 분류(다중분류)의 activation은 softmax, 2진분류는 sigmoid
#                                             #즉 softmax를 사용하려면 OneHotEncoding 해야

#모델 1 (fit 이전에 저장) 로드
from tensorflow.keras.models import load_model
model = load_model('./save/model_test01_1.h5')
model.summary()

#3. 컴파일, 훈련

modelpath = './model/mnist-{epoch:02d}-{val_loss:.4f}.hdf5'

#root 폴더(=Study) 밑의 model 폴더에
#.hd5f라는 확장자 => *.h5 (모델 확장자와 거의 동일하다)
#epoch:두자릿수 정수 표기-val_loss:소수 넷째자리까지 표기 -> 이 파일명으로 생성 : 파일명만 봐도 대략 어느 지점이 좋은지 알 수 있음

from tensorflow.keras.callbacks import EarlyStopping, TensorBoard, ModelCheckpoint

early_stopping = EarlyStopping(monitor='val_loss', patience=10, mode='auto')

#val_loss가 개선이 생길 때마다 history를 저장하고 싶다!
#정의
cp = ModelCheckpoint(filepath=modelpath,
                     monitor='val_loss',
                     save_best_only=True,
                     mode='auto'
) #계속 저장될 것 (계속 좋아질 테니까) 그러다가 튀는 구간 있으면 또 저장 안 되고... 다시 뚫고 내려가는 거 저장되고...
  #좋은 시점 저장됨
#model 폴더 잘 보기!
#04-어쩌고 후에 07-어쩌고가 생겼다면 5, 6은 별로였다는 뜻

#log가 들어갈 폴더='graph'
#여기까지 해서 graph 폴더 생기고 자료들 들어가 있으면 텐서보드 쓸 준비 ok
#단, 로그가 많으면 겹쳐서 보일 수 있으니 그럴 땐 로그 삭제하고
to_hist = TensorBoard(log_dir='graph', histogram_freq=0,
                      write_graph=True, write_images=True
)

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy']) #"mean_squared_error" (풀네임도 가능하다)

#model.fit()은 history 반환
hist = model.fit(x_train, y_train, epochs=100, batch_size=32, verbose=1,
          validation_split=0.2, callbacks=[early_stopping, cp]) #tensorboard 쓰고 싶으면 여기다가 to_hist도...

#fit에 있는 네 가지
loss = hist.history['loss']
val_loss = hist.history['val_loss']
acc = hist.history['accuracy']
val_acc = hist.history['val_accuracy']

#4. 평가, 예측

#얘네는 evaluate에서 나온 애들

result = model.evaluate(x_test, y_test, batch_size=32)
print("loss: ", result[0])
print("acc: ", result[1])

#시각화
#plot에는 x, y가 들어간다 (그래야 그래프가 그려짐)
import matplotlib.pyplot as plt

plt.figure(figsize=(10, 6))
#단위 뭔지 찾아볼 것!!!
#pyplot.figure 는 매개 변수에 주어진 속성으로 새로운 도형을 생성합니다.
#figsize 는 도형 크기를 인치 단위로 정의합니다.

plt.subplot(2, 1, 1) #2, 1, 1 -> 두 장 중의 첫 번째의 첫 번째 (2행 1열에서 첫 번째)
# plt.plot(hist.history['loss'],) #loss값이 순서대로 감
plt.plot(hist.history['loss'], marker='.', c='red', label='loss') #loss값이 순서대로 감, 마커는 점, 색깔은 빨간색, 라벨은 loss
plt.plot(hist.history['val_loss'], marker='.', c='blue', label='val_loss')

plt.grid() #모눈종이 배경
plt.title('loss')
plt.ylabel('loss')
plt.xlabel('epoch')

#위에서 라벨 명시 후 위치 명시
#그림의 위치(location)는 상단: label:loss, label:val_loss 이 둘이 박스로 해서 저 위치에 나올 것
plt.legend(loc='upper right')

plt.subplot(2, 1, 2) #2, 1, 1 -> 2행 1열 중 두 번째 (두 번째 그림)
# plt.plot(hist.history['loss'],) #loss값이 순서대로 감
plt.plot(hist.history['accuracy'], marker='.', c='red') #loss값이 순서대로 감, 마커는 점, 색깔은 빨간색, 라벨은 loss
plt.plot(hist.history['val_accuracy'], marker='.', c='blue')

plt.grid() #모눈종이 배경
plt.title('accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')

#여긴 라벨만 명시
plt.legend(['accuracy', 'val_accuracy'])

#보여 줘
plt.show()

#정답
y_answer = np.argmax(y_answer, axis=1)

#예측값
y_predict = model.predict(x_predict)
y_predict = np.argmax(y_predict, axis=1)

print("예측값: ", y_predict)
print("정답: ", y_answer)

'''
loss:  0.09025763720273972
acc:  0.9763000011444092
'''


#!/usr/bin/env python
# Copyright 2019 NeuroData (http://neurodata.io)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# ndmg_dwi_worker.py
# Repackaged for native space tractography by Derek Pisner in 2019
# Email: dpisner@utexas.edu

import shutil
import time
import warnings

warnings.simplefilter("ignore")

# from ndmg.stats.qa_mri import qa_mri
import nibabel as nib
from dipy.tracking.streamline import Streamlines
from subprocess import Popen
import ndmg
from ndmg import preproc as mgp
from ndmg.utils import gen_utils as mgu
from ndmg.utils import s3_utils
from ndmg.register import gen_reg as mgr
from ndmg.track import gen_track as mgt
from ndmg.graph import gen_graph as mgg
from ndmg.utils.bids_utils import name_resource
from ndmg.stats.qa_tensor import *
from ndmg.stats.qa_fibers import *
from datetime import datetime

os.environ["MPLCONFIGDIR"] = "/tmp/"

def ndmg_dwi_worker(
    dwi,
    bvals,
    bvecs,
    t1w,
    atlas,
    mask,
    labels,
    outdir,
    vox_size,
    mod_type,
    track_type,
    mod_func,
    seeds,
    reg_style,
    clean,
    skipeddy=False,
    skipreg=False,
    buck=None,
    remo=None,
    push=False,
    creds=None,
    debug=False,
    modif="",
):
    """Creates a brain graph from MRI data

    Parameters
    ----------
    dwi : str
        Path for the dwi file(s)
    bvals : str
        Path for the bval file(s)
    bvecs : str
        Path for the bvec file(s)
    t1w : str
        Location of anatomical input file(s)
    atlas : str
        Location of atlas file
    mask : str
        Location of T1w brain mask file, make sure the proper voxel size is used
    labels : list
        Location of file containing the labels for the atlas file(s)
    outdir : str
        The directory where the output files should be stored. Prepopulate this folder with results of participants level analysis if running gorup analysis.
    vox_size : str
        Voxel size to use for template registrations. Default is '2mm'.
    mod_type : str
        Determinstic (det) or probabilistic (prob) tracking. Default is det.
    track_type : str
        Tracking approach: eudx or local. Default is eudx.
    mod_func : str
        Diffusion model: csd, csa, or tensor. Default is tensor.
    seeds : int
        Density of seeding for native-space tractography.
    reg_style : str
        Space for tractography. Default is native.
    clean : bool
        Whether or not to delete intermediates. Default is False.
    skipeddy : bool
        Whether or not to skip the eddy correction if it has already been run. Default is False.
    skipreg : bool
        Whether or not to skip registration. Default is False.
    buck : str, optional
        The name of an S3 bucket which holds BIDS organized data. You musht have build your bucket with credentials to the S3 bucket you wish to access. Default is None
    remo : str, optional
        The path to the data on your S3 bucket. The data will be downloaded to the provided bids_dir on your machine. Default is None.
    push : bool, optional
        Flag to push derivatives back to S3. Default is False
    creds : bool, optional
        Determine if you have S3 credentials. Default is True
    debug : bool, optional
        If False, remove any old filed in the output directory. Default is False
    modif : str, optional
        Name of the folder on s3 to push to. If empty, push to a folder with ndmg's version number. Default is ""

    Raises
    ------
    ValueError
        Raised if downsampling voxel size is not supported
    ValueError
        Raised if bval/bvecs are potentially corrupted
    """
    print("dwi = {}".format(dwi))
    print("bvals = {}".format(bvals))
    print("bvecs = {}".format(bvecs))
    print("t1w = {}".format(t1w))
    print("atlas = {}".format(atlas))
    print("mask = {}".format(mask))
    print("labels = {}".format(labels))
    print("outdir = {}".format(outdir))
    print("vox_size = {}".format(vox_size))
    print("mod_type = {}".format(mod_type))
    print("track_type = {}".format(track_type))
    print("mod_func = {}".format(mod_func))
    print("seeds = {}".format(seeds))
    print("reg_style = {}".format(reg_style))
    print("clean = {}".format(clean))
    print("skip eddy = {}".format(skipeddy))
    print("skip registration = {}".format(skipreg))
    fmt = "_adj.csv"

    assert all(
        [
            dwi,
            bvals,
            bvecs,
            t1w,
            atlas,
            mask,
            labels,
            outdir,
            vox_size,
            mod_type,
            track_type,
            mod_func,
            seeds,
            reg_style,
        ]
    ), "Missing a default argument."

    startTime = datetime.now()

    namer = name_resource(dwi, t1w, atlas, outdir)

    # TODO : do this with shutil instead of an os command
    print("Output directory: " + outdir)
    if not os.path.isdir(outdir):
        cmd = "mkdir -p {}".format(outdir)
        os.system(cmd)

    paths = {
        "prep_dwi": "dwi/preproc",
        "prep_anat": "anat/preproc",
        "reg_anat": "anat/registered",
        "fiber": "dwi/fiber",
        "tensor": "dwi/tensor",
        "conn": "dwi/roi-connectomes",
    }

    label_dirs = ["conn"]  # create label level granularity

    print("Adding directory tree...")
    namer.add_dirs_dwi(paths, labels, label_dirs)

    # Create derivative output file names
    streams = namer.name_derivative(namer.dirs["output"]["fiber"], "streamlines.trk")

    # Again, connectomes are different
    if not isinstance(labels, list):
        labels = [labels]
    connectomes = [
        namer.name_derivative(
            namer.dirs["output"]["conn"][namer.get_label(lab)],
            "{}_{}_measure-spatial-ds{}".format(
                namer.get_mod_source(), namer.get_label(lab), fmt
            ),
        )
        for lab in labels
    ]

    print("Connectomes downsampled to given labels: " + ", ".join(connectomes))

    if vox_size != "1mm" and vox_size != "2mm":
        raise ValueError("Voxel size not supported. Use 2mm or 1mm")

    # -------- Preprocessing Steps --------------------------------- #

    # Perform eddy correction
    dwi_prep = "{}/eddy_corrected_data.nii.gz".format(namer.dirs["output"]["prep_dwi"])

    if len(os.listdir(namer.dirs["output"]["prep_dwi"])) != 0:
        if skipeddy is False:
            try:
                print("Pre-existing preprocessed dwi files found. Deleting these...")
                shutil.rmtree(namer.dirs["output"]["prep_dwi"])
                os.mkdir(namer.dirs["output"]["prep_dwi"])
            except Exception as e:
                print("Exception when trying to delete existing data: {}".format(e))
                pass
            print("Performing eddy correction...")
            cmd = "eddy_correct " + dwi + " " + dwi_prep + " 0"
            print(cmd)
            sts = Popen(cmd, shell=True).wait()
            print(sts)
            ts = time.time()
            st = datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
            print(st)
        else:
            if not os.path.isfile(dwi_prep):
                raise ValueError(
                    "ERROR: Cannot skip eddy correction if it has not already been run!"
                )
    else:
        print("Performing eddy correction...")
        cmd = "eddy_correct " + dwi + " " + dwi_prep + " 0"
        print(cmd)
        sts = Popen(cmd, shell=True).wait()
        print(sts)
        ts = time.time()
        st = datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
        print(st)

    # Instantiate bvec/bval naming variations and copy to derivative director
    bvec_scaled = "{}/bvec_scaled.bvec".format(namer.dirs["output"]["prep_dwi"])
    fbval = "{}/bval.bval".format(namer.dirs["output"]["prep_dwi"])
    fbvec = "{}/bvec.bvec".format(namer.dirs["output"]["prep_dwi"])
    shutil.copyfile(bvecs, fbvec)
    shutil.copyfile(bvals, fbval)

    # Correct any corrupted bvecs/bvals
    from dipy.io import read_bvals_bvecs

    bvals, bvecs = read_bvals_bvecs(fbval, fbvec)
    bvecs[np.where(np.any(abs(bvecs) >= 10, axis=1) == True)] = [1, 0, 0]
    bvecs[np.where(bvals == 0)] = 0
    if (
        len(
            bvecs[
                np.where(
                    np.logical_and(
                        bvals > 50, np.all(abs(bvecs) == np.array([0, 0, 0]), axis=1)
                    )
                )
            ]
        )
        > 0
    ):
        raise ValueError(
            "WARNING: Encountered potentially corrupted bval/bvecs. Check to ensure volumes with a "
            "diffusion weighting are not being treated as B0's along the bvecs"
        )
    np.savetxt(fbval, bvals)
    np.savetxt(fbvec, bvecs)

    # Rescale bvecs
    print("Rescaling b-vectors...")
    mgp.rescale_bvec(fbvec, bvec_scaled)

    # Check orientation (dwi_prep)
    start_time = time.time()
    [dwi_prep, bvecs] = mgu.reorient_dwi(dwi_prep, bvec_scaled, namer)
    print(
        "%s%s%s"
        % ("Reorienting runtime: ", str(np.round(time.time() - start_time, 1)), "s")
    )

    # Check dimensions
    start_time = time.time()
    dwi_prep = mgu.match_target_vox_res(dwi_prep, vox_size, namer, sens="dwi")
    print(
        "%s%s%s"
        % ("Reslicing runtime: ", str(np.round(time.time() - start_time, 1)), "s")
    )

    # Build gradient table
    print("fbval: ", fbval)
    print("bvecs: ", bvecs)
    print("fbvec: ", fbvec)
    print("dwi_prep: ", dwi_prep)
    print("namer.dirs: ", namer.dirs["output"]["prep_dwi"])
    [gtab, nodif_B0, nodif_B0_mask] = mgu.make_gtab_and_bmask(
        fbval, fbvec, dwi_prep, namer.dirs["output"]["prep_dwi"]
    )

    # Get B0 header and affine
    dwi_prep_img = nib.load(dwi_prep)
    hdr = dwi_prep_img.header
    print(
        "%s%s%s"
        % ("Preprocessing runtime: ", str(np.round(time.time() - start_time, 1)), "s")
    )

    # -------- Registration Steps ----------------------------------- #
    if (skipreg is False) and len(os.listdir(namer.dirs["output"]["prep_anat"])) != 0:
        try:
            print("Pre-existing preprocessed t1w files found. Deleting these...")
            shutil.rmtree(namer.dirs["output"]["prep_anat"])
            os.mkdir(namer.dirs["output"]["prep_anat"])
        except:
            pass
    if (skipreg is False) and len(os.listdir(namer.dirs["output"]["reg_anat"])) != 0:
        try:
            print("Pre-existing registered t1w files found. Deleting these...")
            shutil.rmtree(namer.dirs["output"]["reg_anat"])
            os.mkdir(namer.dirs["output"]["reg_anat"])
        except:
            pass
    if (skipreg is False) and (
        (len(os.listdir(namer.dirs["tmp"]["reg_a"])) != 0)
        or (len(os.listdir(namer.dirs["tmp"]["reg_m"])) != 0)
    ):
        try:
            print("Pre-existing temporary files found. Deleting these...")
            shutil.rmtree(namer.dirs["tmp"]["reg_a"])
            os.mkdir(namer.dirs["tmp"]["reg_a"])
            shutil.rmtree(namer.dirs["tmp"]["reg_m"])
            os.mkdir(namer.dirs["tmp"]["reg_m"])
        except:
            pass

    # Check orientation (t1w)
    start_time = time.time()
    t1w = mgu.reorient_img(t1w, namer)
    t1w = mgu.match_target_vox_res(t1w, vox_size, namer, sens="t1w")
    print(
        "%s%s%s"
        % ("Reorienting runtime: ", str(np.round(time.time() - start_time, 1)), "s")
    )

    if reg_style == "native" or reg_style == "native_dsn":

        print("Running tractography in native space...")
        # Instantiate registration
        reg = mgr.dmri_reg(namer, nodif_B0, nodif_B0_mask, t1w, vox_size, simple=False)

        # Perform anatomical segmentation
        start_time = time.time()
        if (skipreg is True) and os.path.isfile(reg.wm_edge):
            print("Found existing gentissue run!")
            pass
        else:
            reg.gen_tissue()
            print(
                "%s%s%s"
                % (
                    "gen_tissue runtime: ",
                    str(np.round(time.time() - start_time, 1)),
                    "s",
                )
            )

        # Align t1w to dwi
        start_time = time.time()
        if (
            (skipreg is True)
            and os.path.isfile(reg.t1w2dwi)
            and os.path.isfile(reg.mni2t1w_warp)
            and os.path.isfile(reg.t1_aligned_mni)
        ):
            print("Found existing t1w2dwi run!")
            pass
        else:
            reg.t1w2dwi_align()
            print(
                "%s%s%s"
                % (
                    "t1w2dwi_align runtime: ",
                    str(np.round(time.time() - start_time, 1)),
                    "s",
                )
            )

        # Align tissue classifiers
        start_time = time.time()
        if (
            (skipreg is True)
            and os.path.isfile(reg.wm_gm_int_in_dwi)
            and os.path.isfile(reg.vent_csf_in_dwi)
            and os.path.isfile(reg.corpuscallosum_dwi)
        ):
            print("Found existing tissue2dwi run!")
            pass
        else:
            reg.tissue2dwi_align()
            print(
                "%s%s%s"
                % (
                    "tissue2dwi_align runtime: ",
                    str(np.round(time.time() - start_time, 1)),
                    "s",
                )
            )

        # -------- Tensor Fitting and Fiber Tractography ---------------- #
        start_time = time.time()
        seeds = mgt.build_seed_list(reg.wm_gm_int_in_dwi, np.eye(4), dens=int(seeds))
        print("Using " + str(len(seeds)) + " seeds...")

        # Compute direction model and track fiber streamlines
        print("Beginning tractography...")
        trct = mgt.run_track(
            dwi_prep,
            nodif_B0_mask,
            reg.gm_in_dwi,
            reg.vent_csf_in_dwi,
            reg.csf_mask_dwi,
            reg.wm_in_dwi,
            gtab,
            mod_type,
            track_type,
            mod_func,
            seeds,
            np.eye(4),
        )
        streamlines = trct.run()
        streamlines = Streamlines([sl for sl in streamlines if len(sl) > 60])
        print("Streamlines complete")

        trk_affine = np.eye(4)
        trk_hdr = nib.streamlines.trk.TrkFile.create_empty_header()
        trk_hdr["hdr_size"] = 1000
        trk_hdr["dimensions"] = hdr["dim"][1:4].astype("float32")
        trk_hdr["voxel_sizes"] = hdr["pixdim"][1:4]
        trk_hdr["voxel_to_rasmm"] = trk_affine
        trk_hdr["voxel_order"] = "RAS"
        trk_hdr["pad2"] = "RAS"
        trk_hdr["image_orientation_patient"] = np.array(
            [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
        ).astype("float32")
        trk_hdr["endianness"] = "<"
        trk_hdr["_offset_data"] = 1000
        trk_hdr["nb_streamlines"] = streamlines.total_nb_rows
        tractogram = nib.streamlines.Tractogram(streamlines, affine_to_rasmm=trk_affine)
        trkfile = nib.streamlines.trk.TrkFile(tractogram, header=trk_hdr)
        nib.streamlines.save(trkfile, streams)
        print(
            "%s%s%s"
            % (
                "Tractography runtime: ",
                str(np.round(time.time() - start_time, 1)),
                "s",
            )
        )

    if reg_style == "native_dsn":

        fa_path = mgt.tens_mod_fa_est(gtab, dwi_prep, nodif_B0_mask)

        # Normalize streamlines
        print("Running DSN...")
        [streamlines_mni, streams_mni] = mgr.direct_streamline_norm(
            streams, fa_path, namer
        )

        # Save streamlines to disk
        print("Saving DSN-registered streamlines: " + streams_mni)

    elif reg_style == "mni":
        # Check dimensions
        start_time = time.time()
        t1w = mgu.match_target_vox_res(
            t1w, vox_size, namer, sens="t1w"
        )  # this is the second time this t1w data has been sent to this function (REMOVE?)
        print(
            "%s%s%s"
            % ("Reslicing runtime: ", str(np.round(time.time() - start_time, 1)), "s")
        )
        print("Running tractography in MNI-space...")
        aligned_dwi = "{}/dwi_mni_aligned.nii.gz".format(
            namer.dirs["output"]["prep_dwi"]
        )

        # Align DWI volumes to Atlas
        print("Aligning volumes...")
        reg = mgr.dmri_reg_old(dwi_prep, gtab, t1w, mask, aligned_dwi, namer, clean)
        print(
            "Registering DWI image at {} to atlas; aligned dwi at {}...".format(
                dwi_prep, aligned_dwi
            )
        )  # alex  # TODO: make sure dwi_prep is what is being registered
        reg.dwi2atlas()

        # -------- Tensor Fitting and Fiber Tractography ---------------- #
        print("Beginning tractography...")
        # Compute tensors and track fiber streamlines
        print("aligned_dwi: {}".format(aligned_dwi))
        print("gtab: {}".format(gtab))
        [tens, streamlines, align_dwi_mask] = mgt.eudx_basic(
            aligned_dwi, gtab, stop_val=0.2
        )
        tensors = "{}/tensors.nii.gz".format(namer.dirs["output"]["tensor"])
        tensor2fa(
            tens,
            tensors,
            aligned_dwi,
            namer.dirs["output"]["tensor"],
            namer.dirs["qa"]["tensor"],
        )

        # Save streamlines to disk
        print("Saving streamlines: " + streams)
        print("streamlines: {}".format(streamlines))
        print("streams: {}".format(streams))
        tractogram_list = [i for i in streamlines]
        trk_affine = np.eye(4)
        tractogram = nib.streamlines.Tractogram(
            tractogram_list, affine_to_rasmm=trk_affine
        )
        nib.streamlines.save(tractogram, streams)
        streamlines = Streamlines(
            streamlines
        )  # alex  # to try to make the streamlines variable be the same thing as the native space one
        print("atlas location: {}".format(atlas))

    # ------- Connectome Estimation --------------------------------- #
    # Generate graphs from streamlines for each parcellation
    for idx, label in enumerate(labels):
        print("Generating graph for {} parcellation...".format(label))
        if reg_style == "native_dsn":
            # align atlas to t1w to dwi
            print("%s%s" % ("Applying native-space alignment to ", labels[idx]))
            labels_im_file = mgu.reorient_img(labels[idx], namer)
            labels_im_file = mgu.match_target_vox_res(
                labels_im_file, vox_size, namer, sens="t1w"
            )
            labels_im_file_mni = reg.atlas2t1w2dwi_align(labels_im_file, dsn=True)
            labels_im = nib.load(labels_im_file_mni)
            g1 = mgg.graph_tools(
                attr=len(np.unique(np.around(labels_im.get_data()).astype("int16")))
                - 1,
                rois=labels_im_file_mni,
                tracks=streamlines_mni,
                affine=np.eye(4),
                namer=namer,
                connectome_path=connectomes[idx],
            )
            g1.g = g1.make_graph()
        elif reg_style == "native":
            # align atlas to t1w to dwi
            print("%s%s" % ("Applying native-space alignment to ", labels[idx]))
            labels_im_file = mgu.reorient_img(labels[idx], namer)
            labels_im_file = mgu.match_target_vox_res(
                labels_im_file, vox_size, namer, sens="t1w"
            )
            labels_im_file_dwi = reg.atlas2t1w2dwi_align(labels_im_file, dsn=False)
            labels_im = nib.load(labels_im_file_dwi)
            g1 = mgg.graph_tools(
                attr=len(np.unique(np.around(labels_im.get_data()).astype("int16")))
                - 1,
                rois=labels_im_file_dwi,
                tracks=streamlines,
                affine=np.eye(4),
                namer=namer,
                connectome_path=connectomes[idx],
            )
            g1.g = g1.make_graph()
        elif reg_style == "mni":
            labels_im_file = mgu.reorient_img(labels[idx], namer)
            labels_im_file = mgu.match_target_vox_res(
                labels_im_file, vox_size, namer, sens="t1w"
            )
            labels_im = nib.load(labels_im_file)
            g1 = mgg.graph_tools(
                attr=len(np.unique(np.around(labels_im.get_data()).astype("int16")))
                - 1,
                rois=labels_im_file,
                tracks=streamlines,
                affine=np.eye(4),
                namer=namer,
                connectome_path=connectomes[idx],
            )
            g1.make_graph_old()
        g1.summary()
        g1.save_graph_png(connectomes[idx])
        g1.save_graph(connectomes[idx])

    exe_time = datetime.now() - startTime

    print("Total execution time: {}".format(exe_time))
    print("NDMG Complete.")

    # TODO : putting this block of code here for now because it wouldn't run in `ndmg_bids`. Figure out how to put it somewhere else.
    if push and buck and remo is not None:
        if not modif:
            modif = "ndmg_{}".format(ndmg.VERSION.replace(".", "-"))
        s3_utils.s3_push_data(buck, remo, outdir, modif, creds, debug=debug)
        print("Pushing Complete!")
        if not debug:
            print("Listing contents of output directory ...")
            print(os.listdir(outdir))
            print("clearing contents of output directory ...")
            shutil.rmtree(outdir)
            print(
                "Clearing complete. Output directory exists: {}".format(
                    os.path.exists(outdir)
                )
            )
            # # Log docker info in EC2 containers, assuming we're using AWS Batch
            # f = subprocess.check_output('docker info', shell=True)
            # info_we_care_about = f[f.find(
            #     'Data Space Used'):f.find('Metadata Space Used')]
            # print("docker info on space: {}".format(info_we_care_about))

def main():
    parser = ArgumentParser(
        description="This is an end-to-end connectome estimation pipeline from sMRI and DTI images"
    )
    parser.add_argument("dwi", action="store", help="Nifti dMRI image stack")
    parser.add_argument("bval", action="store", help="DTI scanner b-values")
    parser.add_argument("bvec", action="store", help="DTI scanner b-vectors")
    parser.add_argument("t1w", action="store", help="Nifti T1w MRI image")
    parser.add_argument("atlas", action="store", help="Nifti T1 MRI atlas")
    parser.add_argument(
        "mask",
        action="store",
        help="Nifti binary mask of \
                        brain space in the atlas",
    )
    parser.add_argument(
        "outdir",
        action="store",
        help="Path to which \
                        derivatives will be stored",
    )
    parser.add_argument(
        "labels",
        action="store",
        nargs="*",
        help="Nifti \
                        labels of regions of interest in atlas space",
    )
    parser.add_argument(
        "--vox",
        action="store",
        nargs="*",
        default="1mm",
        help="Voxel size to use for template registrations \
                        (e.g. '1mm')",
    )
    parser.add_argument(
        "--mod",
        action="store",
        help="Determinstic (det) or probabilistic (prob) tracking. Default is det.",
        default="det",
    )
    parser.add_argument(
        "--tt",
        action="store",
        help="Tracking approach: eudx or local. Default is eudx.",
        default="eudx",
    )
    parser.add_argument(
        "--mf",
        action="store",
        help="Diffusion model: csd, csa, or tensor. Default is tensor.",
        default="tensor",
    )
    parser.add_argument(
        "--sp",
        action="store",
        help="Space for tractography. Default is native.",
        default="native",
    )
    parser.add_argument(
        "-c",
        "--clean",
        action="store_true",
        default=False,
        help="Whether or not to delete intemediates",
    )
    parser.add_argument(
        "-sked",
        "--sked",
        action="store_true",
        default=False,
        help="whether or not to skip eddy correction",
    )
    parser.add_argument(
        "-skreg",
        "--skreg",
        action="store_true",
        default=False,
        help="whether or not to skip registration",
    )
    result = parser.parse_args()

    # Create output directory
    print("Creating output directory: {}".format(result.outdir))
    print("Creating output temp directory: {}/tmp".format(result.outdir))
    mgu.utils.execute_cmd("mkdir -p {} {}/tmp".format(result.outdir, result.outdir))

    ndmg_dwi_worker(
        result.dwi,
        result.bval,
        result.bvec,
        result.t1w,
        result.atlas,
        result.mask,
        result.labels,
        result.outdir,
        result.vox,
        result.mod,
        result.tt,
        result.mf,
        result.sp,
        result.seeds,
        result.clean,
        result.skipeddy,
        result.skipreg,
    )

if __name__ == "__main__":
    main()


import torch.nn as nn
import torch
from collections import OrderedDict

class RagaDetector(nn.Module):
    def __init__(self):
        super(RagaDetector, self).__init__()

        self.norm0 = nn.BatchNorm2d(1)
        self.encoder = nn.Sequential(OrderedDict([
            ('norm0', nn.BatchNorm2d(1)),

            ('conv1', nn.Conv2d(1, 64, 3, padding=1)),
            ('norm1', nn.BatchNorm2d(64)),
            ('elu1', nn.ELU()),
            ('pool1', nn.MaxPool2d(2, 2)),
            ('drop1', nn.Dropout(p=0.1)),

            ('conv2', nn.Conv2d(64, 128, 3, padding=1)),
            ('norm2', nn.BatchNorm2d(128)),
            ('elu2', nn.ELU()),
            ('pool2', nn.MaxPool2d(3, 3)),
            ('drop2', nn.Dropout(p=0.1)),

            ('conv3', nn.Conv2d(128, 128, 3, padding=1)),
            ('norm3', nn.BatchNorm2d(128)),
            ('elu3', nn.ELU()),
            ('pool3', nn.MaxPool2d(4, 4)),
            ('drop3', nn.Dropout(p=0.1)),

            ('conv4', nn.Conv2d(128, 128, 3, padding=1)),
            ('norm4', nn.BatchNorm2d(128)),
            ('elu4', nn.ELU()),
            ('pool4', nn.MaxPool2d(4, 4)),
            ('drop4', nn.Dropout(p=0.1))
        ]))

        self.lstm = nn.LSTM(128, 72, num_layers=2, dropout=0.1)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(
                    m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def forward(self, x, hidden):
        x = x.transpose(0, 1).unsqueeze(1)
        x = self.encoder(x)
        batch_size = x.shape[0]
        x = x.transpose(0, 2).transpose(1, 2).view(-1, batch_size, 128)
        x, hidden = self.lstm(x, hidden)

        return x, hidden

    def init_hidden(self, bsz):
        weight = next(self.parameters())
        return (weight.new_zeros(2, bsz, 72),
                weight.new_zeros(2, bsz, 72))


"""
Unit tests for crossref.py
"""

# pylint: disable=no-member, missing-docstring, len-as-condition
import logging

import pytest

from bibliom.publication_objects import Paper
from bibliom.crossref import *

@pytest.mark.usefixtures('class_manager')
class TestCrossref():
    def test_cr_works(self):
        logging.getLogger('bibliom.pytest').debug('-->TestCrossref.test_cr_works')
        response = cr_works(ids='10.1016/j.ijhydene.2016.06.178')
        assert(
            response['message']['title'][0] ==
            'RETRACTED: Plasma equilibrium reconstruction for the nuclear ' +
            'fusion of magnetically confined hydrogen isotopes'
        )

    def test_parse_response(self):
        logging.getLogger('bibliom.pytsest').debug('-->TestCrossref.test_parse_response')
        response = cr_works(ids='10.1016/j.ijhydene.2016.06.178')
        paper = parse_response(response)
        assert isinstance(paper, Paper)
        assert(
            paper.title ==
            'RETRACTED: Plasma equilibrium reconstruction for the nuclear ' +
            'fusion of magnetically confined hydrogen isotopes'
        )


"""
Author @ Mihir_Srivastava
Dated - 17-07-2020
File - Sentiment_analysis_of_tweets_NB
Aim - To do the sentiment analysis of tweets from the twitter_samples DataSet available in the nltk library and classify
them as positive or negative tweets using sklearn's Multinomial Naive Bayes.
"""

# Import necessary libraries
import numpy as np
import matplotlib.pyplot as plt
import random
import re
import string
import warnings
from nltk.corpus import twitter_samples
from nltk.tokenize import TweetTokenizer
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from sklearn import naive_bayes

# Collect positive and negative tweets
positive_tweets = twitter_samples.strings('positive_tweets.json')
negative_tweets = twitter_samples.strings('negative_tweets.json')

# Append positive and negative tweets to get all the tweets
all_tweets = positive_tweets + negative_tweets

# Split the data into two pieces, one for training and one for testing (validation set)
test_pos = positive_tweets[4000:]
train_pos = positive_tweets[:4000]
test_neg = negative_tweets[4000:]
train_neg = negative_tweets[:4000]

train_x = train_pos + train_neg
test_x = test_pos + test_neg

train_y = np.append(np.ones(len(train_pos)), np.zeros(len(train_neg)))
test_y = np.append(np.ones(len(test_pos)), np.zeros(len(test_neg)))

# A function to visualize the classes
def visualize(all_positive_tweets, all_negative_tweets):
    # Declare a figure with a custom size
    fig = plt.figure(figsize=(5, 5))

    # labels for the two classes
    labels = 'Positives', 'Negative'

    # Sizes for each slide
    sizes = [len(all_positive_tweets), len(all_negative_tweets)]

    # Declare pie chart, where the slices will be ordered and plotted counter-clockwise:
    plt.pie(sizes, labels=labels, autopct='%1.1f%%', shadow=True, startangle=90)

    # Equal aspect ratio ensures that pie is drawn as a circle.
    plt.axis('equal')

    # Display the chart
    plt.show()

# A function to print a sample tweet from each class
def sample_tweets():
    print('\033[92m' + positive_tweets[random.randint(0, 5000)])
    print('\033[91m' + negative_tweets[random.randint(0, 5000)])

# A function to preprocess a tweet before feeding to our ML model
def process_tweet(tweet):
    # remove stock market tickers like $GE
    tweet2 = re.sub(r'\$\w*', '', tweet)

    # remove old style retweet text "RT"
    tweet2 = re.sub(r'^RT[\s]+', '', tweet2)

    # remove hyperlinks
    tweet2 = re.sub(r'https?:\/\/.*[\r\n]*', '', tweet2)

    # remove hashtags (only removing the hash # sign from the word)
    tweet2 = re.sub(r'#', '', tweet2)

    # Create Tweet Tokenizer object
    token = TweetTokenizer(preserve_case=False, reduce_len=False, strip_handles=True)

    # Tokenize the tweet
    tweet_tokenized = token.tokenize(tweet2)

    # Create Porter Stemmer object
    stem = PorterStemmer()

    # Create stop words object
    st = stopwords.words('english')
    tweet_cleaned = []

    # Remove stopwords and punctuations and apply stemming
    for word in tweet_tokenized:
        if word not in st and word not in string.punctuation:
            stemmed_word = stem.stem(word)
            tweet_cleaned.append(stemmed_word)

    return tweet_cleaned

# A function to create a dictionary mapping {(word, label) --> frequency of that word in that label}
def build_freq(tweets, ys):
    yslist = np.squeeze(ys).tolist()
    frequency = {}
    for y, tweet in zip(yslist, tweets):
        for word in process_tweet(tweet):
            pair = (word, y)
            frequency[pair] = frequency.get(pair, 0) + 1
    return frequency

freqs = build_freq(train_x, train_y)

# A function to extract the necessary features from a tweet [positive frequency sum, negative frequency sum]
def extract_features(tweet, freqs):
    # process_tweet tokenizes, stems, and removes stopwords
    word_list = process_tweet(tweet)

    # 2 elements in the form of a 1 x 2 vector
    x = np.zeros((1, 2))

    # loop through each word in the list of words
    for word in word_list:
        # increment the word count for the positive label 1
        x[0, 0] += freqs.get((word, 1.0), 0)
        # increment the word count for the negative label 0
        x[0, 1] += freqs.get((word, 0.0), 0)

    return x

warnings.filterwarnings('ignore')

# collect the features 'x' and stack them into a matrix 'X'
X_train = np.zeros(((len(train_x)), 2))
for i in range(len(train_x)):
    X_train[i, :] = extract_features(train_x[i], freqs)

# training labels corresponding to X
y_train = np.array(train_y)

# Create Multinomial Naive Bayes object
model = naive_bayes.MultinomialNB(alpha=1)

# Train the model
model.fit(X_train, y_train)

# Extract features of the test set
X_test = np.zeros(((len(test_x)), 2))
for i in range(len(test_x)):
    X_test[i, :] = extract_features(test_x[i], freqs)

y_test = np.array(test_y)

tweet = input("Enter your tweet: ")
tweet_final = extract_features(tweet, freqs)

print(model.predict(tweet_final))

if model.predict(tweet_final) == 1.:
    print("Positive sentiment")
else:
    print("Negative sentiment")

accuracy = model.score(X_test, y_test)
print("accuracy: " + str(accuracy))


#! /usr/bin/env python3

'''
Sherlock: Find Usernames Across Social Networks Module

This module contains the main logic to search for usernames at social
networks.
'''

# ==================== Imports ==================== #
import os
import sys
import json
import csv
import re
from concurrent.futures import ThreadPoolExecutor
import requests
from colorama import Fore, Style, init
from torrequest import TorRequest

from watson import *

# TODO: fix tumblr

# ==================== Main ==================== #
class Sherlock:
    def __init__(self, username, index_by_original_query=False, source=os.path.join(os.path.dirname(os.path.realpath(__file__)), "data.json")):
        '''Create an instance of Sherlock that can be used to look up usernames on social media.

        Keyword Arguments:
        username                    -- The username to look up.
        index_by_original_query     -- Index the results by original query, if false, results are indexed by the actual name of the social media site.
        source                      -- The source of the site data JSON used by Sherlock, can be a URL.
        '''
        self.results = dict()
        self.username = username
        self.indexed_by_original_query = index_by_original_query
        if source is None:
            self.source = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data.json")
        else:
            self.source = source

    def __prepare_site_data(self, sites, ranked=False):
        '''Prepares a site data dictionary of all the sites that Sherlock will analyze.

        Keyword Arguments:
        sites       -- a list of sites as strings
        ranked      -- rank the sites by their alexa ranking

        Return Value:
        Dictionary containing all the site data of the given sites. This site data
        is used by Sherlock to check if the username exists at those sites.
        '''
        response_json_online = None
        # Try to load json from website.
        try:
            response_json_online = requests.get(url=self.source)
        except requests.exceptions.MissingSchema:  # In case the schema is wrong it's because it may not be a website
            pass

        # Check if the response is appropriate.
        if response_json_online is not None and response_json_online.status_code == 200:
            # Since we got data from a website, try to load json and exit if parsing fails.
            try:
                site_data_all = response_json_online.json()
                return site_data_all
            except ValueError:
                print(Style.BRIGHT + Fore.RED + "Invalid JSON from website!")
                Loader.stop_all()
                sys.exit(1)
                pass

        # Load the data
        data_file_path = self.source
        with open(data_file_path, "r", encoding="utf-8") as raw:
            site_data_all = json.load(raw)

        if sites is None:
            # Not desired to look at a sub-set of sites
            site_data = site_data_all
        else:
            # User desires to selectively run queries on a sub-set of the site list.
            # Make sure that the sites are supported & build up pruned site database.
            site_data = dict()
            site_missing = []
            for site in sites:
                for existing_site in site_data_all:
                    if site.lower() in existing_site.lower():
                        site_data[existing_site] = site_data_all[existing_site]
                        site_data[existing_site]['original_query'] = site
                if not site_data:
                    # Build up list of sites not supported for future error message.
                    site_missing.append(f"'{site}'")

            if site_missing:
                print(Style.BRIGHT + Fore.RED + f"Error: Desired sites not found: {', '.join(site_missing)}.")
                Loader.stop_all()
                sys.exit(1)

        if ranked:
            # Sort data by rank
            site_dataCpy = dict(site_data)
            ranked_sites = sorted(site_data, key=lambda k: ("rank" not in k, site_data[k].get("rank", sys.maxsize)))
            site_data = {}
            for site in ranked_sites:
                site_data[site] = site_dataCpy.get(site)

        return site_data

    def check(self, sites, verbose=False, tor=False, unique_tor=False, proxy=None, silent_on_error=False, ranked=False):
        '''Run Sherlock Analysis.

        Checks for existence of username on various social media sites.

        Keyword Arguments:
        site_data              -- Dictionary containing all of the site data.
        verbose                -- Boolean indicating whether to give verbose output.
        tor                    -- Boolean indicating whether to use a tor circuit for the requests.
        unique_tor             -- Boolean indicating whether to use a new tor circuit for each request.
        proxy                  -- String indicating the proxy URL.
        silent_on_error        -- Will not print errors to STDOUT.
        ranked                 -- Rank the results by alexa.com rankings.

        Return Value:
        Dictionary containing results from report.  Key of dictionary is the name
        of the social network site, and the value is another dictionary with
        the following keys:
            url_main:      URL of main site.
            url_user:      URL of user on site (if account exists).
            exists:        String indicating results of test for account existence.
            http_status:   HTTP status code of query which checked for existence on
                        site.
            response_text: Text that came back from request.  May be None if
                        there was an HTTP error when checking for existence.
        '''

        # Get site data
        if isinstance(sites, str):
            sites = [sites]
        site_data = self.__prepare_site_data(sites, ranked)

        # A user agent is needed because some sites don't
        # return the correct information since they think that
        # we are bots
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:55.0) Gecko/20100101 Firefox/55.0'
        }

        # Allow 1 thread for each external service, so `len(site_data)` threads total
        executor = ThreadPoolExecutor(max_workers=len(site_data))

        # Create session based on request methodology
        if tor or unique_tor:
            underlying_request = TorRequest()
            underlying_session = underlying_request.session
        else:
            underlying_session = requests.session()
            underlying_request = requests.Request()

        # Create multi-threaded session for all requests. Use our custom FuturesSession that exposes response time
        session = ElapsedFuturesSession(executor=executor, session=underlying_session)

        # Results from analysis of all sites
        results_total = self.results

        # First create futures for all requests. This allows for the requests to run in parallel
        for social_network, net_info in site_data.items():

            # Results from analysis of this specific site
            results_site = {}

            # Record URL of main site
            results_site['url_main'] = net_info.get("urlMain")

            # Don't make request if username is invalid for the site
            regex_check = net_info.get("regexCheck")
            if regex_check and re.search(regex_check, self.username) is None:
                # No need to do the check at the site: this user name is not allowed.
                print((Style.BRIGHT + Fore.WHITE + "[" +
                    Fore.RED + "-" +
                    Fore.WHITE + "]" +
                    Fore.GREEN + " {}:" +
                    Fore.YELLOW + " Illegal Username Format For This Site!").format(social_network))
                print()
                results_site["exists"] = "illegal"
            else:
                # URL of user on site (if it exists)
                url = net_info["url"].format(self.username)
                results_site["url_user"] = url

                request_method = session.get
                if social_network != "GitHub":
                    # If only the status_code is needed don't download the body
                    if net_info["errorType"] == 'status_code':
                        request_method = session.head

                if net_info["errorType"] == "response_url":
                    #Site forwards request to a different URL if username not
                    #found.  Disallow the redirect so we can capture the
                    #http status from the original URL request.
                    allow_redirects = False
                else:
                    #Allow whatever redirect that the site wants to do.
                    #The final result of the request will be what is available.
                    allow_redirects = True

                # This future starts running the request in a new thread, doesn't block the main thread
                if proxy != None:
                    proxies = {"http": proxy, "https": proxy}
                    future = request_method(url=url, headers=headers,
                                            proxies=proxies,
                                            allow_redirects=allow_redirects
                                        )
                else:
                    future = request_method(url=url, headers=headers,
                                            allow_redirects=allow_redirects
                                        )

                # Store future in data for access later
                net_info["request_future"] = future

                # Reset identify for tor (if needed)
                if unique_tor:
                    underlying_request.reset_identity()

            # Add this site's results into final dictionary with all of the other results.
            if self.indexed_by_original_query:
                results_total[site_data[social_network]['original_query']] = results_site
            else:
                results_total[social_network] = results_site

        # Open the file containing account links
        # f = open_file(fname)

        # Core logic: If tor requests, make them here. If multi-threaded requests, wait for responses
        for social_network, net_info in site_data.items():

            # Retrieve results again
            results_site = results_total.get(social_network)

            # Retrieve other site information again
            url = results_site.get("url_user")
            exists = results_site.get("exists")
            if exists is not None:
                # We have already determined the user doesn't exist here
                continue

            # Get the expected error type
            error_type = net_info["errorType"]

            # Default data in case there are any failures in doing a request.
            http_status = "?"
            response_text = ""

            # Retrieve future and ensure it has finished
            future = net_info["request_future"]
            r, error_type, response_time = get_response(request_future=future,
                                                        error_type=error_type,
                                                        social_network=social_network,
                                                        verbose=verbose)

            # Attempt to get request information
            try:
                http_status = r.status_code
            except:
                pass
            try:
                response_text = r.text.encode(r.encoding)
            except:
                pass

            if error_type == "message":
                error = net_info.get("errorMsg")
                # Checks if the error message is in the HTML
                if not error in r.text:
                    # print_found(social_network, url, response_time, verbose)
                    exists = "yes"
                else:
                    # print_not_found(social_network, response_time, verbose)
                    exists = "no"

            elif error_type == "status_code":
                # Checks if the status code of the response is 2XX
                if not (r.status_code >= 300 or r.status_code < 200):
                    # print_found(social_network, url, response_time, verbose)
                    exists = "yes"
                else:
                    # print_not_found(social_network, response_time, verbose)
                    exists = "no"

            elif error_type == "response_url":
                # For this detection method, we have turned off the redirect.
                # So, there is no need to check the response URL: it will always
                # match the request.  Instead, we will ensure that the response
                # code indicates that the request was successful (i.e. no 404, or
                # forward to some odd redirect).
                if (r.status_code >= 200) and (r.status_code < 300):
                    # print_found(social_network, url, response_time, verbose)
                    exists = "yes"
                else:
                    # print_not_found(social_network, response_time, verbose)
                    exists = "no"

            elif error_type == "":
                exists = "error"

            # Save exists flag
            results_site['exists'] = exists

            # Save results from request
            results_site['http_status'] = http_status
            results_site['error_type'] = error_type
            results_site['response_object'] = r
            results_site['response_text'] = response_text
            results_site['response_time_ms'] = response_time

            # Add this site's results into final dictionary with all of the other results.
            results_total[social_network] = results_site

        if not silent_on_error:
            dump_errors()
        return results_total


import requests
import json
import pandas as pd

url = "https://ecshweb.pchome.com.tw/search/v3.3/all/results?q=IPHONE12&page=1&sort=sale/dc"
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 Edg/92.0.902.67"}

res = requests.get(url,headers=headers)
json_string = json.loads(res.text)
prodids = []
print(json_string)
names =[]
for prodid in json_string['prods']:
    prodids.append(prodid['Id'])
    names.append(prodid['name'])
prodids = list(set(prodids))
names = list(set(names))
# print(prodids)
price = []
price_cleaner = []
qty = []
price_url = "https://ecapi.pchome.com.tw/ecshop/prodapi/v2/prod/button&id={}&fields=Id,Price,Qty".format(','.join(prodids))
# print(price_url)
price_jres = requests.get(price_url,headers=headers)
price_json = json.loads(price_jres.text)
for k in price_json:
    k['Id'] = k['Id'].split('-0')[0]
    if k['Id'] not in price_cleaner:
        price_cleaner.append(k['Id'])
        price.append(k['Price']['P'])
        qty.append(k['Qty'])
htmls = []
for i in prodids:
    htmls.append("https://24h.pchome.com.tw/prod/"+i)
if prodids == price_cleaner:
    df_ID = pd.DataFrame(prodids,columns=['ID'])
    df_Name = pd.DataFrame(names,columns=['Title'])
    df_price = pd.DataFrame(price,columns=['price'])
    df_qty = pd.DataFrame(qty,columns=['qty'])

    df = pd.DataFrame(
        {'ID':prodids,
        'Title':names,
        'Price':price,
        'Qty':qty,
        'Html':htmls}
    )
    df.to_excel('excel_output.xlsx',sheet_name='Sheet1')
print(df)


#   numth/primality.py

from numth.main import gcd, jacobi, padic
from numth.rational import integer_sqrt

from random import randint
from functools import reduce
from concurrent.futures import ProcessPoolExecutor, wait

##############################

def _default_values(cat):
    if cat == 'mr_wit':
        return 20
    elif cat == 'l_wit':
        return 20
    elif cat == 'sieve primes':
        return [2, 3, 5, 7]

############################################################
############################################################
#       Primes
############################################################
############################################################

def is_prime(num, mr_wit=None, l_wit=None, mp=True):
    """
    Primality test (with multiprocessing).

    Args:   int:    num
            int:    mr_wit      number of Miller-Rabin witnesses
            int:    l_wit       number of Lucas witness pairs
            bool:   mp          whether to use multiprocessing

    Return: bool

    Notes:  1. If num < 341_550_071_728_321:
                a. only pre-designated Miller-Rabin witnesses are used.
                b. the result is completely deterministic.
            2. Otherwise:
                a. the result is probabalistic, with probability of incorrect
                    < ( (1/4)**mr_wit ) * ( (4/15)**l_wit )
            3. If mp=False, Lucas tests are slower than Miller-Rabin tests,
                so it is prudent to use a smaller l_wit.
    """
    if num < 2:
        return False
    prime = ('prime', 'probable prime', 'strong probable prime')
    if num < miller_rabin_cutoffs()[-1][0]:
        return (miller_rabin_test(num, mr_wit) in prime)
    else:
        if mp:
            with ProcessPoolExecutor() as pool:
                futures = [
                    pool.submit( miller_rabin_test, num, mr_wit ),
                    pool.submit( lucas_test, num, l_wit )
                ]
                wait(futures)
            mr, l = (x.result() for x in futures)
        else:
            mr = miller_rabin_test(num, mr_wit)
            l = lucas_test(num, l_wit)
        return (mr in prime) and (l in prime)

##############################

def prime_to(nums, mr_wit=None, l_wit=None):
    """
    List of numbers relatively prime to a given list.

    Args:   list:   nums    preferrably prime numbers
            int:    mr_wit, l_wit

    Return: list:   [ n for n < product of nums if n relatively prime to nums ]
    """
    nums = sorted(nums)
    val = nums[0]
    already_used = [val]
    if is_prime(val, mr_wit, l_wit):
        result = [ x for x in range(1, val) ]
    else:
        result = [ x for x in range(1, val) if gcd(x, val) == 1 ]

    for num in nums[1:]:
        orig = [ x for x in result ]
        for i in range(1, num):
            result += [ i * val + x for x in orig ]
        if num not in already_used:
            for x in orig:
                result.remove(num * x)
            already_used.append(num)
        val *= num

    return result

##############################

def next_prime_gen(num, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Generator of the next primes.

    Args:   int:        num, mr_wit, l_wit
            list:       sieve_primes

    Return: generator:  int:    next primes after num
    """
    if num < 1:
        num = 1
    if sieve_primes is None:
        sieve_primes = _default_values('sieve primes')
    sieve, diam = _generate_block(sieve_primes)
    block = _shift_block(num, sieve, diam)
    preblock = _restrict_block(block, l_bd=num+1)

    for p in [x for x in sieve_primes if num < x]:
        yield p

    if preblock:
        for x in preblock:
            if is_prime(x, mr_wit, l_wit):
                yield x

    while True:
        block = _next_block(block, diam)
        for x in block:
            if is_prime(x, mr_wit, l_wit):
                yield x

##############################

def next_prime(num, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Find the next prime.

    Args:   int:    num, mr_wit, l_wit
            list:   sieve_primes

    Return: int:    next prime after num
    """
    gen = next_prime_gen(num, sieve_primes, mr_wit, l_wit)
    return next(gen)

##############################

def next_primes(num, k, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Find the next primes.

    Args:   int:    num, k, mr_wit, l_wit       k: number of primes
            list:   sieve_primes

    Return: list:   k primes after num
    """
    if num < 2:
        primes = [2]
        num = 2
    else:
        primes = []

    gen = next_prime_gen(num, sieve_primes, mr_wit, l_wit)
    while len(primes) < k:
        primes.append( next(gen) )

    return primes

##############################

def prev_prime_gen(num, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Generator of the previous primes.

    Args:   int:        num, mr_wit, l_wit
            list:       sieve_primes        primes used for sieving

    Return: generator:  int                 previous primes before num
    """
    if sieve_primes is None:
        sieve_primes = _default_values('sieve primes')

    if num <= next_prime(max(sieve_primes)):
        for y in reversed(sieve_primes):
            if y < num:
                yield y
        raise StopIteration('Iterator is empty')

    block, diam = _generate_block(sieve_primes)
    block = _shift_block(num, block, diam)
    preblock = _restrict_block(block, u_bd=num)

    if preblock:
        for x in reversed(preblock):
            if is_prime(x, mr_wit, l_wit):
                yield x

    while True:
        block = _prev_block(block, diam)
        for x in reversed(block):
            if block[0] == 1 and x < max(sieve_primes):
                for y in reversed(sieve_primes):
                    yield y
                raise StopIteration('Iterator is empty')
            elif is_prime(x, mr_wit, l_wit):
                yield x

##############################

def prev_prime(num, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Find the previous prime.

    Args:   int:    num, mr_wit, l_wit
            list:   sieve_primes

    Return: int:    next prime after num
    """
    if num < 3:
        return None
    elif num == 3:
        return 2
    else:
        gen = prev_prime_gen(num, sieve_primes, mr_wit, l_wit)
        return next(gen)

##############################

def prev_primes(num, k, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Find the previous primes.

    Args:   int:    num, k, mr_wit, l_wit       k: number of primes
            list:   sieve_primes

    Return: list:   k primes before num
    """
    if num < 6:
        return [ x for x in [5, 3, 2] if x < num ]
    gen = prev_prime_gen(num, sieve_primes, mr_wit, l_wit)

    primes = []
    while len(primes) < k and 2 not in primes:
        primes.append( next(gen) )

    return primes

##############################

def primes_in_range(l_bd, u_bd, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Find primes in a range.

    Args:   int:    l_bd, u_bd, mr_wit, l_wit       l_bd: lower bound
                                                    u_bd: upper bound
            list:   sieve_primes

    Return: list:   primes in range(l_bd, u_bd)
    """
    if sieve_primes is None:
        sieve_primes = _default_values('sieve primes')
    gen = next_prime_gen(l_bd - 1, sieve_primes, mr_wit, l_wit)

    primes = []
    while True:
        p = next(gen)
        if p < u_bd:
            primes.append(p)
        else:
            break

    return primes

##############################

def prime_count_gen(yield_values, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    The prime-counting function pi(x).

    Args:   generator/tuple:    yield_values
            int:                mr_wit, l_wit

    Return: generator:  int

    Notes:  1. pi(x) is the number of primes <= x
            2. the returned generator is (pi(x) for x in yield_values)
    """

    start = 1
    count = 0

    if isinstance(yield_values, tuple):
        yield_values = (y for y in yield_values)

    end = next(yield_values)
    yield_ability = True
    while True:
        if not yield_ability:
            yield None
        else:
            count += len(primes_in_range(
                start, end, sieve_primes, mr_wit, l_wit))
            try:
                yield end, count
                start, end = end, next(yield_values)
            except StopIteration:
                print('Iterator is empty')
                yield_ability = False

##############################

def prime_count(num, sieve_primes=None, mr_wit=None, l_wit=None):
    pcg = prime_count_gen((num,), sieve_primes, mr_wit, l_wit)
    return next(pcg)[1]

############################################################
############################################################
#       Twin primes
############################################################
############################################################

def next_twin_primes_gen(num, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Generator of next pairs of twin primes.

    Args:   int:        num, mr_wit, l_wit

    Return: generator:  tuple(int p, int p+2)       p, p+2 both prime
                                                    p+1 >= num

    Note:   This is a 'dangerous' function until the
            Twin Prime Conjecture is proved ;]
    """
    if num < 4:
        yield 3, 5

    gen = next_prime_gen(num - 2, sieve_primes, mr_wit, l_wit)
    p = next(gen)
    q = next(gen)
    while True:
        while q - p > 2:
            p, q = q, next(gen)
        yield p, q
        p, q = q, next(gen)

##############################

def next_twin_primes(num, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Find the next pair of twin primes.

    Args:   int:    num, mr_wit, l_wit

    Return: tuple:  (p, p+2)            p, p+2 both prime
                                        p+1 >= num
    """
    gen = next_twin_primes_gen(num, sieve_primes, mr_wit, l_wit)
    return next(gen)

##############################

def prev_twin_primes(num, sieve_primes=None, mr_wit=None, l_wit=None):
    """
    Find previous pair of twin primes.

    Args:   int:    num, mr_wit, l_wit

    Return: tuple:  (p-2, p)            p-2, p both prime
                                        p-1 <= num
    """
    if num < 4:
        return None

    gen = prev_prime_gen(num + 2, sieve_primes, mr_wit, l_wit)
    q = next(gen)
    p = next(gen)
    while q - p > 2:
        q, p = p, next(gen)

    return p, q

############################################################
############################################################
#       Goldbach partitions
############################################################
############################################################

def goldbach_partition(num,
        sieve_primes=None, mr_wit=None, l_wit=None, RANDOM=True):
    """
    Express number as a sum of 2 or 3 primes.

    Args:   int:    num, mr_wit, l_wit

    Return: tuple   (p, q [, r])        num = p + q [+ r]
    """
    if num < 4:
        raise ValueError('num must be at least 4')
    if num == 4:
        return 2, 2
    if num == 5:
        return 2, 3
    if num % 2 == 1:
        return weak_goldbach_partition(num, sieve_primes, mr_wit, l_wit, RANDOM)

    if RANDOM:
        start = randint(1, num//2 - 1)
    else:
        start = num//2 - 1

    p_gen = next_prime_gen(start, sieve_primes, mr_wit, l_wit)
    p = next(p_gen)
    q = num - p
    while not is_prime(q, mr_wit, l_wit):
        p = next(p_gen)
        q = num - p

    return tuple(sorted([p, q], reverse=True))

##############################

def weak_goldbach_partition(num,
        sieve_primes=None, mr_wit=None, l_wit=None, RANDOM=True):
    """
    Express number as a sum of 3 primes.

    Args:   int:    num, mr_wit, l_wit

    Return: tuple:  (p, q, r)           num = p + q + r
    """
    if num < 6:
        raise ValueError('number must be at least 6')

    if num == 7:
        return (3, 2, 2)

    if RANDOM:
        p_start = randint(1, num//3 - 1)
        q_start = randint(num//3, 2*num//3)
    else:
        p_start = num//3 - 1
        q_start = num//3 + 1

    p_gen = next_prime_gen(p_start, sieve_primes, mr_wit, l_wit)
    q_gen = prev_prime_gen(q_start, sieve_primes, mr_wit, l_wit)
    p = next(p_gen)
    q = next(q_gen)
    r = num - p - q
    while not is_prime(r, mr_wit, l_wit):
        try:
            p = next(p_gen)
            r = num - p - q
            if is_prime(r, mr_wit, l_wit):
                return p, q, r
        except StopIteration:
            pass
        try:
            q = next(q_gen)
            r = num - p - q
        except StopIteration:
            pass
    return tuple(sorted([p,q,r], reverse=True))

############################################################
############################################################
#       Naive primality test
############################################################
############################################################

def naive_primality_test(num):
    """
    Naive primality test.

    Args:   int:    num

    Return: str:    'composite' <- if num is not prime
                    'prime'     <- if num is prime
    """
    all_nums = [ x for x in range(2, num) if x**2 <= num ]
    while all_nums != []:
        sm = all_nums[0]
        if num % sm == 0:
            return 'composite'
        else:
            all_nums = [ x for x in all_nums if x % sm != 0 ]
    return 'prime'

############################################################
############################################################
#       Miller-Rabin primality testing
############################################################
############################################################

def miller_rabin_witness(num, wit):
    """
    Miller-Rabin witness for primality.

    Args:   int:    num, wit        wit: witness

    Return: str:    'composite'         <- if wit detects num is composite
                    'probable prime'    <- otherwise
    """
    exp, init = padic(num - 1, 2)
    x = pow(wit, init, num)

    if x in [1, num-1]:
        return 'probable prime'

    for j in range(exp):
        x = (x*x) % num
        if x == num - 1:
            return 'probable prime'
        elif x == 1:
            return 'composite'

    return 'composite'

##############################

def miller_rabin_cutoffs():
    """Cutoff values for witnesses needed in Miller-Rabin primality test."""
    return ((1, 2),
            (2047, 3),
            (1373653, 5),
            (25326001, 7),
            (3215031751, 11),
            (2152302898747, 13),
            (3474749660383, 17),
            (341550071728321, None))

##############################

def _generate_miller_rabin_witnesses(num, num_wit=None):
    """
    Generate a list of witnesses for Miller-Rabin primality test.

    Args:   int:    num, num_wit        num_wit: number of witnesses

    Return: list:   witnesses
    """
    cutoffs = miller_rabin_cutoffs()
    max_val = cutoffs[-1][0]
    if num > max_val:
        if num_wit is None:
            num_wit = _default_values('mr_wit')
        if num_wit > num:
            witnesses = [ x for x in range(2, num-1) ]
        else:
            witnesses = []
            while len(witnesses) < num_wit:
                x = randint(2, num-1)
                if x not in witnesses:
                    witnesses.append(x)
    else:
        witnesses = []
        for val, p in cutoffs[:-1]:
            if num >= val:
                witnesses.append(p)

    return witnesses

##############################

def miller_rabin_test(num, num_wit=None):
    """
    Miller-Rabin primality test.

    Args:   int:    num, num_wit

    Return: str:    'composite' <- if any witness returns 'composite'
                    'prime'     <- if num < 341550071728321 and
                                      every witness returns 'probable prime'
                    'strong probable prime'
                                <- otherwise

    Note:   'strong probable prime' is correct
            with probability > 1 - (.25)**num_wit
    """
    if num == 2:
        return 'prime'
    witnesses = _generate_miller_rabin_witnesses(num, num_wit)

    for wit in witnesses:
        if miller_rabin_witness(num, wit) == 'composite':
            return 'composite'

    if num < miller_rabin_cutoffs()[-1][0]:
        return 'prime'
    else:
        return 'strong probable prime'

############################################################
############################################################
#       Lucas sequence primality testing
############################################################
############################################################

def _lucas_double_index(U, V, Q, mod):
    """
    Shortcut to double the index of a Lucas sequence.

    Args:   int:    U, V, Q, mod

    Return: tuple
    """
    return (
        (U*V) % mod,
        (V*V - 2*Q) % mod,
        (Q**2) % mod
    )

##############################

def _lucas_index_plus_one(U, V, P, Q, Q1, mod):
    """
    Shortcut to increase the index of a Lucas sequence by one.

    Args:   int:    U, V, P, Q, Q1, mod

    Return: tuple
    """
    return (
        ((P*U + V) * (mod + 1)//2) % mod,
        (((P**2 - 4*Q1)*U + P*V) * (mod + 1)//2) % mod,
        (Q*Q1) % mod
    )

##############################

def _lucas_sequence_by_index(k, P, Q, mod):
    """
    Explicit formula for any element of a Lucas sequence.

    Args:   int:    k, P, Q, mod

    Return: tuple:  (U[k], V[k], Q**k)
    """
    if k == 0:
        return (0, 2, 1)
    elif k == 1:
        return (1, P, Q)
    elif k % 2 == 0:
        U_, V_, Q_ = _lucas_sequence_by_index(k//2, P, Q, mod)
        return _lucas_double_index(U_, V_, Q_, mod)
    else:
        U_, V_, Q_ = _lucas_sequence_by_index(k-1, P, Q, mod)
        return _lucas_index_plus_one(U_, V_, P, Q_, Q, mod)

##############################

def _lucas_sequence(P, Q, mod):
    """
    Lucas sequence.

    Args:   int:        P, Q, mod

    Return: generator:  tuple
    """
    U0, V0, Q0 = (0, 2, 1)
    yield (U0, V0, Q0)
    U1, V1, Q1 = (1, P, Q)
    yield (U1, V1, Q1)
    while True:
        U0, U1 = U1, (P*U0 - Q*U1) % mod
        V0, V1 = V1, (P*V0 - Q*V1) % mod
        Q0, Q1 = Q1, (Q*Q1) % mod
        yield (U1, V1, Q1)

##############################

def lucas_witness(num, P, Q):
    """
    Lucas sequence witness for primality.

    Args:   int:    num, P, Q       P, Q: Lucas parameters

    Return: str:    'composite' <- if Lucas pair U(P,Q), V(P,Q) detects it
                    '(strong) probable prime' <- otherwise

    Note:   the 'strength' of a probable prime is based on
            some additional conditions that make it more likely to be prime.
    """
    D = P**2 - 4*Q
    gcd_D = gcd(D, num)
    gcd_P = gcd(P, num)
    gcd_Q = gcd(Q, num)
    for g in (gcd_D, gcd_P, gcd_Q):
        if g == num:
            raise ValueError('Bad parameters: {}, {}'.format(P, Q))
        elif g > 1:
            return 'composite'

    delta = num - jacobi(D, num)
    s, d = padic(delta, 2)
    strong = False

    U, V, Q_ = _lucas_sequence_by_index(d, P, Q, num)
    if U == 0:
        strong = True
    for j in range(s - 1):
        U, V, Q_ = _lucas_double_index(U, V, Q_, num)
        if V == 0:
            strong = True
    U, V = _lucas_double_index(U, V, Q_, num)[:2]

    if U == 0:
        if delta == num + 1:
            if V != (2*Q) % num:
                return 'composite'
            if Q_ != (Q * jacobi(Q, num)) % num:
                return 'composite'
        else:
            if Q_ != jacobi(Q, num) % num:
                return 'composite'
        if strong:
            return 'strong probable prime'
        else:
            return 'probable prime'

    return 'composite'

##############################

def _generate_lucas_witness_pairs(num, num_wit=None):
    """
    Generate a list of witness pairs for Lucas primality test.

    Args:   int:    num, num_wit

    Return: list    of tuples   (P, Q)
    """
    if num_wit is None:
        num_wit = _default_values('l_wit')
    witnesses = []

    if integer_sqrt(num)**2 != num:
        D = 5
        sgn = 1
        while len(witnesses) < num_wit // 2 + 1:
            if (jacobi(D*sgn, num) == -1):
                witnesses.append( (1, (1 - D)//4) )
            D += 2
            sgn *= 1

    while len(witnesses) < num_wit:
        P = randint(1, 100*num_wit)
        Q = randint(1, 100*num_wit)
        D = P**2 - 4*Q
        if (P, Q) not in witnesses:
            witnesses.append( (P, Q) )

    return witnesses

##############################

def lucas_test(num, num_wit=None):
    """
    Lucas test for primality.

    Args:   int:    num, num_wit        num_wit: number of witnesses

    Return: str:    'composite' <- if any pair returns 'composite'
                    'probable prime' <- if each pair returns 'probable prime'
                    'strong probable prime' <- otherwise

    Note:   '(strong) probable prime' is correct
            with probability > 1 - (4/15)**num_wit
    """
    if num <= 2:
        raise ValueError('Num should be at least 3')
    if num % 2 == 0:
        return 'composite'
    witnesses = _generate_lucas_witness_pairs(num, num_wit)

    strong = False
    for P, Q in witnesses:

        primality = lucas_witness(num, P, Q)
        if primality == 'composite':
            return 'composite'
        if primality == 'strong probable prime':
            strong = True
    if strong:
        return 'strong probable prime'
    else:
        return 'probable prime'

############################################################
############################################################
#       Auxiliary functions
############################################################
############################################################

def _generate_block(sieve_primes):
    """
    Generate a block of offsets for prime searching.

    Args:   list:   sieve_primes

    Return: list:   offsets for sieving
    """
    diam = reduce(lambda x, y : x*y, sieve_primes, 1)
    return prime_to(sieve_primes, mr_wit=5, l_wit=1), diam

##############################

def _shift_block(num, block, diam):
    """Shift a block of offsets for prime searching."""
    block_min = num - num % diam

    return [ block_min + x for x in block ]

##############################

def _restrict_block(block, l_bd=None, u_bd=None):
    """Restrict a block to range(l_bd, u_bd)."""
    if l_bd:
        block = [ x for x in block if x >= l_bd ]
    if u_bd:
        block = [ x for x in block if x < u_bd ]

    return block

##############################

def _next_block(block, diam):
    """Find next block."""
    return [ x + diam for x in block ]

##############################

def _prev_block(block, diam):
    """Find previous block."""
    return [ x - diam for x in block ]

############################################################
############################################################
#       End
############################################################
############################################################


import pandas as pd
import numpy as np
from nltk.tokenize import TreebankWordTokenizer
import string
from konlpy.tag import Komoran,Kkma,Okt,Hannanum,Mecab,Twitter
from wordcloud import STOPWORDS
myframe = pd.read_csv('../concat_data/updatae_csv.csv')
komoran = Komoran()
tokenizer = TreebankWordTokenizer()
stopword = set(STOPWORDS)
review = myframe['text'][0]

tokens = komoran.pos(review)

token_list = set([ x[0] for x in tokens if x[1] in ('NNG','NNP')])

# print(token_list)
# print(tokenizer.tokenize(review))
import re
regex1 = '^[\d가-힣a-zA-Z]*$'
regex2 = '^[\d]*$'
pattern1 = re.compile(regex1)
pattern2 = re.compile(regex2)
token_set = set()
# for idx in range(0,61):
#     review = myframe['text'][idx]
#     review = review.replace(u'\\xa0','')
#
#     # review = review.replace(u'\\n','').encode('utf-8')
#     tknize = tokenizer.tokenize(review)
#
#
#     tknize_st = [x for x in tknize if pattern1.match(x) and not x.isnumeric()]
#     print(idx, tknize_st)
#     token_set.update(tknize_st)
# print(token_set)
# print(len(token_set))

kkma = Kkma()
okt = Okt()
hananum = Hannanum()
twit = Twitter()
# mecab = Mecab()

hangul_stopword = [x.strip() for x in open('../concat_data/hangle_stop_word.txt', 'r', encoding='utf-8').readlines()]

token_set = set()
print(string.punctuation)  # !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
pt = re.compile("!\"#")
pattern = re.compile('[\U00010000-\U0010ffff]',flags=re.UNICODE)
for idx in range(0,61):
    review = myframe['text'][idx]
    review = pattern.sub(r' ',review)
    tokens = twit.pos(review,norm=True)
    # 불용어 처리
    tokens = [ x[0].replace(u'\u3000',u' ').replace(u'\xa0',u' ').translate(str.maketrans('', '', string.punctuation)) for x in tokens if x not in hangul_stopword]
    tokens = [ x for x in tokens if len(x)>1]
    token_set.update(tokens)
    print(idx,tokens)

print(len(token_set))
pat = re.compile('^[\d\r가-힣]*$')
bul = [x for x in token_set if not pat.match(x) or len(x)==1]
print(bul)

print(string.punctuation)
print(string.ascii_letters)
# ['※', '&', '!', "'", ')', '–', '-', '=', 'i', '(', '.', 'ㄴ', '☞', '~', '·', 'ㅁ', 'ㄹ', ':', '/', 'P', ',', '%', '"', '→']
# ['4월\xa0밸런스', '매', '얻고\u3000\xa0', '스테이지\u3000·', '그랩,', '힘', '넘', '3초\xa0→', "30초'\xa0게", "증가\xa0'3\xa0→",
#  '·\xa0업데이트', '돕', "%'-", '모드와\xa0새로운', '‘1.6초', '모델\xa0(특수', '회전기\u3000\u3000회전하', "50'\u3000커브볼(스타",
#  '승리\xa0', '램프\u3000\u3000주위', '폭풍\u3000\u300012초', '‘300', '스니커즈\u3000\u3000발', '증가\xa0‘4,600', '150\xa0트로피',
#  '쉘리\u3000쉘', '질주\u3000\u3000리코', '·\xa0미스터', '폭풍이\xa0아군', '–', '피루엣\u3000\u3000특수', '끊', '5초’-', '새', '·\xa0희귀',
#  '수정\u3000·', '엠즈\u3000후원', '상점\u3000·', '기준으로\xa0매치메이킹', '·\xa0재키\xa0-', '변경\u3000특수', '프랭크\u3000노이즈', '별',
#  '700’-', '틱\u3000미니', '휘', '수정\u3000·\xa0새로운', '페니\u3000약탈', "'1840\xa0→\xa0", "'640\xa0→", '자세(스타', '포크아웃\u3000·',
#  "'70\xa0→", '그랩(스타', '증폭\u3000\u3000특수', '요', '지역\u3000·', '길', '선', '0초’\u3000오토매틱', '29일(화)부터', '0.3초',
#  '보석\xa0\u3000\xa0에이전트', '증가\xa0‘40', '타', "'3500\xa0→", '테마\u3000·', '보석\xa0\u3000\xa0늑대', '로사의\xa0몸',
#  "'260\xa0→", "'420\xa0→", '25%’-', '‘100%', '0.5개', '은신술\u3000\u3000은신', "'240\xa0→", '리모델링\u3000·\xa0페니',
#  '내역\u3000·\xa0모티스', '600’\u3000그림자', '-\xa0모티스\u3000전속력', '얻고\xa0', '‘8%', '‘격려', '잃', '시\xa0재장전',
#  '곁', '론', '하이스트\u3000\u3000삭제', '·\xa08비트\xa0-', '5월\xa0밸런스', '프랭크\u3000파워', '수정\u3000·\xa0바운티', '→',
#  '지뢰\u3000\u3000지면', '·\xa0니타\xa0-', '파이퍼\u3000팡팡팡(특수', '리코\u3000통통탄(일반', '브롤러\u3000\u3000로사',
#  '3,200’\xa0오류', '(빅', '피루엣(스타', '0.5초', '밸런스\xa0-', '감소\xa0‘5,500\xa0→', '에', '1,000’-\xa0대릴\u3000강철',
#


import chi

env = 'CartPole-v0'
# env = 'OneRoundDeterministicReward-v0'

@chi.experiment
def dqn_atari(env=env, logdir=""):
  # from PIL import Image
  import numpy as np
  import gym
  import tensorflow as tf
  from gym import wrappers
  from tensorflow.contrib import layers
  from tensorflow.contrib.framework import arg_scope
  from chi.util import in_collections

  chi.set_loglevel('debug')

  env = gym.make(env)
  if hasattr(env.observation_space, 'shape'):
    oshape = env.observation_space.shape
    obsp = env.observation_space
    print(f'Observations in [{obsp.low}, {obsp.high}]')
  else:
    oshape = [1]

  env = wrappers.Monitor(env, logdir+'/monitor')
  # wrappers.SkipWrapper(4)

  if hasattr(env, 'spec') and hasattr(env.spec, 'reward_threshold'):
    print(f'reward threshold: {env.spec.reward_threshold}')

  @chi.model(tracker=tf.train.ExponentialMovingAverage(1-.01))
  def deep_q_network(x: [oshape]):
    with arg_scope([layers.fully_connected],
                   outputs_collections=[tf.GraphKeys.ACTIVATIONS],
                   weights_regularizer=layers.l1_regularizer(.00001)):
      # x = layers.conv2d(x, 32, 8, 4)
      # x = layers.conv2d(x, 64, 4, 2)
      # x = layers.conv2d(x, 64, 3, 1)
      x = layers.fully_connected(x, 32)
      x = layers.fully_connected(x, 32)
      x = layers.fully_connected(x, env.action_space.n, activation_fn=None)
      x = tf.identity(x, name='Q')
      print(f'dqn activations: {chi.activations()}')
      return x

  deep_q_network.optimizer = tf.train.AdamOptimizer(.1)  # .00025
  m = chi.rl.ReplayMemory(500000, oshape, 1)

  @chi.function
  def act(x: [oshape]):
    qs = deep_q_network(x)
    a = tf.argmax(qs, axis=1)
    qm = tf.reduce_max(qs, axis=1)
    layers.summarize_tensor(a)
    return a, qm

  al = "\n".join([f"{v.name} - {in_collections(v)}" for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)])
  print('global vars \n' + al + '\n')

  @chi.function
  def train(o, a: (tf.int32, (None, 1)), r: (None,), t: (tf.bool, (None,)), o2):
    asq = tf.squeeze(a, axis=1)

    q = deep_q_network(o)
    # ac = tf.argmax(q, axis=1)

    with tf.name_scope('compute_targets'):
      q2 = deep_q_network.tracked(o2)
      q_target = tf.where(t, r, r + 0.99 * tf.reduce_max(q2, axis=1))
      q_target = tf.stop_gradient(q_target)

    oh = tf.one_hot(asq, env.action_space.n, 1.0, 0.0, axis=1)
    qs = tf.reduce_sum(q * oh, axis=1, name='q_max')
    td = tf.subtract(q_target, qs, name='td')
    # td = tf.clip_by_value(td, -10, 10)
    mse = tf.reduce_mean(tf.abs(td), axis=0, name='mae')
    # mse = tf.where(tf.abs(td) < 1.0, 0.5 * tf.square(td), tf.abs(td) - 0.5, name='mse_huber')
    # mse = tf.reduce_mean(tf.square(td), axis=0, name='mse')

    with tf.name_scope('minimize'):
      loss = deep_q_network.minimize(mse)

    with tf.name_scope('summarize'):
      print(f'train activations: {chi.activations()}')
      layers.summarize_tensors([td, mse, r, o, a,
                                tf.subtract(o2, o, name='state_dif'),
                                tf.reduce_mean(tf.cast(t, tf.float32), name='frac_terminal'),
                                tf.subtract(tf.reduce_max(q, 1, True), q, name='av_advantage')])
      layers.summarize_tensors(chi.activations())
      layers.summarize_tensors(chi.gradients())
    return loss

  @chi.function
  def log_weigths():
    v = deep_q_network.trainable_variables()
    print(f'log weights {v}')

    f = deep_q_network.tracker_variables
    print(f'log weights EMA {f}')

    difs = []
    for g in v:
      a = deep_q_network.tracker.average(g)
      difs.append(tf.subtract(g, a, name=f'ema/dif{g.name[:-2]}'))

    layers.summarize_tensors(v+f+difs)

  @chi.function
  def log_returns(R, qs):

    layers.summarize_tensors([R, qs, tf.subtract(R, qs, name='R-Q')])

  print('\n VARIABLES: \n')
  al = "\n".join([f"{v.name} - {in_collections(v)}" for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)])
  print(al)

  print('\n START TRAINING: \n')

  training_time = 1000000
  t = 0
  Rs = [0]
  for ep in range(1000000):
    ob = env.reset()
    done = False
    R = 0
    qs = []
    while not done:
      anneal = 1 - t / training_time
      if np.random.rand() < .05 * anneal:
        a = np.random.randint(0, env.action_space.n)
      else:
        # ob = [ob]
        a, qm = act(ob)
        qs.append(qm)

      ob2, r, done, _ = env.step(a)
      r = r
      m.enqueue(ob, a, r, done)

      ob = ob2
      R += r
      if t > 1000:
        train(*m.minibatch(32)[:-1])

      # if (t+1) % 100 == 0:
      #   print(m)
      #   exit()

      if t % 5000 == 0:
        print(f'Return average after {t} timesteps: {np.average(Rs)}')
        Rs = []

      if t % 20000 == 0:
        log_weigths()

      t += 1

    Rs.append(R)
    log_returns(R, qs)

    if t > training_time:
      break

      # dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, window_length=WINDOW_LENGTH, memory=memory,
#                processor=processor, nb_steps_warmup=50000, gamma=.99, delta_range=(-1., 1.),
#                target_model_update=10000, train_interval=4)

#
# INPUT_SHAPE = (84, 84)
# WINDOW_LENGTH = 4
#
#
# class AtariProcessor(Processor):
#     def process_observation(self, observation):
#         assert observation.ndim == 3  # (height, width, channel)
#         img = Image.fromarray(observation)
#         img = img.resize(INPUT_SHAPE).convert('L')  # resize and convert to grayscale
#         processed_observation = np.array(img)
#         assert processed_observation.shape == INPUT_SHAPE
#         return processed_observation.astype('uint8')  # saves storage in experience memory
#
#     def process_state_batch(self, batch):
#         # We could perform this processing step in `process_observation`. In this case, however,
#         # we would need to store a `float32` array instead, which is 4x more memory intensive than
#         # an `uint8` array. This matters if we store 1M observations.
#         processed_batch = batch.astype('float32') / 255.
#         return processed_batch
#
#     def process_reward(self, reward):
#         return np.clip(reward, -1., 1.)
#
# parser = argparse.ArgumentParser()
# parser.add_argument('--mode', choices=['train', 'test'], default='train')
# parser.add_argument('--env-name', type=str, default='Breakout-v0')
# parser.add_argument('--weights', type=str, default=None)
# args = parser.parse_args()
#
# # Get the environment and extract the number of actions.
# np.random.seed(123)
# env.seed(123)
# nb_actions = env.action_space.n

# We patch the environment to be closer to what Mnih et al. actually do: The environment
# repeats the action 4 times and a game is considered to be over during training as soon as a live
# is lost.
# def _step(a):
#     reward = 0.0
#     action = env._action_set[a]
#     lives_before = env.ale.lives()
#     for _ in range(4):
#         reward += env.ale.act(action)
#     ob = env._get_obs()
#     done = env.ale.game_over() or (args.mode == 'train' and lives_before != env.ale.lives())
#     return ob, reward, done, {}
# env._step = _step

# Select a policy. We use eps-greedy action selection, which means that a random action is selected
# with probability eps. We anneal eps from 1.0 to 0.1 over the course of 1M steps. This is done so that
# the agent initially explores the environment (high eps) and then gradually sticks to what it knows
# (low eps). We also set a dedicated eps value that is used during testing. Note that we set it to 0.05
# so that the agent still performs some random actions. This ensures that the agent cannot get stuck.
# policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=.05,
#                               nb_steps=1000000)

# dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, window_length=WINDOW_LENGTH, memory=memory,
#                processor=processor, nb_steps_warmup=50000, gamma=.99, delta_range=(-1., 1.),
#                target_model_update=10000, train_interval=4)

# The trade-off between exploration and exploitation is difficult and an on-going research topic.
# If you want, you can experiment with the parameters or use a different policy. Another popular one
# is Boltzmann-style exploration:
# policy = BoltzmannQPolicy(tau=1.)
# Feel free to give it a try!


from urllib import request
import io
import gzip

__all__ = [
    'get_content_by_url'
]

def get_content_by_url(url):
    page = request.Request(url)
    headers = {
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36'
    }
    data = request.urlopen(page,headers=headers)
    encoding = data.getheader('Content-Encoding')
    content = data.read()
    if encoding == 'gzip':
        buf = io.BytesIO(content)
        gf = gzip.GzipFile(fileobj=buf)
        content = gf.read()
    return content.decode("gb2312", 'ignore')


from basic.select import get_files, select_folder
from FRET.cluster_FRET import get_col, gen_random_code
import scipy.io as sio
import pandas as pd
from EM_Algorithm.EM import *
import matplotlib.pyplot as plt
from matplotlib import rcParams
rcParams["font.family"] = "sans-serif"
rcParams["font.sans-serif"] = ["Arial"]
rcParams.update({'font.size': 12})

def remove_steps(step, dwell, criteria):
    booleans = step < criteria
    return step[booleans], dwell[booleans]

if __name__ == '__main__':
    all_gauss = []
    all_survival = []
    all_results = []

    n_sample = []
    # conc = ['0.0', '0.5', '1.0', '1.5', '2.0', '3.0']  ## S5S1
    conc = ['1.0', '1.2', '1.5', '1.8', '2.0', '3.0', '4.0'] ## m51 only
    # conc = ['0.10', '0.20', '0.25', '0.50', '0.70', '0.80', '1.10', '1.20', '2.00']  ## EcRecA
    # conc = ['0.25']
    # path_folder = select_folder()
    # path_folder = r'/home/hwligroup/Desktop/Data/step-dwell time/m51'
    # path_folder = r'/home/hwligroup/Desktop/Data/step-dwell time/m51 + mSS'

    path_folder = r'C:\Users\pine\Desktop\Data\step-dwell time\m51'
    # path_folder = r'C:\Users\pine\Desktop\Data\step-dwell time\EcRecA'
    # path_folder = r'C:\Users\pine\Desktop\Data\step-dwell time\m51 + mSS'
    # path_folder = r'C:\Users\pine\Desktop\Data\step-dwell time\m51 + mSS - v3(0621)'

    for i in range(1):

        tolerance = 1e-3

        for c in conc:
            path_data = get_files(f'*{c}*.mat', dialog=False, path_folder=path_folder)
            step = []
            dwell = []
            for path in path_data:
                data = sio.loadmat(path)
                step = np.append(step, data['step'])
                dwell = np.append(dwell, [data['dwell']])
                step, dwell = remove_steps(step, dwell, criteria=15)
            n_sample += [len(step)]

            # ## get Gaussian EM results
            # EM_g = EM(step)
            # n_components_g = EM_g.opt_components_iter(tolerance=1e-2, mode='GMM', criteria='BIC', figure=False)
            # f, m, s, converged_g = EM_g.GMM(n_components_g, rand_init=True, tolerance=1e-2)
            # all_gauss += [np.array([f, m, s, converged_g]).T]

            ## get poisson EM results
            EM_p = EM(dwell)
            # n_components_p = EM_p.opt_components_iter(tolerance=1e-2, mode='PEM', criteria='BIC', figure=False)
            f_tau, tau, s_tau, converged_p, LLE = EM_p.PEM(1)
            # all_survival += [np.array([f_tau, tau, converged_p]).T]

            ##  2D clustering
            step_dwell = np.array([step, dwell]).T
            EM_gp = EM(step_dwell, dim=2)
            # opt_components = EM_gp.opt_components_iter(tolerance=1e-2, mode='GPEM', criteria='BIC', figure=False)
            f1, m1, s1, tau1, converged_gp, LLE_gp = EM_gp.GPEM(n_components=1, tolerance=1e-2, rand_init=True)

            ##  plot figure
            # EM_g.plot_fit_gauss(scatter=False, xlim=[0, 16], save=False, path=f'{c}_gauss.png', figsize=(7,2))
            # EM_p.plot_fit_exp(xlim=[0, 10], save=False, path=f'{c}_survival.png', figsize=(7,2))
            para = EM_gp.para_final
            EM_p.plot_fit_exp(xlim=[0, 10], save=True, path=f'mRAD51_{c}_survival.png', figsize=(7,2),
                              para=[para[0]]+[para[3]], fontsize=12, remove_ytick=True, ylabel='', line_width=2.5)

            # EM_gp.plot_gp_contour(xlim=[0, 20], ylim=[0, 10.], save=True, path=f'{c}_2D.png')
            # EM_gp.plot_gp_surface()
            EM_gp.plot_gp_contour_2hist(xlim=[0,12], ylim=[0,10], bins_x=np.log2(len(step)).astype('int'),
                                        save=True, path=f'mRAD51_{c}_scatter.png', figsize=(6,5))

            all_results += [np.array([f1, m1, s1, tau1, converged_gp, LLE_gp]).T]
            print(all_results)

        # df_f = pd.DataFrame(f1, columns=['f'])
        # df_m = pd.DataFrame(m1, columns=['center'])
        # df_s = pd.DataFrame(s1, columns=['std'])
        # df_tau = pd.DataFrame(tau1, columns=['tau'])
        # df_LLE = pd.DataFrame(LLE_gp, columns=['LLE'])
        # df = pd.concat([df_f, df_m, df_s, df_tau, df_LLE], axis=1)
        #
        # ## save to disk
        # writer = pd.ExcelWriter( f'{gen_random_code(3)}_EM_results.xlsx')
        # df.to_excel(writer, sheet_name='EM', index=True)
        # writer.save()