import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
import project.data.preprocess_data as preprocess
import torch
from sklearn.preprocessing import LabelEncoder
from pytorch_pretrained_bert import BertTokenizer

def tokenize_input(baseline_text, context_text, split, tokenizer=None, max_len=509):
    if tokenizer == None:
        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    baseline = tokenizer.tokenize(baseline_text)
    context = tokenizer.tokenize(context_text)
    baseline_size = int(split*max_len)
    context_size = max_len - baseline_size

    baseline = preprocess.preprocess_tokens(baseline, baseline_size)
    context = preprocess.preprocess_tokens(context, context_size)

    final_tokens = ["[CLS]"]
    classifications = [0, 0]

    for token in baseline:
        final_tokens.append(token)
        classifications.append(0)

    final_tokens.append("[SEP]")
    classifications.append(1)

    for token in context:
        final_tokens.append(token)
        classifications.append(1)

    final_tokens.append("[SEP]")
    for i in range(max_len-(len(context) + len(baseline))):
        final_tokens.append("[MASK]")
        classifications.append(0)
    return tokenizer.convert_tokens_to_ids(final_tokens), classifications

def one_hot_encode(labels):
    out = np.zeros((labels.shape[0], 4))
    out[np.arange(labels.shape[0]), labels] = 1
    return out

def learn_bow(reports, min_df=1, ngram_range=(1, 3), max_features=5000):
    stopwords = ['mm', 'dd', '2017', '2016', '2015', '2014', '2013', '2012', 'date', 'md']
    countVec = CountVectorizer(min_df = min_df, \
                               ngram_range = ngram_range, \
                               max_features = max_features, \
                               stop_words = stopwords)
    countVec.fit(reports)
    return countVec.transform(reports)

def prepare_y(data_y):
    label_enc = LabelEncoder()
    label_enc_y = label_enc.fit(data_y)

    return label_enc_y.transform(data_y)

def createTextFeatures(reports, max_base_feats, max_prog_feats):
    baseline_text, progress_text, _, __ = reports

    baseline_bow = np.array(learn_bow(baseline_text['clean_report_text'].tolist(), max_features=max_base_feats).todense())
    progress_bow = np.array(learn_bow(progress_text['clean_report_text'].tolist(), max_features=max_prog_feats).todense())
    print(baseline_bow.shape)
    print(progress_bow.shape)
    overallTextFeatures = np.hstack([baseline_bow, progress_bow])
    return overallTextFeatures

def make_id(patient_id):
    if patient_id < 10:
        return "MSK_00" + str(patient_id)
    elif patient_id < 100:
        return "MSK_0" + str(patient_id)
    else:
        return "MSK_" + str(patient_id)

def pad_vectors(feats, max_len, feat_lens):
    for val in (True, False):
        for i in range(len(feats[val])):
            for j in range(max_len):
                if j >= len(feats[val][i]):
                    feats[val][i].append(np.zeros(feat_lens))
    return feats

def setupFeatureVectors(df, desired_features, max_before, max_after):
    FEAT_LENS = len(desired_features) + max_before + max_after

    patients = df.groupby("Patient ID")
    max_len = 0

    train_feats = {True: [], False: []}

    train_labels = []
    id_list = set()

    count = -1

    before_text = np.array(learn_bow(df["before_text"], max_features = max_before).todense())
    after_text = np.array(learn_bow(df["after_text"], max_features = max_after).todense())

    train_features = {True: [], False: []}

    for patient_id in sorted([int(key[-3:]) for key in patients.groups.keys()]):
        count += 1
        patient = make_id(patient_id)
        context = {True: [], False: []}

        checker = {True: False, False: False}
        len_counter = {True: 0, False: 0}

        count2 = -1
        for i in patients.groups[patient]:
            count2 += 1

            checker[df["is_baseline"][i]] = True
            len_counter[df["is_baseline"][i]] += 1

            context[df["is_baseline"][i]].append(np.concatenate((np.array([df[desired_feat][i] for desired_feat in desired_features]), \
                                                            before_text[i], after_text[i])))

        if not(checker[True] or checker[False]):
            continue
        elif not checker[True]:
            context[True].append(np.zeros(FEAT_LENS))
        elif not checker[False]:
            context[False].append(np.zeros(FEAT_LENS))

        max_len = max(max_len, len_counter[True], len_counter[False])

        id_list.add(patient)
        for val in (True, False):
            train_features[val].append(context[val])

        train_labels.append(df["labels"][i])

    train_features  = pad_vectors(train_features, max_len, FEAT_LENS)

    return np.array(train_features[False]), np.array(train_features[True]),  one_hot_encode(prepare_y(train_labels)), id_list

def create_data(max_base, max_prog, max_before, max_after, desired_features):
    df = preprocess.load_reports()
    df_extraction = preprocess.extractFeatures(df)
    baseX, progX, labs, id_list = setupFeatureVectors(df_extraction, desired_features, max_before, max_after)
    reports = preprocess.extractText(df, id_list)
    df_text = createTextFeatures(reports, max_base, max_prog)
    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
    id_vals = torch.tensor(list(map(lambda x: tokenize_input(x[0], x[1], split=0.4, tokenizer=tokenizer), zip(reports[2]['bert_text'], \
                                                                                reports[3]['bert_text']))))

    return torch.from_numpy(baseX), torch.from_numpy(progX), torch.from_numpy(df_text), torch.from_numpy(labs), id_vals[:,0,:], id_vals[:, 1,:]


# encoding: utf-8
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models

SETTINGS_RENAMES = (
    ("SHOP_SSL_ENABLED", "SSL_ENABLED"),
    ("SHOP_SSL_FORCE_HOST", "SSL_FORCE_HOST"),
)

class Migration(DataMigration):

    def forwards(self, orm):
        "Write your forwards methods here."
        for name_from, name_to in SETTINGS_RENAMES:
            try:
                setting = orm.Setting.objects.get(name=name_from)
            except orm.Setting.DoesNotExist:
                pass
            else:
                setting.name = name_to
                setting.save()

    def backwards(self, orm):
        "Write your backwards methods here."
        for name_to, name_from in SETTINGS_RENAMES:
            try:
                setting = orm.Setting.objects.get(name=name_from)
            except orm.Setting.DoesNotExist:
                pass
            else:
                setting.name = name_to
                setting.save()

    models = {
        'conf.setting': {
            'Meta': {'object_name': 'Setting'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'site': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['sites.Site']"}),
            'value': ('django.db.models.fields.CharField', [], {'max_length': '2000'})
        },
        'sites.site': {
            'Meta': {'ordering': "('domain',)", 'object_name': 'Site', 'db_table': "'django_site'"},
            'domain': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        }
    }

    complete_apps = ['conf']


import csv
import json
import sys
import traceback
from bs4 import BeautifulSoup
from time import sleep

import os

from get_restaurants_list import check_element_by_xpath, wait_to_display, load_restaurants, save_restaurants
from utils import VERSION, log, launch_driver, save_file

# 主方法
def get_restaurant_info(r):
    driver = launch_driver()

    try:
        # 请求链接
        url = r.get('url')
        log(url)
        driver.get(url)

        # 重定向判定
        if driver.current_url == url:
            pass
        else:
            log('该网页进行了重定向,不再获取.')
            return 0

        # 首先分析参观信息,更新到new_info, 如果评论数量为0,直接跳过,不再点击评论
        # 从script中解析出新信息
        sleep(2)
        new_info = parse_new_info(driver)
        if int(new_info.get('count_rating')) == 0:
            log('餐馆评论为0, 可以直接跳过')
            return 0

        # 开始点击加载评论
        show_rate = check_element_by_xpath(driver, '//*[@id="showRating"]/a')
        show_rate.click()

        # 增加判断,如果js加载失败或点击加载失败,没有active表示没有加载出评论,直接跳过
        parent_tag = check_element_by_xpath(show_rate, '..')
        if parent_tag.get_attribute('class') != 'active':
            log('评论加载失败,跳过')
            return -1

        # 拉取评论
        log('拉取评论中...')
        while True:
            driver.execute_script("$('html,body').animate({'scrollTop':'700000000'},1000)")
            ver_mais = check_element_by_xpath(driver, '//*[@id="ratingContent"]/div[3]/div[2]')
            # 没有ver_mais, 就结束. 实际正常应该是ver_mais不可见了
            if ver_mais == -1:
                break
            else:
                # log('没找到评论加载完毕,说明等待加载')
                wait_to_display(ver_mais)
                if ver_mais.is_displayed() is False:
                    break
                ver_mais.click()
                # log('加载更多评论。。。')
                # 等待三秒页面加载
                sleep(2)

        # 加载结束, 解析源代码
        source_page = BeautifulSoup(driver.page_source, 'html.parser')

        # 解析出评论
        comments = parse_comments(source_page, new_info)

        # 更新餐馆信息，保存评论信息
        # 获得的新的信息，不再更新到源文件，直接输出到csv文件吧

        # r.update(new_info)
        new_info['state'] = r.get('state')
        new_info['id'] = r.get('id')
        r_to_csv(new_info)
        save_comments(comments, r)
        return 0
    except Exception as e:
        log(traceback.format_exc())
        return -1
    finally:
        driver.close()

def parse_new_info(driver):
    try:
        source_page = BeautifulSoup(driver.page_source, 'html.parser')
        geo_content = source_page.find('script', type='application/ld+json')

        # 异常修复5次，还不行就跳过
        r = geo_content.text.strip()
        loop_mark = 5
        r_info = None
        while loop_mark > 0:
            try:
                r_info = json.loads(r)
                break
            except Exception as e1:
                log('修复json数据')
                code = int(str(e1).split(' ')[-1].replace(')', ''))
                error_c_index = r.rfind(',', 0, code)
                r = r[:error_c_index] + r[error_c_index + 1:]
                loop_mark -= 1
                ...

        if r_info is None:
            new_info = None
            pass
        else:
            new_info = {
                'url': r_info.get('url'),
                'name': r_info.get('name'),
                'cuisine': r_info.get('servesCuisine'),
                'rating_summary': r_info.get('aggregateRating').get('ratingValue'),
                'price': r_info.get('priceRange'),
                'latitude': r_info.get('geo').get('latitude'),
                'longitude': r_info.get('geo').get('longitude'),
                'address_locality': r_info.get('address').get('addressLocality'),
                'address_region': r_info.get('address').get('addressRegion'),
                'address_street': r_info.get('address').get('streetAddress'),
                'postalcode': r_info.get('address').get('postalCode'),
                'count_rating': r_info.get('aggregateRating').get('reviewCount'),
                'max_rating_date': None,
                'min_rating_date': None,
            }
            # 增加log,和下面统计的评论进行比对
        return new_info
    except Exception as e:
        log('获取餐馆坐标出错')
        log(traceback.format_exc())
        raise e

def parse_comments(source_page, new_info):  # 11
    log('解析评论中。。。')
    # eval_boxes = web_driver.find_elements_by_class_name('evaluation')
    # eval_boxes = source_page.find_all('.evaluation')
    eval_boxes = source_page.findAll('article', {'class': 'evaluation'})
    # 不能这样判断,因为expect_count只是没有内容的评级的数量
    log('全部评论{}条'.format(len(eval_boxes)))
    # log(type(expect_count))
    # if int(expect_count) != len(eval_boxes):
    #     log('评论加载没有结束,需要跳过重新加载')
    #     return None.get('len')
    comments = {}
    count = 0
    for box in eval_boxes:
        count += 1
        # log(count)
        # print(box.find('div', {'class':'evaluation-header'}))
        if box.find('div', {'class': 'evaluation-header'}):
            source = 'C'
        else:
            source = 'R'
        try:
            if source == 'C':
                comment_name = box.find('h1', {'class': 'user-name'}).text
                user_score_str = box.find('div', {'class': 'user-score'}).text
                comment_rating = int(user_score_str.replace(',', '')) / 10.0
                comment_date = box.find('div', {'class': 'comment-date'}).text
                comment_desc = box.find('div', {'class': 'user-comment'}).text
                try:
                    # 店家没有评论，返回None
                    comment_reply = box.find('div', {'class': 'restaurant-comment'}).text
                except:
                    comment_reply = None
                tmp_r = {
                    'source': source,
                    'comment_name': comment_name,
                    'comment_date': comment_date,
                    'comment_rating': comment_rating,
                    'comment_desc': comment_desc,
                    'comment_reply': comment_reply,
                }
                comments[count] = tmp_r
            elif source == 'R':
                rating_name = box.find('h1', {'class': 'user-name'}).text
                user_score_str = box.find('div', {'class': 'user-score'}).text
                rating_value = int(user_score_str.replace(',', '')) / 10.0
                rating_date = box.find('div', {'class': 'comment-date'}).text
                if new_info.get('max_rating_date') is None:
                    new_info['max_rating_date'] = rating_date
                new_info['min_rating_date'] = rating_date
                tmp_r = {
                    'source': source,
                    'rating_name': rating_name,
                    'rating_value': rating_value,
                    'rating_date': rating_date,
                }
                comments[count] = tmp_r
            # log(tmp_r)
        except:
            log(traceback.format_exc())
            log('获取评论细节出错')
    log('解析评论结束')
    return comments

def save_comments(comments, r):
    state = r.get('state')
    name = r.get('id')
    file_path = '../data/{}/comments/{}/{}.csv'.format(VERSION, state, name)
    header = [
        'source',
        'comment_name',
        'comment_date',
        'comment_rating',
        'comment_desc',
        'comment_reply',
        'rating_name',
        'rating_value',
        'rating_date'
    ]
    file_dir = os.path.split(file_path)[0]
    if not os.path.exists(file_dir):
        os.makedirs(file_dir)
    with open(file_path, 'w', newline='', encoding='utf-8') as f:
        csv_f = csv.writer(f)
        csv_f.writerow(header)
        for comment in comments.values():
            tmp_row = []
            for h in header:
                tmp_content = comment.get(h)
                if tmp_content is None:
                    pass
                elif type(tmp_content) == str:
                    tmp_content = tmp_content.encode("gbk", 'ignore').decode("gbk", "ignore")
                tmp_row.append(tmp_content)
            csv_f.writerow(tmp_row)

def r_to_csv(r):
    # rs_info_path = '../data/{}/restaurants/{}.txt'.format(VERSION, state)
    header = [
        'url',
        'name',
        'cuisine',
        'rating_summary',
        'price',
        'latitude',
        'longitude',
        'address_locality',
        'address_region',
        'address_street',
        'postalcode',
        'max_rating_date',
        'min_rating_date',
        'count_rating',
        'id'
    ]
    state = r.get('state')
    csv_path = '../data/{}/restaurants/{}.csv'.format(VERSION, state)
    csv_dir = os.path.split(csv_path)[0]
    if not os.path.exists(csv_dir):
        os.makedirs(csv_dir)
    if not os.path.exists(csv_path):
        with open(csv_path, 'w', newline='', encoding='utf-8') as f:
            csv_f = csv.writer(f)
            csv_f.writerow(header)
    with open(csv_path, 'a+', newline='', encoding='utf-8') as f:
        csv_f = csv.writer(f)
        tmp_raw = []
        for h in header:
            try:
                tmp_content = r.get(h)
                if tmp_content is not None:
                    tmp_content = tmp_content.encode("gbk", 'ignore').decode("gbk", "ignore")
                # print(r.get(h))
                tmp_raw.append(tmp_content)
            # tmp_raw.append(r.get(h))
            except:
                print(r.get(h))
                pass
        csv_f.writerow(tmp_raw)

def get_restaurants_info_by_state(state, start_num=1, end_num=0):

    rs = load_restaurants(state)

    # 开始获取数据, 允许程序意外中断10次
    count = 10
    while count > 0:
        # 确定运行数据范围
        total = len(list(rs.keys()))

        if start_num == 0:
            start_num = 1
        if end_num == 0 or end_num > total:
            end_num = total + 1
        log('脚本将要获取州:{}, id为{}至{}的餐馆的数据'.format(state, start_num, end_num - 1))

        # 统计目前进度
        has_get_count = 0
        for i in range(start_num, end_num):
            r = rs.get(str(i))
            if r.get('has_get') is True:
                has_get_count += 1
        log('脚本重新启动, 目前进展：{}/{}'.format(has_get_count, end_num - start_num))

        try:
            for r_id in range(start_num, end_num):
                r = rs.get(str(r_id))
                if r.get('has_get') is True:
                    # log(r_name, "已经获取了数据")
                    continue
                log('开始进行', r_id)
                return_mark = get_restaurant_info(r)
                if return_mark == 0:
                    r['has_get'] = True
                    save_restaurants(rs, state)
                    has_get_count += 1
                    # 输出下当前进展
                    log('目前进展：{}/{}'.format(has_get_count, end_num - start_num))
                else:
                    log(r_id, '的地址出现错误，要查看异常原因')
                sleep(5)

            # check
            rs = load_restaurants(state)
            for i_id in range(start_num, end_num):
                r = rs.get(str(i_id))
                if r.get('has_get') is None:
                    count = -1
                    break
                count = 0
        except Exception as e:
            log('异常退出一次, 原因：', e)
            count -= 1
    return 0

# 分段统计数据, 后两个参数是起始和末尾id
# 包含start, 不包含end
if __name__ == '__main__':

    states = ['SP', 'PR', 'RS', 'RJ', 'MG', 'DF']

    start = 1
    end = 0
    # 获取输入
    if len(sys.argv) == 2:
        index = int(sys.argv[1])
    elif len(sys.argv) == 4:
        index = int(sys.argv[1])
        start = int(sys.argv[2])
        end = int(sys.argv[3]) + 1
    else:
        index = 0

    state = states[index]
    get_restaurants_info_by_state(state, start, end)


import json

from lemon.app import Lemon
from lemon.const import HTTP_METHODS
from lemon.request import HttpHeaders

class ASGIResponse:
    def __init__(self, status, headers: HttpHeaders, content):
        self.status = status
        self.headers = headers
        self.content = content

    @property
    def status_code(self):
        return self.status

    @property
    def text(self):
        return self.content.decode('utf-8')

    def json(self):
        return json.loads(self.content)

class ASGIHttpTestCase:
    app = None
    scope = {}
    address = '127.0.0.1'
    port = 9999

    def setup_method(self, method):
        self.app = Lemon(
            config={
                'LEMON_CORS_ENABLE': True,
            },
            debug=True
        )
        self.scope = {
            'type': 'http.request',
            'server': (self.address, self.port),
            'client': ('127.0.0.1', 55555),
            'scheme': 'http',
            'http_version': '1.1',
            'headers': [
                [b'user-agent', b'lemon-pytest'],
                [b'accept', b'*/*'],
                [b'host', '{0}:{1}'.format(self.address, self.port).encode()],
                [b'connection', b'keep-alive'],
            ],
        }

    def set_scope(self, method, path, query_string='', headers=None):
        self.scope['method'] = method
        self.scope['path'] = path
        self.scope['query_string'] = query_string.encode('utf-8')

        if headers:
            self.scope['headers'].extend(headers)

    async def asgi_request(
            self, app, method, path, query_string='',
            data=None, body=None, headers=None,
    ):
        raw_body = body or b''
        is_start = False
        resp = {}

        if data:
            raw_body = json.dumps(data).encode()

        _headers = headers or []
        _headers.append(
            [b'content-length', f'{len(raw_body)}'.encode()],
        )
        self.set_scope(method, path, query_string, _headers)

        async def receive():
            return {
                'body': raw_body,
            }

        async def send(msg):
            global is_start
            if msg['type'] == 'http.response.start':
                resp['status'] = msg['status']
                resp['headers'] = msg['headers']
                is_start = True
            if msg['type'] == 'http.response.body':
                assert is_start is True
                resp['body'] = msg['body']

        await app.application(self.scope)(receive, send)
        return ASGIResponse(resp['status'], HttpHeaders(raw_headers=resp['headers']), resp['body'])

    async def get(self, path, params=None):
        query_string = ''
        if params:
            params_pairs = []
            for p in params:
                params_pairs.append('{0}={1}'.format(p, params[p]))
            query_string = '&'.join(params_pairs)

        return await self.asgi_request(
            app=self.app,
            method=HTTP_METHODS.GET,
            path=path,
            query_string=query_string,
        )

    async def post(self, path, data=None):
        return await self.asgi_request(
            app=self.app,
            method=HTTP_METHODS.POST,
            path=path,
            data=data,
        )

    async def put(self, path, data=None):
        return await self.asgi_request(
            app=self.app,
            method=HTTP_METHODS.PUT,
            path=path,
            data=data,
        )

    async def delete(self, path, data=None):
        return await self.asgi_request(
            app=self.app,
            method=HTTP_METHODS.DELETE,
            path=path,
            data=data,
        )

    async def options(self, path, data=None):
        return await self.asgi_request(
            app=self.app,
            method=HTTP_METHODS.OPTIONS,
            path=path,
            data=data,
        )


"""citeseer_rnm.py
The domain knowledge here is represented in paper connections p1 and p2 which tend to be about
the same topic. For example:
∀p1 ∀p2 AG(p1 ) ∧ Cite(p1 , p2 ) → AG(p2 )

Where Cite is an evidence predicate (value over the groundings is known a priori), determining whether
a pattern cites another one.
"""
import mme
import tensorflow as tf
import datasets
import numpy as np
import os
from itertools import product
import argparse

"""Cuda visible devices"""
os.environ["CUDA_VISIBLE_DEVICES"] = "1"

"""Get logger"""
tf.get_logger().setLevel("ERROR")

"""Savings"""
base_savings = os.path.join("savings", "citeseer")
pretrain_path = os.path.join(base_savings, "pretrain")
posttrain_path = os.path.join(base_savings, "posttrain")

def main(
    lr,
    seed,
    lambda_0,
    l2w,
    test_size,
    valid_size,
    run_on_test=False,
    map_steps=20,
    em_cycles=4,
):
    """Main function of the citeseer with RNM

    Args:
        lr: learning rate
        seed: seed
        lambda_0: lambda_0
        l2w: weight of the l2 regularizer
        valid_size: validation size
        run_on_test: run on test
        map_steps: maximum number of steps
        em_cycles: em cycles

    Returns:
        acc_nn: accuracy of the neural network
        acc_map: accuracy of the maximum a posteriori
    """
    # get data from citeseer
    (
        (x_train, hb_train),
        (x_valid, hb_valid),
        (x_test, hb_test),
        (x_all, hb_all),
        labels,
        mask_train_labels,
        trid,
        vaid,
        teid,
    ) = datasets.citeseer_em(test_size, valid_size, seed)
    num_examples = len(x_all)
    num_classes = 6

    # set the seeds
    np.random.seed(seed)
    tf.random.set_seed(seed)

    indices = np.reshape(
        np.arange(num_classes * len(x_all)), [num_classes, len(x_all)]
    ).T  # T because we made classes as unary potentials

    indices_train = indices[trid]
    if run_on_test:
        x_to_test = x_test
        hb_to_test = hb_test
        num_examples_to_test = len(x_test)
        indices_to_test = indices[teid]
    else:
        x_to_test = x_valid
        hb_to_test = hb_valid
        num_examples_to_test = len(x_valid)
        indices_valid = np.reshape(
            np.arange(num_classes * len(x_valid)), [num_classes, len(x_valid)]
        ).T  # T because we made classes as unary potentials
        indices_to_test = indices[vaid]

    y_to_test = tf.gather(hb_all[0], indices_to_test)

    """Logic Program Definition"""
    o = mme.Ontology()

    # Domains
    docs = mme.Domain("Documents", data=x_all)
    o.add_domain([docs])

    # Predicates

    preds = ["ag", "ai", "db", "ir", "ml", "hci"]
    for name in preds:
        p = mme.Predicate(name, domains=[docs])
        o.add_predicate(p)

    cite = mme.Predicate("cite", domains=[docs, docs], given=True)
    o.add_predicate(cite)

    """MME definition"""
    potentials = []
    # Supervision
    indices = np.reshape(
        np.arange(num_classes * docs.num_constants), [num_classes, docs.num_constants]
    ).T  # T because we made classes as unary potentials

    """Network defintion"""
    nn = tf.keras.Sequential()
    nn.add(tf.keras.layers.Input(shape=(x_train.shape[1],)))
    nn.add(
        tf.keras.layers.Dense(
            50, activation=tf.nn.relu, kernel_regularizer=tf.keras.regularizers.l2(l2w)
        )
    )  # up to the last hidden layer
    nn.add(
        tf.keras.layers.Dense(
            50, activation=tf.nn.relu, kernel_regularizer=tf.keras.regularizers.l2(l2w)
        )
    )  # up to the last hidden layer
    nn.add(
        tf.keras.layers.Dense(
            50, activation=tf.nn.relu, kernel_regularizer=tf.keras.regularizers.l2(l2w)
        )
    )  # up to the last hidden layer
    nn.add(tf.keras.layers.Dense(num_classes, use_bias=False))
    tf.keras.utils.plot_model(nn)

    # logical with supervision given the indices and the nn
    p1 = mme.potentials.SupervisionLogicalPotential(nn, indices)
    potentials.append(p1)

    # Mutual Exclusivity (needed for inference , since SupervisionLogicalPotential already subsumes it during training)
    # p2 = mme.potentials.MutualExclusivityPotential(indices=indices)
    # potentials.append(p2)

    # Logical
    np.ones_like(hb_all)
    evidence_mask = np.zeros_like(hb_all)
    evidence_mask[:, num_examples * num_classes :] = 1
    for name in preds:
        # for name in prediction
        # formmula
        c = mme.Formula(
            definition="%s(x) and cite(x,y) -> %s(y)" % (name, name), ontology=o
        )
        # get the evidence logic potential
        p3 = mme.potentials.EvidenceLogicPotential(
            formula=c,
            logic=mme.logic.BooleanLogic,
            evidence=hb_all,
            evidence_mask=evidence_mask,
        )
        potentials.append(p3)

    # get the Global Potential from the potentials
    P = mme.potentials.GlobalPotential(potentials)

    def pretrain_step():
        """pretrain rete"""
        y_train = tf.gather(hb_all[0], indices_train)

        adam = tf.keras.optimizers.Adam(lr=0.001)

        def training_step():
            with tf.GradientTape() as tape:
                neural_logits = nn(x_train)

                total_loss = tf.reduce_mean(
                    tf.nn.softmax_cross_entropy_with_logits(
                        labels=y_train, logits=neural_logits
                    )
                ) + tf.reduce_sum(nn.losses)

            grads = tape.gradient(target=total_loss, sources=nn.variables)
            grad_vars = zip(grads, nn.variables)
            adam.apply_gradients(grad_vars)

        epochs_pretrain = 200
        for e in range(epochs_pretrain):
            training_step()
            y_nn = nn(x_to_test)
            acc_nn = tf.reduce_mean(
                tf.cast(
                    tf.equal(tf.argmax(y_to_test, axis=1), tf.argmax(y_nn, axis=1)),
                    tf.float32,
                )
            )
            #  print(acc_nn)

        y_new = tf.gather(tf.eye(num_classes), tf.argmax(nn(x_all), axis=1), axis=0)

        new_labels = tf.where(mask_train_labels > 0, labels, y_new)
        new_hb = tf.concat(
            (
                tf.reshape(tf.transpose(new_labels, [1, 0]), [1, -1]),
                hb_all[:, num_examples * num_classes :],
            ),
            axis=1,
        )

        return new_hb

    def em_step(new_hb):

        pwt = mme.PieceWiseTraining(global_potential=P)
        hb = new_hb

        """BETA TRAINING"""
        pwt.compute_beta_logical_potentials(y=hb)
        #  for p in potentials:
        #      print(p, p.beta)

        """NN TRAINING"""
        epochs = 50

        for _ in range(epochs):
            pwt.maximize_likelihood_step(new_hb, x=x_all)
            y_nn = nn(x_to_test)
            acc_nn = tf.reduce_mean(
                tf.cast(
                    tf.equal(tf.argmax(y_to_test, axis=1), tf.argmax(y_nn, axis=1)),
                    tf.float32,
                )
            )
            #  print("TRAINING", acc_nn.numpy())

        """Fix the training hb for inference"""
        new_labels = tf.where(mask_train_labels > 0, labels, 0.5 * tf.ones_like(labels))
        evidence = tf.concat(
            (
                tf.reshape(tf.transpose(new_labels, [1, 0]), [1, -1]),
                hb_all[:, num_examples * num_classes :],
            ),
            axis=1,
        )
        evidence = tf.cast(evidence, tf.float32)
        evidence_mask = (
            tf.concat(
                (
                    tf.reshape(
                        tf.transpose(mask_train_labels.astype(np.float32), [1, 0]),
                        [1, -1],
                    ),
                    tf.ones_like(hb_all[:, num_examples * num_classes :]),
                ),
                axis=1,
            )
            > 0
        )

        """MAP Inference"""
        dict = {
            "var": tf.Variable(initial_value=nn(x_all)),
            "labels": labels,
            "mask_train_labels": mask_train_labels,
            "hb_all": hb_all,
            "num_examples": num_examples,
            "num_classes": num_classes,
        }

        steps_map = map_steps
        map_inference = mme.inference.FuzzyMAPInference(
            y_shape=hb.shape,
            potential=P,
            logic=mme.logic.LukasiewiczLogic,
            evidence=evidence,
            evidence_mask=evidence_mask,
            learning_rate=lr,
            external_map=dict,
        )

        P.potentials[0].beta = lambda_0
        y_map = tf.gather(map_inference.map()[0], indices_to_test)
        acc_map = tf.reduce_mean(
            tf.cast(
                tf.equal(tf.argmax(y_to_test, axis=1), tf.argmax(y_map, axis=1)),
                tf.float32,
            )
        )
        print("MAP", acc_map.numpy())
        for i in range(steps_map):
            map_inference.infer_step(x_all)
            y_map = tf.gather(map_inference.map()[0], indices_to_test)
            acc_map = tf.reduce_mean(
                tf.cast(
                    tf.equal(tf.argmax(y_to_test, axis=1), tf.argmax(y_map, axis=1)),
                    tf.float32,
                )
            )
            print("MAP", i, acc_map.numpy())
            if mme.utils.heardEnter():
                break

        new_hb = tf.cast(map_inference.map() > 0.5, tf.float32)

        y_new_fuzzy = tf.gather(map_inference.map()[0], indices)
        new_labels = tf.where(mask_train_labels > 0, labels, y_new_fuzzy)
        new_hb_fuzzy = tf.concat(
            (
                tf.reshape(tf.transpose(new_labels, [1, 0]), [1, -1]),
                hb_all[:, num_examples * num_classes :],
            ),
            axis=1,
        )

        return new_hb, new_hb_fuzzy

    em_cycles = em_cycles
    for i in range(em_cycles):
        if i == 0:
            new_hb = hb_pretrain = pretrain_step()
        else:
            old_hb = new_hb
            new_hb, new_hb_fuzzy = em_step(old_hb)

    y_map = tf.gather(new_hb[0], indices_to_test)
    y_pretrain = tf.gather(hb_pretrain[0], indices_to_test)
    acc_pretrain = tf.reduce_mean(
        tf.cast(
            tf.equal(tf.argmax(y_to_test, axis=1), tf.argmax(y_pretrain, axis=1)),
            tf.float32,
        )
    )
    acc_map = tf.reduce_mean(
        tf.cast(
            tf.equal(tf.argmax(y_to_test, axis=1), tf.argmax(y_map, axis=1)), tf.float32
        )
    )
    return acc_pretrain, acc_map

if __name__ == "__main__":
    """Main function"""
    seed = 0  # set the seed
    """Write the file header"""
    with open(f"res_dlm_10_splits_{seed}", "w") as file:
        file.write("seed, test, acc_map, acc_nn\n")

    res = []

    """Set the GPU to copute the data"""
    os.environ["CUDA_VISIBLE_DEVICES"] = "0"
    np.random.seed(0)

    print(tf.config.list_physical_devices("GPU"))
    # [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
    print(tf.test.is_built_with_cuda())
    # <function is_built_with_cuda at 0x7f4f5730fbf8>
    print(tf.test.gpu_device_name())
    # /device:GPU:0
    print(tf.config.get_visible_devices())

    seeds = np.random.choice(np.arange(1000), [10], replace=False)
    # seeds=[0]
    for a in product(
        seeds,
        [
            (0.1, 0.0, 4, 30),
            (0.25, 0.001, 4, 30),
            (0.5, 0.001, 4, 30),
            (0.75, 0.001, 4, 30),
            (0.9, 0.001, 4, 30),
        ],
    ):
        """Once again looping over all the hyperparameters"""
        # get seed, test-size, lambda_0, em_cycles, steps
        seed, (test_size, lambda_0, em_cycles, steps) = a
        # get the accuracy of the map and the accuracy of the nn
        acc_map, acc_nn = main(
            lr=1.0,
            seed=seed,
            lambda_0=lambda_0,
            l2w=0.006,
            test_size=test_size,
            valid_size=0.0,
            run_on_test=True,
            map_steps=steps,
            em_cycles=em_cycles,
        )
        # to numpy
        acc_map, acc_nn = acc_map.numpy(), acc_nn.numpy()
        # format
        res.append(
            "\t".join([str(a) for a in [seed, test_size, acc_map, str(acc_nn) + "\n"]])
        )
        # print
        for i in res:
            print(i)
    # append to the csv file
    with open(f"res_rnm_10_splits", "a") as file:
        file.write(res[-1])


"""Training module for nondefaced-detector."""

import os
import math

import tensorflow as tf
import pandas as pd
import numpy as np

from sklearn.utils import class_weight
from tensorflow.keras import backend as K
from tensorflow.keras import metrics
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping

from nondefaced_detector.models import model as _model
from nondefaced_detector.dataloaders.dataset import get_dataset

def scheduler(epoch):
    if epoch < 3:
        return 0.001
    else:
        return 0.001 * tf.math.exp(0.1 * (10 - epoch))

def train(
    csv_path,
    model_save_path,
    tfrecords_path,
    volume_shape=(128, 128, 128),
    image_size=(128, 128),
    dropout=0.2,
    batch_size=16,
    n_classes=2,
    n_epochs=15,
    mode="CV",
):
    """Train a model.

    Parameters
    ----------
    csv_path: str - Path
        Path to the csv file containing training volume paths, labels (X, Y).
    model_save_path: str - Path
        Path to where the save model and model weights.
    tfrecords_path: str - Path
        Path to preprocessed training tfrecords.
    volume_shape: tuple of size 3, optional, default=(128, 128, 128)
        The shape of the preprocessed volumes.
    image_size: tuple of size 2, optional, default=(128, 128)
        The shape of a 2D slice along each volume axis.
    dropout: float, optional, default=0.4
         Float between 0 and 1. Fraction of the input units to drop.
    batch_size: int, optional, default=16
        No. of training examples utilized in each iteration.
    n_classes: int, optional, default=2
        No. of unique classes to train the model on. Default assumption is a
        binary classifier.
    n_epochs: int, optional, default=15
        No. of complete passes through the training dataset.
    mode: str, optional, default=15
        One of "CV" or "full". Indicates the type of training to perform.

    Returns
    -------
    `tf.keras.callbacks.History`
        A History object that records several metrics such as training/validation loss/metrics
        at successive epochs.
    """

    train_csv_path = os.path.join(csv_path, "training.csv")
    train_paths = pd.read_csv(train_csv_path)["X"].values
    train_labels = pd.read_csv(train_csv_path)["Y"].values

    if mode == "CV":
        valid_csv_path = os.path.join(csv_path, "validation.csv")
        valid_paths = pd.read_csv(valid_csv_path)["X"].values
        # valid_labels = pd.read_csv(valid_csv_path)["Y"].values

    weights = class_weight.compute_class_weight(
        "balanced", np.unique(train_labels), train_labels
    )
    weights = dict(enumerate(weights))

    planes = ["axial", "coronal", "sagittal", "combined"]

    global_batch_size = batch_size

    os.makedirs(model_save_path, exist_ok=True)
    cp_save_path = os.path.join(model_save_path, "weights")
    logdir_path = os.path.join(model_save_path, "tb_logs")
    metrics_path = os.path.join(model_save_path, "metrics")

    os.makedirs(metrics_path, exist_ok=True)

    for plane in planes:

        logdir = os.path.join(logdir_path, plane)
        os.makedirs(logdir, exist_ok=True)

        tbCallback = TensorBoard(log_dir=logdir)

        os.makedirs(os.path.join(cp_save_path, plane), exist_ok=True)

        model_checkpoint = ModelCheckpoint(
            os.path.join(cp_save_path, plane, "best-wts.h5"),
            monitor="val_loss",
            save_weights_only=True,
            mode="min",
        )

        if not plane == "combined":
            lr = 1e-3
            model = _model.Submodel(
                input_shape=image_size,
                dropout=dropout,
                name=plane,
                include_top=True,
                weights=None,
            )
        else:
            lr = 5e-4
            model = _model.CombinedClassifier(
                input_shape=image_size,
                dropout=dropout,
                trainable=True,
                wts_root=cp_save_path,
            )

        print("Submodel: ", plane)

        METRICS = [
            metrics.TruePositives(name="tp"),
            metrics.FalsePositives(name="fp"),
            metrics.TrueNegatives(name="tn"),
            metrics.FalseNegatives(name="fn"),
            metrics.BinaryAccuracy(name="accuracy"),
            metrics.Precision(name="precision"),
            metrics.Recall(name="recall"),
            metrics.AUC(name="auc"),
        ]

        model.compile(
            loss=tf.keras.losses.binary_crossentropy,
            optimizer=Adam(learning_rate=lr),
            metrics=METRICS,
        )

        dataset_train = get_dataset(
            file_pattern=os.path.join(tfrecords_path, "data-train_*"),
            n_classes=n_classes,
            batch_size=global_batch_size,
            volume_shape=volume_shape,
            plane=plane,
            shuffle_buffer_size=global_batch_size,
        )

        steps_per_epoch = math.ceil(len(train_paths) / batch_size)

        if mode == "CV":
            earlystopping = EarlyStopping(monitor="val_loss", patience=3)

            dataset_valid = get_dataset(
                file_pattern=os.path.join(tfrecords_path, "data-valid_*"),
                n_classes=n_classes,
                batch_size=global_batch_size,
                volume_shape=volume_shape,
                plane=plane,
                shuffle_buffer_size=global_batch_size,
            )

            validation_steps = math.ceil(len(valid_paths) / batch_size)

            history = model.fit(
                dataset_train,
                epochs=n_epochs,
                steps_per_epoch=steps_per_epoch,
                validation_data=dataset_valid,
                validation_steps=validation_steps,
                callbacks=[tbCallback, model_checkpoint, earlystopping],
                class_weight=weights,
            )

            hist_df = pd.DataFrame(history.history)

        else:
            earlystopping = EarlyStopping(monitor="loss", patience=3)
            print(model.summary())
            print("Steps/Epoch: ", steps_per_epoch)
            history = model.fit(
                dataset_train,
                epochs=n_epochs,
                steps_per_epoch=steps_per_epoch,
                callbacks=[tbCallback, model_checkpoint, earlystopping],
                class_weight=weights,
            )

        hist_df = pd.DataFrame(history.history)
        jsonfile = os.path.join(metrics_path, plane + ".json")

        with open(jsonfile, mode="w") as f:
            hist_df.to_json(f)

    return history


from aiohttp import web

from chessviz import main

async def handle(request):
    gid = request.match_info.get("gid")
    ans = main(gid)
    return web.Response(body=ans, content_type="image/svg+xml")

async def make_app():
    app = web.Application(client_max_size=10 * 1024**2)
    app.add_routes([web.get("/{gid}", handle)])
    return app

if __name__ == "__main__":
    app = make_app()
    web.run_app(app)


from selenium import webdriver
from time import sleep
import os
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import urllib.request
import random
import pandas as pd

browser  = webdriver.Chrome(executable_path="./chromedriver.exe")
browser.maximize_window()
sleep(5)
browser.get("https://images.google.com/")
sleep(10)

txtKeys = browser.find_element(By.ID, "APjFqb")
txtKeys.send_keys("Cat")
txtKeys.send_keys(Keys.ENTER)

sleep(15)

def scrollPage(browser):
    # scroll page
    scroll_pause_time = random.randint(3, 16)
    i = 0
    while i < 2:
        if (i > 0):
            browser.execute_script("window.scrollTo(0, 0);")
            sleep(random.randint(1, 3))
            browser.execute_script(
                "window.scrollTo(0, document.body.scrollHeight);")
            # Wait to load page
            sleep(scroll_pause_time)
        else:
            browser.execute_script(
                "window.scrollTo(0, document.body.scrollHeight);")
            sleep(scroll_pause_time)
        i += 1
        #print(i)

scrollPage(browser)
sleep(5)
soup = BeautifulSoup(browser.page_source, 'html.parser')
images = soup.findAll('div', {'class': "bRMDJf islir"})
print(len(images))

'''for image in images:
    img = image.find('img')
    if img.has_attr('src'):
        #print(img['src'])
        img_name = str(i) + ".png"
        urllib.request.urlretrieve(img['src'], "./images/"+img_name)
        print(img_name)
        i = i+1
'''
for i in range(30):
    img = images[i].find('img')
    if img.has_attr('src'):
        #print(img['src'])
        img_name = str(i) + ".png"
        urllib.request.urlretrieve(img['src'], "./images/"+img_name)
        print(img_name)

browser.close()


import re
import json

from lib.configLoader import config
from lib.xlsGetter import request_table

template = {
    'weeks': config['weeks'],
    'lessons': config['lessons'],
    'blank': config['blank'],
    'time': config['time'],
    'courses': config['courses']
}

footer_match_list = [
    '(体育).+?◇(.+?)◇(\\d+-\\d+)',
    '(博雅课程).+?◇(.+?)◇(\\d+-\\d+)',
    '(实践与展示).+?◇(.+?)◇(\\d+-\\d+)'
]

course_name_format_list = [
    '体育\\(\\d+\\)\\((.+?)\\d*?\\)'
]

place_lesson_format_list = [
    '计算机教学实验中心\\(新主楼([a-zA-Z0-9]+)\\)'
]

def format_course_name(course_name):
    for i in course_name_format_list:
        match = re.search(i, course_name)
        if match is not None:
            course_name = match.group(1)
    return course_name

def format_teacher_time(teacher_time):
    teachers, times = [], [False for _ in range(20)]
    for i in teacher_time.split('周，'):
        split = i.split('[')
        if len(split) == 2:
            teacher, time = split[0].replace('|', ''), split[1]
            teachers.append(teacher)
            for j in time.split('，'):
                if re.search('(\\d+)-(\\d+)', j) is not None:
                    match = re.search('(\\d+)-(\\d+)', j)
                    for k in range(int(match.group(1)), int(match.group(2)) + 1):
                        times[k] = True
                elif re.search('(\\d+)', j) is not None:
                    match = re.search('(\\d+)', j)
                    times[int(match.group(1))] = True
                else:
                    print('warning: parse \"' + j + '\" failed')
    teacher, time, start = '多名教师' if len(''.join(teachers)) > 7 else '/'.join(teachers), [], -1
    for i in range(len(times)):
        if times[i]:
            if start == -1:
                start = i
        else:
            if start != -1:
                time.append(str(i - 1) if start == i - 1 else str(start) + '-' + str(i - 1))
            start = -1
    return teacher + ' [' + '/'.join(time) + ']周'

def format_place_lesson(place_lesson):
    for i in place_lesson_format_list:
        match = re.search(i + '(.*)', place_lesson)
        if match is not None:
            place_lesson = match.group(1) + match.group(2)
    return place_lesson.replace('，', '/')

def format_header(data):
    match = re.search('(^\\d+)(.+?学期)', data)
    return match.group(1) + ' ' + match.group(2) + '个人课表'

def format_footer(data):
    result = []
    for i in footer_match_list:
        match = re.search(i, data)
        if match is not None:
            name, teacher, time = match.group(1), match.group(2), match.group(3)
            result.append(name + ' ' + teacher + ' [' + time + ']周')
    return '其它课程: ' + ('无' if len(result) == 0 else ' / '.join(result))

def format_item(data):
    for i in range(len(data)):
        data[i] = data[i].replace('（', '(').replace('）', ')').replace(' ', '')
    split = data[1].split('周')
    return [format_course_name(data[0]), format_teacher_time('周'.join(split[:-1])),
            format_place_lesson(' '.join([split[-1]] + data[2:]))]

def create_settings():
    table = request_table()
    template['header'] = format_header(table.row_values(0)[0])
    template['footer'] = format_footer(table.row_values(table.nrows - 1)[0])
    unique = set(list(map(lambda x: (x['x'], x['y']), template['courses'])))
    for i in range(2, table.nrows - 1):
        row = table.row_values(i)
        for j in range(2, len(row)):
            if row[j] and (i - 2, j - 2) not in unique:
                unique.add((i - 2, j - 2))
                template['courses'].append({
                    'x': i - 2, 'y': j - 2,
                    'info': format_item(row[j].split('\n'))
                })
    return json.dumps(template, ensure_ascii=False)