#coding=utf-8
from collections import defaultdict
import torch.nn.functional as F
from loss import calc_loss
import time
import torch
from torch.utils.data import Dataset, DataLoader
from utils.lr_scheduler import adjust_learning_rate_poly
from utils.label2color import label_img_to_color, diff_label_img_to_color
from data import GaofenTrain, GaofenVal

from tqdm import tqdm
from evaluate import Evaluator
import numpy as np
import argparse

import cv2
cv2.setNumThreads(1)
import os
from tensorboardX import SummaryWriter
from model.seg_hrnet import hrnet18
from ipdb import set_trace

parser = argparse.ArgumentParser()
parser.add_argument('--root', default='/media/yr/新加卷/syz/gaofen')               # 根目录
parser.add_argument('--train_list_path', default='/media/yr/新加卷/syz/gaofen/makeData/train.txt')                   # 训练集id路径
parser.add_argument('--val_list_path', default='/media/yr/新加卷/syz/gaofen/makeData/val.txt')                       # 验证集
parser.add_argument('--test_list_path', default='/media/yr/新加卷/syz/gaofen/makeData/test.txt')                     # 测试
parser.add_argument('--backbone', default='resnest50', type=str,help='xception|resnet|resnest101|resnest200|resnest50|resnest26')
parser.add_argument('--n_cls', default=2, type=int)                                    # 类别
parser.add_argument('--batchsize', default=4, type=int)                                # batchsize
parser.add_argument('--lr', default=0.01, type=float)                                  # 学习率
parser.add_argument('--num_epochs', default=170, type=int)                             # epoch
parser.add_argument('--warmup', default=100, type=int)
parser.add_argument('--multiplier', default=100, type=int)
parser.add_argument('--eta_min', default=0.0005, type=float)
parser.add_argument('--num_workers', default=8, type=int)                              # 线程
parser.add_argument('--decay_rate', default=0.8, type=float)
parser.add_argument('--decay_epoch', default=200, type=int)                            # 学习率衰减设置
parser.add_argument('--vis_frequency', default=1, type=int)                           # 验证可视化的频率
parser.add_argument('--save_path', default='./results')                                # 保存路径
parser.add_argument('--gpu-id', default='0,1', type=str,
                    help='id(s) for CUDA_VISIBLE_DEVICES')                             # GPU
parser.add_argument('--is_resume', default=False, type=bool)                           # 是否重新训练模型
parser.add_argument('--resume', default='', type=str, help='./results/checkpoint.pth') # 参数保存路径
args = parser.parse_args()

folder_path = '/backbone={}/warmup={}_lr={}_multiplier={}_eta_min={}_num_epochs={}_batchsize={}'.format(args.backbone, args.warmup, args.lr, args.multiplier, args.eta_min, args.num_epochs,args.batchsize)
isExists = os.path.exists(args.save_path + folder_path)              # 判断文件是否存在
if not isExists:
    os.makedirs(args.save_path + folder_path)                        # 创建文件夹
isExists = os.path.exists(args.save_path +folder_path+'/vis')        # 创建 vis文件夹
if not isExists: os.makedirs(args.save_path + folder_path+'/vis')

isExists = os.path.exists(args.save_path +folder_path+'/log')        # 创建 log文件夹
if not isExists: os.makedirs(args.save_path + folder_path+'/log')

os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') # 显卡管理

def train_model():
    F_txt = open('./opt_results.txt', 'w')  # 创建保存结果文本
    evaluator = Evaluator(args.n_cls)       # 指标类实例化
    writer = SummaryWriter(args.save_path + folder_path + '/log')  # 日志，记录训练过程
    model = hrnet18(pretrained=True).to(device)                 # 实例化模型
    optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=0.0001)  # 随机梯度下降优化器
    # 参数初始化
    best_miou = 0.    # miou初始参数
    best_fwiou = 0.
    best_AA = 0.
    best_OA = 0.
    best_loss = 0.
    lr = args.lr
    epoch_index = 0

    if args.is_resume:  # 用于恢复训练
        args.resume = args.save_path + folder_path + '/checkpoint_fwiou.pth'
        if os.path.isfile(args.resume):
            checkpoint = torch.load(args.resume)
            epoch_index = checkpoint['epoch']
            best_miou = checkpoint['miou']
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            lr = optimizer.param_groups[0]['lr']
            print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch']))
            F_txt.write("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch']) + '\n')
            # print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch']), file=F_txt)
        else:
            print('EORRO: No such file!!!!!')

    train_root = args.root                     # 训练的根路径
    train_list_path = args.train_list_path     # 文件名路径

    val_root = args.root
    val_list_path = args.val_list_path
    # 实例化dataloader
    dataloaders = {
        "train": DataLoader(GaofenTrain(train_root, train_list_path), batch_size=args.batchsize,
                            shuffle=True, num_workers=args.num_workers, pin_memory=True, drop_last=True),
        "val": DataLoader(GaofenVal(val_root, val_list_path), batch_size=args.batchsize,
                          num_workers=args.num_workers, pin_memory=True)
    }

    evaluator.reset()     # 设置2*2的0矩阵
    print('config: ' + folder_path)   # folder_path (包含训练参数)
    print('config: ' + folder_path, file=F_txt,flush=True)  # 将打印的内容存入F_txt中，flush=True刷新内容，新的打印内容会持续存入，这里写入的是参数

    for epoch in range(epoch_index, args.num_epochs):       # 开始迭代150代
        print('Epoch [{}]/[{}] lr={:6f}'.format(epoch + 1, args.num_epochs, lr))
        print('Epoch [{}]/[{}] lr={:4f}'.format(epoch + 1, args.num_epochs, lr), file=F_txt, flush=True) # 这里写入训练过程
        since = time.time()
        # 训练一代、验证一代
        for phase in ['train', 'val']:
            evaluator.reset()
            if phase == 'train':
                model.train()   # dropout层会按照设置好的失活概率进行失活，batchnorm会继续计算数据的均值和方差等参数并在每个batch size之间不断更新
            else:
                model.eval()    # eval主要是用来影响网络中的dropout层和batchnorm层的行为

            metrics = defaultdict(float)     # 默认字典
            epoch_samples = 0

            for i, (inputs, labels, edge, _, datafiles) in enumerate(tqdm(dataloaders[phase], ncols=50)):
                inputs = inputs.to(device)   # img
                edge = edge.to(device, dtype=torch.float)    # edg
                labels = labels.to(device, dtype=torch.long) # labels

                optimizer.zero_grad()
                with torch.set_grad_enabled(phase == 'train'):   # 设置梯度反传，默认为false
                    outputs = model(inputs)   # outputs[1].shape----torch.Size([4, 2, 321, 321])  outputs[0].shape----torch.Size([4, 2, 81, 81])
                    outputs[1] = F.interpolate(input=outputs[1], size=(labels.shape[1], labels.shape[2]), mode='bilinear', align_corners=True)   # 输出双线性插值采样为label大小
                    loss = calc_loss(outputs, labels, edge, metrics)    # 损失
                    pred = outputs[1].data.cpu().numpy()      # 模型输出
                    pred = np.argmax(pred, axis=1)            # 输出预测标签
                    labels = labels.data.cpu().numpy()     # 标签
                    evaluator.add_batch(labels, pred)  # 评估

                    if phase == 'val' and (epoch + 1) % args.vis_frequency == 0 and inputs.shape[0] == args.batchsize:  # 验证每隔30代
                        for k in range(args.batchsize // 2):   # k = 0, 1
                            name = datafiles['name'][k][:-4]   # 当前batch图片名
                            writer.add_image('{}/img'.format(name), cv2.cvtColor(cv2.imread(datafiles["img"][k], cv2.IMREAD_COLOR),cv2.COLOR_BGR2RGB), global_step=int((epoch + 1)), dataformats='HWC')
                            # 图片格式转换（RGB）,添加到日志中
                            writer.add_image('{}/gt'.format(name), label_img_to_color(labels[k])[:, :, ::-1],global_step=int((epoch + 1)), dataformats='HWC')
                            # gt格式转换保存于日志中
                            pred_label_img = pred.astype(np.uint8)[k]    # batch_size中的pred的第一张图（512，512）
                            # 预测图（pred）
                            pred_label_img_color = label_img_to_color(pred_label_img)   # (512, 512, 3)
                            # 预测图
                            writer.add_image('{}/mask'.format(name), pred_label_img_color[:, :, ::-1],global_step=int((epoch + 1)), dataformats='HWC')
                            # mask 标签转化为三通道，代表预测标签
                            softmax_pred = F.softmax(outputs[1][k],dim=0)      # 上采样后的预测图通道维度softmax  torch.Size([2, 512, 512])
                            softmax_pred_np = softmax_pred.data.cpu().numpy()  # torch.Size([2, 512, 512])
                            probility = softmax_pred_np[0]
                            probility = probility*255          # 概率值乘255
                            probility = probility.astype(np.uint8)
                            probility = cv2.applyColorMap(probility,cv2.COLORMAP_HOT)   # 伪彩图  可以显示网络的关注点
                            writer.add_image('{}/prob'.format(name),cv2.cvtColor(probility,cv2.COLOR_BGR2RGB),global_step=int((epoch+1)),dataformats='HWC')
                            # 差分图
                            diff_img = np.ones((pred_label_img.shape[0], pred_label_img.shape[1]), dtype=np.int32)*255   # 制造全1
                            mask = (labels[k] != pred_label_img)          # 得到gt不等于1的mask
                            diff_img[mask] = labels[k][mask]
                            diff_img_color = diff_label_img_to_color(diff_img)  #  得到diff图
                            writer.add_image('{}/different_image'.format(name), diff_img_color[:, :, ::-1], global_step=int((epoch + 1)), dataformats='HWC')
                    if phase == 'train':
                        loss.backward()   # 反向传播
                        optimizer.step()  # 梯度更新
                        adjust_learning_rate_poly(args.lr,optimizer, epoch * len(dataloaders['train']) + i,args.num_epochs * len(dataloaders['train']))
                        lr = optimizer.param_groups[0]['lr']
                        writer.add_scalar('lr', lr, global_step=epoch * len(dataloaders['train']) + i)

                epoch_samples += 1
            epoch_loss = metrics['loss'] / epoch_samples    # 一个epoch的平均损失
            ce_loss = metrics['ce_loss'] / epoch_samples
            ls_loss = metrics['ls_loss'] / epoch_samples
            fwiou = evaluator.Frequency_Weighted_Intersection_over_Union()
            miou = evaluator.Mean_Intersection_over_Union()
            AA = evaluator.Pixel_Accuracy_Class()
            OA = evaluator.Pixel_Accuracy()
            if phase == 'val':
                miou_mat = evaluator.Mean_Intersection_over_Union_test()
                writer.add_scalar('val/val_loss', epoch_loss, global_step=epoch)   # 数据标识符，要保存的数值，全局步值
                writer.add_scalar('val/ce_loss', ce_loss, global_step=epoch)
                writer.add_scalar('val/ls_loss', ls_loss, global_step=epoch)
                writer.add_scalar('val/val_fwiou', fwiou, global_step=epoch)
                writer.add_scalar('val/val_miou', miou, global_step=epoch)
                for index in range(args.n_cls):
                    writer.add_scalar('class/{}'.format(index + 1), miou_mat[index], global_step=epoch)
                    print('[val]------fwiou: {:4f}, miou: {:4f}, OA:{:4f}, AA: {:4f}, loss: {:4f}'.format(fwiou, miou, OA, AA, epoch_loss))  # 打印
                    # 写入
                    print('[val]------fwiou: {:4f}, miou: {:4f}, OA:{:4f}, AA: {:4f}, loss: {:4f}'.format(fwiou, miou, OA, AA, epoch_loss),file=F_txt, flush=True)
            if phase == 'train':
                writer.add_scalar('train/train_loss', epoch_loss, global_step=epoch)
                writer.add_scalar('train/ce_loss', ce_loss, global_step=epoch)
                writer.add_scalar('train/ls_loss', ls_loss, global_step=epoch)
                writer.add_scalar('train/train_fwiou', fwiou, global_step=epoch)
                writer.add_scalar('train/train_miou', miou, global_step=epoch)
                print(
                    '[train]------fwiou: {:4f}, miou: {:4f}, OA: {:4f}, AA: {:4f}, loss: {:4f}'.format(fwiou, miou, OA, AA, epoch_loss))
                # 下面的print是写入
                print(
                    '[train]------fwiou: {:4f}, miou: {:4f}, OA: {:4f}, AA: {:4f}, loss: {:4f}'.format(fwiou, miou, OA, AA, epoch_loss), file=F_txt, flush=True)
            if phase == 'val' and fwiou > best_fwiou:
                print("\33[91msaving best model miou\33[0m")
                print("saving best model fwiou", file=F_txt, flush=True)
                best_miou = miou
                best_fwiou = fwiou
                best_OA = OA
                best_AA = AA
                best_loss = epoch_loss
                torch.save({
                    'name': 'resnest50_lovasz_edge_rotate',
                    'epoch': epoch + 1,
                    'state_dict': model.state_dict(),
                    'best_miou': best_miou
                }, args.save_path + folder_path + '/model_best.pth')

                torch.save({
                    'optimizer': optimizer.state_dict(),
                }, args.save_path + folder_path + '/optimizer.pth')

        time_elapsed = time.time() - since                         # 一个epoch运行时间
        print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))      # 打印运行时间
        print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60), file=F_txt, flush=True)  # 记录日志

    print('[Best val]------fwiou: {:4f}; miou: {:4f}; OA: {:4f}; AA: {:4f}; loss: {:4f}'.format(best_fwiou, best_miou, best_OA, best_AA, best_loss))
    print('[Best val]------fwiou: {:4f}; miou: {:4f}; OA: {:4f}; AA: {:4f}; loss: {:4f}'.format(best_fwiou, best_miou, best_OA, best_AA, best_loss),file=F_txt,flush=True)
    F_txt.close()

if __name__ == '__main__':
    train_model()


#!/usr/bin/env python

import logging
from math import floor
import time

from csvkit import CSVKitReader
from django.conf import settings
from django.utils.translation import ugettext
from livesettings import config_value

from panda import solr, utils
from panda.exceptions import DataImportError
from panda.tasks.import_file import ImportFileTask
from panda.utils.typecoercion import DataTyper

SOLR_ADD_BUFFER_SIZE = 500

class ImportCSVTask(ImportFileTask):
    """
    Task to import all data for a dataset from a CSV.
    """
    name = 'panda.tasks.import.csv'

    def _count_lines(self, filename):
        """
        Efficiently count the number of lines in a file.
        """
        with open(filename) as f:
            for i, l in enumerate(f):
                pass
        return i + 1

    def run(self, dataset_slug, upload_id, external_id_field_index=None, *args, **kwargs):
        """
        Execute import.
        """
        from panda.models import Dataset, DataUpload

        log = logging.getLogger(self.name)
        log.info('Beginning import, dataset_slug: %s' % dataset_slug)

        try:
            dataset = Dataset.objects.get(slug=dataset_slug)
        except Dataset.DoesNotExist:
            log.warning('Import failed due to Dataset being deleted, dataset_slug: %s' % dataset_slug)

            return

        upload = DataUpload.objects.get(id=upload_id)

        task_status = dataset.current_task
        task_status.begin(ugettext('Preparing to import'))

        line_count = self._count_lines(upload.get_path())

        if self.is_aborted():
            task_status.abort('Aborted during preperation')

            log.warning('Import aborted, dataset_slug: %s' % dataset_slug)

            return

        f = open(upload.get_path(), 'r')

        reader = CSVKitReader(f, encoding=upload.encoding, **upload.dialect_as_parameters())
        reader.next()

        add_buffer = []
        data_typer = DataTyper(dataset.column_schema)
        throttle = config_value('PERF', 'TASK_THROTTLE')

        i = 0

        while True:
            # The row number which is about to be read, for error handling and indexing
            i += 1

            try:
                row = reader.next()
            except StopIteration:
                i -= 1
                break
            except UnicodeDecodeError:
                raise DataImportError(ugettext('This CSV file contains characters that are not %(encoding)s encoded in or after row %(row)i. You need to re-upload this file and input the correct encoding in order to import data from this file.') % { 'encoding': upload.encoding, 'row': i })

            external_id = None

            if external_id_field_index is not None:
                external_id = row[external_id_field_index]

            data = utils.solr.make_data_row(dataset, row, data_upload=upload, external_id=external_id)
            data = data_typer(data, row)

            add_buffer.append(data)

            if i % SOLR_ADD_BUFFER_SIZE == 0:
                solr.add(settings.SOLR_DATA_CORE, add_buffer)

                add_buffer = []

                task_status.update(ugettext('%.0f%% complete (estimated)') % floor(float(i) / float(line_count) * 100))

                if self.is_aborted():
                    task_status.abort(ugettext('Aborted after importing %.0f%% (estimated)') % floor(float(i) / float(line_count) * 100))

                    log.warning('Import aborted, dataset_slug: %s' % dataset_slug)

                    return

                time.sleep(throttle)

        if add_buffer:
            solr.add(settings.SOLR_DATA_CORE, add_buffer)
            add_buffer = []

        solr.commit(settings.SOLR_DATA_CORE)

        f.close()

        task_status.update('100% complete')

        # Refresh dataset from database so there is no chance of crushing changes made since the task started
        try:
            dataset = Dataset.objects.get(slug=dataset_slug)
        except Dataset.DoesNotExist:
            log.warning('Import could not be completed due to Dataset being deleted, dataset_slug: %s' % dataset_slug)

            return

        if not dataset.row_count:
            dataset.row_count = i
        else:
            dataset.row_count += i

        dataset.column_schema = data_typer.schema

        dataset.save()

        # Refres
        upload = DataUpload.objects.get(id=upload_id)

        upload.imported = True
        upload.save()

        log.info('Finished import, dataset_slug: %s' % dataset_slug)

        return data_typer


"""
Analyzing the Disputed Federalist Papers
"""

import pandas as pd
from sklearn import tree
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

# Import csv
df = pd.read_csv('federalist_papers_data.csv')

a = df[df['class'] == 1].count()
print(a)

# Separate the entire dataframe into two subsets
# One for training, containing the papers of known authorship
# The other for testing, which is the papers of unknown authorship
train_df = df[df['class'] != 3]
test_df = df[df['class'] == 3]

print(test_df)
print(train_df)

# Extract the target variable
y = train_df.pop('class').values
print(y)
print(train_df)

# Create the Decision TreeClassifier object
# Fit the decision tree
clf = tree.DecisionTreeClassifier(max_depth=50)
clf.fit(train_df, y)

# Visualize the decision tree,
tree.export_graphviz(clf,
                     out_file='tree.dot',
                     feature_names=list(train_df.columns),
                     class_names=['Hamilton', 'Madison'],
                     filled=True,
                     rounded=True,
                     special_characters=True,
                     proportion=True)

# Use the fitted model to predict the class of the unknown papers
y = test_df.pop('class').values
predicted_labels = clf.predict(test_df)
print(predicted_labels)

#######
# Writeup
# In this problem, the data regarding the collection of short essays written, known as "The Federalist Papers", by Alexander Hamilton or James Madison was analyzed to identify the authors of the disputed papers in our dataset by using a decision tree.
# Our dataset contains 118 data points, 56 of them belonging to Hamilton's papers, 50 of them belonging to Madison's papers, and 12 of them being unknown disputed papers. The data points in the dataset also contain frequency of occurence of the 70 function words in the text, scaled to units of estimated occurences per 1000 words to eliminate bias caused by the length of the essays. To train the model, a dataframe called "train_df" was initialized, having all the data points that have a stated author (class 1 (Hamilton) & class 2 (Madison)). The values of class (1 or 2) are extracted from this dataframe in order to use it as a target variable in our decision tree, and the "train_df" was used to build the decision tree in order to identify the words that distinguish these two authors the most.
# As the decision tree that is built based on our training data suggests, the word that gives our model the biggest information gain is the frequency of the word "upon". This can be seen in the top node of our decision tree that contains 100% of our training data. The nodes of the first split proves this point, as it shows that more than 98% of essays that have a frequency of more than 1.34 for the word "upon" belongs to Hamilton, and more than 94% of essays that have a frequency of less than 1.34 for the word "upon" belongs to Madison. The other words that were chosen as significant by our model were "it", "is", "our", and "info", leading to our decision tree growing until it reaches the gini coefficient of 0 for every node.
# According to Bosch, Smith, and Fung, all of the 12 disputed papers were works of Madison, but there are still opinions of other scholars that believe that some of these works are Hamilton's pieces or collaborative efforts. To solve this uncertainty, we used our decision tree classifier model to test our "test_df" data consisting of all 12 disputed papers. As a results, our model attributed 10/12 papers to Madison, and 2/12 papers to Hamilton. Results: [2 2 1 2 2 2 2 2 1 2 2 2]
# In conclusion, the purpose of our study was to train a decision tree model based on the certain existing data to predict the authors of the 12 disputed papers in our dataset. To do this, a decision tree with a max_depth of 3 is visualized, and "upon" is identified as the word frequency that distinguishes these two authors the most. Finally, the results of our study were 10 papers belonging to Madison, and 2 papers belonging to Hamilton, unlike the theories of Bosch, Smith, and Fung.


from itertools import product
import sqlite3

# class Card:
#
#     def __init__(self, name, rules_text, color, type_, subtype, cmc, power,
#                  toughness, mana_cost, rarity):
#         self.name = name
#         self.rules_text = rules_text
#         self.color = color
#         self.type_ = type_
#         self.subtype = subtype
#         self.cmc = cmc
#         self.power = power
#         self.toughness = toughness
#         self.mana_cost = mana_cost
#         self.rarity = rarity

sqlite_file = 'db.sqlite'
conn = sqlite3.connect(sqlite_file)
c = conn.cursor()

class Card:

    def __init__(self, cost):
        self.cost = cost

class Creature(Card):

    def __init__(self, power, toughness, *args, **kwargs):
        super(Creature, self).__init__(*args, **kwargs)
        self.power = power
        self.toughness = toughness

class Deck:

    def __init__(self):
        self.cards = []

    def take_top_card(self):
        # last item is top
        card = self.cards.pop()
        return card

class Player:

    def __init__(self, deck):
        self.deck = deck
        self.hand = []

    def draw(self, size):
        for i in range(size):
            card = self.deck.take_top_card()
            self.hand.append(card)

class Engine:

    STARTING_HAND_SIZE = 7

    def __init__(self, player1, player2):
        self.player1 = player1
        self.player2 = player2
        self.player1.draw(7)
        self.player2.draw(7)
        self.turn = 1

def run_simulation():
    deck1 = Deck()
    player1 = Player(deck1)
    deck2 = Deck()
    player2 = Player(deck2)
    engine = Engine(player1, player2)

def generate_cards():
    setup_db()

    res = list(product(range(0, 6), range(1, 6)))
    print(len(res))
    print(res)

def setup_db():
    """
    INTEGER: A signed integer up to 8 bytes depending on the magnitude of the value.
    REAL: An 8-byte floating point value.
    TEXT: A text string, typically UTF-8 encoded (depending on the database encoding).
    BLOB: A blob of data (binary large object) for storing binary data.
    NULL: A NULL value, represents missing data or an empty cell.
    """
    c.execute("""
    CREATE TABLE creature (
        id INTEGER PRIMARY KEY,
        power INTEGER NOT NULL,
        toughness INTEGER NOT NULL,
        UNIQUE (power, toughness)
    )
    """)
    conn.commit()

if __name__ == '__main__':
    generate_cards()
    # run_simulation()


# IMPORTING CONFIGURATIONS
import yaml

try:
    with open('config.yaml', 'r') as configfile:
        cfg = yaml.load(configfile)
except:
    with open('../config.yaml', 'r') as configfile:
        cfg = yaml.load(configfile)

# IMPORTING NECESSARY PACKAGES
from microsoftbotframework import ReplyToActivity
from helpers.db_class import DB
from helpers.dialogflow_connector import detect_intent_texts
import json
import os.path
import apiai
from numpy.random import randint
from datetime import datetime
from helpers.rephrase import check_rephrase, check_override
from helpers.log_conversations import log_response, log_message, get_conversation_status, get_participantid_invalid

db = DB()

## TO DO:
## Logging
## Check participant ID
## Create and/or update conversation
## Check conversation status
## Standard messages

def echo_response(message):
    conditionid = None
    participantid = None
    # CONTACT RELATION UPDATE: Agent is added to Skype
    if message["type"] == "contactRelationUpdate":
        if 'action' in message.keys():
            if message['action'] == 'add':
                resp_speech = '[STANDARDWELCOMEMESSAGE]'
                resp_speech = check_rephrase(resp_speech)

                log_response(db, message, resp_speech, resp_speech)
                if 'channelData' in message.keys():
                    del message['channelData']['clientActivityID']
                ReplyToActivity(fill=message,
                        text=resp_speech).send()
                return

    # MESSAGE: Default type of conversation
    if message["type"] == "message":
        if 'text' not in message.keys():
            message['text'] = '-'

        message['text'] = str(message['text']).replace("'", '‘')
        conversationid = message['conversation']['id']
        conversationid_trunc = conversationid[0:35]

        resp_speech = None

        resp = detect_intent_texts(conversationid_trunc, str(message['text']))

        try:
            # resp_speech = resp_dialogflow.query_result.fulfillment_text
            resp_speech = resp['fulfillmentText']
        except:
            resp_speech = 'ERROR: ' + str(resp_dialogflow)

        if not resp_speech:
            try:
                resp_speech  = resp['result']['fulfillment']['speech']
            except:
                resp_speech = resp['result']['fulfillment']['messages'][0]['speech']

        conversation_status, conversation_code, conditionid, participantid  = get_conversation_status(db, conversationid, message, resp)
        print(conversation_code, conditionid, conversation_status)

        message = log_message(db, message, participantid=participantid)
        override = check_override(message['text'])

        ## Performing manual connection if intents in DialogFlow
        connect_intent = None
        # If the conversation_status is considered invalid (due to invalid participant id)
        if conversation_status == 'conditionid_invalid':
            override_invalid_participantid = get_participantid_invalid()
            if override_invalid_participantid:
                connect_intent = '[' + str(override_invalid_participantid) + ']'
                log_response(db, message, connect_intent, conversation_status, participantid = participantid)

        # If the response in DialogFlow needs to connect manually to a new intent
        elif 'connect_intents' in cfg.keys():
            for source_intent, target_intent in cfg['connect_intents'].items():
                if resp_speech == '[' + str(source_intent) + ']':
                    print('check 1:', source_intent)
                    connect_intent = '[' + str(target_intent) + ']'
                    log_response(db, message, connect_intent, conversation_status, participantid = participantid)

        if connect_intent:
            ai = apiai.ApiAI(cfg['other']['dialogflow_access_token'])
            request = ai.text_request()
            request.lang = 'en'  # optional, default value equal 'en'
            request.session_id = conversationid_trunc
            request.query = connect_intent

            respAPI = request.getresponse()
            resp = respAPI.read().decode('utf-8')
            resp = json.loads(resp)
            try:
                resp_speech  = resp['result']['fulfillment']['speech']
            except:
                resp_speech = resp['result']['fulfillment']['messages'][0]['speech']

            resp_speech = check_rephrase(resp_speech, conversation_code = conversation_code, conditionid=conditionid)
            log_response(db, message, resp_speech, resp, participantid=participantid)
            if 'channelData' in message.keys():
                del message['channelData']['clientActivityID']
            ReplyToActivity(fill=message, text=resp_speech).send()

        elif override['override'] == True:
            resp_speech = check_rephrase(override['resp_speech'], conversation_code = conversation_code)
            reason_override = override['reason']
            log_response(db, message, resp_speech, reason_override, participantid=participantid)
            ReplyToActivity(fill=message,
                        text=resp_speech).send()

        else:
            resp_speech = check_rephrase(resp_speech, conversation_code = conversation_code, conditionid=conditionid)
            message['textFormat'] = 'markdown'
            log_response(db, message, resp_speech, resp, participantid=participantid)
            #override for disappearing messages
            if 'channelData' in message.keys():
                del message['channelData']['clientActivityID']
            ReplyToActivity(fill=message, text=resp_speech).send()


import torch

from conf import agents_cfgs
from fairdiplomacy.game import sort_phase_key
from fairdiplomacy.agents.base_agent import BaseAgent
from fairdiplomacy.agents.model_wrapper import ModelWrapper
from fairdiplomacy.models.consts import POWERS
from fairdiplomacy.utils.thread_pool_encoding import FeatureEncoder

_DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

class ModelSampledAgent(BaseAgent):
    def __init__(self, cfg: agents_cfgs.ModelSampledAgent, device=_DEFAULT_DEVICE):
        self.model = ModelWrapper(
            cfg.model_path, device=device, half_precision=cfg.half_precision
        )
        self.temperature = cfg.temperature
        self.top_p = cfg.top_p
        self.device = device
        self.input_encoder = FeatureEncoder()

    def get_orders(self, game, power, **kwargs):
        if len(game.get_orderable_locations().get(power, [])) == 0:
            return []
        return self.get_orders_many_powers(game, [power], **kwargs)[power]

    def get_orders_many_powers(self, game, powers, *, temperature=None, top_p=None):
        temperature = temperature if temperature is not None else self.temperature
        top_p = top_p if top_p is not None else self.top_p
        encode_fn = (
            self.input_encoder.encode_inputs_all_powers
            if self.model.is_all_powers()
            else self.input_encoder.encode_inputs
        )
        inputs = encode_fn([game])

        inputs = inputs.to(self.device)
        actions, _, _ = self.model.do_model_request(inputs, temperature=temperature, top_p=top_p)
        actions = actions[0]  # batch dim
        return {p: a for p, a in zip(POWERS, actions) if p in powers}


s = "Keep it short and simple!"

def make_n_grams(n, s):
    rangeCount = max(len(s) - n + 1, 0)
    # use list comprehensive to assemble list of ngrams (represented as tuples)
    return [tuple(s[i:i+n]) for i in range(rangeCount)]

three_grams = make_n_grams(3, s)

# could be printed directly, or as on assignment:
for gram in three_grams:
    print("{} {} {}".format(gram[0], gram[1], gram[2]))

# Or as in forum post
for gram in three_grams:
    s = ""
    for l in gram:
        if l == " ":
            s += "_"
        else:
            s += l
        s += " "
    print(s)


# /!usr/bin/env python3

"""
Plotting functionality
"""

import copy
import logging
import os
from os.path import join as pjoin
from multiprocessing import Process

import matplotlib.markers as mmarkers
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib.colors import ListedColormap
from matplotlib.ticker import FormatStrFormatter
from tqdm import tqdm

from . import analyses as a
from . import decision_makers as dm
from .colors import rgb
from .. import constants as consts
from ..datasets.experiments import get_experiments
from ..helper import plotting_helper as ph
from ..utils import load_dataset

logger = logging.getLogger(__name__)

##################################################################
# PLOTTING-SPECIFIC CONSTANTS AND GLOBAL SETTINGS
##################################################################

# global default boundary settings for thin gray transparent
# boundaries to avoid not being able to see the difference
# between two partially overlapping datapoints of the same color:
PLOTTING_EDGE_COLOR = (0.3, 0.3, 0.3, 0.3)
PLOTTING_EDGE_WIDTH = 0.02

METRICS = {"OOD accuracy": (a.SixteenClassAccuracy(), "16-class-accuracy"),
           "accuracy difference": (a.SixteenClassAccuracyDifference(),
                                   "16-class-accuracy-difference"),
           "observed consistency": (a.ErrorConsistency(), "observed-consistency"),
           "error consistency": (a.ErrorConsistency(), "error-consistency")}

# exclusion criteria:
# - not OOD: control condition without manipulation (e.g. 100% contrast)
# - mean human accuracy < 0.2 (error consistency etc. not meaningful)
EXCLUDE_CONDITIONS = {
    "colour": ["cr"],
    "contrast": ["c100", "c03", "c01"],
    "high-pass": ["inf", "0.55", "0.45", "0.4"],
    "low-pass": ["0", "15", "40"],
    "phase-scrambling": ["0", "150", "180"],
    "power-equalisation": ["0"],
    "false-colour": ["True"],
    "rotation": ["0"],
    "eidolonI": ["1-10-10", "64-10-10", "128-10-10"],
    "eidolonII": ["1-3-10", "32-3-10", "64-3-10", "128-3-10"],
    "eidolonIII": ["1-0-10", "16-0-10", "32-0-10", "64-0-10", "128-0-10"],
    "uniform-noise": ["0.0", "0.6", "0.9"]
}

EXCLUDE = True

##################################################################
# Main plot function to be called by user
##################################################################

def plot(plot_types,
         plotting_definition,
         dataset_names=None,
         figure_directory_name="example-figures",
         crop_PDFs=True):
    """Start different processes for different plotting types to speed up plotting."""

    processes = []
    for plot_type in plot_types:
        process = Process(target = plot_nonparallel, kwargs={"plot_types": [plot_type],
                                                             "plotting_definition": plotting_definition,
                                                             "dataset_names": dataset_names,
                                                             "figure_directory_name": figure_directory_name,
                                                             "crop_PDFs": crop_PDFs})
        process.start()
        processes.append(process)

    for process in processes:
        process.join()

    if crop_PDFs:
        ph.crop_pdfs_in_directory(os.path.join(consts.FIGURE_DIR, figure_directory_name))

def plot_nonparallel(plot_types,
                     plotting_definition,
                     dataset_names=None,
                     figure_directory_name="example-figures",
                     crop_PDFs=True):
    for plot_type in plot_types:
        assert plot_type in consts.PLOT_TYPE_TO_DATASET_MAPPING.keys(), "please select plot_types from: " + str(
            consts.PLOT_TYPE_TO_DATASET_MAPPING.keys())
        if dataset_names is not None:
            for d_name in dataset_names:
                assert d_name in consts.PLOT_TYPE_TO_DATASET_MAPPING[plot_type], f"plot_type '{plot_type}' is not available for dataset '{d_name}'. The following datasets can be plotted for this plot_type: {consts.PLOT_TYPE_TO_DATASET_MAPPING[plot_type]}"

    result_dir = os.path.join(consts.FIGURE_DIR, figure_directory_name)
    if not os.path.exists(result_dir):
        os.makedirs(result_dir)

    for plot_type in plot_types:
        if dataset_names is None:
            current_dataset_names = get_dataset_names(plot_type)
        else:
            current_dataset_names = dataset_names

        datasets = get_experiments(current_dataset_names)

        if plot_type == "confusion-matrix":
            confusion_matrices_dir = os.path.join(result_dir, "confusion-matrices/")
            if not os.path.exists(confusion_matrices_dir):
                os.makedirs(confusion_matrices_dir)
            plot_confusion_matrix(datasets=datasets,
                                  decision_maker_fun=plotting_definition,
                                  result_dir=confusion_matrices_dir)
            if crop_PDFs:
                ph.crop_pdfs_in_directory(confusion_matrices_dir)

        elif plot_type == "accuracy":
            plot_accuracy(datasets=datasets,
                          decision_maker_fun=plotting_definition,
                          result_dir=result_dir)

        elif plot_type == "entropy":
            plot_entropy(datasets=datasets,
                         decision_maker_fun=plotting_definition,
                         result_dir=result_dir)

        elif plot_type == "shape-bias":
            plot_shape_bias_matrixplot(datasets=datasets,
                                       decision_maker_fun=plotting_definition,
                                       result_dir=result_dir)
            plot_shape_bias_boxplot(datasets=datasets,
                                    decision_maker_fun=plotting_definition,
                                    result_dir=result_dir)

        elif plot_type == "error-consistency":
            plot_error_consistency(datasets=datasets,
                                   decision_maker_fun=plotting_definition,
                                   result_dir=result_dir)

        elif plot_type == "benchmark-barplot":
            plot_benchmark_barplot(datasets=datasets,
                                   decision_maker_fun=plotting_definition,
                                   result_dir=result_dir)

        elif plot_type == "nonparametric-benchmark-barplot":
            for dataset_name in current_dataset_names:
                datasets = get_experiments([dataset_name])
                plot_benchmark_barplot(datasets=datasets,
                                       decision_maker_fun=plotting_definition,
                                       result_dir=result_dir,
                                       print_to_latex=False,
                                       metrics_to_plot=["OOD accuracy"],
                                       single_dataset_name=dataset_name)

        elif plot_type == "scatterplot":
            plot_scatterplot(datasets=datasets,
                             decision_maker_fun=plotting_definition,
                             result_dir=result_dir)

        elif plot_type == "error-consistency-lineplot":
            plot_error_consistency_lineplot(datasets=datasets,
                                            decision_maker_fun=plotting_definition,
                                            result_dir=result_dir)

        else:
            raise NotImplementedError("unknown plot_type: " + plot_type)

##################################################################
# Helper methods
##################################################################

def get_datasets(dataset_names, *args, **kwargs):
    dataset_list = []
    for dataset in dataset_names:
        dataset = load_dataset(dataset, *args, **kwargs)
        dataset_list.extend(dataset) if isinstance(dataset, list) else dataset_list.append(dataset)
    return dataset_list

def get_dataset_names(plot_type):
    """Given plot_type, return suitable dataset(s).

    In this regard, 'suitable' means:
    - valid plot_type <-> dataset combination
    - data is available
    """

    dataset_names = []
    dataset_candidates = consts.PLOT_TYPE_TO_DATASET_MAPPING[plot_type]

    for candidate in dataset_candidates:
        if os.path.exists(pjoin(consts.RAW_DATA_DIR, candidate)):
            dataset_names.append(candidate)

    if len(dataset_names) == 0:
        raise ValueError("No data found for the specified plot_types.")

    return dataset_names

def get_permutations(elements):
    """Return permutation of elements.

    Return value: list of tuples, where tuples are
    unique combinations of elements
    """

    permutations = []
    for i, elem1 in enumerate(elements):
        for j, elem2 in enumerate(elements):
            if i < j:
                permutations.append((elem1, elem2))
    return permutations

def exclude_conditions(dataset):
    dataset = copy.deepcopy(dataset)
    if len(dataset.experiments) > 0:
        assert dataset.name in EXCLUDE_CONDITIONS.keys()
        for c in EXCLUDE_CONDITIONS[dataset.name]:
            assert len(dataset.experiments) == 1
            assert c in dataset.experiments[0].data_conditions, f"{c} not found for {dataset.name}"
            idx = dataset.experiments[0].data_conditions.index(c)
            dataset.experiments[0].data_conditions.remove(c)
            del dataset.experiments[0].plotting_conditions[idx]
            #print(f"Dataset {dataset.name}: removing condition {c}")
    return dataset

def log(plot_type, dataset_name):
    """Print logging info for plotting to console"""

    logging_info = f"Plotting {plot_type} for dataset {dataset_name}"
    logger.info(logging_info)
    print(logging_info)

def get_human_and_CNN_subjects(subjects):
    """Split subjects into 2 lists: human, CNNs subjects."""

    assert type(subjects) is list
    human_subjects = []
    CNN_subjects = []
    for s in subjects:
        if s.startswith("subject-"):
            human_subjects.append(s)
        else:
            CNN_subjects.append(s)
    return human_subjects, CNN_subjects

def get_raw_matrix(dataset,
                   decision_maker_fun,
                   analysis,
                   value="error-consistency"):
    """Return NxN data frame of error consistencies."""

    df = ph.get_experimental_data(dataset)
    decision_makers = decision_maker_fun(df)
    subjects = dm.get_individual_decision_makers(decision_makers)

    num_subjects = len(subjects)
    matrix = np.ones([num_subjects, num_subjects])
    for i in tqdm(range(num_subjects)):
        s1 = subjects[i]
        df1 = df.loc[(df["subj"] == s1)]
        for j in range(i, num_subjects):
            s2 = subjects[j]
            df2 = df.loc[(df["subj"] == s2)]
            a = analysis.analysis(df1, df2)[value]
            matrix[i, j] = a
            matrix[j, i] = a

    plotting_names = []
    colors = []
    for s in subjects:
        attr = dm.decision_maker_to_attributes(s, decision_makers)
        if s.startswith("subject-"):
            plotting_names.append(s)
        else:
            plotting_names.append(attr["plotting_name"])
        if attr["color"] == (230 / 255.0, 230 / 255.0, 230 / 255.0):
            # supervised models: too bright for error consistency matrices
            colors.append((150 / 255.0, 150 / 255.0, 150 / 255.0))
        else:
            colors.append(attr["color"])

    assert len(colors) == matrix.shape[0] == matrix.shape[1]

    return {"matrix": pd.DataFrame(data=matrix,
                                   columns=plotting_names,
                                   index=plotting_names),
            "colors": colors}

def plotting_names_to_data_subjects(plotting_names,
                                    decision_makers):
    subjects = []
    plotting_set = set()
    for d in decision_makers:
        if d.plotting_name in plotting_names:
            subjects += d.decision_makers
            plotting_set.add(d.plotting_name)
    missing_subjects = plotting_set.symmetric_difference(set(plotting_names))
    if len(missing_subjects) > 0:
        print("Missing subjects: ")
        print(missing_subjects)
        raise ValueError("subjects missing")
    return subjects

def get_mean_over_datasets(colname,
                           metric_fun,
                           metric_name,
                           datasets,
                           decision_maker_fun):
    """Compute the mean result of metric_fun applied to datasets.

    Returns data frame with columns as follows:
    <plotting_name> (name of decision maker)
    <colname> (name of numerical column with metric results)
    <color> (plotting color)
    """

    result_df = pd.DataFrame(columns=['model', 'plotting_name', 'dataset', colname, 'color'])
    for d_orig in datasets:
        df_raw = ph.get_experimental_data(d_orig)
        if EXCLUDE:
            d = exclude_conditions(d_orig)
        else:
            d = d_orig
        for dmaker in decision_maker_fun(df_raw):
            if len(d.experiments) == 1:
                df_selection = df_raw.loc[(df_raw["subj"].isin(dmaker.decision_makers)) &
                                          (df_raw["condition"].isin(d.experiments[0].data_conditions))]
            elif len(d.experiments) == 0:
                df_selection = df_raw.loc[(df_raw["subj"].isin(dmaker.decision_makers))]
            else:
                raise ValueError("unknown")
            r1 = metric_fun.analysis(df=df_selection)
            result_df = result_df.append([{"plotting_name": dmaker.plotting_name,
                                           "dataset": d.name,
                                           colname: r1[metric_name],
                                           "color": dmaker.color}],
                                         ignore_index=True)

    # average over datasets
    result_df = result_df.groupby(['plotting_name', 'color'], as_index=False)[colname].mean()
    return result_df

##################################################################
# Individual plotting methods
##################################################################

def x_y_plot(figure_path,
             analysis,
             decision_makers,
             experiment,
             result_df):
    """Plot experimental data on an x-y plot."""

    plt.rcParams.update({'font.size': 20})

    fig, ax = plt.subplots(figsize=analysis.figsize)
    plt.xlabel(experiment.xlabel)
    plt.ylabel(analysis.ylabel)
    margin = analysis.ylim[1] * 0.01  # avoid cropping datapoints
    plt.ylim((analysis.ylim[0] - margin, analysis.ylim[1] + margin))

    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)

    for ID in result_df["decision-maker-ID"].unique():
        decision_maker = None
        for d in decision_makers:
            if ID == d.ID:
                decision_maker = d
                break
        assert decision_maker is not None, "no matching decision maker found"
        result_list = result_df.loc[result_df["decision-maker-ID"] == ID]["yvalue"]
        plt.plot(experiment.plotting_conditions, result_list,
                 marker=decision_maker.marker, color=decision_maker.color,
                 markersize=12, linewidth=1,
                 markeredgecolor=PLOTTING_EDGE_COLOR,
                 markeredgewidth=PLOTTING_EDGE_WIDTH)

    if analysis.height_line_for_chance is not None:
        axes = plt.gca()
        x_vals = np.array(ax.get_xlim())
        y_vals = analysis.height_line_for_chance + 0 * x_vals
        plt.plot(x_vals, y_vals, ':', color="grey")

    plt.savefig(figure_path)
    plt.close()

def confusion_matrix_helper(data, output_filename,
                            plot_cbar=False, plot_labels=False):
    """Plot confusion matrix using heatmap.

    Args:
        data (list of list): List of lists with confusion matrix data.
        labels (list): Labels which will be plotted across x and y axis.
        output_filename (str): Path to output file.

    """
    sns.set(color_codes=True)
    plt.figure(1, figsize=(9, 6))

    sns.set(font_scale=1.4)

    N = 256
    vals = np.ones((N, 4))
    vals[:, 0] = np.linspace(165 / 256, 1, N)
    vals[:, 1] = np.linspace(30 / 256, 1, N)
    vals[:, 2] = np.linspace(55 / 256, 1, N)
    newcmp = ListedColormap(vals)

    ax = sns.heatmap(data, cmap=newcmp.reversed(),
                     vmin=0.0, vmax=100.0, linecolor="black", linewidths=1.0,
                     square=True, cbar=plot_cbar,
                     xticklabels=plot_labels, yticklabels=plot_labels)

    if plot_labels:
        colnames = list(data.columns)
        rownames = list(data.index.values)
        ax.set_xticklabels(colnames)
        ax.set_yticklabels(rownames)
        ax.set(xlabel="Presented category", ylabel="Decision")
    else:
        ax.set(xlabel="", ylabel="")

    plt.savefig(output_filename, bbox_inches='tight', dpi=300)
    plt.close()
    sns.reset_defaults()
    sns.reset_orig()
    plt.style.use('default')

def plot_shape_bias_matrixplot(datasets,
                               decision_maker_fun,
                               result_dir,
                               analysis=a.ShapeBias(),
                               order_by='humans'):
    assert len(datasets) == 1
    ds = datasets[0]
    assert ds.name == "cue-conflict"

    log(plot_type="shape-bias-matrixplot", dataset_name=ds.name)

    df = ph.get_experimental_data(ds)

    fontsize = 25
    ticklength = 10
    markersize = 250

    classes = df["category"].unique()
    num_classes = len(classes)

    # plot setup
    fig = plt.figure(1, figsize=(12, 12), dpi=300.)
    ax = plt.gca()

    ax.set_xlim([0, 1])
    ax.set_ylim([-.5, num_classes - 0.5])

    # secondary reversed x axis
    ax_top = ax.secondary_xaxis('top', functions=(lambda x: 1 - x, lambda x: 1 - x))

    # labels, ticks
    plt.tick_params(axis='y',
                    which='both',
                    left=False,
                    right=False,
                    labelleft=False)
    ax.set_ylabel("Shape categories", labelpad=60, fontsize=fontsize)
    ax.set_xlabel("Fraction of 'texture' decisions", fontsize=fontsize, labelpad=25)
    ax_top.set_xlabel("Fraction of 'shape' decisions", fontsize=fontsize, labelpad=25)
    ax.xaxis.set_major_formatter(FormatStrFormatter('%g'))
    ax_top.xaxis.set_major_formatter(FormatStrFormatter('%g'))
    ax.get_xaxis().set_ticks(np.arange(0, 1.1, 0.1))
    ax_top.set_ticks(np.arange(0, 1.1, 0.1))
    ax.tick_params(axis='both', which='major', labelsize=fontsize, length=ticklength)
    ax_top.tick_params(axis='both', which='major', labelsize=fontsize, length=ticklength)

    # arrows on x axes
    plt.arrow(x=0, y=-1.75, dx=1, dy=0, fc='black',
              head_width=0.4, head_length=0.03, clip_on=False,
              length_includes_head=True, overhang=0.5)
    plt.arrow(x=1, y=num_classes + 0.75, dx=-1, dy=0, fc='black',
              head_width=0.4, head_length=0.03, clip_on=False,
              length_includes_head=True, overhang=0.5)

    # icons besides y axis
    ## determine order of icons
    for dmaker in decision_maker_fun(df):
        if dmaker.plotting_name == order_by:
            df_selection = df.loc[(df["subj"].isin(dmaker.decision_makers))]
            class_avgs = []
            for cl in classes:
                df_class_selection = df_selection.query("category == '{}'".format(cl))
                class_avgs.append(1 - analysis.analysis(df=df_class_selection)['shape-bias'])
            sorted_indices = np.argsort(class_avgs)
            classes = classes[sorted_indices]
            break

    # icon placement is calculated in axis coordinates
    WIDTH = 1 / num_classes  #
    XPOS = -1.25 * WIDTH  # placement left of yaxis (-WIDTH) plus some spacing (-.25*WIDTH)
    YPOS = -0.5
    HEIGHT = 1
    MARGINX = 1 / 10 * WIDTH  # vertical whitespace between icons
    MARGINY = 1 / 10 * HEIGHT  # horizontal whitespace between icons

    left = XPOS + MARGINX
    right = XPOS + WIDTH - MARGINX

    for i in range(num_classes):
        bottom = i + MARGINY + YPOS
        top = (i + 1) - MARGINY + YPOS
        iconpath = pjoin(consts.ICONS_DIR, "{}.png".format(classes[i]))
        plt.imshow(plt.imread(iconpath), extent=[left, right, bottom, top], aspect='auto', clip_on=False)

    # plot horizontal intersection lines
    for i in range(num_classes - 1):
        plt.plot([0, 1], [i + .5, i + .5], c='gray', linestyle='dotted', alpha=0.4)

    # plot average shapebias + scatter points
    for dmaker in decision_maker_fun(df):
        df_selection = df.loc[(df["subj"].isin(dmaker.decision_makers))]
        result_df = analysis.analysis(df=df_selection)
        avg = 1 - result_df['shape-bias']
        ax.plot([avg, avg], [-1, num_classes], color=dmaker.color)
        class_avgs = []
        for cl in classes:
            df_class_selection = df_selection.query("category == '{}'".format(cl))
            class_avgs.append(1 - analysis.analysis(df=df_class_selection)['shape-bias'])

        ax.scatter(class_avgs, classes,
                   color=dmaker.color,
                   marker=dmaker.marker,
                   label=dmaker.plotting_name,
                   s=markersize,
                   clip_on=False,
                   edgecolors=PLOTTING_EDGE_COLOR,
                   linewidths=PLOTTING_EDGE_WIDTH,
                   zorder=3)

    figure_path = pjoin(result_dir, f"{ds.name}_shape-bias_matrixplot.pdf")
    fig.savefig(figure_path)
    plt.close()

def plot_shape_bias_boxplot(datasets,
                            decision_maker_fun,
                            result_dir,
                            analysis=a.ShapeBias(),
                            order_by='humans'):
    assert len(datasets) == 1
    ds = datasets[0]
    assert ds.name == "cue-conflict"

    log(plot_type="shape-bias-boxplot", dataset_name=ds.name)

    df = ph.get_experimental_data(ds)

    # plot setup
    fig = plt.figure(1, figsize=(15, 4), dpi=300.)
    ax = plt.gca()
    plt.xticks(rotation=90)
    ax.set_ylabel("shape bias", fontsize=12)

    decision_maker_to_shape_bias_dict = {}
    colors = []
    labels = []
    label_colors = []
    for dmaker in decision_maker_fun(df):
        if len(dmaker.decision_makers) > 1:
            decision_maker_to_shape_bias_humans_dict = {}
            for dmaker_human in dmaker.decision_makers:
                df_selection = df.loc[(df["subj"].isin([dmaker_human]))]
                class_avgs = df_selection.groupby(["category"]).apply(lambda x: analysis.analysis(df=x)["shape-bias"])
                decision_maker_to_shape_bias_humans_dict[dmaker_human] = class_avgs.tolist()
            df_results_humans = pd.DataFrame(decision_maker_to_shape_bias_humans_dict)
            df_results_humans["humans"] = df_results_humans.mean(axis=1)

        else:
            subject_name = dmaker.decision_makers[0]
            df_selection = df.loc[(df["subj"].isin(dmaker.decision_makers))]
            class_avgs = df_selection.groupby(["category"]).apply(lambda x: analysis.analysis(df=x)["shape-bias"])
            decision_maker_to_shape_bias_dict[subject_name] = class_avgs.tolist()
        colors.append(dmaker.color)
        if subject_name in consts.TORCHVISION_MODELS:
            label_colors.append(rgb(150, 150, 150))
        else:
            label_colors.append(dmaker.color)
        labels.append(dmaker.plotting_name)

    decision_maker_to_shape_bias_dict["humans"] = df_results_humans.humans.tolist()
    df_results = pd.DataFrame(decision_maker_to_shape_bias_dict)
    boxplot = ax.boxplot(df_results,
                         vert=True,  # vertical box alignment
                         patch_artist=True,  # fill with color
                         labels=labels,
                         showfliers=False
                         )
    for element, x_axis, color, label_color in zip(boxplot["boxes"], ax.xaxis.get_ticklabels(), colors, label_colors):
        element.set(color=color)
        x_axis.set_color(label_color)
    plt.subplots_adjust(bottom=0.55)
    figure_path = pjoin(result_dir, f"{ds.name}_shape-bias_boxplot.pdf")
    fig.savefig(figure_path)
    plt.close()

def plot_error_consistency(datasets, decision_maker_fun, result_dir,
                           analysis=a.ErrorConsistency()):
    plot_matrix(datasets=datasets, analysis=analysis,
                decision_maker_fun=decision_maker_fun,
                result_dir=result_dir, plot_type="error-consistency")

def plot_matrix(datasets, analysis,
                decision_maker_fun,
                result_dir, plot_type,
                sort_by_mean=False):
    """Plot a matrix of NxN analysis values.

    Adapted from:
    https://seaborn.pydata.org/examples/many_pairwise_correlations.html
    """

    for dataset in datasets:

        log(plot_type=plot_type, dataset_name=dataset.name)

        res = get_raw_matrix(dataset,
                             decision_maker_fun,
                             analysis)
        if sort_by_mean:
            #res = sort_matrix_by_models_mean(res)
            res = sort_matrix_by_subjects_mean(res)
            by_mean_str = "_by-mean"
        else:
            by_mean_str = ""

        colors = res["colors"]
        sns.set(font_scale=1.5)
        sns.set_style("whitegrid", {'axes.grid': False})
        f, ax = plt.subplots(figsize=(22, 18))
        cmap = sns.diverging_palette(230, 20, as_cmap=True)

        sns.heatmap(res["matrix"], ax=ax, mask=None, cmap=cmap, vmax=1.0, center=0,
                              square=True, linewidths=2.0, cbar_kws={"shrink": .5},
                              xticklabels=True, yticklabels=True)

        for i, tick_label in enumerate(ax.axes.get_yticklabels()):
            tick_label.set_color(colors[i])
        for i, tick_label in enumerate(ax.axes.get_xticklabels()):
            tick_label.set_color(colors[i])

        figure_path = pjoin(result_dir,
                            f"{dataset.name}_{analysis.plotting_name.replace(' ', '-')}_matrix{by_mean_str}.pdf")
        f.savefig(figure_path, bbox_inches='tight', pad_inches=0)
        f.clear()
    plt.cla()
    plt.clf()
    plt.close()
    sns.reset_defaults()
    sns.reset_orig()
    plt.style.use('default')

def sort_matrix_by_models_mean(result_dict):
    """Helper function: Given error consistency results, sort matrix.

    The matrix will be sorted in descending order
    according to mean agreement with models.
    """

    matrix = result_dict["matrix"]
    colors = result_dict["colors"]
    # link models to colors
    models_colors_df = pd.DataFrame.from_dict({"colors": colors, "models": matrix.index})
    models_colors_df.set_index(["models"], inplace=True)

    models_only = matrix.drop(columns=[column for column in matrix.columns if column.startswith("subject-")])
    models_only["mean"] = models_only.mean(axis=1)
    models_only = models_only.sort_values(by=["mean"], ascending=False)
    name_order = models_only.index.tolist()

    # sort columns
    matrix = matrix[name_order]
    # sort rows
    matrix = matrix.reindex(name_order)
    # sort colors
    models_colors_df = models_colors_df.reindex(name_order)

    return {"matrix": matrix,
            "colors": models_colors_df.colors.tolist()}

def sort_matrix_by_subjects_mean(result_dict):
    """Helper function: Given error consistency results, sort matrix.

    The matrix will be sorted in descending order
    according to mean agreement with subjects.
    """

    matrix = result_dict["matrix"]
    colors = result_dict["colors"]
    # link models to colors
    models_colors_df = pd.DataFrame.from_dict({"colors": colors, "models": matrix.index})
    models_colors_df.set_index(["models"], inplace=True)

    humans_only = matrix[[column for column in matrix.columns if column.startswith("subject-")]]
    humans_only["mean"] = humans_only.mean(axis=1)
    models_only = humans_only.sort_values(by=["mean"], ascending=False)
    name_order = models_only.index.tolist()

    # sort columns
    matrix = matrix[name_order]
    # sort rows
    matrix = matrix.reindex(name_order)
    # sort colors
    models_colors_df = models_colors_df.reindex(name_order)

    return {"matrix": matrix,
            "colors": models_colors_df.colors.tolist()}

def plot_confusion_matrix(datasets,
                          decision_maker_fun,
                          result_dir,
                          analysis=a.ConfusionAnalysis()):
    analysis = a.ConfusionAnalysis()
    for d in datasets:

        log(plot_type="confusion-matrix", dataset_name=d.name)

        df = ph.get_experimental_data(d)

        for e in d.experiments:
            for i, cond in enumerate(e.data_conditions):
                for dmaker in decision_maker_fun(df):
                    figure_path = pjoin(result_dir,
                                        f"{analysis.plotting_name.replace(' ', '-')}_{e.name}\
                                        _{dmaker.file_name}_\
                                        {e.plotting_conditions[i]}.pdf")
                    df_selection = df.loc[(df["subj"].isin(dmaker.decision_makers)) & \
                                          (df["condition"] == cond)]
                    result_df = analysis.analysis(df=df_selection)
                    confusion_matrix_helper(output_filename=figure_path,
                                            data=result_df)

def plot_accuracy(datasets, decision_maker_fun, result_dir):
    plot_general_analyses(datasets=datasets, analysis=a.SixteenClassAccuracy(),
                          decision_maker_fun=decision_maker_fun,
                          result_dir=result_dir, plot_type="accuracy")

def plot_entropy(datasets, decision_maker_fun, result_dir):
    plot_general_analyses(datasets=datasets, analysis=a.Entropy(),
                          decision_maker_fun=decision_maker_fun,
                          result_dir=result_dir, plot_type="entropy")

def plot_error_consistency_lineplot(datasets, decision_maker_fun, result_dir):
    plot_general_analyses(datasets=datasets, analysis=a.ErrorConsistency(),
                          decision_maker_fun=decision_maker_fun,
                          result_dir=result_dir, plot_type="error-consistency-lineplot")

def plot_general_analyses(datasets, analysis, decision_maker_fun,
                          result_dir, plot_type):
    for d in datasets:
        log(plot_type=plot_type, dataset_name=d.name)

        df = ph.get_experimental_data(d)

        for e in d.experiments:
            figure_path = pjoin(result_dir,
                                f"{e.name}_{analysis.plotting_name.replace(' ', '-')}.pdf")
            decision_makers = decision_maker_fun(df)
            result_df = analysis.get_result_df(df=df,
                                               decision_makers=decision_makers,
                                               experiment=e)

            x_y_plot(figure_path=figure_path,
                     result_df=result_df,
                     decision_makers=decision_makers,
                     analysis=analysis,
                     experiment=e)

def get_raw_benchmark_df(datasets, metric_names, decision_maker_fun,
                         include_humans=True):
    def _analyse_inner(df, model, human, all_humans, dataset, colname,
                       condition):

        analysis, metric_name = METRICS[colname]

        df1 = df.loc[(df["subj"] == model)]
        df2 = df.loc[(df["subj"] == human)]
        if condition is not None:
            df1 = df1.loc[(df1["condition"] == condition)]
            df2 = df2.loc[(df2["condition"] == condition)]

        res = analysis.analysis(df1, df2)[metric_name]

        row = {'model': model,
               'human': human,
               'dataset': dataset,
               'metric': colname,
               'condition': condition,
               'value': res}
        return row

    def _analyse_outer(group_1, group_2, metric_names, dataset,
                       df, result_df):

        rows_list = []
        is_same_group = group_1 == group_2

        for i, g1 in enumerate(group_1):
            # print(i, g1)
            for colname in metric_names:

                if len(dataset.experiments) == 0:
                    experiments = ["default-experiment"]
                else:
                    experiments = dataset.experiments

                for experiment in experiments:

                    if type(experiment) is str:
                        conditions = [None]
                        dataset_name = dataset.name
                    else:
                        conditions = experiment.data_conditions
                        dataset_name = experiment.name

                    for condition in conditions:
                        for j, g2 in enumerate(group_2):
                            if is_same_group and j <= i:  # avoid 2x evaluation
                                continue
                            elif g1 == g2:  # don't compare against yourself
                                continue
                            row = _analyse_inner(df=df, model=g1, human=g2,
                                                 all_humans=sorted(group_2),
                                                 dataset=dataset_name,
                                                 colname=colname,
                                                 condition=condition)
                            rows_list.append(row)
        result_df = result_df.append(rows_list, ignore_index=True)
        return result_df

    result_df = pd.DataFrame(columns=['model', 'human', 'dataset', 'metric',
                                      'condition', 'value'])
    for dataset_orig in tqdm(datasets):
        df = ph.get_experimental_data(dataset_orig)
        decision_makers = decision_maker_fun(df)

        humans, models = dm.get_human_and_model_decision_makers(decision_makers)
        assert len(humans) > 0, "no human data found"
        assert len(models) > 0, "no model data found"

        if EXCLUDE:
            dataset = exclude_conditions(dataset_orig)
        else:
            dataset = dataset_orig

        result_df = _analyse_outer(group_1=models,
                                   group_2=humans,
                                   metric_names=metric_names,
                                   dataset=dataset,
                                   df=df, result_df=result_df)

        if include_humans:
            result_df = _analyse_outer(group_1=humans,
                                       group_2=humans,
                                       metric_names=metric_names,
                                       dataset=dataset,
                                       df=df, result_df=result_df)
    return result_df

def print_benchmark_table_accuracy_to_latex(df):
    df = copy.deepcopy(df[df["plotting_name"] != "humans"])
    df['rank'] = df['OOD accuracy'].rank(ascending=False)
    df = df.sort_values(by="rank", ascending=True)
    df = df.reset_index(drop=True)

    df = df[["plotting_name", "OOD accuracy", "rank"]]
    df.rename(columns={"plotting_name": "model",
                       "OOD accuracy": "OOD accuracy $\\uparrow$",
                       "rank": "rank $\\downarrow$"}, inplace=True)
    df["model"] = df["model"].apply(lambda x: x.replace("_", "\_"))

    # formatting cells
    formatters = dict()
    cols_bold_mapping = {"OOD accuracy $\\uparrow$": max,
                         "rank $\\downarrow$": min}

    def format_numbers(y, num_digits=3):
        return ("{:." + str(num_digits) + "f}").format(y)

    for c, func in cols_bold_mapping.items():
        m = func(df[c])
        formatters[c] = lambda y, m=m: "\\textbf{" + format_numbers(y) + "}" if y == m else format_numbers(y)

    with open(pjoin(consts.REPORT_DIR, "assets/", "benchmark_table_accuracy.tex"), 'w') as f:
        print(df.to_latex(escape=False, formatters=formatters,
                          float_format="%.3f", index=False), file=f)

def print_benchmark_table_humanlike_to_latex(df):
    df = copy.deepcopy(df[df["plotting_name"] != "humans"])
    df['acc rank'] = df['accuracy difference'].rank()
    df['err rank'] = df['error consistency'].rank(ascending=False)
    df['obs rank'] = df['observed consistency'].rank(ascending=False)
    df['mean rank'] = (df['acc rank'] + df['err rank'] + df['obs rank']) / 3.0
    df = df.sort_values(by="mean rank", ascending=True)
    df = df.reset_index(drop=True)

    df = df[["plotting_name", "accuracy difference", "observed consistency",
             "error consistency", "mean rank"]]

    df.rename(columns={"plotting_name": "model",
                       "accuracy difference": "accuracy diff. $\\downarrow$",
                       "observed consistency": "obs. consistency $\\uparrow$",
                       "error consistency": "error consistency $\\uparrow$",
                       "mean rank": "mean rank $\\downarrow$"}, inplace=True)
    df["model"] = df["model"].apply(lambda x: x.replace("_", "\_"))

    # formatting cells
    formatters = dict()
    cols_bold_mapping = {"accuracy diff. $\\downarrow$": min,
                         "obs. consistency $\\uparrow$": max,
                         "error consistency $\\uparrow$": max,
                         "mean rank $\\downarrow$": min}

    def format_numbers(y, num_digits=3):
        return ("{:." + str(num_digits) + "f}").format(y)

    for c, func in cols_bold_mapping.items():
        m = func(df[c])
        formatters[c] = lambda y, m=m: "\\textbf{" + format_numbers(y) + "}" if y == m else format_numbers(y)

    with open(pjoin(consts.REPORT_DIR, "assets/", "benchmark_table_humanlike.tex"), 'w') as f:
        print(df.to_latex(escape=False, formatters=formatters,
                          float_format="%.3f", index=False), file=f)

def plot_benchmark_barplot(datasets, decision_maker_fun, result_dir,
                           print_to_latex=True, include_humans = True,
                           metrics_to_plot=METRICS.keys(),
                           single_dataset_name=None):

    # data frame formatting; printing humanlike benchmark table to LaTeX
    metric_names = ["accuracy difference",
                    "observed consistency",
                    "error consistency"]
    df = get_raw_benchmark_df(datasets=copy.deepcopy(datasets),
                              metric_names=metric_names,
                              decision_maker_fun=decision_maker_fun,
                              include_humans=include_humans)
    decision_makers = decision_maker_fun(ph.get_experimental_data(datasets[0]))
    df_formatted = format_benchmark_df(df=df,
                                       decision_makers=decision_makers,
                                       metric_names=metric_names,
                                       include_humans=include_humans)
    if print_to_latex:
        print_benchmark_table_humanlike_to_latex(df_formatted)

    # plotting
    for colname in metrics_to_plot:

        metric_fun, metric_name = METRICS[colname]
        logging_dataset = f"and dataset {single_dataset_name}" if single_dataset_name else ""
        logging_info = f"Plotting benchmark-barplot for metric {metric_name} {logging_dataset}"
        logger.info(logging_info)
        print(logging_info)

        if metric_fun.num_input_models == 1:
            assert metric_name == "16-class-accuracy"
            df1 = get_mean_over_datasets(colname=colname,
                                         metric_fun=metric_fun,
                                         metric_name=metric_name,
                                         datasets=copy.deepcopy(datasets),
                                         decision_maker_fun=decision_maker_fun)
            if print_to_latex:
                print_benchmark_table_accuracy_to_latex(df1)
        else:
            df1 = copy.deepcopy(df_formatted)
            df1["color"] = df1["model"].apply(lambda y: dm.decision_maker_to_attributes(y, decision_makers)["color"])

        df1 = df1.sort_values(by=colname, ascending=colname != "accuracy difference")
        df1 = df1.reset_index()
        values = df1[colname]
        names = df1["plotting_name"]
        colors = df1["color"]

        humans_add_string = "" if include_humans else "_no-humans"

        if single_dataset_name:
            dataset_add_string = f"{single_dataset_name}_" if single_dataset_name else ""
            barplot(path=pjoin(result_dir, f"{single_dataset_name}_{colname.replace(' ', '-')}{humans_add_string}.pdf"),
                    names=names, values=values, colors=colors, ylabel=colname)
        else:
            barplot(path=pjoin(result_dir, f"benchmark_{metric_name.replace(' ', '-')}{humans_add_string}.pdf"),
                    names=names, values=values, colors=colors, ylabel=colname)

def format_benchmark_df(df, decision_makers, metric_names,
                        include_humans):
    ### formatting & averaging
    df["model"] = df["model"].str.replace("_", "-")
    df["dataset"] = df["dataset"].str.replace("_", "-")

    if include_humans:
        # human analysis
        dfh = df.loc[(df["model"].str.startswith('subject-', na=False))]
        # average over conditions:
        dfh = dfh.groupby(['model', 'human', 'dataset', 'metric'], as_index=False)["value"].mean()
        # average over human subjects:
        dfh = dfh.groupby(['dataset', 'metric'], as_index=False)["value"].mean()
        # average over datasets:
        dfh = dfh.groupby(['metric'], as_index=False)["value"].mean()

    # CNN-to-human analysis
    df = df.loc[(~df["model"].str.startswith('subject-', na=False))]
    # average over conditions:
    df = df.groupby(['model', 'human', 'dataset', 'metric'], as_index=False)["value"].mean()
    # average over human subjects:
    df = df.groupby(['model', 'dataset', 'metric'], as_index=False)["value"].mean()
    # average over datasets:
    df = df.groupby(['model', 'metric'], as_index=False)["value"].mean()
    # reshape to make metrics -> columns
    df = df.pivot(index='model', columns='metric', values='value').reset_index()

    if include_humans:
        human_dict = {"model": "humans"}
        for colname in metric_names:
            analysis, metric_name = METRICS[colname]
            scalar = dfh.loc[dfh["metric"] == colname]["value"].values[0]
            human_dict[colname] = scalar
        df = df.append(human_dict, ignore_index=True)

    # replace model name with the model's plotting name
    df["plotting_name"] = df["model"].apply(
        lambda y: dm.decision_maker_to_attributes(y, decision_maker_list=decision_makers)["plotting_name"])

    return df

def barplot(path, names, values, colors, ylabel=None,
            figsize=(10, 6), ylim=None):
    fig, ax = plt.subplots(figsize=figsize)
    for s in ["right", "top"]:  # , "bottom"]:
        ax.spines[s].set_visible(False)
    ax.xaxis.set_ticks_position('none')
    ax.xaxis.set_ticklabels([])
    if ylim is None:
        limit = np.ceil(max(values) * 10) / 10.0
        ylim = (min(0, min(values)),
                max(0, limit))
        if "benchmark_16-class-accuracy-difference" in path:
            ylim = (0, 0.15)
    plt.ylim(ylim)
    plt.ylabel(ylabel, fontsize=17)

    bar1 = plt.bar(np.arange(len(names)), values,
                   color=colors, align="center")

    for i, rect in enumerate(bar1):
        height = rect.get_height()
        width = rect.get_width()

        plot_arrow_for_models_trained_on_large_datasets = False
        if "M)" in names[i] and plot_arrow_for_models_trained_on_large_datasets:
            plt.plot(rect.get_x() + rect.get_width() / 2.0 - 0.1, rect.get_height() + 0.02 * limit,
                     marker="$\u2193$", linestyle="", color="r")
        plt.text(rect.get_x() + rect.get_width() / 2.0, 0.01 * limit,
                 names[i], ha='center',
                 va='bottom', rotation=90)

    plt.xlim([-0.8, len(names) - 1 + 0.5 * width])
    plt.savefig(path)
    plt.close()

def plot_scatterplot(datasets,
                     decision_maker_fun, result_dir,
                     metric_x=None, metric_y=None):
    """Plot scatter plot for metric_x vs. metric_y"""

    if (metric_x and metric_y is None) or (metric_x is None and metric_y):
        raise ValueError("Please specify either both or none of the metrics.")

    if metric_x and metric_y:
        assert metric_x in METRICS.keys()
        assert metric_y in METRICS.keys()

    colname = "OOD accuracy"
    analysis, metric_name = METRICS[colname]
    df_acc = get_mean_over_datasets(colname=colname,
                                    metric_fun=a.SixteenClassAccuracy(),
                                    metric_name=metric_name,
                                    datasets=copy.deepcopy(datasets),
                                    decision_maker_fun=decision_maker_fun)

    metric_names = ["accuracy difference",
                    "observed consistency",
                    "error consistency"]
    df_raw = get_raw_benchmark_df(datasets=copy.deepcopy(datasets),
                                  metric_names=metric_names,
                                  decision_maker_fun=decision_maker_fun,
                                  include_humans=True)
    decision_makers = decision_maker_fun(ph.get_experimental_data(datasets[0]))
    df_ec = format_benchmark_df(df=df_raw,
                                decision_makers=decision_makers,
                                metric_names=metric_names,
                                include_humans=True)

    df = df_acc.merge(df_ec, on="plotting_name", how="inner")
    df["marker"] = df["model"].apply(lambda y: mmarkers.MarkerStyle(
        dm.decision_maker_to_attributes(y, decision_maker_list=decision_makers)["marker"]))

    def scatterplot_log(metric_x, metric_y):
        logging_info = f"Plotting scatterplot for {metric_x} vs. {metric_y}"
        logger.info(logging_info)
        print(logging_info)

    if len(datasets) == 1:
        dataset_name = datasets[0].name
    else:
        dataset_name = "multiple-datasets"

    if metric_x and metric_y:
        scatter_plot_helper(df, metric_x, metric_y, result_dir, dataset_name)
        scatterplot_log(metric_x, metric_y)
    else:
        perm = get_permutations(METRICS.keys())
        for metric_x, metric_y in perm:
            scatter_plot_helper(df, metric_x, metric_y, result_dir, dataset_name)
            scatterplot_log(metric_x, metric_y)

def scatter_plot_helper(df, metric_x, metric_y, result_dir, dataset_name):
    fig, ax = plt.subplots(figsize=(5, 5))
    for s in ["right", "top"]:
        ax.spines[s].set_visible(False)

    for _, row in df.iterrows():
        plt.plot(row[metric_x], row[metric_y],
                 color=row["color"], marker=row["marker"],
                 markeredgecolor=PLOTTING_EDGE_COLOR,
                 markeredgewidth=PLOTTING_EDGE_WIDTH)

    # plt.xlim(left=min(0, row[metric_x]))
    # plt.xlim(left=min(df[metric_x])
    if metric_y == "error consistency" and metric_x == "OOD accuracy":
        bottom_min_val = -0.03
    else:
        bottom_min_val = 0
    plt.ylim(bottom=min(bottom_min_val, min(df[metric_y])),
             top=np.ceil(max(df[metric_y]) * 10) / 10.0)
    plt.xlabel(metric_x)
    plt.ylabel(metric_y)

    # plot dashed line
    xlim = ax.get_xlim()
    h_acc = df.loc[(df["model"] == "humans")]["OOD accuracy"].values[0]
    l1 = np.linspace(start=xlim[0], stop=xlim[1], num=100)
    l2 = np.linspace(start=xlim[0], stop=xlim[1], num=100)
    x_text = xlim[0] + (xlim[1] - xlim[0]) / 2.0
    if metric_y == "observed consistency" and metric_x == "OOD accuracy":
        for i, _ in enumerate(l1):
            l2[i] = l1[i] * h_acc + (1 - l1[i]) * (1 - h_acc)
        plt.plot(l1, l2, linestyle='dashed', color="gray", linewidth=0.8)
        y_text = x_text * h_acc + (1 - x_text) * (1 - h_acc) - 0.03
        plt.text(x=x_text, y=y_text, s="expected", color="gray", ha='center')
    elif metric_y == "error consistency" and metric_x == "OOD accuracy":
        for i, _ in enumerate(l1):
            l2[i] = 0
        plt.plot(l1, l2, linestyle='dashed', color="gray", linewidth=0.8)
        plt.text(x=x_text, y=0.01, s="expected", color="gray", ha='center')

    plt.savefig(pjoin(result_dir,
                      f"scatter-plot_{metric_x.replace(' ', '-')}_vs_{metric_y.replace(' ', '-')}_{dataset_name}.pdf"))
    plt.close()


from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import time
import json
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
import re
# Set up the Selenium WebDriver
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service)

main_page_url = 'https://www.immobilier.ch/fr/louer/appartement-maison/geneve/geneve/page-1?t=rent&c=1;2&p=c10377&pn=8000&nb=false'

# Navigate to the main page
driver.get(main_page_url)

# Find an apartment listing in the "Location" category and click on it
apartment_links = driver.find_elements_by_css_selector('a[category="Location"]')
if apartment_links:
    apartment_links[0].click()  # Click on the first apartment in the list
    time.sleep(5)  # Wait for the apartment page to load
# URL of the individual apartment listing
url = 'https://www.immobilier.ch/fr/louer/appartement/geneve/geneve/nsr-group-1607/magnifique-appartement-vue-parc-bertrand-999481'

# Open the URL
driver.get(url)

# Wait for the dynamic content to load
time.sleep(5)  # Adjust this time based on your internet speed

# Get the page source and parse it
soup = BeautifulSoup(driver.page_source, 'html.parser')

# Extract details
address_elements = driver.find_elements(By.CLASS_NAME, 'object-address')
price_element = driver.find_element(By.CLASS_NAME, 'im__postDetails__price')
price_text = price_element.text
price_numeric = int(''.join(filter(str.isdigit, price_text)))

addresses = [element.text for element in address_elements]

match = re.search(r'(\d+)$', url)
if match:
    apartment_id = match.group(1)
else:
    apartment_id = None
apartment_id = apartment_id

# Construct the XPath for the "Read More" button using the extracted apartment ID
button_xpath = f'//a[contains(@href, "javascript:sDetail.readMore({apartment_id})")]'

# Wait for the "Read More" button to become clickable
try:
    button = WebDriverWait(driver, 10).until(
        EC.element_to_be_clickable((By.XPATH, button_xpath))
    )
    button.click()

    # Explicit wait for the table to load
    # You can adjust the timeout as needed

    # Find the specific list items using the refined CSS selector
    elements = WebDriverWait(driver, 10).until(
        EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".im__table.im__table--responsive.im__row .im__table__row.js-last-col .im__assets__title.im__assets__title--big"))
    )
except NoSuchElementException as e:
    print(f"Element not found: {e}")

# Explicit wait for the table to load
# You can adjust the timeout as needed

# Find the specific list items using the refined CSS selector
elements = driver.find_elements(By.CSS_SELECTOR, ".im__table.im__table--responsive.im__row .im__table__row.js-last-col .im__assets__title.im__assets__title--big")

# Get the textual content of the element without HTML tags
description_element = driver.find_element(By.CLASS_NAME, 'im__postContent__body')
description_text = description_element.text

# Create the apartment_data dictionary

# Convert the dictionary to a JSON string

# Print the JSON data

image_elements = driver.find_elements(By.CSS_SELECTOR, '.im__banner__slide img')

# URL to exclude
exclude_url = "https://www.immobilier.ch/Images/Loading/Detail.jpg?v=2"

# Extract URLs, ensuring uniqueness and excluding the specific URL
image_urls_set = set()
for img in image_elements:
    # Check if 'data-lazy' attribute exists
    lazy_load_url = img.get_attribute('data-lazy')
    if lazy_load_url:
        if lazy_load_url != exclude_url and lazy_load_url not in image_urls_set:
            image_urls_set.add(lazy_load_url)
    else:
        src_url = img.get_attribute('src')
        if src_url and src_url != exclude_url and src_url not in image_urls_set:
            image_urls_set.add(src_url)

# Convert the set to a list for further processing if necessary
image_urls = list(image_urls_set)

# Print or process the image URLs
apartment_details = {
    'price': price_numeric,
    'addresses': addresses,
    'description': description_text,
    'elements': [element.text for element in elements],
    'url': image_urls
}

# Nesting the apartment_details under 'apartment'
apartment_data = {'apartment': apartment_details}

# Convert the dictionary to a JSON string
json_data = json.dumps(apartment_data, indent=4)

# Print the JSON data
print(json_data)

# Write JSON data to a file named 'estate.json'
with open('estate.json', 'w', encoding='utf-8') as file:
    file.write(json_data)

print("JSON data has been saved to estate.json")

# Close the browser
driver.quit()


#-*- coding:utf-8 -*-
import json, urllib2

API_KEY    = os.environ["REKOGNITION_API_KEY"]
API_SECRET = os.environ["REKOGNITION_API_SECRET"]

jobs = 'face_age_beauty'
pic_url = 'http://pic.prepics-cdn.com/smapphoto/12756846.jpeg'
url = 'http://rekognition.com/func/api/?api_key=' + API_KEY + '&api_secret=' + API_SECRET + '&jobs=' + jobs + '&urls=' + pic_url

r = urllib2.urlopen(url)
root = json.loads(r.read())

print(root)
print(root[u'face_detection'][0][u'age'])
print(root[u'face_detection'][0][u'beauty'])


import sys
input = lambda: sys.stdin.readline().rstrip()

def resolve():
    S = input()

    cnt = [0]*2019
    cnt[0] = 1

    # Sの右側から1ずつ左に伸ばして、cntの対応するところに加算
    rmd = 0
    num_rate = 1
    L = len(S)
    for i in range(L):
        rmd = (int(S[L-i-1])*num_rate + rmd) % 2019
        num_rate = num_rate*10%2019
        cnt[rmd] += 1

    ans = sum([i*(i-1)//2 for i in cnt])
    print(ans)

if __name__ == '__main__':
    resolve()