# -*- coding: utf-8 -*-
"""
Created on Tue Jul 25 19:09:30 2017

@author: Achin Jain (achinj@seas.upenn.edu)

"""

from __future__ import division

import numpy as np
from utils import load_data
import matplotlib.pyplot as plt
from plot_data import plot_experiment_design
import cPickle as pickle

building_type = 'LargeHotel'
n_samples = 2000

data_type = 'unconstrained'

load_model = False
#model_type = 'constrained'
model_type = 'unconstrained'
#model_type = 'rulebased'

# load model
if load_model:
    filename = ('../models/gp-%s-%s-numsamples-%s.pickle' %(model_type, building_type, n_samples))
    with open(filename, 'rb') as f:
        (model, buildingType, features_c, features_d, output,
         normalizer_c, normalizer_d, normalizer_y) = pickle.load(f)

Xd_test, Xc_test, y_test = load_data(data_type, building_type, features_d, features_c, output)

# select a sample
idx = 3005

Xd_test = Xd_test.iloc[idx,:]
Xc_test = Xc_test.iloc[idx,:].as_matrix()
print(Xc_test)
y_test = y_test.iloc[idx,:].as_matrix()
Xd_test = normalizer_d.transform(Xd_test.reshape(1,-1))
#Xc_test = normalizer_c.transform(Xc_test.reshape(1,-1))

# vary control variables from min to max
ClgSP = np.linspace(22,32,50).reshape(-1,1)
KitchenClgSP = np.linspace(24,32,50).reshape(-1,1)
GuestClgSP = np.linspace(22,26,50).reshape(-1,1)
SupplyAirSP = np.linspace(12,14,50).reshape(-1,1)
ChwSP = np.linspace(3.7,9.7,50).reshape(-1,1)

#Xc_design = np.concatenate((ClgSP,KitchenClgSP,GuestClgSP,SupplyAirSP,ChwSP), axis=1)
Xc_design = np.concatenate((ClgSP, np.tile(Xc_test[1:], (50,1))), axis=1)
Xc_design = normalizer_c.transform(Xc_design)

Xd_design = np.tile(Xd_test, (50,1))

# predict using GP
X_design = np.concatenate((Xd_design, Xc_design), axis=1)
y_mean, y_std = model.predict(X_design, return_std=True)
y_mean = normalizer_y.inverse_transform(y_mean.reshape(-1, 1))
y_mean = y_mean.flatten()
y_std =y_std*(normalizer_y.data_max_-normalizer_y.data_min_)/2

plt.close('all')
plot_experiment_design(ClgSP.flatten(), y_mean, y_std, Xc_test[0], y_test[0])
#plot_experiment_design(KitchenClgSP.flatten(), y_mean, y_std)
#plot_experiment_design(GuestClgSP.flatten(), y_mean, y_std)
#plot_experiment_design(SupplyAirSP.flatten(), y_mean, y_std)
#plot_experiment_design(ChwSP.flatten(), y_mean, y_std)


# Create your views here.
from django.shortcuts import render, get_object_or_404
from django.views.generic import ListView, DetailView
from .models import Company, Comment, Recommendations
from .forms import CommentForm, RecommendForm

def company_detail(request, slug):
    template_name = 'company_detail.html'
    company = get_object_or_404(Company, slug=slug)
    comments = company.comments.filter(active=True)
    new_comment = None
    # Comment posted
    if request.method == 'POST':
        comment_form = CommentForm(data=request.POST)
        # if comment_form.is_valid(): this can be used to add CSRF token validation

        # Create Comment object but don't save to database yet
        new_comment = comment_form.save(commit=False)
            # Assign the current post to the comment
        new_comment.company = company
            # Save the comment to the database
        new_comment.save()
    else:
        comment_form = CommentForm()

    return render(request, template_name, {'company': company,
                                           'comments': comments,
                                           'new_comment': new_comment,
                                           'comment_form': comment_form})

class companyListView(ListView):
    model = Company
    template_name = 'company_list.html'

class companyDetailView(DetailView):
    model = Company
    template_name = 'company_detail.html'

def recommend_detail(request):
    template_name = 'index.html'
    new_recommend = None
    if request.method == 'POST':
        recommendation_form = RecommendForm(data=request.POST)
        new_recommend = recommendation_form.save(commit=False)
        new_recommend.save()
    else:
        recommendation_form = RecommendForm()
    return render(request, template_name, {'new_recommendation': new_recommend,
                                           'recommendation_form': recommendation_form})


# -*- coding: utf-8 -*-
import PIL.Image as Image
import io
import subprocess
import sys
import os
import tempfile
tesseract_cmd = 'tesseract-bin\\tesseract'

class TesseractError(Exception):
    def __init__(self, status, message):
        self.status = status
        self.message = message
        self.args = (status, message)

def run_tesseract(input_filename, output_filename_base, tess_lang=None, boxes=False):
    command = [tesseract_cmd, input_filename, output_filename_base]
    if tess_lang is not None:
        command += ['-l', tess_lang]
    if boxes:
        command += ['batch.nochop', 'makebox']
    proc = subprocess.Popen(command, stderr=subprocess.PIPE)
    return proc.wait(), proc.stderr.read()

def cleanup(temp_filename):
    try:
        os.remove(temp_filename)
    except OSError:
        pass

def get_errors(error_string):
    lines = error_string.splitlines()
    error_lines = tuple(line for line in lines if line.find('Error') >= 0)
    if len(error_lines) > 0:
        return '\n'.join(error_lines)
    else:
        return error_string.strip()

def tempnam():
    stderr = sys.stderr
    try:
        sys.stderr = io.StringIO()
        tmpfile = tempfile.TemporaryFile(prefix='tess_')
        return tmpfile.name
    finally:
        sys.stderr = stderr

def image_to_string(image_inp, tess_lang=None, boxes=False):
    input_file_name = '%s.bmp' % tempnam()
    output_file_name_base = tempnam()
    if not boxes:
        output_file_name = '%s.txt' % output_file_name_base
    else:
        output_file_name = '%s.box' % output_file_name_base
    try:
        image_inp.save(input_file_name)
        status, error_string = run_tesseract(input_file_name,
                                             output_file_name_base,
                                             tess_lang=tess_lang,
                                             boxes=boxes)
        if status:
            errors = get_errors(error_string)
            raise TesseractError(status, errors)
        f = open(output_file_name, 'r')
        try:
            return f.read().strip()
        finally:
            f.close()
    finally:
        cleanup(input_file_name)
        cleanup(output_file_name)

if __name__ == '__main__':
    if len(sys.argv) == 2:
        filename = sys.argv[1]
        try:
            image = Image.open(filename)
        except IOError:
            raise Exception('ERROR: Could not open file "%s"\n' % filename)
        print(image_to_string(image))
    elif len(sys.argv) == 4 and sys.argv[1] == '-l':
        lang = sys.argv[2]
        filename = sys.argv[3]
        try:
            image = Image.open(filename)
        except IOError:
            raise Exception('ERROR: Could not open file "%s"\n' % filename)
        print(image_to_string(image, tess_lang=lang))
    else:
        sys.stderr.write('Usage: python tesseract.py [-l language] input_file\n')
        exit(2)


from flask import Flask, request, render_template, send_from_directory
from protolib.python import document_pb2, rpc_pb2
from utils.rpc.grpcapi import GrpcInterface
from utils.helper import DocHelper
import json

app = Flask(__name__, static_url_path='/brat', static_folder='brat')

def process_one_document(request):
    interface = GrpcInterface(host='128.4.20.169')
    # interface = grpcapi.GrpcInterface(host='localhost')
    response = interface.process_document(request)
    assert len(response.document) == 1
    return response.document[0]

def parse_using_bllip(doc):
    request = rpc_pb2.Request()
    request.request_type = rpc_pb2.Request.PARSE_BLLIP
    request.document.extend([doc])
    return process_one_document(request)

def parse_using_stanford(doc):
    request = rpc_pb2.Request()
    request.request_type = rpc_pb2.Request.PARSE_STANFORD
    request.document.extend([doc])
    return process_one_document(request)

def split_sentence_using_stanford(doc):
    request = rpc_pb2.Request()
    request.request_type = rpc_pb2.Request.SPLIT
    request.document.extend([doc])
    return process_one_document(request)

def get_brat_data(parsed_doc):
    helper = DocHelper(parsed_doc)
    # Convert the dependency parse to json, which can be visualized by brat.
    # Support multiple sentences.
    brat_sentences = {}
    for sentence in parsed_doc.sentence:
        brat_data = helper.dependency_for_brat(sentence)
        brat_sentences[sentence.index] = brat_data
    return brat_sentences

@app.route('/brat/<path:path>')
def send_js(path):
    return send_from_directory('brat', path)

@app.route("/", methods=['GET', 'POST'])
def parse():
    if request.method == 'POST':
        text = request.form['text']
        split = request.form.getlist('split')

        doc = document_pb2.Document()
        doc.text = text

        parse_doc_bllip_brat = ''
        parse_doc_stanford_brat = ''
        split_doc_brat = ''
        cst_parses = ''

        if len(split) == 0:
            parse_doc_bllip = parse_using_bllip(doc)
            parse_doc_stanford = parse_using_stanford(doc)
            parse_doc_bllip_brat = json.dumps(get_brat_data(parse_doc_bllip))
            parse_doc_stanford_brat = json.dumps(get_brat_data(parse_doc_stanford))
            parses = {}
            for sentence in parse_doc_bllip.sentence:
                parses[sentence.index] = sentence.parse
            cst_parses = json.dumps(parses)
        else:
            split_doc = split_sentence_using_stanford(doc)
            split_doc_brat = json.dumps(get_brat_data(split_doc))

        return render_template('index.html', text=text,
                               parse_bllip=parse_doc_bllip_brat,
                               parse_stanford=parse_doc_stanford_brat,
                               split_stanford=split_doc_brat,
                               bllip_cst_parses=cst_parses)
    else:
        return render_template('index.html')

@app.route('/highlight', methods=['GET', 'POST'])
def highlight():
    if request.method == 'POST':
        text = request.form['text'].strip()
        text = text.replace('<Gene>', '')
        text = text.replace('</Gene>', '')
        text = text.replace('<MiRNA>', '')
        text = text.replace('</MiRNA>', '')
        text = text.replace('<Subcellular_location>', '')
        text = text.replace('</Subcellular_location>', '')

        lines = text.split('\n')
        return render_template('highlight.html', lines=lines)
    else:
        return render_template('highlight.html')

if __name__ == "__main__":
    app.debug = False
    app.run(host='0.0.0.0')


# Date 2018-03-12
# Author VanLiuZhi

from django.db import models
from base.fields import verbose_name as _
from django.utils import timezone
from django.utils.functional import cached_property
import datetime

class BaseModel(models.Model):
    ordering = models.IntegerField(_('排序权值'), default=0, db_index=True, editable=False)
    created = models.DateTimeField(_('创建时间'), auto_now_add=True)
    date = models.DateField(_('创建日期'), default=timezone.now)  # titimezone 是系统设置时区
    updated = models.DateTimeField(_('修改时间'), auto_now=True)

    class Meta:
        abstract = True
        ordering = ['-ordering', '-id']

    @cached_property
    def base_fields(self):
        """
        基本字段list（BaseModel的字段 + id）
        :return:
        """
        return [item.attname for item in BaseModel()._meta.concrete_fields]

    def get_fields(self, not_add_fields=True):
        """
        返回模型字典名称的list（不会返回id和基本字段，需要则手动添加）
        :param not_add_fields: 决定是否添加模型的add_fields属性里面的字段
        :return:
        """
        fields = [item.attname for item in self._meta.concrete_fields if item.attname not in self.base_fields]
        if getattr(self, 'add_fields', None) and not not_add_fields:
            add_fields = getattr(self, 'add_fields', [])
            fields += add_fields
        return fields

    def get_dict(self, not_add_fields=True, other_fields=[]):
        """
        返回模型实例的字典
        :return:
        """
        _dict = {}
        _field = self.get_fields(not_add_fields)
        _field += other_fields
        for item in _field:
            _dict[item] = getattr(self, item, '')
        return _dict


# -*-encoding:utf-8 -*-

import os
import flask
import simplejson
from bson.objectid import ObjectId
from bson.json_util import dumps
from flask_breadcrumbs import register_breadcrumb, default_breadcrumb_root
from flask_login import current_user
from flask import (
    current_app,
    flash,
    request,
    render_template,
    redirect,
    send_file,
    url_for,
    g,
    jsonify,
)

from flask_babel import lazy_gettext, gettext
from ercc.ercc_bps.erc.models import Entity, SubjectArea, Model, EntityWithPyMongo
# from ercc.ercc_bps.project.models import Project
from ercc.ercc_bps.glossary.models import GlossaryBase
from ercc.ercc_bps.report.forms import ModelScriptForm, ModelValidateForm
from ercc.ercc_bps.report.models import (
    ATTR_REPORT_COLUMNS,
    # PATTR_REPORT_COLUMNS,
    ENTITY_REPORT_COLUMNS,
    MODEL_VALIDATE_RESULT_COLUMNS,
)
from ercc.ercc_bps.report import bpapp
from ercc.utils.excel import ExcelHelper
from ercc.utils.portlet import Portlet

default_breadcrumb_root(bpapp, '.')

@bpapp.route('/p/<mbj:project_id>/model_integrity', methods=['GET'])
@register_breadcrumb(bpapp, '.model_integrity', lazy_gettext(u'모델 레포트'))
def model_integrity(project_id):
    return render_template(
        'project/portlets.htm.j2',
        active_page='integrity',
        portlets=[
            Portlet('integrity._model_summary_report', project_id=project_id),
            # Portlet('._model_glossary_report', project_id=project_id),
            # Portlet('integrity._model_schema_report', project_id=project_id),
        ]
    )

@bpapp.route('/p/<mbj:project_id>/model_report', methods=['GET'])
@register_breadcrumb(bpapp, '.model_report', lazy_gettext(u'모델 레포트'))
def model_report(project_id):
    return render_template(
        'project/portlets.htm.j2',
        active_page='report',
        portlets=[
            # Portlet('._model_glossary_report', project_id=project_id),
            Portlet('._model_report_info', project_id=project_id),
        ]
    )

@bpapp.route('/p/<mbj:project_id>/_model_report_info', methods=['GET'])
def _model_report_info(project_id):
    ms_form = ModelScriptForm()

    ms_form.model.label_attr = 'name'
    ms_form.model.queryset = Model.head_objects(
        prjId=project_id).order_by('name')
    ms_form.model.allow_blank = True  # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.model.blank_text = lazy_gettext(u'--선택--')

    ms_form.subjectarea.label_attr = 'escaped_name'
    ms_form.subjectarea.queryset = SubjectArea.head_objects(
        prjId=project_id).order_by('name')
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.subjectarea.allow_blank = True
    ms_form.subjectarea.blank_text = lazy_gettext(u'--선택--')

    mv_form = ModelValidateForm()

    mv_form.glossary.label_attr = 'glossary_name'
    mv_form.glossary.queryset = GlossaryBase.head_objects(project=g.project)\
        .order_by('glossary_name')
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    mv_form.glossary.allow_blank = True
    mv_form.glossary.blank_text = lazy_gettext(u'--전체--')

    mv_form.model.label_attr = 'name'
    mv_form.model.queryset = Model.head_objects(
        prjId=project_id).order_by('name')
    mv_form.model.allow_blank = True  # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    mv_form.model.blank_text = lazy_gettext(u'--선택--')

    mv_form.subjectarea.label_attr = 'escaped_name'
    mv_form.subjectarea.queryset = SubjectArea.head_objects(
        prjId=project_id).order_by('name')
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    mv_form.subjectarea.allow_blank = True
    mv_form.subjectarea.blank_text = lazy_gettext(u'--선택--')

    return g.project.project_group.render_template(
        'report/_model_report_info.htm.j2',
        project=g.project,
        ms_form=ms_form,
        mv_form=mv_form,)

@bpapp.route('/p/<mbj:project_id>/_model_report', methods=['GET'])
def _model_glossary_report(project_id):
    return render_template(
        'report/_model_glossary_report.htm.j2',
        project=g.project)

@bpapp.route('/p/<mbj:project_id>/_model_report_run', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/_model_report_run/<mbj:model_id>', methods=['GET', 'POST'])
def _model_glossary_report_run(project_id, model_id=None):
    from ercc.models import Model, Project

    project = g.project
    if not project:
        project = Project.objects.get(id=project_id)
    if model_id:
        model = Model.objects.get_or_404(id=model_id)
        model.inspect_glossary(delay=current_app.config['CELERY_USE'])
        return jsonify(ret=True)
    else:
        for model in project.queryset_model:
            model.inspect_glossary(delay=current_app.config['CELERY_USE'])
        return jsonify(ret=True)

@bpapp.route('/p/<mbj:project_id>/model_entities', methods=['GET'])
@register_breadcrumb(bpapp, '.model_report.model_entities', lazy_gettext(u'엔티티 조회'))
def model_entities(project_id):
    return render_template(
        'project/portlets.htm.j2',
        active_page='report',
        portlets=[
            Portlet(
                '._model_report_info',
                project_id=project_id
            ),
            Portlet(
                '._model_entities',
                project_id=project_id
            ),
        ]
    )

@bpapp.route('/p/<mbj:project_id>/model_attributes', methods=['GET'])
@register_breadcrumb(bpapp, '.model_report.model_attributes', lazy_gettext(u'속성 조회'))
def model_attributes(project_id):
    return render_template(
        'project/portlets.htm.j2',
        active_page='report',
        portlets=[
            Portlet(
                '._model_report_info',
                project_id=project_id
            ),
            Portlet(
                '._model_attributes',
                project_id=project_id
            ),
        ]
    )

@bpapp.route('/p/<mbj:project_id>/model_scripts', methods=['GET'])
@register_breadcrumb(bpapp, '.model_report.model_scripts', lazy_gettext(u'스크립트'))
def model_scripts(project_id):
    return render_template(
        'project/portlets.htm.j2',
        active_page='report',
        portlets=[
            Portlet(
                '._model_report_info',
                project_id=project_id
            ),
            Portlet(
                '._model_scripts',
                project_id=project_id
            ),
        ]
    )

@bpapp.route('/p/<mbj:project_id>/model_validate_with_glossary', methods=['GET'])
@register_breadcrumb(bpapp, '.model_report.model_validate_with_glossary', lazy_gettext(u'표준점검'))
def model_validate_with_glossary(project_id):
    return render_template(
        'project/portlets.htm.j2',
        active_page='report',
        portlets=[
            Portlet(
                '._model_report_info',
                project_id=project_id
            ),
            Portlet(
                '._model_validate_with_glossary',
                project_id=project_id
            ),
        ]
    )

@bpapp.route('/p/<mbj:project_id>/_model_entities', methods=['GET'])
def _model_entities(project_id):

    ms_form = ModelScriptForm()

    ms_form.model.label_attr = 'name'
    ms_form.model.queryset = Model.head_objects(
        prjId=project_id).order_by('name')
    ms_form.model.allow_blank = True  # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.model.blank_text = lazy_gettext(u'--선택--')

    ms_form.subjectarea.label_attr = 'escaped_name'
    ms_form.subjectarea.queryset = SubjectArea.head_objects(
        prjId=project_id).order_by('name')
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.subjectarea.allow_blank = True
    ms_form.subjectarea.blank_text = lazy_gettext(u'--선택--')

    return render_template(
        'report/_model_entities.htm.j2',
        project=g.project,
        ms_form=ms_form)

@bpapp.route('/p/<mbj:project_id>/_model_attributes', methods=['GET'])
def _model_attributes(project_id):

    ms_form = ModelScriptForm()

    ms_form.model.label_attr = 'name'
    ms_form.model.queryset = Model.head_objects(
        prjId=project_id).order_by('name')
    ms_form.model.allow_blank = True  # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.model.blank_text = lazy_gettext(u'--선택--')

    ms_form.subjectarea.label_attr = 'escaped_name'
    ms_form.subjectarea.queryset = SubjectArea.head_objects(
        prjId=project_id).order_by('name')
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.subjectarea.allow_blank = True
    ms_form.subjectarea.blank_text = lazy_gettext(u'--선택--')

    return render_template(
        'report/_model_attributes.htm.j2',
        project=g.project,
        ms_form=ms_form)

@bpapp.route('/p/<mbj:project_id>/_model_scripts', methods=['GET'])
def _model_scripts(project_id):

    ms_form = ModelScriptForm()

    ms_form.model.label_attr = 'name'
    ms_form.model.queryset = Model.head_objects(
        prjId=project_id).order_by('name')
    ms_form.model.allow_blank = True  # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.model.blank_text = lazy_gettext(u'--선택--')

    ms_form.subjectarea.label_attr = 'escaped_name'
    ms_form.subjectarea.queryset = SubjectArea.head_objects(
        prjId=project_id).order_by('name')
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.subjectarea.allow_blank = True
    ms_form.subjectarea.blank_text = lazy_gettext(u'--선택--')

    return render_template(
        'report/_model_scripts.htm.j2',
        project=g.project,
        ms_form=ms_form)

@bpapp.route('/p/<mbj:project_id>/_model_validate_with_glossary', methods=['GET'])
def _model_validate_with_glossary(project_id):

    mv_form = ModelValidateForm()

    mv_form.glossary.label_attr = 'glossary_name'
    mv_form.glossary.queryset = GlossaryBase.head_objects(project=g.project)\
        .order_by('glossary_name')
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    mv_form.glossary.allow_blank = True
    mv_form.glossary.blank_text = lazy_gettext(u'--전체--')

    mv_form.model.label_attr = 'name'
    mv_form.model.queryset = Model.head_objects(
        prjId=project_id).order_by('name')
    mv_form.model.allow_blank = True  # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    mv_form.model.blank_text = lazy_gettext(u'--선택--')

    mv_form.subjectarea.label_attr = 'escaped_name'
    mv_form.subjectarea.queryset = SubjectArea.head_objects(
        prjId=project_id).order_by('name')
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    mv_form.subjectarea.allow_blank = True
    mv_form.subjectarea.blank_text = lazy_gettext(u'--선택--')

    return render_template(
        'report/_model_validate_with_glossary.htm.j2',
        project=g.project,
        mv_form=mv_form)

# # @bpapp.route('/<project_id>', methods=['GET'])
# @bpapp.route('/p/<mbj:project_id>/_report', methods=['GET'])
# @register_breadcrumb(bpapp, '._get_report_main_view', lazy_gettext(u'모델 레포트'))
# def _get_report_main_view(project_id):

#     project = Project.objects.get_or_404(id=project_id)

#     # 모델 스크립트 생성 화면 추가
#     ms_form = ModelScriptForm()

#     ms_form.model.label_attr = 'name'
#     ms_form.model.queryset = Model.head_objects.filter(
#         prjId=project_id).order_by('name')
#     ms_form.model.allow_blank = True  # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
#     ms_form.model.blank_text = lazy_gettext(u'--선택--')

#     ms_form.subjectarea.label_attr = 'escaped_name'
#     ms_form.subjectarea.queryset = SubjectArea.head_objects.filter(
#         prjId=project_id).order_by('name')
#     # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
#     ms_form.subjectarea.allow_blank = True
#     ms_form.subjectarea.blank_text = lazy_gettext(u'--선택--')

#     # 모델 표준 점검 화면 추가
#     mv_form = ModelValidateForm()

#     mv_form.glossary.label_attr = 'glossary_name'
#     mv_form.glossary.queryset = Glossary.objects.filter(project=project)\
#         .order_by('glossary_name')
#     # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
#     mv_form.glossary.allow_blank = True
#     mv_form.glossary.blank_text = lazy_gettext(u'--전체--')

#     mv_form.model = ms_form.model
#     mv_form.subjectarea = ms_form.subjectarea

#     return render_template('report/report_main.html', project=project, ms_form=ms_form, mv_form=mv_form)

@bpapp.route('/p/<mbj:project_id>/s_lst', methods=['GET'])
@bpapp.route('/p/<mbj:project_id>/m/<mbj:model_id>/lst_subj', methods=['GET'])
def get_subjectareas_list(project_id, model_id=None):

    ms_form = ModelScriptForm()
    if model_id:
        # model = Model.head_objects(id=ObjectId(model_id)).first()
        model = Model.newest_model(model_id)
        subjectarea_object_id_list = []
        for subjectarea_dict in model.subjectareas:
            subjectarea_object_id_list.append(
                SubjectArea.newest_subjectarea(subjectarea_dict.get('id')).id)

        queryset = SubjectArea.head_objects(
            id__in=subjectarea_object_id_list).order_by('name')

    else:
        queryset = SubjectArea.head_objects(prjId=project_id).order_by('name')

    ms_form.subjectarea.label_attr = 'escaped_name'
    ms_form.subjectarea.queryset = queryset
    # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.subjectarea.allow_blank = True
    ms_form.subjectarea.blank_text = lazy_gettext(u'-선택-')

    return render_template('report/_option_subject.html', ms_form=ms_form)

@bpapp.route('/p/<mbj:project_id>/m_lst', methods=['GET'])
@bpapp.route('/p/<mbj:project_id>/g/<mbj:glossary_base_id>/m_lst', methods=['GET'])
def get_model_list(project_id, glossary_base_id=None):
    ms_form = ModelScriptForm()

    # glossary = Glossary.objects(id=ObjectId(glossary_id)).first()

    ms_form.model.label_attr = 'name'

    if glossary_base_id is None:
        ms_form.model.queryset = Model.head_objects(
            prjId=project_id).order_by('name')
    else:
        # import pdb; pdb.set_trace()
        ms_form.model.queryset = Model.head_objects(
            glossaryId=ObjectId(glossary_base_id)).order_by('name')

    ms_form.model.allow_blank = True  # 기본값은 False인데 넣어주면 blank_text를 상위에 추가해줌
    ms_form.model.blank_text = lazy_gettext(u'--선택--')

    return render_template('report/_option_model.html', ms_form=ms_form)

@bpapp.route('/p/<mbj:project_id>/s/<mbj:subject_id>/entities', methods=['GET'])
def get_entities(project_id, subject_id):

    subjectarea = SubjectArea.head_objects(id=ObjectId(subject_id)).first()
    entity_list = []

    for ent_dict in subjectarea.entities:
        entity = Entity.head_objects(
            id=ObjectId(ent_dict['id']),
            entTyp__in=['A', 'P']).first()
        if entity is not None:
            entity_list.append(entity)

    entity_list = sorted(entity_list, key=lambda entity: entity.entNm)
    return render_template('report/_option_entities.html', entity_list=entity_list)

def get_report(project_id, func_name, model_id=None, subject_id=None, searching_name=None):
    report = []
    if subject_id:
        # subjectarea = SubjectArea.head_objects.get(id=ObjectId(subject_id))
        subjectarea = SubjectArea.newest_subjectarea(subject_id)
        model = Model.head_objects(
            subjectareas__Oid=str(subjectarea.Oid)).first()
        report = getattr(subjectarea, func_name)(model, searching_name)
    elif model_id:
        # model = Model.head_objects.get(id=ObjectId(model_id))
        model = Model.newest_model(model_id)
        report = getattr(model, func_name)(searching_name)
    else:
        models = Model.head_objects(prjId=project_id)
        for model in models:
            report += getattr(model, func_name)(searching_name)
    # print("func get_report = [{}]".format(report))
    return report

@bpapp.route('/p/<mbj:project_id>/e_lst/', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/e_lst/<searching_name>', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/m/<mbj:model_id>/e_lst/', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/m/<mbj:model_id>/e_lst/<searching_name>', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/s/<mbj:subject_id>/e_lst/', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/s/<mbj:subject_id>/e_lst/<searching_name>', methods=['GET', 'POST'])
def entity_report(project_id, model_id=None, subject_id=None, searching_name=None):

    report = get_report(
        project_id, 'entity_report', model_id, subject_id, searching_name)
    if request.method == 'POST':
        user_id = str(current_user.to_dbref().id)
        filename = "ENTITIES_{0}_{1}.xlsx".format(project_id[8:], user_id)

        filepath = os.path.join(
            current_app.config['EXCEL_UPLOAD_DIR'], filename)
        excel_helper = ExcelHelper(filepath)
        excel_helper.write_sheet(
            ENTITY_REPORT_COLUMNS, report, gettext(u'엔티티목록'))
        excel_helper.save()

        if os.path.isfile(filepath):
            return flask.send_file(filepath,
                                   attachment_filename="ENTITIES.xlsx",
                                   as_attachment=True)
        else:
            flask.abort(404)
    else:
        return dumps(report, ensure_ascii=False)

@bpapp.route('/p/<mbj:project_id>/e_attrs/', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/e_attrs/<string:searching_name>', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/s/<mbj:subject_id>/e_attrs/', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/s/<mbj:subject_id>/e_attrs/<string:searching_name>', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/m/<mbj:model_id>/e_attrs/', methods=['GET', 'POST'])
@bpapp.route('/p/<mbj:project_id>/m/<mbj:model_id>/e_attrs/<string:searching_name>', methods=['GET', 'POST'])
def attr_report(project_id, subject_id=None, model_id=None, searching_name=None):

    report = get_report(
        project_id, 'attr_report', model_id, subject_id, searching_name)

    if request.method == 'POST':
        filename = "ATTRIBUTES_{0}_{1}.xlsx".format(
            project_id[8:],
            str(current_user.to_dbref().id)
        )

        filepath = os.path.join(
            current_app.config['EXCEL_UPLOAD_DIR'], filename)
        excel_helper = ExcelHelper(filepath)
        excel_helper.write_sheet(ATTR_REPORT_COLUMNS, report, gettext(u'속성목록'))
        # excel_helper.write_sheet(PATTR_REPORT_COLUMNS, report, gettext(u'물리속성'))
        excel_helper.save()

        if os.path.isfile(filepath):
            return flask.send_file(filepath,
                                   attachment_filename="ATTRIBUTES.xlsx",
                                   as_attachment=True)
        else:
            flask.abort(404)
    else:
        return dumps(report, ensure_ascii=False)

# @bpapp.route('/attr/<entity_id>', methods=['GET'])
# @bpapp.route('/report/attr/<entity_id>', methods=['GET'])
@bpapp.route('/p/<mbj:project_id>/e/<mbj:entity_id>/attrs', methods=['GET'])
def entity_attr(project_id, entity_id):

    entity = EntityWithPyMongo(id=ObjectId(entity_id))
    attrs = entity.attrs()
    # import pdb;pdb.set_trace()
    return simplejson.dumps([attr.__dict__ for attr in attrs], ensure_ascii=False)

# @bpapp.route('/report/attr_info/<entity_id>/<attr_nm>/<subject_nm>/<model_nm>', methods=['GET'])
@bpapp.route('/p/<mbj:project_id>/e/<mbj:entity_id>/attr/<attr_name>/<subject_nm>/<model_nm>', methods=['GET'])
def attr_detail(project_id, entity_id, attr_name, subject_nm, model_nm):

    entity = EntityWithPyMongo(id=ObjectId(entity_id))
    attr = entity.attr(attr_name)
    return render_template('report/_attribute_detail.html',
                           attr=attr,
                           subject_nm=subject_nm,
                           model_nm=model_nm)

# @bpapp.route('/create_script/<project_id>', methods=['POST'])
@bpapp.route('/p/<mbj:project_id>/s_create_script', methods=['POST'])
@register_breadcrumb(bpapp, '.create_script', lazy_gettext(u'모델 레포트'))
def create_script(project_id):

    ms_form = ModelScriptForm(request.form)

    selected_entity_list = request.form.getlist('entity_list')

    # if ms_form.validate():
    if ms_form.subjectarea.data:
        subjectArea = SubjectArea.newest_subjectarea(ms_form.subjectarea.data.id)
        script_data = subjectArea.create_script(
            ms_form.model.data,
            ms_form.idx_tblspce.data,
            ms_form.tbl_tblspce.data,
            ms_form.db_owner_use_yn.data,
            ms_form.db_owner.data,
            ms_form.use_attr_nm_as_cmnt.data,
            ms_form.comment_src.data,
            selected_entity_list,
            ms_form.pk_idx_nm.raw_data
        )
    else:
        # current_app.logger.critical(str(ms_form.errors))
        script_data = ''

    return dumps({'script_data': script_data}, ensure_ascii=False)

# @bpapp.route('/validate_subjectarea/<project_id>', methods=['POST'])
@bpapp.route('/p/<mbj:project_id>/s_validate/<is_excel>', methods=['POST'])
@bpapp.route('/p/<mbj:project_id>/s_validate', methods=['POST'])
@register_breadcrumb(bpapp, '.validate_strd', lazy_gettext(u'모델 레포트'))
def validate_strd(project_id, is_excel=False):

    # project = Project.objects.get_or_404(id=project_id)

    mv_form = ModelValidateForm(request.form)

    try:

        if mv_form.subjectarea.data:
            subjectArea = SubjectArea.newest_subjectarea(mv_form.subjectarea.data.id)
            non_strd_attr_list = subjectArea.validate_strd(
                mv_form.model.data, mv_form.except_suffix_num_yn.data)
        else:
            model = Model.newest_model(mv_form.model.data.id)
            non_strd_attr_list = model.validate_strd(
                mv_form.except_suffix_num_yn.data)

        if is_excel:

            filename = "VALIDATIONS_{0}_{1}.xlsx".format(
                project_id[8:],
                str(current_user.to_dbref().id))

            filepath = os.path.join(
                current_app.config['EXCEL_UPLOAD_DIR'], filename)
            excel_helper = ExcelHelper(filepath)
            excel_helper.write_sheet(
                MODEL_VALIDATE_RESULT_COLUMNS, non_strd_attr_list, gettext(u"점검결과"))
            excel_helper.save()

            return send_file(excel_helper.filepath,
                             attachment_filename="VALIDATIONS.xlsx",
                             as_attachment=True)

        else:
            return dumps(non_strd_attr_list, ensure_ascii=False)

    except Exception as e:
        current_app.logger.warning('report.views.validate_strd', exc_info=True)
        if is_excel:
            flash(e.message)
            return redirect(url_for(
                '._get_report_main_view',
                project_id=project_id))
        else:
            return dumps({
                "message": getattr(e, 'message', ''),
                "result": "error",
            }, ensure_ascii=False)


import json
import os
import tarfile
import tempfile

import dask.dataframe as dd
import numpy as np
import pandas as pd

from featuretools.entityset.relationship import Relationship
from featuretools.entityset.serialize import FORMATS
from featuretools.utils.gen_utils import check_schema_version, import_or_raise
from featuretools.utils.s3_utils import get_transport_params, use_smartopen_es
from featuretools.utils.wrangle import _is_local_tar, _is_s3, _is_url
from featuretools.variable_types import LatLong, find_variable_types

def description_to_variable(description, entity=None):
    '''Deserialize variable from variable description.

    Args:
        description (dict) : Description of :class:`.Variable`.
        entity (Entity) : Instance of :class:`.Entity` to add :class:`.Variable`. If entity is None, :class:`.Variable` will not be instantiated.

    Returns:
        variable (Variable) : Returns :class:`.Variable`.
    '''
    is_type_string = isinstance(description['type'], str)
    variable = description['type'] if is_type_string else description['type'].pop('value')
    if entity is not None:
        variable_types = find_variable_types()
        variable_class = variable_types.get(variable, variable_types.get('unknown'))
        kwargs = {} if is_type_string else description['type']
        variable = variable_class(description['id'], entity, **kwargs)
        interesting_values = pd.read_json(description['properties']['interesting_values'], typ='series')
        variable.interesting_values = interesting_values
        variable_description = description['properties'].get('description')
        if variable_description is not None and variable_description != 'the "{}"'.format(variable.name):
            variable.description = variable_description

    return variable

def description_to_entity(description, entityset, path=None):
    '''Deserialize entity from entity description and add to entityset.

    Args:
        description (dict) : Description of :class:`.Entity`.
        entityset (EntitySet) : Instance of :class:`.EntitySet` to add :class:`.Entity`.
        path (str) : Root directory to serialized entityset.
    '''
    if path:
        dataframe = read_entity_data(description, path=path)
    else:
        dataframe = empty_dataframe(description)
    variable_types = {variable['id']: (description_to_variable(variable), variable)
                      for variable in description['variables']}
    es = entityset.entity_from_dataframe(
        description['id'],
        dataframe,
        index=description.get('index'),
        time_index=description.get('time_index'),
        secondary_time_index=description['properties'].get('secondary_time_index'),
        variable_types={variable: variable_types[variable][0] for variable in variable_types})
    for variable in es[description['id']].variables:
        interesting_values = variable_types[variable.id][1]['properties']['interesting_values']
        interesting_values = pd.read_json(interesting_values, typ="series")
        variable.interesting_values = interesting_values
        variable_description = variable_types[variable.id][1]['properties'].get('description')
        if variable_description is not None and variable_description != 'the "{}"'.format(variable.name):
            variable.description = variable_description

def description_to_entityset(description, **kwargs):
    '''Deserialize entityset from data description.

    Args:
        description (dict) : Description of an :class:`.EntitySet`. Likely generated using :meth:`.serialize.entityset_to_description`
        kwargs (keywords): Additional keyword arguments to pass as keywords arguments to the underlying deserialization method.

    Returns:
        entityset (EntitySet) : Instance of :class:`.EntitySet`.
    '''
    check_schema_version(description, 'entityset')

    from featuretools.entityset import EntitySet

    # If data description was not read from disk, path is None.
    path = description.get('path')
    entityset = EntitySet(description['id'])

    last_time_index = []
    for entity in description['entities'].values():
        entity['loading_info']['params'].update(kwargs)
        # If path is None, an empty dataframe will be created for entity.
        description_to_entity(entity, entityset, path=path)
        if entity['properties']['last_time_index']:
            last_time_index.append(entity['id'])

    for relationship in description['relationships']:
        relationship = Relationship.from_dictionary(relationship, entityset)
        entityset.add_relationship(relationship)

    if len(last_time_index):
        entityset.add_last_time_indexes(updated_entities=last_time_index)

    return entityset

def empty_dataframe(description):
    '''Deserialize empty dataframe from entity description.

    Args:
        description (dict) : Description of :class:`.Entity`.

    Returns:
        df (DataFrame) : Empty dataframe for entity.
    '''
    columns = [variable['id'] for variable in description['variables']]
    dtypes = description['loading_info']['properties']['dtypes']
    return pd.DataFrame(columns=columns).astype(dtypes)

def read_entity_data(description, path):
    '''Read description data from disk.

    Args:
        description (dict) : Description of :class:`.Entity`.
        path (str): Location on disk to read entity data.

    Returns:
        df (DataFrame) : Instance of dataframe.
    '''
    file = os.path.join(path, description['loading_info']['location'])
    kwargs = description['loading_info'].get('params', {})
    load_format = description['loading_info']['type']
    entity_type = description['loading_info'].get('entity_type', 'pandas')
    dtypes = description['loading_info'].get('properties', {}).get('dtypes')
    read_kwargs = {}
    if entity_type == 'dask':
        lib = dd
        read_kwargs['dtype'] = dtypes
    elif entity_type == 'koalas':
        import_error = 'Cannot load Koalas entityset - unable to import Koalas. ' \
                       'Consider doing a pip install with featuretools[koalas] to install Koalas with pip'
        lib = import_or_raise('databricks.koalas', import_error)
        read_kwargs['multiline'] = True
        kwargs['compression'] = str(kwargs['compression'])
    else:
        lib = pd
    if load_format == 'csv':
        dataframe = lib.read_csv(
            file,
            engine=kwargs['engine'],
            compression=kwargs['compression'],
            encoding=kwargs['encoding'],
            **read_kwargs
        )
    elif load_format == 'parquet':
        dataframe = lib.read_parquet(file, engine=kwargs['engine'])
    elif load_format == 'pickle':
        dataframe = pd.read_pickle(file, **kwargs)
    else:
        error = 'must be one of the following formats: {}'
        raise ValueError(error.format(', '.join(FORMATS)))
    if entity_type == 'koalas':
        for col, dtype in dtypes.items():
            if dtype == 'object':
                dtypes[col] = 'str'
            if dtype == 'datetime64[ns]':
                dtypes[col] = np.datetime64
    dataframe = dataframe.astype(dtypes)

    if load_format in ['parquet', 'csv']:
        latlongs = []
        for var_description in description['variables']:
            if var_description['type']['value'] == LatLong.type_string:
                latlongs.append(var_description["id"])

        def parse_latlong_tuple(x):
            return tuple(float(y) for y in x[1:-1].split(","))

        def parse_latlong_list(x):
            return list(float(y) for y in x[1:-1].split(","))

        for column in latlongs:
            if entity_type == 'dask':
                meta = (column, tuple([float, float]))
                dataframe[column] = dataframe[column].apply(parse_latlong_tuple,
                                                            meta=meta)
            elif entity_type == 'koalas':
                dataframe[column] = dataframe[column].apply(parse_latlong_list)

            else:
                dataframe[column] = dataframe[column].apply(parse_latlong_tuple)

    return dataframe

def read_data_description(path):
    '''Read data description from disk, S3 path, or URL.

        Args:
            path (str): Location on disk, S3 path, or URL to read `data_description.json`.

        Returns:
            description (dict) : Description of :class:`.EntitySet`.
    '''

    path = os.path.abspath(path)
    assert os.path.exists(path), '"{}" does not exist'.format(path)
    filepath = os.path.join(path, 'data_description.json')
    with open(filepath, 'r') as file:
        description = json.load(file)
    description['path'] = path
    return description

def read_entityset(path, profile_name=None, **kwargs):
    '''Read entityset from disk, S3 path, or URL.

        Args:
            path (str): Directory on disk, S3 path, or URL to read `data_description.json`.
            profile_name (str, bool): The AWS profile specified to write to S3. Will default to None and search for AWS credentials.
                Set to False to use an anonymous profile.
            kwargs (keywords): Additional keyword arguments to pass as keyword arguments to the underlying deserialization method.
    '''
    if _is_url(path) or _is_s3(path) or _is_local_tar(str(path)):
        with tempfile.TemporaryDirectory() as tmpdir:
            local_path = path
            transport_params = None

            if _is_s3(path):
                transport_params = get_transport_params(profile_name)

            if _is_s3(path) or _is_url(path):
                local_path = os.path.join(tmpdir, "temporary_es")
                use_smartopen_es(local_path, path, transport_params)

            with tarfile.open(str(local_path)) as tar:
                tar.extractall(path=tmpdir)

            data_description = read_data_description(tmpdir)
            return description_to_entityset(data_description, **kwargs)
    else:
        data_description = read_data_description(path)
        return description_to_entityset(data_description, **kwargs)


from telclass import TelClass
from blessings import Terminal
import multiprocessing,time,sys

# multiprocessor function. Reads data.

def mp_handler(client, e):
    if e.is_set():
        data = client.read()
        if data.startswith("Error:"):
            e.clear()
            return 1
        else:
            return 1

# multiprocessor does not work outside of __main__

if __name__ != "__main__":
    print("Hestia: Must be run as script or command. Not in __main__.")
    f = open('/var/log/hestia.log','a')
    f.write("Hestia @ " + time.localtime(time.clock()) + ": Must be run as script or command. Not in __main__.")
    f.close()
    sys.exit(1)

# We are running in __main__

if __name__ == "__main__":
    interface = Terminal()
    should_read = multiprocessing.Event()
    conn = TelClass()
    if interface.does_styling:
        x256=1
    else:
        x256=0

    with interface.location(0,interface.height-1):
        print(":")
    with interface.location(0,interface.height-2):
        if x256:
            print(interface.black_on_white+"Status Bar"+interface.normal)
        else:
            print("Status Bar")
    for i in range(1,interface.width):
        with interface.location(interface.width-i,interface.height-3):
            print("-")
    print(interface.move(1,interface.height-1))

    should_read.clear()

    time.sleep(5)
    sys.exit(1)


__author__ = 'cvaughn'

import sys
import pygame
import random
import time

pygame.init()

pixelscale = 2

width, height = 96, 64

size = (width * pixelscale, height * pixelscale)

screen = pygame.display.set_mode(size)
fader = pygame.Surface((width, height))
fader.fill((0,0,0))
fader.set_alpha(255)
class RainDrop:
    def __init__(self):
        self.x = random.randrange(width)
        self.y = 0
        self.speed = random.uniform(0.3, 0.7)
        self.color = pygame.Color(1, 1, 1, 1)
        self.color.hsva = (random.uniform(270, 290), random.uniform(90, 100), random.uniform(95, 100), 100)

drops = []

for ct in range(width * 3 / 2):
    drops.append(RainDrop())

darken=pygame.Surface(pygame.display.get_surface().get_size())
darken.fill((0,0,0))
darken.set_alpha(15)

while True:
    #screen.fill((0, 0, 0))

    for event in pygame.event.get():
        if event.type == pygame.QUIT: sys.exit()

    new_drops = []

    for d in drops:
        pygame.draw.rect(screen, d.color,
                     (int(d.x) * pixelscale, int(d.y) * pixelscale,
                      pixelscale, pixelscale))
        d.y += d.speed

        if d.y < height + 1:
            new_drops.append(d)
        else:
            new_drops.append(RainDrop())

    drops = new_drops

    old_clip=pygame.display.get_surface().get_clip()
    pygame.display.get_surface().blit(darken, (0, 0))
    pygame.display.flip()
    pygame.display.get_surface().set_clip(old_clip)

    pygame.display.flip()

    time.sleep(1/60.0)


from matplotlib import pyplot as plt
import numpy as np

def przepisFunkcji(x):
    return (1 / (1 + (25 * (x**2))))

def przepisWezlowInterpol(x):
    return (-1 + 2 *(x / n))

def obliczXi(wezlyX, wezlyY):
    # wektory do zapamietania macierzy
    u0 = []; u1 = []; l1 = []
    x = [] #prawa strona rownania
    h = wezlyX[1] - wezlyX[0]

    #dodaje pusty znak do kazdego wektora, bo chce miec indeksy od 1 (potrzebne dla xi[0])
    u0.append(''); u1.append(''); l1.append(''); x.append('')

    for i in range(1, n):
        x.append((6 / (h**2)) * (wezlyY[i-1] - 2*wezlyY[i] + wezlyY[i+1]))
        u0.append(4)
        if i < n-1:
            u1.append(1)
            l1.append(1)

    # inicjalizacja
    l1[1] = l1[1]/u0[1]

    #obliczam macierz LU
    for i in range(2, n):
        u0[i] = u0[i] - l1[i-1] * u1[i-1]
        if i < n-1:
            l1[i] = l1[i]/u0[i]

    #obliczam wektor 'z' z rownania Lz = x forwardsubstitution
    for i in range(2, n):
        x[i] = x[i] - (l1[i-1] * x[i-1])

    #obliczam wektor 'y' rownania Uy = z backsubstitution
    x[n-1] = x[n-1]/u0[n-1]
    x[n-2] = (x[n-2] - (u1[n-2] * x[n-1])) / u0[n-2]
    for i in reversed(range(1, n-2)):
        x[i] = (x[i] - (u1[i] * x[i+1])) / u0[i]

    #wynik xi jest w wektorze 'x' + warunek naturalnego splajnu kubicznego
    x[0] = 0
    x.append(0)
    return x

def splajnyKubiczne(wezlyX, wezlyY, xi, wezelKoncaPrzedzialu):
    def S(x):
        A = (wezlyX[wezelKoncaPrzedzialu] - x) / (wezlyX[wezelKoncaPrzedzialu] - wezlyX[wezelKoncaPrzedzialu-1])
        B = (x - wezlyX[wezelKoncaPrzedzialu-1]) / (wezlyX[wezelKoncaPrzedzialu] - wezlyX[wezelKoncaPrzedzialu-1])
        C = 1/6 * ((A**3) - A) * ((wezlyX[wezelKoncaPrzedzialu] - wezlyX[wezelKoncaPrzedzialu-1])**2)
        D = 1/6 * ((B**3) - B) * ((wezlyX[wezelKoncaPrzedzialu] - wezlyX[wezelKoncaPrzedzialu-1])**2)
        wzorWielomianu = (A * wezlyY[wezelKoncaPrzedzialu-1]) + (B * wezlyY[wezelKoncaPrzedzialu]) + (C * xi[wezelKoncaPrzedzialu-1]) + (D * xi[wezelKoncaPrzedzialu])
        return wzorWielomianu
    return S

def plot(wezlyX, wezlyY, xi):
    x1 = np.arange(-1, 1, 0.01)
    funkcjaY = []
    for i in x1:
        funkcjaY.append(przepisFunkcji(i))

    plt.axis([-1.1,1.1,-0.5,1.1])
    plt.plot(x1, funkcjaY, color="orangered", linewidth=0.5)

    for i in range(1, n+1):
        S = splajnyKubiczne(wezlyX, wezlyY, xi, i)
        x = np.arange(wezlyX[i-1], wezlyX[i], 0.01)
        y = list(map(S, x))
        plt.plot(x, y, linewidth=2, color="tab:blue")

    plt.plot(wezlyX, wezlyY, 'ro')

def main():
    wezlyX = []
    wezlyY = []
    xi = []
    global n, h

    n = int(input("Podaj n = "))

    for i in range(n+1):
        wezlyX.append(przepisWezlowInterpol(i))
        wezlyY.append(przepisFunkcji(wezlyX[i]))

    xi = obliczXi(wezlyX, wezlyY)
    print(xi)
    plot(wezlyX, wezlyY, xi)

    plt.show()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Apr  2 22:12:38 2019

First part is the V2 analysis (not block-wise)

@author: Greta
"""

import pickle
from matplotlib import pyplot as plt
import os
import statistics
import numpy as np
from scipy import stats
import seaborn as sns
os.chdir('C:\\Users\\Greta\\Documents\\GitHub\\ClosedLoop\\Scripts\\')
from responseTime_func import findRTsBlocks
import random

saveDir = 'P:\\closed_loop_data\\beh_analysis\\'
os.chdir(saveDir)

plt.style.use('seaborn-pastel')

font = {'family' : 'sans-serif',
       'weight' : 1,
       'size'   : 12}

#%%
with open('Beh_subjID_07.pkl', "rb") as fin:
    sub07 = (pickle.load(fin))[0]
with open('Beh_subjID_08.pkl', "rb") as fin:
    sub08 = (pickle.load(fin))[0]
with open('Beh_subjID_11.pkl', "rb") as fin:
    sub11 = (pickle.load(fin))[0]
with open('Beh_subjID_13.pkl', "rb") as fin:
    sub13 = (pickle.load(fin))[0]
with open('Beh_subjID_14.pkl', "rb") as fin:
    sub14 = (pickle.load(fin))[0]
with open('Beh_subjID_15.pkl', "rb") as fin:
    sub15 = (pickle.load(fin))[0]
with open('Beh_subjID_16.pkl', "rb") as fin:
    sub16 = (pickle.load(fin))[0]
with open('Beh_subjID_17.pkl', "rb") as fin:
    sub17 = (pickle.load(fin))[0]
with open('Beh_subjID_18.pkl', "rb") as fin:
    sub18 = (pickle.load(fin))[0]
with open('Beh_subjID_19.pkl', "rb") as fin:
    sub19 = (pickle.load(fin))[0]
with open('Beh_subjID_21.pkl', "rb") as fin:
    sub21 = (pickle.load(fin))[0]
with open('Beh_subjID_22.pkl', "rb") as fin:
    sub22 = (pickle.load(fin))[0]
with open('Beh_subjID_23.pkl', "rb") as fin:
    sub23 = (pickle.load(fin))[0]
with open('Beh_subjID_25.pkl', "rb") as fin:
    sub25 = (pickle.load(fin))[0]
with open('Beh_subjID_27.pkl', "rb") as fin:
    sub27 = (pickle.load(fin))[0]
with open('Beh_subjID_30.pkl', "rb") as fin:
    sub30 = (pickle.load(fin))[0]
with open('Beh_subjID_31.pkl', "rb") as fin:
    sub31 = (pickle.load(fin))[0]
with open('Beh_subjID_32.pkl', "rb") as fin:
    sub32 = (pickle.load(fin))[0]
with open('Beh_subjID_33.pkl', "rb") as fin:
    sub33 = (pickle.load(fin))[0]
with open('Beh_subjID_34.pkl', "rb") as fin:
    sub34 = (pickle.load(fin))[0]

#%% Measures

def computeStats(subjID,expDay):

    with open('Beh_subjID_' + subjID + '.pkl', "rb") as fin:
        sub = (pickle.load(fin))[0]

    TP = sub['no_Inhibitions_nonlure_day_'+expDay]
    FN = sub['inhibitions_nonlure_day_'+expDay]

    TN = sub['inhibitions_lure_day_'+expDay]
    FP = sub['no_Inhibitions_lure_day_'+expDay]

    sensitivity = TP/(TP+FN)
    specificity = TN/(TN+FP)

    false_positive_rate = FP/(FP+TN)

    accuracy = (TP+TN)/(TP+TN+FP+FN)

    return sensitivity,specificity,false_positive_rate,accuracy

#%% Analyze 4 days
dayLst = ['1','3','4','5']

f07=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f07[count,:]=computeStats('07',day)
f08=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f08[count,:]=computeStats('08',day)
f11=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f11[count,:]=computeStats('11',day)
f13=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f13[count,:]=computeStats('13',day)
f14=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f14[count,:]=computeStats('14',day)
f15=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f15[count,:]=computeStats('15',day)
f16=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f16[count,:]=computeStats('16',day)
f17=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f17[count,:]=computeStats('17',day)
f18=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f18[count,:]=computeStats('18',day)
f19=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f19[count,:]=computeStats('19',day)
f21=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f21[count,:]=computeStats('21',day)
f22=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f22[count,:]=computeStats('22',day)
f23=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f23[count,:]=computeStats('23',day)
f24=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f24[count,:]=computeStats('24',day)
f25=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f25[count,:]=computeStats('25',day)
f26=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f26[count,:]=computeStats('26',day)
f27=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f27[count,:]=computeStats('27',day)
f30=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f30[count,:]=computeStats('30',day)
f31=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f31[count,:]=computeStats('31',day)
f32=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f32[count,:]=computeStats('32',day)
f33=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f33[count,:]=computeStats('33',day)
f34=np.zeros((4,4))
for count,day in enumerate(dayLst):
    f34[count,:]=computeStats('34',day)

#%%
sub_fb=[f07,f08,f11,f13,f14,f16,f19,f22,f26,f27,f30]
sub_c=[f15,f17,f18,f21,f23,f24,f25,f31,f32,f33,f34]

#%%
sen_feedback=np.asarray([f[:,0] for f in sub_fb])
sen_control=np.asarray([f[:,0] for f in sub_c])

spec_feedback=np.asarray([f[:,1] for f in sub_fb])
spec_control=np.asarray([f[:,1] for f in sub_c])

fpr_feedback=np.asarray([f[:,2] for f in sub_fb])
fpr_control=np.asarray([f[:,2] for f in sub_c])

acc_feedback=np.asarray([f[:,3] for f in sub_fb])
acc_control=np.asarray([f[:,3] for f in sub_c])

#%% Sensitivity
plt.figure(1)
plt.bar(1,np.mean(sen_feedback[:,0]),color=(0,0,0,0),edgecolor='tomato',yerr=np.std(sen_feedback[:,0]))
plt.bar(2,np.mean(sen_feedback[:,1]),color=(0,0,0,0),edgecolor='brown',yerr=np.std(sen_feedback[:,1]))
plt.bar(3,np.mean(sen_control[:,0]),color=(0,0,0,0),edgecolor='dodgerblue',yerr=np.std(sen_control[:,0]))
plt.bar(4,np.mean(sen_control[:,1]),color=(0,0,0,0),edgecolor='navy',yerr=np.std(sen_control[:,1]))

plt.scatter(np.full(11,1),sen_feedback[:,0],color='tomato')
plt.scatter(np.full(11,2),sen_feedback[:,1],color='brown')
plt.scatter(np.full(11,3),sen_control[:,0],color='dodgerblue')
plt.scatter(np.full(11,4),sen_control[:,1],color='navy')

plt.ylabel('Response sensitivity')
plt.ylim([0.9,1])
plt.xticks([1,2,3,4],['NF day1','NF day3', 'Control day1', 'Control day3'])
plt.title('Sensitivity')

# Connect lines
plt.figure(10)
plt.ylabel('Response sensitivity')
plt.ylim([0.9,1])
plt.xticks([1,2,3,4],['NF day1','NF day3', 'Control day1', 'Control day3'])
plt.title('Sensitivity')

plt.bar(1,np.mean(sen_feedback[:,0]),color=(0,0,0,0),edgecolor='tomato')
plt.bar(2,np.mean(sen_feedback[:,1]),color=(0,0,0,0),edgecolor='brown')
plt.bar(3,np.mean(sen_control[:,0]),color=(0,0,0,0),edgecolor='dodgerblue')
plt.bar(4,np.mean(sen_control[:,1]),color=(0,0,0,0),edgecolor='navy')

plt.scatter(np.full(11,1),sen_feedback[:,0],color='tomato')
plt.scatter(np.full(11,2),sen_feedback[:,1],color='brown')
plt.scatter(np.full(11,3),sen_control[:,0],color='dodgerblue')
plt.scatter(np.full(11,4),sen_control[:,1],color='navy')

for i in range(11):
    plt.plot([(np.full(11,1))[i],(np.full(11,2))[i]], [(sen_feedback[:,0])[i],(sen_feedback[:,1])[i]],color='gray')
    plt.plot([(np.full(11,3))[i],(np.full(11,4))[i]], [(sen_control[:,0])[i],(sen_control[:,1])[i]],color='gray')

# Day 4 and 5
#plt.figure(3)
#plt.bar(1,np.mean(sen_feedback[:,2]))
#plt.bar(2,np.mean(sen_feedback[:,3]))
#plt.bar(3,np.mean(sen_control[:,2]))
#plt.bar(4,np.mean(sen_control[:,3]))

#%% Specificity
plt.figure(2)
plt.bar(1,np.mean(spec_feedback[:,0]),color=(0,0,0,0),edgecolor='tomato',yerr=np.std(spec_feedback[:,0]))
plt.bar(2,np.mean(spec_feedback[:,1]),color=(0,0,0,0),edgecolor='brown',yerr=np.std(spec_feedback[:,1]))
plt.bar(3,np.mean(spec_control[:,0]),color=(0,0,0,0),edgecolor='dodgerblue',yerr=np.std(spec_control[:,0]))
plt.bar(4,np.mean(spec_control[:,1]),color=(0,0,0,0),edgecolor='navy',yerr=np.std(spec_control[:,1]))

plt.scatter(np.full(11,1),spec_feedback[:,0],color='tomato')
plt.scatter(np.full(11,2),spec_feedback[:,1],color='brown')
plt.scatter(np.full(11,3),spec_control[:,0],color='dodgerblue')
plt.scatter(np.full(11,4),spec_control[:,1],color='navy')

plt.ylabel('Response specificity')
plt.ylim([0.4,1])
plt.xticks([1,2,3,4],['NF day1','NF day3', 'Control day1', 'Control day3'])
plt.title('Specificity')

# Connect lines
plt.figure(12)
plt.bar(1,np.mean(spec_feedback[:,0]),color=(0,0,0,0),edgecolor='tomato')
plt.bar(2,np.mean(spec_feedback[:,1]),color=(0,0,0,0),edgecolor='brown')
plt.bar(3,np.mean(spec_control[:,0]),color=(0,0,0,0),edgecolor='dodgerblue')
plt.bar(4,np.mean(spec_control[:,1]),color=(0,0,0,0),edgecolor='navy')

plt.scatter(np.full(11,1),spec_feedback[:,0],color='tomato')
plt.scatter(np.full(11,2),spec_feedback[:,1],color='brown')
plt.scatter(np.full(11,3),spec_control[:,0],color='dodgerblue')
plt.scatter(np.full(11,4),spec_control[:,1],color='navy')

plt.ylabel('Response specificity')
plt.ylim([0.4,1])
plt.xticks([1,2,3,4],['NF day1','NF day3', 'Control day1', 'Control day3'])
plt.title('Specificity')

for i in range(11):
    plt.plot([(np.full(11,1))[i],(np.full(11,2))[i]], [(spec_feedback[:,0])[i],(spec_feedback[:,1])[i]],color='gray')
    plt.plot([(np.full(11,3))[i],(np.full(11,4))[i]], [(spec_control[:,0])[i],(spec_control[:,1])[i]],color='gray')

#%% FPR
plt.figure(3)
plt.bar(1,np.mean(fpr_feedback[:,0]),color=(0,0,0,0),edgecolor='tomato',yerr=np.std(fpr_feedback[:,0]))
plt.bar(2,np.mean(fpr_feedback[:,1]),color=(0,0,0,0),edgecolor='brown',yerr=np.std(fpr_feedback[:,1]))
plt.bar(3,np.mean(fpr_control[:,0]),color=(0,0,0,0),edgecolor='dodgerblue',yerr=np.std(fpr_control[:,0]))
plt.bar(4,np.mean(fpr_control[:,1]),color=(0,0,0,0),edgecolor='navy',yerr=np.std(fpr_control[:,1]))

plt.scatter(np.full(11,1),fpr_feedback[:,0],color='tomato')
plt.scatter(np.full(11,2),fpr_feedback[:,1],color='brown')
plt.scatter(np.full(11,3),fpr_control[:,0],color='dodgerblue')
plt.scatter(np.full(11,4),fpr_control[:,1],color='navy')

plt.ylabel('False positive rate')
plt.ylim([0,0.6])
plt.xticks([1,2,3,4],['NF day1','NF day3', 'Control day1', 'Control day3'])
plt.title('FPR')

# Connect
plt.figure(13)
plt.bar(1,np.mean(fpr_feedback[:,0]),color=(0,0,0,0),edgecolor='tomato')
plt.bar(2,np.mean(fpr_feedback[:,1]),color=(0,0,0,0),edgecolor='brown')
plt.bar(3,np.mean(fpr_control[:,0]),color=(0,0,0,0),edgecolor='dodgerblue')
plt.bar(4,np.mean(fpr_control[:,1]),color=(0,0,0,0),edgecolor='navy')

plt.scatter(np.full(11,1),fpr_feedback[:,0],color='tomato')
plt.scatter(np.full(11,2),fpr_feedback[:,1],color='brown')
plt.scatter(np.full(11,3),fpr_control[:,0],color='dodgerblue')
plt.scatter(np.full(11,4),fpr_control[:,1],color='navy')

plt.ylabel('False positive rate')
plt.ylim([0,0.6])
plt.xticks([1,2,3,4],['NF day1','NF day3', 'Control day1', 'Control day3'])
plt.title('FPR')

for i in range(11):
    plt.plot([(np.full(11,1))[i],(np.full(11,2))[i]], [(fpr_feedback[:,0])[i],(fpr_feedback[:,1])[i]],color='gray')
    plt.plot([(np.full(11,3))[i],(np.full(11,4))[i]], [(fpr_control[:,0])[i],(fpr_control[:,1])[i]],color='gray')

#%% Accuracy
plt.figure(4)
plt.bar(1,np.mean(acc_feedback[:,0]),color=(0,0,0,0),edgecolor='tomato',yerr=np.std(acc_feedback[:,0]))
plt.bar(2,np.mean(acc_feedback[:,1]),color=(0,0,0,0),edgecolor='brown',yerr=np.std(acc_feedback[:,1]))
plt.bar(3,np.mean(acc_control[:,0]),color=(0,0,0,0),edgecolor='dodgerblue',yerr=np.std(acc_control[:,0]))
plt.bar(4,np.mean(acc_control[:,1]),color=(0,0,0,0),edgecolor='navy',yerr=np.std(acc_control[:,1]))

plt.scatter(np.full(11,1),acc_feedback[:,0],color='tomato')
plt.scatter(np.full(11,2),acc_feedback[:,1],color='brown')
plt.scatter(np.full(11,3),acc_control[:,0],color='dodgerblue')
plt.scatter(np.full(11,4),acc_control[:,1],color='navy')

plt.ylabel('Accuracy')
plt.ylim([0.9,1])
plt.xticks([1,2,3,4],['NF day1','NF day3', 'Control day1', 'Control day3'])
plt.title('Accuracy')

# Connect
plt.figure(14)
plt.bar(1,np.mean(acc_feedback[:,0]),color=(0,0,0,0),edgecolor='tomato')
plt.bar(2,np.mean(acc_feedback[:,1]),color=(0,0,0,0),edgecolor='brown')
plt.bar(3,np.mean(acc_control[:,0]),color=(0,0,0,0),edgecolor='dodgerblue')
plt.bar(4,np.mean(acc_control[:,1]),color=(0,0,0,0),edgecolor='navy')

plt.scatter(np.full(11,1),acc_feedback[:,0],color='tomato')
plt.scatter(np.full(11,2),acc_feedback[:,1],color='brown')
plt.scatter(np.full(11,3),acc_control[:,0],color='dodgerblue')
plt.scatter(np.full(11,4),acc_control[:,1],color='navy')

plt.ylabel('Accuracy')
plt.ylim([0.9,1])
plt.xticks([1,2,3,4],['NF day1','NF day3', 'Control day1', 'Control day3'])
plt.title('Accuracy')

for i in range(11):
    plt.plot([(np.full(11,1))[i],(np.full(11,2))[i]], [(acc_feedback[:,0])[i],(acc_feedback[:,1])[i]],color='gray')
    plt.plot([(np.full(11,3))[i],(np.full(11,4))[i]], [(acc_control[:,0])[i],(acc_control[:,1])[i]],color='gray')

#%% T tests

# Sensitivity
diff_fb=np.mean(sen_feedback[:,1]-sen_feedback[:,0])
diff_c=np.mean(sen_control[:,1]-sen_control[:,0])

diff_fb_p=stats.ttest_rel(sen_feedback[:,0],sen_feedback[:,1])
diff_c_p=stats.ttest_rel(sen_control[:,0],sen_control[:,1])

# Acc
#diff_fb_acc=np.mean(acc_feedback[:,1]-acc_feedback[:,0])
#diff_c_acc=np.mean(acc_control[:,1]-acc_control[:,0])

p_acc_fb=stats.ttest_rel(acc_feedback[:,0],acc_feedback[:,1])
p_acc_c=stats.ttest_rel(acc_control[:,0],acc_control[:,1])

#%% Plots for RT surrounding lures
def extractRTs(subjID):

    with open('Beh_subjID_' + subjID + '.pkl', "rb") as fin:
        sub = (pickle.load(fin))[0]

    c1 = sub['surrounding_CR_Lst_day_1']
    c3 = sub['surrounding_CR_Lst_day_3']

    f1 = sub['surrounding_FR_Lst_day_1']
    f3 = sub['surrounding_FR_Lst_day_3']

    # Computing day 1 and 3 together
    c13m = [statistics.mean(k) for k in zip(c1, c3)]
    f13m = [statistics.mean(k) for k in zip(f1, f3)]

    # Computing day 1 and 3 separately

    return np.asarray(c1), np.asarray(c3), np.asarray(f1), np.asarray(f3), np.asarray(c13m), np.asarray(f13m)

all_RTs = np.zeros((22,6),dtype=np.ndarray)

sub_all = ['07','08','11','13','14','15','16','17','18','19','21','22','23','24','25','26','27','30','31','32','33','34']

for idx, subj in enumerate(sub_all):
    all_RTs[idx,:] = extractRTs(subj)

# Mean over c13m and f13m for all subjects
all_c1 = all_RTs[:,0]
all_c3 = all_RTs[:,1]
all_f1 = all_RTs[:,2]
all_f3 = all_RTs[:,3]

all_c13 = all_RTs[:,4]
all_f13 = all_RTs[:,5]
# Make into np arrays?
all_c1m = np.mean(all_c1)
all_c3m = np.mean(all_c3)
all_f1m = np.mean(all_f1)
all_f3m = np.mean(all_f3)
all_c13m = np.mean(all_c13)
all_f13m = np.mean(all_f13)

# Standard error of the mean for errorbars
c1_yerr=(np.std(all_c1))/np.sqrt(22)
c3_yerr=(np.std(all_c3))/np.sqrt(22)
f1_yerr=(np.std(all_f1))/np.sqrt(22)
f3_yerr=(np.std(all_f3))/np.sqrt(22)
c13_yerr=(np.std(all_c13))/np.sqrt(22)
f13_yerr=(np.std(all_f13))/np.sqrt(22)

#%% Plots
ticks = ['-3','-2','-1','lure','1','2','3']

# Both days
#plt.plot(all_c13m,color='green',)
#plt.plot(all_f13m,color='red')

plt.figure(5)
plt.errorbar(np.arange(0,7),all_c13m,yerr=c13_yerr,color='green')
plt.errorbar(np.arange(0,7),all_f13m,yerr=f13_yerr,color='red')
plt.title('Day 1 and 3 - all participants')
plt.xticks(np.arange(0,7,1),ticks)
plt.xlabel('Trials from lure')
plt.ylabel('RT (ms)')

# Day 1
plt.figure(6)
#plt.plot(all_c1m,color='green')
#plt.plot(all_f1m,color='red')
plt.errorbar(np.arange(0,7),all_c1m,yerr=c1_yerr,color='green')
plt.errorbar(np.arange(0,7),all_f1m,yerr=f1_yerr,color='red')
plt.title('Day 1, all participants')
plt.xticks(np.arange(0,7,1),ticks)
plt.xlabel('Trials from lure')
plt.ylabel('RT (ms)')

# Day 3
plt.figure(7)
#plt.plot(all_c3m,color='green')
#plt.plot(all_f3m,color='red')
plt.errorbar(np.arange(0,7),all_c3m,yerr=c3_yerr,color='green')
plt.errorbar(np.arange(0,7),all_f3m,yerr=f3_yerr,color='red')
plt.title('Day 3, all participants')
plt.xticks(np.arange(0,7,1),ticks)
plt.xlabel('Trials from lure')
plt.ylabel('RT (ms)')

#%% ################## V3 analysis ####################
def computeStatsBlocks(subjID):

    with open('BehV3_subjID_' + subjID + '.pkl', "rb") as fin:
        sub = (pickle.load(fin))[0]

    statsDay = np.zeros((5,8))

    statsBlock = np.zeros((4,16,5))
    statsBlock_day2 = np.zeros((48,5))

    idx_c = 0

    for idx, expDay in enumerate(['1','2','3','4','5']):

        # Load catFile
        catFile = 'P:\\closed_loop_data\\' + str(subjID) + '\\createIndices_'+subjID+'_day_'+expDay+'.csv'

        if subjID == '11' and expDay == '2':
            responseTimes = [np.nan]
            nKeypress = np.nan
        else:
            responseTimes = sub['responseTimes_day'+expDay]
            nKeypress = sub['TOTAL_keypress_day'+expDay]

        if subjID == '11' and expDay == '2':
            CI_lure, NI_lure, I_nlure, NI_nlure, lure_RT_mean, nonlure_RT_mean, RT_mean, nNaN = [np.nan]*8
        else:
            CI_lure, NI_lure, I_nlure, NI_nlure, lure_RT_mean, nonlure_RT_mean, RT_mean, nNaN = findRTsBlocks(catFile,responseTimes,block=False)

        # Compute stats (for visibility)
        TP = NI_nlure
        FP = NI_lure
        TN = CI_lure
        FN = I_nlure

        sensitivity = TP/(TP+FN)
        specificity = TN/(TN+FP)

        accuracy = (TP+TN)/(TP+TN+FP+FN)
#        accuracy_all = (TP+TN)/nKeypress # Out of all the keypresses that day

        statsDay[idx,:] = sensitivity, specificity, accuracy, lure_RT_mean, nonlure_RT_mean, RT_mean, nKeypress, nNaN

        # Block-wise
        for block in range(1,int(len(responseTimes)/50)+1):
            print(block)

            if subjID == '11' and expDay == '2':
                TN, FP, FN, TP, lure_RT_mean, nonlure_RT_mean, RT_mean, nNaN = [np.nan]*8
            else:
                TN, FP, FN, TP, lure_RT_mean, nonlure_RT_mean, RT_mean, nNaN = findRTsBlocks(catFile,responseTimes,block=block)

            sensitivity = TP/(TP+FN)
            specificity = TN/(TN+FP)

            accuracy = (TP+TN)/(TP+TN+FP+FN)

            if expDay == '2':
                statsBlock_day2[block-1,:] = sensitivity, specificity, accuracy, RT_mean, nNaN

            if expDay != '2':
                statsBlock[idx_c,block-1,:] = sensitivity, specificity, accuracy, RT_mean, nNaN

        if expDay != '2':
            idx_c += 1

    return statsDay, statsBlock, statsBlock_day2

#%%
# Test for subj 11
#statsDay11, statsBlock11, statsBlock_day211 = computeStatsBlocks('11')
#statsDay, statsBlock, statsBlock_day2 = computeStatsBlocks('07')

sub_all = ['07','08','11','13','14','15','16','17','18','19','21','22','23','24','25','26','27','30','31','32','33','34']

statsAll = {}

for idx,subjID in enumerate(sub_all):
    statsDay, statsBlock, statsBlock_day2 = computeStatsBlocks(subjID)

    statsAll[subjID] = statsBlock

# Mean response time per block, day 2
plt.plot(statsBlock_day2[:,3])

# Mean response time per block, day 1 and 3
plt.plot(statsBlock[0,:,3])
plt.plot(statsBlock[1,:,3])

# Sensitivity across blocks
plt.plot(statsBlock[0,:,0])
plt.plot(statsBlock[1,:,0])

# Accuracy across blocks
plt.plot(statsBlock[0,:,2])
plt.plot(statsBlock[1,:,2])

#%%
subjIDs_NF = ['07','08','11','13','14','16','19','22','26','27','30']
subjIDs_C = ['15','17','18','21','23','24','25','31','32','33','34']

def extractBehVal(day,wanted_measure):
    '''
    Day is
    1=0
    3=1

    for statsBlock

    and wanted_measure is
    0 = sens
    1 = spec
    2 = acc
    3 = RT mean
    4 = nNaN

    '''

    if day == 1:
        day_idx = 0
    if day == 3:
        day_idx = 1
    if day == 4:
        day_idx = 2

    if wanted_measure == 'sen':
        w_idx = 0
    if wanted_measure == 'spec':
        w_idx = 1
    if wanted_measure == 'acc':
        w_idx = 2
    if wanted_measure == 'rt':
        w_idx = 3
    if wanted_measure == 'nan':
        w_idx = 4

    subsAll = []
    subsNF = []
    subsC = []

    for key, value in statsAll.items():
        result = value[day_idx,:,w_idx]
        print(result.shape)
        subsAll.append(result)

        if key in subjIDs_NF:
            subsNF.append(result)
        if key in subjIDs_C:
            subsC.append(result)

    return subsAll, subsNF, subsC

#%% NOT block-wise, statsDay. Check whether this corresponds to the other (V2) beh measures.
os.chdir(saveDir)

statsDayAll = {}

for idx,subjID in enumerate(sub_all):
    statsDay, statsBlock, statsBlock_day2 = computeStatsBlocks(subjID)

    statsDayAll[subjID] = statsDay
#%%
def extractStatsDayAll(day_idx,wanted_measure):
    #sensitivity, specificity, accuracy, lure_RT_mean, nonlure_RT_mean, RT_mean, nKeypress, nNaN
    day_idx = day_idx-1

    if wanted_measure == 'sen':
        w_idx = 0
    if wanted_measure == 'spec':
        w_idx = 1
    if wanted_measure == 'acc':
        w_idx = 2
    if wanted_measure == 'rt_lure':
        w_idx = 3
    if wanted_measure == 'rt_nlure':
        w_idx = 4
    if wanted_measure == 'rt':
        w_idx = 5
    if wanted_measure == 'nkeypress':
        w_idx = 6
    if wanted_measure == 'nan':
        w_idx = 7

    subsAll = []
    subsNF = []
    subsC = []

    for key, value in statsDayAll.items():
#        print(value.shape)
        result = value[day_idx,w_idx]
#        print(result.shape)
        subsAll.append(result)

        if key in subjIDs_NF:
            subsNF.append(result)
        if key in subjIDs_C:
            subsC.append(result)

    return subsAll, subsNF, subsC
#%%
# Reality check
senAll_d1, senNF_d1, senC_d1 = extractStatsDayAll(1,'sen')
senAll_d3, senNF_d3, senC_d3 = extractStatsDayAll(3,'sen')

sen_feedback[:,0] # Sensitivity, feedback group, day1
sen_control[:,0]

sen_feedback[:,1]

# Checked 29April, legit. V2 and V3 analysis corresponds.

#%%
def make4Bars(wanted_measure,title,ylabel):
    '''
    Comparing day 1 and 3, NF and controls, using the statsDayAll structure and extract func.

    Connected lines bar plot

    '''
    all_d1, NF_d1, C_d1 = extractStatsDayAll(1,wanted_measure)
    all_d3, NF_d3, C_d3 = extractStatsDayAll(3,wanted_measure)

    print('NF mean day 1: ',round(np.mean(NF_d1),3))
    print('NF mean day 3: ',round(np.mean(NF_d3),3))
    print('C mean day 1: ',round(np.mean(C_d1),3))
    print('C mean day 3: ',round(np.mean(C_d3),3))

    y_min = np.min([np.min(all_d1),np.min(all_d3)])
    y_max = np.max([np.max(all_d1),np.max(all_d3)])

    # Connect lines
    plt.figure(random.randint(0,100))
    plt.ylabel(ylabel)
    plt.ylim([y_min-0.05,y_max+0.05])
#    plt.ylim([y_min-5,y_max+5])
    plt.xticks([1,2,3,4],['NF day 1','NF day 3', 'Control day 1', 'Control day 3'])
    plt.title(title)

    plt.bar(1,np.mean(NF_d1),color=(0,0,0,0),edgecolor='tomato')
    plt.bar(2,np.mean(NF_d3),color=(0,0,0,0),edgecolor='brown')
    plt.bar(3,np.mean(C_d1),color=(0,0,0,0),edgecolor='dodgerblue')
    plt.bar(4,np.mean(C_d3),color=(0,0,0,0),edgecolor='navy')

    plt.scatter(np.full(11,1),NF_d1,color='tomato')
    plt.scatter(np.full(11,2),NF_d3,color='brown')
    plt.scatter(np.full(11,3),C_d1,color='dodgerblue')
    plt.scatter(np.full(11,4),C_d3,color='navy')

    for i in range(11):
        plt.plot([(np.full(11,1))[i],(np.full(11,2))[i]], [(NF_d1)[i],(NF_d3)[i]],color='gray')
        plt.plot([(np.full(11,3))[i],(np.full(11,4))[i]], [(C_d1)[i],(C_d3)[i]],color='gray')

    t_fb = stats.ttest_rel(NF_d1,NF_d3)
    t_c = stats.ttest_rel(C_d1,C_d3)

    return plt, t_fb, t_c

def make2Bars(wanted_measure,title,ylabel):
    '''
    Comparing day 2, NF and controls, using the statsDayAll structure and extract func.
    '''

    all_d2, NF_d2, C_d2 = extractStatsDayAll(2,wanted_measure)

    print('NF mean day 2: ',round(np.nanmean(NF_d2),3))
    print('C mean day 2: ',round(np.mean(C_d2),3))

    y_min = np.nanmin(all_d2)
    y_max = np.nanmax(all_d2)

    # Connect lines
    plt.figure(random.randint(0,100))
    plt.ylabel(ylabel)
    plt.ylim([y_min-0.01,y_max+0.01])
#    plt.ylim([y_min-5,y_max+5])
    plt.xticks([1,2],['NF day 2','Control day 2'])
    plt.title(title)

    plt.bar(1,np.nanmean(NF_d2),color=(0,0,0,0),edgecolor='tomato')
    plt.bar(2,np.mean(C_d2),color=(0,0,0,0),edgecolor='dodgerblue')

    plt.scatter(np.full(11,1),NF_d2,color='tomato')
    plt.scatter(np.full(11,2),C_d2,color='dodgerblue')

    # Omit one subject from C?
    t = stats.ttest_ind(NF_d2,C_d2,nan_policy='omit')

    return plt, t, all_d2

#%%
pl, t_fb, t_c = make4Bars('rt','Overall response time','Response time (s)')
pl, t_fb, t_c = make4Bars('rt_lure','Response time for lures','Response time (s)')
pl, t_fb, t_c = make4Bars('rt_nlure','Response time for non lures','Response time (s)')
pl, t_fb, t_c = make4Bars('nkeypress','Number of total keypresses','Number keypresses')
pl, t_fb, t_c = make4Bars('nan','Number of total NaNs','Number ') # Not saved

pl, t_fb, t_c = make4Bars('sen','x','x')
pl, t_fb, t_c = make4Bars('spec','x','x')
pl, t_fb, t_c = make4Bars('acc','x','x')

#diff_fb_p=stats.ttest_rel(sen_feedback[:,0],sen_feedback[:,1])
#diff_c_p=stats.ttest_rel(sen_control[:,0],sen_control[:,1])

# Investigate day 2, using make2Bars and StatsDayAll (NOT block-wise)
pl, t, sen_all_d2 = make2Bars('sen','Sensitivity day 2','Sensitivity')
pl, t, acc_all_d2 = make2Bars('acc','Accuracy day 2','Accuracy')
pl, t, rt_all_d2 = make2Bars('rt','RT day 2','RT')
pl, t = make2Bars('nkeypress','Number of total keypresses','Number keypresses')

# Check whether this looks reasonable by checking one day 2 file manually. DONE.

#%% Is good decoding accuracy correlated with a good day 2 behavioral response?
# Omit 11
sen_all_d2_c = np.copy(sen_all_d2)
sen_all_d2_c = sen_all_d2_c[~np.isnan(sen_all_d2_c)]

# Acc
acc_all_d2_c = np.copy(acc_all_d2)
acc_all_d2_c = acc_all_d2_c[~np.isnan(acc_all_d2_c)]

# RT
rt_all_d2_c = np.copy(rt_all_d2)
rt_all_d2_c = rt_all_d2_c[~np.isnan(rt_all_d2_c)]

subsAll_c = np.copy(subsAll) # From EEG inv 18April
subsAll_c = np.delete(subsAll_c, 2)

plt.scatter(subsAll_c,sen_all_d2_c)
plt.ylabel('Sensitivity day 2')
plt.xlabel('Real time decoding accuracy, bias corrected')
plt.title('Sensitivity day 2 vs. RT decoding accuracy, N=21')

plt.scatter(subsAll_c,acc_all_d2_c)
plt.ylabel('Accuracy day 2')
plt.xlabel('Real time decoding accuracy, bias corrected')
plt.title('Accuracy day 2 vs. RT decoding accuracy, N=21')

# Plot RT vs decoding acc
plt.scatter(subsAll_c,rt_all_d2_c)
plt.ylabel('Response time day 2')
plt.xlabel('Real time decoding accuracy, bias corrected')
plt.title('Response time day 2 vs. RT decoding accuracy, N=21')

stats.linregress(acc_all_d2_c,subsAll_c)

np.corrcoef(sen_all_d2_c,subsAll_c)

stats.linregress(sen_all_d2_c,subsAll_c)

np.corrcoef(acc_all_d2_c,subsAll_c)

#%% Plot sensitivity block-wise, with individual lines for subjects, day1
subsAll, subsNF, subsC = extractBehVal(1,'sen')

colorC = sns.color_palette("Blues",11)
colorNF = sns.color_palette("Reds",11)

# Plot NF subjects
plt.figure(20)
for j in range(len(subsNF)):
    plt.plot(subsNF[j],color=colorNF[j],linewidth=0.8)

plt.plot(np.mean(subsNF,axis=0),label='Mean NF group',color='red',linewidth=2.0)

# Plot C subjects
for i in range(len(subsC)):
    plt.plot(subsC[i],color=colorC[i],linewidth=0.8)

plt.plot(np.mean(subsC,axis=0),label='Mean control group',color='blue',linewidth=2.0)

plt.plot(np.mean(subsAll,axis=0),label='Mean all participants',color='black',linewidth=2.0)
plt.title('Sensitivity across blocks, all participants, day 1')
plt.xticks(np.arange(0,16),[str(item) for item in np.arange(1,17)])
plt.xlabel('Block number, day 1')
plt.ylabel('Sensitivity')
plt.legend()

#%% sensitivity block-wise, with individual lines for subjects, day3
subsAll, subsNF, subsC = extractBehVal(3,'sen')

colorC = sns.color_palette("Blues",11)
colorNF = sns.color_palette("Reds",11)

# Plot NF subjects
plt.figure(21)
for j in range(len(subsNF)):
    plt.plot(subsNF[j],color=colorNF[j],linewidth=0.8)

plt.plot(np.mean(subsNF,axis=0),label='Mean NF group',color='red',linewidth=2.0)

# Plot C subjects
for i in range(len(subsC)):
    plt.plot(subsC[i],color=colorC[i],linewidth=0.8)

plt.plot(np.mean(subsC,axis=0),label='Mean control group',color='blue',linewidth=2.0)

plt.plot(np.mean(subsAll,axis=0),label='Mean all participants',color='black',linewidth=2.0)
plt.title('Sensitivity across blocks, all participants, day 3')
plt.xticks(np.arange(0,16),[str(item) for item in np.arange(1,17)])
plt.xlabel('Block number, day 3')
plt.ylabel('Sensitivity')
plt.legend()

#%% spec block-wise, with individual lines for subjects, day1
subsAll, subsNF, subsC = extractBehVal(1,'spec')

colorC = sns.color_palette("Blues",11)
colorNF = sns.color_palette("Reds",11)

# Plot NF subjects
plt.figure(23)
for j in range(len(subsNF)):
    plt.plot(subsNF[j],color=colorNF[j],linewidth=0.8)

plt.plot(np.mean(subsNF,axis=0),label='Mean NF group',color='red',linewidth=2.0)

# Plot C subjects
for i in range(len(subsC)):
    plt.plot(subsC[i],color=colorC[i],linewidth=0.8)

plt.plot(np.mean(subsC,axis=0),label='Mean control group',color='blue',linewidth=2.0)

plt.plot(np.mean(subsAll,axis=0),label='Mean all participants',color='black',linewidth=2.0)
plt.title('Specificity across blocks, all participants, day 1')
plt.xticks(np.arange(0,16),[str(item) for item in np.arange(1,17)])
plt.xlabel('Block number, day 1')
plt.ylabel('Specificity')
plt.legend()

#%% spec block-wise, with individual lines for subjects, day3
subsAll, subsNF, subsC = extractBehVal(3,'spec')

colorC = sns.color_palette("Blues",11)
colorNF = sns.color_palette("Reds",11)

# Plot NF subjects
plt.figure(24)
for j in range(len(subsNF)):
    plt.plot(subsNF[j],color=colorNF[j],linewidth=0.8)

plt.plot(np.mean(subsNF,axis=0),label='Mean NF group',color='red',linewidth=2.0)

# Plot C subjects
for i in range(len(subsC)):
    plt.plot(subsC[i],color=colorC[i],linewidth=0.8)

plt.plot(np.mean(subsC,axis=0),label='Mean control group',color='blue',linewidth=2.0)

plt.plot(np.mean(subsAll,axis=0),label='Mean all participants',color='black',linewidth=2.0)
plt.title('Specificity across blocks, all participants, day 3')
plt.xticks(np.arange(0,16),[str(item) for item in np.arange(1,17)])
plt.xlabel('Block number, day 3')
plt.ylabel('Specificity')
plt.legend()

#%% acc block-wise, with individual lines for subjects, day1
subsAll, subsNF, subsC = extractBehVal(1,'acc')

colorC = sns.color_palette("Blues",11)
colorNF = sns.color_palette("Reds",11)

# Plot NF subjects
plt.figure(23)
for j in range(len(subsNF)):
    plt.plot(subsNF[j],color=colorNF[j],linewidth=0.8)

plt.plot(np.mean(subsNF,axis=0),label='Mean NF group',color='red',linewidth=2.0)

# Plot C subjects
for i in range(len(subsC)):
    plt.plot(subsC[i],color=colorC[i],linewidth=0.8)

plt.plot(np.mean(subsC,axis=0),label='Mean control group',color='blue',linewidth=2.0)

plt.plot(np.mean(subsAll,axis=0),label='Mean all participants',color='black',linewidth=2.0)
plt.title('Accuracy across blocks, all participants, day 1')
plt.xticks(np.arange(0,16),[str(item) for item in np.arange(1,17)])
plt.xlabel('Block number, day 1')
plt.ylabel('Accuracy')
plt.legend()

#%% acc block-wise, with individual lines for subjects, day1
subsAll, subsNF, subsC = extractBehVal(3,'acc')

colorC = sns.color_palette("Blues",11)
colorNF = sns.color_palette("Reds",11)

# Plot NF subjects
plt.figure(23)
for j in range(len(subsNF)):
    plt.plot(subsNF[j],color=colorNF[j],linewidth=0.8)

plt.plot(np.mean(subsNF,axis=0),label='Mean NF group',color='red',linewidth=2.0)

# Plot C subjects
for i in range(len(subsC)):
    plt.plot(subsC[i],color=colorC[i],linewidth=0.8)

plt.plot(np.mean(subsC,axis=0),label='Mean control group',color='blue',linewidth=2.0)

plt.plot(np.mean(subsAll,axis=0),label='Mean all participants',color='black',linewidth=2.0)
plt.title('Accuracy across blocks, all participants, day 3')
plt.xticks(np.arange(0,16),[str(item) for item in np.arange(1,17)])
plt.xlabel('Block number, day 3')
plt.ylabel('Accuracy')
plt.legend()


import http
from unittest.mock import patch

import pytest

from django.core.signing import Signer
from django.core.urlresolvers import reverse

@pytest.mark.django_db
def test_create_anonymous_unsubscribe_create_bad_signature(client):
    url = reverse('anonymous-unsubscribe')
    response = client.post(url, {'email': 'test@example.com'})

    assert response.status_code == http.client.BAD_REQUEST

@pytest.mark.django_db
@patch('core.tasks.send_email')
def test_create_anonymous_unsubscribe_create_good_signature(mock_task, client):
    url = reverse('anonymous-unsubscribe')
    email = 'test@example.com'
    response = client.post(url, {'email': Signer().sign(email)})

    assert response.status_code == http.client.CREATED
    assert mock_task.delay.called is True

@pytest.mark.django_db
@patch('core.tasks.send_email')
def test_create_anonymous_unsubscribe_multiple_times(mock_task, client):
    url = reverse('anonymous-unsubscribe')
    email = 'test@example.com'
    client.post(url, {'email': Signer().sign(email)})

    response = client.post(url, {'email': Signer().sign(email)})

    assert response.status_code == http.client.OK
    assert mock_task.delay.called is True

@pytest.mark.django_db
@patch('notifications.notifications.anonymous_unsubscribed')
def test_create_anonymous_unsubscribe_email_confirmation(
    mock_anonymous_unsubscribed, client
):
    url = reverse('anonymous-unsubscribe')
    email = 'test@example.com'
    client.post(url, {'email': Signer().sign(email)})

    mock_anonymous_unsubscribed.assert_called_once_with(
        recipient_email=email
    )


"""
Automatically segments an audio file based on silence.

Marks an utterance for speech and silence.  Then, via
markTranscriptForAnnotations, chunks those segments into small,
manageable chunks.  The code assumes that the speaker is
talking most of the time.

Probably only useful in limited circumstances.
"""

from os.path import join
import math

from praatio import tgio
from praatio import praat_scripts
from praatio.utilities import utils

def markTranscriptForAnnotations(tgFN, tierName, outputTGFN, proportion=1 / 5.0):
    """
    Prep a noisy silence annotation for an annotation task

    Voice activity detectors are liable to segment speech into very small
    chunks (fragments of speech and silence).  The point of this code is
    to segment a recording into larger units that could be used in a
    speech transcription task.

    Assumes the speaker is speaking for most of the recording.
    """
    tg = tgio.openTextgrid(tgFN)

    duration = tg.maxTimestamp
    numEntries = int(math.ceil(duration * proportion))
    entryList = tg.tierDict[tierName].entryList

    # Get all silent intervals
    entryList = [
        (stop - start, start, stop, label)
        for start, stop, label in entryList
        if label == "silent"
    ]

    # Remove silent intervals at the start or end of the file
    entryList = [entry for entry in entryList if entry[1] != 0 and entry[2] != duration]

    # Put longest intervals first
    entryList.sort(reverse=True)

    # Get the mid point of the longest n intervals and convert them
    # into intervals to be transcribed
    entryList = entryList[:numEntries]
    pointList = [start + ((stop - start) / 2.0) for _, start, stop, _ in entryList]
    pointList.sort()

    pointList = (
        [
            0.0,
        ]
        + pointList
        + [
            duration,
        ]
    )

    newEntryList = []
    for i in range(len(pointList) - 1):
        newEntryList.append((pointList[i], pointList[i + 1], "%d" % i))

    outputTG = tgio.Textgrid()
    tier = tgio.IntervalTier("toTranscribe", newEntryList, 0, duration)
    outputTG.addTier(tier)

    outputTG.save(outputTGFN)

def autoSegmentSpeech(praatEXE, inputWavPath, rawTGPath, finalTGPath):

    utils.makeDir(finalTGPath)

    praat_scripts.annotateSilences(praatEXE, inputWavPath, rawTGPath)

    for tgFN in utils.findFiles(rawTGPath, filterExt=".TextGrid"):
        markTranscriptForAnnotations(
            join(rawTGPath, tgFN), "silences", join(finalTGPath, tgFN)
        )

_praatEXE = r"C:\praat.exe"
_root = join(".", "files")
_inputWavPath = _root
_rawTGPath = join(_root, "silence_marked_textgrids")
_finalTGPath = join(_root, "ready-to-transcribe_textgrids")
autoSegmentSpeech(_praatEXE, _inputWavPath, _rawTGPath, _finalTGPath)


"""Gets covid data"""
# External Imports
import requests

def get_data(url):
    """Gets a JSON from a url

    'url' is any full url string
    Returns a dictionary
    """
    request = requests.get(url)
    json_ = request.json()

    return json_['data']


import logging
import re
from urllib.parse import unquote_plus

import markdown
from markdown.preprocessors import Preprocessor
from markdown.inlinepatterns import Pattern
from markdown.extensions import Extension
from flask import url_for

from . import db

class QuotedExtension(Extension):
    def extendMarkdown(self, md, md_globals):
        md.preprocessors.add('inlinezip', InlineZipPreprocessor(), "_begin")
        md.preprocessors.add('inlineimage', InlinePreprocessor(), "_begin")
        md.preprocessors.add('inlinequotation', InlineQuotationPreprocessor(), "_begin")
        md.preprocessors.add('checklist', ChecklistPreprocessor(), "_begin")
        md.preprocessors.add('poem', PoemPreprocessor(), "_begin")
        md.inlinePatterns.add('sourceurl', ObjectUrlPattern("source", db.Source), "_begin")
        md.inlinePatterns.add('seriesurl', ObjectUrlPattern("series", db.Series), "_begin")
        md.inlinePatterns.add('termurl', ObjectUrlPattern("term", db.Term), "_begin")
        md.inlinePatterns.add('scite', ObjectCitePattern("s", db.Source, markdown_instance=md), "_begin")
        md.inlinePatterns.add('secite', ObjectCitePattern("se", db.Series, markdown_instance=md), "_begin")
        md.inlinePatterns.add('ncite', ObjectCitePattern("n", db.Name, markdown_instance=md), "_begin")

class InlinePreprocessor(Preprocessor):
    regex = re.compile(r"\[inline (?P<file_id>\d+)(?:\|(?P
.+?))?\]$")
    def run(self, lines):
        new_lines = []
        for line in lines:
            m = self.regex.fullmatch(line)
            if m:
                try:
                    file_id = int(m.group('file_id'))
                    the_file = db.Session.query(db.File).get(file_id)
                    if not the_file:
                        raise ValueError("File not found.")
                    new_lines.append('
')
                    new_lines.append(the_file.html_display)
                    if m.group('caption'):
                        new_lines.append('
{}
'.format(m.group('caption')))
                    new_lines.append('
')
                except ValueError:
                    new_lines.append(line)
            else:
                new_lines.append(line)
        return new_lines

class InlineQuotationPreprocessor(Preprocessor):
    regex = re.compile(r"\[inlinequotation (?P<quotation_id>\d+)(?:\|(?P
.+?))?\]$")
    def run(self, lines):
        new_lines = []
        for line in lines:
            m = self.regex.fullmatch(line)
            if m:
                try:
                    qlines = []
                    quotation_id = int(m.group('quotation_id'))
                    the_quotation = db.Session.query(db.Quotation).get(quotation_id)
                    if not the_quotation:
                        raise ValueError("Quotation not found.")
                    qlines.append('
')
                    q = ('
'
                         '{}'
                         '{}'
                         '
').format(
                            ''.join(markdown.markdown(line) for line in the_quotation.quotation.splitlines()),
                            "{}".format(the_quotation.source.__class__.__name__, the_quotation.source.internal_url, the_quotation.source))
                    qlines.append(q)
                    if m.group('caption'):
                        qlines.append('
{}
'.format(m.group('caption')))
                    qlines.append('
')
                    new_lines.extend(qlines)
                except ValueError:
                    new_lines.append(line)
            else:
                new_lines.append(line)
        return new_lines

class InlineZipPreprocessor(Preprocessor):
    regex = re.compile(r"\[inlinezip (?P<file_id>\d+)\|(?P<file_name>.+?)(?:\|(?P
.+?))?\]$")
    def run(self, lines):
        new_lines = []
        for line in lines:
            m = self.regex.fullmatch(line)
            if m:
                try:
                    file_id = int(m.group('file_id'))
                    file_name = unquote_plus(m.group('file_name'))
                    the_file = db.Session.query(db.File).get(file_id)
                    if not the_file:
                        raise ValueError("File not found.")
                    new_lines.append('
')
                    new_lines.append(''.format(url_for('viewers.view_cbz_file', file_id=file_id, file=file_name)))
                    if m.group('caption'):
                        new_lines.append('
{}
'.format(m.group('caption')))
                    new_lines.append('
')
                except ValueError:
                    logging.exception("inlinezip failed")
                    new_lines.append(line)
            else:
                new_lines.append(line)
        return new_lines

class ObjectUrlPattern(Pattern):
    def __init__(self, text, object_type):
        Pattern.__init__(self, r"\[{} (\d+)\]".format(text))
        self.text = text
        self.object_type = object_type

    def handleMatch(self, m):
        try:
            object_id = int(m.group(2))
            the_object = db.Session.query(self.object_type).get(object_id)
            if not the_object:
                raise ValueError("Object not found.")
            return the_object.internal_url
        except ValueError:
            return "[{} {}]".format(self.text, m.group(2))

class ObjectCitePattern(Pattern):
    def __init__(self, text, object_type, markdown_instance=None):
        Pattern.__init__(self, r"\[{}cite (\d+)\]".format(text), markdown_instance=markdown_instance)
        self.text = text
        self.object_type = object_type

    def handleMatch(self, m):
        try:
            object_id = int(m.group(2))
            the_object = db.Session.query(self.object_type).get(object_id)
            if not the_object:
                raise ValueError("Object not found.")
            return self.markdown.htmlStash.store("{}".format(the_object.__class__.__name__, the_object.internal_url, the_object))
        except ValueError:
            return "[{}cite {}]".format(self.text, m.group(2))

class ChecklistPreprocessor(Preprocessor):
    start_regex = re.compile(r"```(?:CHECKLIST|checklist)(?: (.+))?$")
    end_regex = re.compile(r"```$")
    cl_regex = re.compile(r"(-|X) (.+)$")

    def run(self, lines):
        checklist_mode = False
        new_lines = []
        checkboxes = 0
        for line in lines:
            if not checklist_mode:
                start_match = self.start_regex.fullmatch(line)
                if start_match:
                    checklist_mode = True
                    new_lines.append('<fieldset class="checklist">')
                    if start_match.group(1):
                        new_lines.append("<legend>{}</legend>".format(start_match.group(1)))
                    continue
                else:
                    new_lines.append(line)
                    continue
            else:
                end_match = self.end_regex.fullmatch(line)
                if end_match:
                    new_lines.append("</fieldset>")
                    checklist_mode = False
                    continue
                else:
                    # this should be a checklist item
                    cl_match = self.cl_regex.fullmatch(line)
                    if cl_match:
                        checkboxes = checkboxes + 1
                        checked = 'checked="checked"' if (cl_match.group(1) == "X") else ''
                        item = cl_match.group(2)
                        new_lines.append('
<input class="autoCheckbox" onchange="toggleCheckbox(event)" type="checkbox" name="checkbox_{}" id="checkbox_{}" {} />{}
'.format(checkboxes, checkboxes, checked, checkboxes, checkboxes, item))
                    elif line[:4] == "    ":
                        new_lines.append("
" + line[4:] + "
")
                    else:
                        # throw away lines not in the right format
                        pass
        return new_lines

class PoemPreprocessor(Preprocessor):
    start_regex = re.compile(r"```(?:POEM|poem)(?: (.+))?$")
    end_regex = re.compile(r"```$")

    def run(self, lines):
        poem_mode = False
        new_lines = []
        caption = None
        for line in lines:
            if not poem_mode:
                start_match = self.start_regex.fullmatch(line)
                if start_match:
                    poem_mode = True
                    new_lines.append('
')
                    if start_match.group(1):
                        caption = start_match.group(1)
                    new_lines.append('
')
                    continue
                else:
                    new_lines.append(line)
                    continue
            else:
                end_match = self.end_regex.fullmatch(line)
                if end_match:
                    new_lines.append("
")
                    if caption:
                        new_lines.append('
{}
'.format(caption))
                    new_lines.append("
")
                    poem_mode = False
                    continue
                else:
                    # this is a line of the poem
                    new_lines.append(line)
        return new_lines