from flask import Blueprint, render_template, send_from_directory, url_for, current_app, request, Response, send_file, jsonify
from flask_login import login_required, current_user
from flask_uploads import UploadSet, IMAGES
from flask_wtf import FlaskForm
from flask_wtf.file import FileField, FileRequired, FileAllowed
from wtforms import SubmitField
from . import db
from . import imgedit
from . import videoedit
from flask_cors import CORS
from moviepy.editor import VideoFileClip

import base64
import os

main = Blueprint('main', __name__)

@main.route('/')
def index():
    return render_template('index.html')

@main.route('/profile')
@login_required
def profile():
    return render_template('profile.html', name=current_user.name)

photos = UploadSet('photos', IMAGES)

class UploadForm(FlaskForm):
    photo = FileField(
        validators=[
            FileAllowed(photos, 'Only images are allowed.'),
            FileRequired('File field should not be empty.')
        ]
    )
    submit = SubmitField('Upload')

@main.route('/uploads/<filename>')
def get_file(filename):
    return send_from_directory(current_app.config['UPLOADED_PHOTOS_DEST'], filename)

@main.route('/takeimage', methods=['POST'])
def takeimage():
    # Nhận dữ liệu hình ảnh từ yêu cầu POST
    data_url = request.json['image']

    # Loại bỏ phần đầu của chuỗi dữ liệu URL (prefix "data:image/jpeg;base64,")
    img_data = data_url.split(',')[1]

    # Chuyển đổi dữ liệu hình ảnh từ base64 sang binary
    binary_data = base64.b64decode(img_data)

    # Lưu hình ảnh vào thư mục static/images với tên duy nhất và định dạng jpg
    image_name = f'image.jpg'
    image_path = os.path.join(image_name)
    with open(image_path, 'wb') as f:
        f.write(binary_data)

    print(f'Image saved at: {image_path}')

    return Response(status=200)

@main.route('/RGBtoGray', methods = ['GET', 'POST'])
@login_required
def RGBtoGray():
    form = UploadForm()
    if form.validate_on_submit():
        filename = photos.save(form.photo.data)
        file_url = url_for('main.get_file', filename=filename)
        gray_filename = imgedit.RGBtoGray(filename, file_url)
        gray_file_url = url_for('main.get_file', filename=gray_filename)
    else:
        file_url = None
        gray_file_url = None
    return render_template('imgedit_RGBtoGray.html', form = form, file_url = file_url, gray_file_url = gray_file_url)

@main.route('/face_detection', methods = ['GET', 'POST'])
@login_required
def face_detection():
    form = UploadForm()
    if form.validate_on_submit():
        filename = photos.save(form.photo.data)
        file_url = url_for('main.get_file', filename=filename)
        face_filename = imgedit.face_detection(filename, file_url)
        face_file_url = url_for('main.get_file', filename=face_filename)
    else:
        file_url = None
        face_file_url = None
    return render_template('imgedit_face_detection.html', form = form, file_url = file_url, face_file_url = face_file_url)

@main.route('/crop', methods = ['GET', 'POST'])
@login_required
def crop():
    form = UploadForm()
    if form.validate_on_submit():
        x1 = int(request.form.get('x1'))
        x2 = int(request.form.get('x2'))
        y1 = int(request.form.get('y1'))
        y2 = int(request.form.get('y2'))
        filename = photos.save(form.photo.data)
        file_url = url_for('main.get_file', filename=filename)
        face_filename = imgedit.crop(filename, file_url, x1, x2, y1, y2)
        face_file_url = url_for('main.get_file', filename=face_filename)
    else:
        file_url = None
        face_file_url = None
    print(file_url)
    return render_template('imgedit_crop.html', form = form, file_url = file_url, crop_file_url = face_file_url)

@main.route('/rotate', methods = ['GET', 'POST'])
@login_required
def rotate():
    form = UploadForm()
    if form.validate_on_submit():
        degree = int(request.form.get('degree'))
        x = int(request.form.get('x'))
        y = int(request.form.get('y'))
        filename = photos.save(form.photo.data)
        file_url = url_for('main.get_file', filename=filename)
        face_filename = imgedit.rotate(filename, file_url, degree, x, y)
        face_file_url = url_for('main.get_file', filename=face_filename)
    else:
        file_url = None
        face_file_url = None
    return render_template('imgedit_rotate.html', form = form, file_url = file_url, rotate_file_url = face_file_url)

@main.route('/resize', methods = ['GET', 'POST'])
@login_required
def resize():
    form = UploadForm()
    if form.validate_on_submit():
        size = int(request.form.get('size'))
        filename = photos.save(form.photo.data)
        file_url = url_for('main.get_file', filename=filename)
        face_filename = imgedit.resize(filename, file_url, size)
        face_file_url = url_for('main.get_file', filename=face_filename)
    else:
        file_url = None
        face_file_url = None
    return render_template('imgedit_resize.html', form = form, file_url = file_url, resize_file_url = face_file_url)

@main.route('/hsv', methods = ['GET', 'POST'])
@login_required
def hsv():
    form = UploadForm()
    if form.validate_on_submit():
        h = int(request.form.get('hue'))
        s = int(request.form.get('saturation'))
        l = int(request.form.get('light'))
        filename = photos.save(form.photo.data)
        file_url = url_for('main.get_file', filename=filename)
        face_filename = imgedit.hsv(filename, file_url, h, s, l)
        face_file_url = url_for('main.get_file', filename=face_filename)
    else:
        file_url = None
        face_file_url = None
    return render_template('imgedit_resize.html', form = form, file_url = file_url, face_file_url = face_file_url)

@main.route('/trim_video', methods = ['GET', 'POST'])
def trim_video():
    return render_template('trim_video.html')

@main.route('/uploads', methods=['GET'])
def get_uploaded_videos():
    uploaded_videos = [filename for filename in os.listdir(current_app.config["UPLOAD_FOLDER"])]
    return jsonify(uploaded_videos)

@main.route('/clips/<filename>')
def render_clip(filename):
    return send_file(os.path.join(current_app.config["UPLOAD_FOLDER"], filename))

@main.route('/playback/<filename>')
def playback(filename):
    video_path = os.path.join(current_app.config["UPLOAD_FOLDER"], filename)

    # You may want to add additional security checks here before sending the file
    # For example, check user authentication, permissions, etc.

    return send_file(video_path)

@main.route('/upload_video', methods=['POST'])
def upload_video():
    if not os.path.exists(current_app.config["UPLOAD_FOLDER"]):
        os.makedirs(current_app.config["UPLOAD_FOLDER"], exist_ok=True)

    try:
        videofile = request.files['videofile']
        filepath = os.path.join(current_app.config["UPLOAD_FOLDER"], videofile.filename)
        videofile.save(filepath)
        return {"filename": videofile.filename}
    except FileNotFoundError:
        return {"error": "File not found"}

@main.route('/edit_video/trim', methods=['POST'])
def trim_video_edit():
    try:
        videofile = request.json['videofile']
        trim_start = int(request.json['trim_start'])
        trim_end = int(request.json['trim_end'])
        trimmed_filename = request.json['trimmed_filename']

        # Append the video extension (e.g., ".mp4") to the filename
        trimmed_filename_with_extension = f"{trimmed_filename}.mp4"

        edited_videopath = trim_video_function(videofile, trim_start, trim_end, trimmed_filename_with_extension)
        return {
            "status": "success",
            "message": "video edit success",
            "edited_videopath": edited_videopath
        }
    except Exception as e:
        return {
            "status": "error",
            "message": "video edit failure: " + str(e),
        }

def trim_video_function(videofile, start_time, end_time, trimmed_filename):
    clip = VideoFileClip(os.path.join(current_app.config["UPLOAD_FOLDER"], videofile))
    trimmed_filename_with_extension = f"{trimmed_filename}"
    trimpath = os.path.join(current_app.config["UPLOAD_FOLDER"], trimmed_filename_with_extension)
    trimmed_clip = clip.subclip(start_time, end_time)
    trimmed_clip.write_videofile(trimpath)
    return trimpath


from django.shortcuts import render, redirect, get_object_or_404
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from .forms import CreateEventForm, JoinEvent, LikeEvent
from django.urls import reverse, reverse_lazy
from .models import Event, Participant, Like
from membership.models import Membership
from django.contrib.auth.models import User
import datetime

# Create your views here.
@login_required
def post_event(request):
    """Renders Create Event Form"""
    event_form = CreateEventForm()
    user = request.user
    if request.method == "POST":
        event_form = CreateEventForm(request.POST, request.FILES)
        if event_form.is_valid():
            event = event_form.save(commit=False)
            event.event_host = user
            event.save()
            membership = Membership.objects.get(user=user)
            membership.posts_remaining -= 1
            membership.save()
            current_places = event.max_participants
            full_address = (event.address + ' ' + event.town +
                            ' ' + event.post_code)
            context = {
                'event': event,
                'current_places': current_places,
                'full_address': full_address,
                'event_host': event.event_host,
            }
            messages.success(
                request, f"You have posted {request.POST['title']}!")
            return render(request, 'view_one_event.html', context)

    context = {
        'event_form': event_form
    }

    return render(request, 'post_event.html', context)

@login_required
def edit_event(request, pk):
    """Allows Event Host to make changes to an upcoming event
       and re-publish a finished event"""
    event = get_object_or_404(Event, pk=pk)
    event_form = CreateEventForm(instance=event)
    context = {
        'event_form': event_form
    }
    user = request.user
    if request.method == "POST":
        event_form = CreateEventForm(request.POST,
                                     request.FILES,
                                     instance=event)
        if event_form.is_valid():
            edit = event_form.save(commit=False)
            edit.event_host = user
            edit.save(force_update=True)
            if event.event_date_begins < datetime.date.today():
                membership = Membership.objects.get(user=user)
                if membership.posts_remaining == 0:
                    messages.error(
                        request, "Sorry, you have no posts remaining!")
                    return redirect('user_profile')
                else:
                    membership.posts_remaining -= 1
                    membership.save()
            messages.success(request, "You have updated your event!")
            return redirect('user_profile')
    return render(request, 'post_event.html', context)

@login_required
def delete_event(request, pk):
    """Allows Event Host to delete event"""
    event = get_object_or_404(Event, pk=pk)
    event.delete()
    messages.success(request, "You have successfully deleted this event!")
    return redirect('user_profile')

def view_one_event(request, pk):
    """Displays event information for site user"""
    event = get_object_or_404(Event, pk=pk)
    user = request.user
    join_form = JoinEvent(request.POST or None)
    like_form = LikeEvent(request.POST or None)
    if user.is_authenticated:
        user_liked = Like.objects.filter(event=event).filter(user=user)
        user_joined = Participant.objects.filter(event=event).filter(user=user)
    else:
        user_liked = None
        user_joined = None
    user_queue = Participant.objects.filter(event=event).count()
    user_list_id = Participant.objects.filter(event=event).values('user')
    user_list = User.objects.filter(id__in=user_list_id)
    date = datetime.datetime.now()
    current_places = event.max_participants - user_queue
    event_host = User.objects.get(id=event.event_host.id)
    # Ensures map renders at correct location
    full_address = event.address + ' ' + event.town + ' ' + event.post_code
    context = {
        'join_form': join_form,
        'like_form': like_form,
        'event': event,
        'current_places': str(current_places),
        'full_address': full_address,
        'user_joined': bool(user_joined),
        'user_liked': bool(user_liked),
        'event_host': event_host,
        'user_list': user_list,
        'date': date
    }

    # Allow user to join an event

    if request.method == "POST" and 'join_form' in request.POST:
        if join_form.is_valid():
            join_form.instance.user = user
            join_form.instance.event = event
            join_form.save()
            messages.success(
                request, f"You have signed up for { event.title }")
            return redirect(view_one_event, pk)
        else:
            messages.error(request, f"Error: Try again later!")

    # Allow user to like and save an event

    elif request.method == "POST" and 'like_form' in request.POST:
        if like_form.is_valid():
            like_form.instance.user = user
            like_form.instance.event = event
            like_form.save()
            messages.success(
                request, f"You have saved { event.title } to your like list")
            return redirect(view_one_event, pk)
        else:
            messages.error(request, f"Error: Try again later!")
    return render(request, 'view_one_event.html', context)

def delete_participant(request, pk):
    """Removes user from event"""
    event = get_object_or_404(Event, pk=pk)
    user = request.user
    participant = Participant.objects.filter(event=event).filter(user=user)
    participant.delete()
    messages.success(request, f"You are no longer booked for { event.title }")
    return redirect('search')

def delete_like(request, pk):
    """Removes event from user like list"""
    event = get_object_or_404(Event, pk=pk)
    user = request.user
    like = Like.objects.get(event=event, user=user)
    like.delete()
    return redirect(view_one_event, pk)


#!/home/zeyu/anaconda3/bin/python3.6

"""
    Preprocessing

    Author:
        Zeyu Li <zyli@cs.ucla.edu> or <zeyuli@ucla.edu>

    Description:
        Take in the Very raw data and produce a ready-to-use version
"""

import sys, os
import re
import logging
import numpy as np
from lxml import etree
from bs4 import BeautifulSoup

import nltk
nltk.data.path.append("/workspace/nltk_data")
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

import string, random
from collections import Counter

try:
    import ujson as json
except:
    import json

part_user = set()
    # Users participated in Asking and Answering

# count how many questions an users asked
# count how many questions an answerer responded
count_Q, count_A = {}, {}

qa_map = {}
test_candidates = set()

def clean_html(x):
    return BeautifulSoup(x, 'lxml').get_text()

def clean_str(string):
    """Clean up the string

    Cleaning strings of content or title
    Original taken from [https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py]

    Args:
        string - the string to clean

    Return:
        _ - the cleaned string
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

def clean_str2(s):
    """Clean up the string

    * New version, removing all punctuations

    Cleaning strings of content or title
    Original taken from [https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py]

    Args:
        string - the string to clean

    Return:
        _ - the cleaned string
    """
    ss = s
    translator = str.maketrans("", "", string.punctuation)
    ss = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", ss)
    ss = re.sub(r"\'s", "s", ss)
    ss = re.sub(r"\'ve", "ve", ss)
    ss = re.sub(r"n\'t", "nt", ss)
    ss = re.sub(r"\'re", "re", ss)
    ss = re.sub(r"\'d", "d", ss)
    ss = re.sub(r"\'ll", "ll", ss)
    ss = re.sub(r"\s{2,}", " ", ss)
    ss = ss.translate(translator)
    return ss.strip().lower()

def remove_stopwords(string, stopword_set):
    """Removing Stopwords

    Args:
        string - the input string to remove stopwords
        stopword_set - the set of stopwords

    Return:
        _ - the string that has all the stopwords removed
    """
    word_tokens = word_tokenize(string)
    filtered_string = [word for word in word_tokens
                       if word not in stopword_set]
    return " ".join(filtered_string)

def split_post(raw_dir, data_dir):
    """ Split the post

    Split post to question and answer,
    keep all information, output to file

    Args:
        raw_dir - raw data directory
        data_dir - parsed data directory
    """
    if os.path.exists(data_dir + "Posts_Q.json") \
        and os.path.exists(data_dir + "Posts_A.json"):
        print("\t\tPosts_Q.json, Posts_A.json already exists."
              "Skipping the split_post.")
        return

    with open(data_dir + "Posts_Q.json", "w") as fout_q, \
            open(data_dir + "Posts_A.json", "w") as fout_a:
        parser = etree.iterparse(raw_dir + 'Posts.xml',
                                 events=('end',), tag='row')
        for event, elem in parser:
            attr = dict(elem.attrib)
            attr['Body'] = clean_html(attr['Body'])

            # Output to separate files
            if attr['PostTypeId'] == '1':
                fout_q.write(json.dumps(attr) + "\n")
            elif attr['PostTypeId'] == '2':
                fout_a.write(json.dumps(attr) + "\n")
    return

def process_QA(data_dir):
    """Process QA

    Extract attributes used in this project
    Get rid of the text information,
    only record the question-user - answer-user relation

    Args:
        data_dir - the dir where primitive data is stored
    """
    POST_Q = "Posts_Q.json"
    POST_A = "Posts_A.json"
    OUTPUT = "Record_All.json"
    RAW_STATS = "question.stats.raw"

    # Get logger to log exceptions
    logger = logging.getLogger(__name__)

    no_acc_question = 0

    raw_question_stats = []

    if not os.path.exists(data_dir + POST_Q):
        raise IOError("file {} does NOT exist".format(data_dir + POST_Q))

    if not os.path.exists(data_dir + POST_A):
        raise IOError("file {} does NOT exist".format(data_dir + POST_A))

    # Process question information
    with open(data_dir + POST_Q, 'r') as fin_q:
        for line in fin_q:
            data = json.loads(line)
            try:
                qid, rid = data.get('Id', None), data.get('OwnerUserId', None)
                # If such
                if qid and rid:
                    acc_id = data.get('AcceptedAnswerId', None)
                    answer_count = int(data.get('AnswerCount', -1))
                    if acc_id:
                        qa_map[qid] = {
                            'QuestionId': qid,
                            'QuestionOwnerId': rid,
                            'AcceptedAnswerId': acc_id,
                            'AcceptedAnswererId': None,
                            'AnswererIdList': [],
                            'AnswererAnswerTuples': []
                        }
                        count_Q[rid] = count_Q.get(rid, 0) + 1
                    else:
                        no_acc_question += 1

                    if answer_count >= 0:
                        raw_question_stats.append(answer_count)
            except:
                logger.error("Error at process_QA 1: " + str(data))
                continue
    print("\t\t{} questions do not have accepted answer!"
          .format(no_acc_question))

    # Count raw question statistics
    raw_question_stats_cntr = Counter(raw_question_stats)
    with open(data_dir + RAW_STATS, "w") as fout:
        for x in sorted(list(raw_question_stats_cntr.keys())):
            print("{}\t{}".format(x, raw_question_stats_cntr[x]), file=fout)
        print("Total\t{}".format(sum(raw_question_stats)), file=fout)

    # Process answer information
    with open(data_dir + POST_A, 'r') as fin_a:
        for line in fin_a:
            data = json.loads(line)
            try:
                answer_id = data.get('Id', None)
                aid = data.get('OwnerUserId', None)
                qid = data.get('ParentId', None)
                entry = qa_map.get(qid, None)
                if answer_id and aid and qid and entry:
                    entry['AnswererAnswerTuples'].append((aid, answer_id))
                    entry['AnswererIdList'].append(aid)
                    count_A[aid] = count_A.get(aid, 0) + 1

                    # Check if we happen to hit the accepted answer
                    if answer_id == entry['AcceptedAnswerId']:
                        entry['AcceptedAnswererId'] = aid
                else:
                    logger.error(
                        "Answer {} belongs to unknown Question {} at Process QA"
                        .format(answer_id, qid))
            except IndexError as e:
                logger.error(e)
                logger.info("Error at process_QA 2: " + str(data))
                continue

    # Fill in the blanks of `AcceptedAnswererId`
    # for qid in qa_map.keys():
    #    acc_id = qa_map[qid]['AcceptedAnswerId']
    #    for aid, answer_id in qa_map[qid]['AnswererAnswerTuples']:
    #        if answer_id == acc_id:
    #            qa_map[qid]['AcceptedAnswererId'] = aid
    #            break

    print("\t\tWriting the Record for ALL to disk.")
    with open(data_dir + OUTPUT, 'w') as fout:
        for q in qa_map.keys():
            fout.write(json.dumps(qa_map[q]) + "\n")

def question_stats(data_dir):
    """Find the question statistics for `Introduction`

    Args:
        data_dir -
    Return
    """
    OUTPUT = "question.stats"
    count = []
    for qid in qa_map.keys():
        ans_count = len(qa_map[qid]['AnswererIdList'])
        count.append(ans_count)
        if ans_count == 0:
            print("0 answer id list", qid)
    question_stats_cntr = Counter(count)

    with open(data_dir + OUTPUT, "w") as fout:
        for x in sorted(list(question_stats_cntr.keys())):
            print("{}\t{}".format(x, question_stats_cntr[x]), file=fout)
        print("Total\t{}".format(sum(count), file=fout), file=fout)
    return

def build_test_set(data_dir, parsed_dir, threshold, test_sample_size,
                   test_proportion):
    """
    Building test datase,
    test_proportiont
    Args:
        parse_dir - the directory to save parsed set.
        threshold - the selection threshold

    Return:
    """
    TEST = "test.txt"
    OUTPUT_TRAIN = "Record_Train.json"

    accept_no_answerer = 0

    ordered_count_A = sorted(
        count_A.items(), key=lambda x:x[1], reverse=True)
    ordered_aid = [x[0] for x in ordered_count_A]
    ordered_aid = ordered_aid[: int(len(ordered_aid) * 0.1)]

    question_count = len(qa_map)

    for qid in qa_map.keys():
        accaid = qa_map[qid]['AcceptedAnswererId']
        rid = qa_map[qid]['QuestionOwnerId']
        if not accaid:
            accept_no_answerer += 1
            continue
        if count_Q[rid] >= threshold and count_A[accaid] >= threshold:
            test_candidates.add(qid)

    print("\t\tSample table size {}. Using {} instances for test."
          .format(len(test_candidates), int(question_count * test_proportion)))

    test = np.random.choice(list(test_candidates),
                            size=int(question_count * test_proportion),
                            replace=False)

    print("\t\tAccepted answer without Answerer {}".format(accept_no_answerer))

    print("\t\tWriting the sampled test set to disk")
    with open(parsed_dir + TEST, "w") as fout:
        for qid in test:
            rid = qa_map[qid]['QuestionOwnerId']
            accaid = qa_map[qid]['AcceptedAnswererId']
            aid_list = qa_map[qid]['AnswererIdList']
            if len(aid_list) <= test_sample_size:
                neg_sample_size = test_sample_size - len(aid_list)
                neg_samples = random.sample(ordered_aid, neg_sample_size)
                samples = neg_samples + aid_list
            else:
                samples = random.sample(aid_list, test_sample_size)
                if accaid not in samples:
                    samples.pop()
                    samples.append(accaid)
            samples = " ".join(samples)
            print("{} {} {} {}".format(rid, qid, accaid, samples),
                  file=fout)

    # if qid is a test instance or qid doesn't have an answer
    qid_list = list(qa_map.keys())
    for qid in qid_list:
        if qid in test\
            or not len(qa_map[qid]['AnswererIdList'])\
            or not qa_map[qid]['AcceptedAnswererId']:
            del qa_map[qid]

    # Write QA pair to file
    print("\t\tWriting the Record for training to disk")
    with open(data_dir + OUTPUT_TRAIN, 'w') as fout:
        for q in qa_map.keys():
            fout.write(json.dumps(qa_map[q]) + "\n")
    return

def extract_question_user(data_dir, parsed_dir):
    """Extract Question User pairs and output to file.
    Extract "Q" and "R". Format:
        <Qid> <Rid>
    E.g.
        101 40
        145 351

    Args:
        data_dir - data directory
        parsed_dir - parsed file directory
    """
    # INPUT = "Record_Train.json"
    INPUT = "Record_All.json"
    OUTPUT = "Q_R.txt"

    if not os.path.exists(data_dir + INPUT):
        IOError("Can NOT find {}".format(data_dir + INPUT))

    with open(data_dir + INPUT, "r") as fin:
        with open(parsed_dir + OUTPUT, "w") as fout:
            for line in fin:
                data = json.loads(line)
                qid = data['QuestionId']
                rid = data['QuestionOwnerId']
                part_user.add(int(rid))  # Adding participated questioners
                print("{} {}".format(str(qid), str(rid)), file=fout)

def extract_question_answer_user(data_dir, parsed_dir):
    """Extract Question, Answer User pairs and output to file.

    (1) Extract "Q" - "A"
        The list of AnswerOwnerList contains <aid>-<owner_id> pairs
        Format:
            <Qid> <Aid>
        E.g.
            100 1011
            21 490

    (2) Extract "Q" - Accepted answerer
        Format:
            <Qid> <Acc_Aid>
    Args:
        data_dir - data directory
        parsed_dir - parsed file directory
    """
    INPUT = "Record_Train.json"
    OUTPUT_A = "Q_A.txt"
    OUTPUT_ACC = "Q_ACC.txt"

    if not os.path.exists(data_dir + INPUT):
        IOError("Can NOT find {}".format(data_dir + INPUT))

    with open(data_dir + INPUT, "r") as fin, \
            open(parsed_dir + OUTPUT_A, "w") as fout_a, \
            open(parsed_dir + OUTPUT_ACC, "w") as fout_acc:
        for line in fin:
            data = json.loads(line)
            qid = data['QuestionId']
            aid_list = data['AnswererIdList']
            accaid = data['AcceptedAnswererId']
            for aid in aid_list:
                part_user.add(int(aid))
                print("{} {}".format(str(qid), str(aid)), file=fout_a)
            print("{} {}".format(str(qid), str(accaid)), file=fout_acc)

def extract_question_content(data_dir, parsed_dir):
    """Extract questions, content pairs from question file

    Question content pair format:
        <qid> <content>
    We extract both with and without stop-word version
        which is signified by "_nsw"

    Args:
        data_dir - data directory
        parsed_dir - parsed file directory
    """
    INPUT = "Posts_Q.json"
    OUTPUT_T = "Q_title.txt"  # Question title
    OUTPUT_T_NSW = "Q_title_nsw.txt"  # Question title, no stop word
    OUTPUT_C = "Q_content.txt"  # Question content
    OUTPUT_C_NSW = "Q_content_nsw.txt"  # Question content, no stop word

    logger = logging.getLogger(__name__)

    if not os.path.exists(data_dir + INPUT):
        IOError("Can NOT locate {}".format(data_dir + INPUT))

    sw_set = set(stopwords.words('english'))  # Create the stop word set

    # We will try both with or without stopwords to
    # check out the performance.
    with open(data_dir + INPUT, "r") as fin, \
            open(parsed_dir + OUTPUT_T, "w") as fout_t, \
            open(parsed_dir + OUTPUT_T_NSW, "w") as fout_t_nsw, \
            open(parsed_dir + OUTPUT_C, "w") as fout_c, \
            open(parsed_dir + OUTPUT_C_NSW, "w") as fout_c_nsw:
        for line in fin:
            data = json.loads(line)
            try:
                qid = data.get('Id')
                if qid not in qa_map:
                    continue
                title = data.get('Title')
                content = data.get('Body')

                content, title = clean_str2(content), clean_str2(title)
                content_nsw = remove_stopwords(content, sw_set)
                title_nsw = remove_stopwords(title, sw_set)

                print("{} {}".format(qid, content_nsw),
                      file=fout_c_nsw)  # Without stopword
                print("{} {}".format(qid, content),
                      file=fout_c)  # With stopword
                print("{} {}".format(qid, title_nsw),
                      file=fout_t_nsw)  # Without stopword
                print("{} {}".format(qid, title),
                      file=fout_t)  # With stopword
            except:
                logger.info("Error at Extracting question content and title: "
                            + str(data))
                continue

def extract_answer_score(data_dir, parsed_dir):
    """Extract the answers vote, a.k.a. Scores.

    This information might be useful when
        the accepted answer is not selected.

    Args:
        data_dir - Input data dir
        parsed_dir - Output data dir
    """
    INPUT = "Posts_A.json"
    OUTPUT = "A_score.txt"

    logger = logging.getLogger(__name__)

    if not os.path.exists(data_dir + INPUT):
        IOError("Cannot find file{}".format(data_dir + INPUT))

    with open(data_dir + INPUT, "r") as fin, \
        open(parsed_dir + OUTPUT, "w") as fout:
        for line in fin:
            data = json.loads(line)
            try:
                aid = data.get('Id')
                score = data.get('Score')
                print("{} {}".format(aid, score), file=fout)
            except:
                logging.info("Error at Extracting answer score: "
                             + str(data))
                continue

def extract_question_best_answerer(data_dir, parsed_dir):
    """Extract the question-best-answerer relation

    Args:
        data_dir  - as usual
        parsed_dir  -  as usual
    """
    INPUT_A = "Posts_A.json"
    INPUT_MAP = "Record_Train.json"
    OUTPUT = "Q_ACC_A.txt"

    if not os.path.exists(data_dir + INPUT_A):
        IOError("Cannot find file {}".format(data_dir + INPUT_A))
    if not os.path.exists(data_dir + INPUT_MAP):
        IOError("Cannot find file {}".format(data_dir + INPUT_MAP))

    accanswerid_uaid = {}  # Accepted answer id to Answering user id
    answerid_score = {}  # Answer id to answer scores
    with open(data_dir + INPUT_A, "r") as fin_a, \
        open(data_dir + INPUT_MAP, "r") as fin_map, \
        open(parsed_dir + OUTPUT, "w") as fout:

        # build acc-a dict
        for line in fin_a:
            data = json.loads(line)
            try:
                answerid = data.get("Id")
                if answerid == "5":
                    print(100)
                score = data.get("Score")
                uaid = data.get("OwnerUserId")
                answerid_score[answerid] = score
                accanswerid_uaid[answerid] = uaid  # uaid is rid
            except:
                logging.info(
                    "Error at Extracting question, best answer user: "
                    + str(data))

        print(len(accanswerid_uaid))
        for line in fin_map:
            data = json.loads(line)
            try:
                qid = data.get('QuestionId')
                if "AcceptedAnswerID" in data:  # If acc answer exists
                    acc_answerid = data.get('AcceptedAnswerId')
                else:
                # If acc answer doesn't exist, choose highest score answer
                    ans = data.get('AnswerIdList')
                    ans = list(zip(*ans))[0]
                    scores = [answerid_score[answerid] for answerid in ans]
                    max_ind = scores.index(max(scores))
                    acc_answerid = ans[max_ind]
                uaccid = accanswerid_uaid[acc_aid]
                print("{} {}".format(qid, uaccid), file=fout)
            except:
                print(1)
                logging.info(
                    "Error at Extracting question, best answer user: "
                     + str(data))

def extract_question_best_answerer_2(data_dir, parsed_dir):
    """Extract the question-best-answerer relation

    Args:
        data_dir  - as usual
        parsed_dir  -  as usual
    """
    INPUT_A = "Posts_A.json"
    # INPUT_MAP = "Record_Train.json"
    # Uncomment this when running NeRank
    INPUT_MAP = "Record_All.json"
    OUTPUT = "Q_ACC_A.txt"

    if not os.path.exists(data_dir + INPUT_A):
        IOError("Cannot find file {}".format(data_dir + INPUT_A))
    if not os.path.exists(data_dir + INPUT_MAP):
        IOError("Cannot find file {}".format(data_dir + INPUT_MAP))

    accanswerid_uaid = {}  # Accepted answer id to Answering user id
    answerid_score = {}  # Answer id to answer scores
    with open(data_dir + INPUT_MAP, "r") as fin_map, \
        open(parsed_dir + OUTPUT, "w") as fout:

        for line in fin_map:
            data = json.loads(line)
            try:
                qid = data.get('QuestionId')
                acc_aid = data.get("AcceptedAnswererId")
                if qid and acc_aid:
                    print("{} {}".format(qid, acc_aid), file=fout)
            except:
                print(1)
                logging.info(
                    "Error at Extracting question, best answer user: "
                     + str(data))

def write_part_users(parsed_dir):
    OUTPUT = "QA_ID.txt"
    with open(parsed_dir + OUTPUT, "w") as fout:
        IdList = list(part_user)
        IdList.sort()
        for index, user_id in enumerate(IdList):
            print("{} {}".format(index + 1, user_id), file=fout)

def preprocess_(dataset, threshold, prop_test, sample_size):
    DATASET = dataset
    RAW_DIR = os.getcwd() + "/raw/{}/".format(DATASET)
    DATA_DIR= os.getcwd() + "/data/{}/".format(DATASET)
    PARSED_DIR = os.getcwd() + "/data/parsed/{}/".format(DATASET)

    print("Preprocessing {} ...".format(dataset))

    if not os.path.exists(RAW_DIR):
        print("{} dir or path doesn't exist.\n"
              "Please download the raw data set into the /raw."
              .format(RAW_DIR), file=sys.stderr)
        sys.exit()

    if not os.path.exists(DATA_DIR):
        print("{} data dir not found.\n"
              " Creating a folder for that."
              .format(DATA_DIR))
        os.makedirs(DATA_DIR)

    if not os.path.exists(PARSED_DIR):
        print("{} dir or path NOT found.\n"
              "Creating a folder for that."
              .format(PARSED_DIR))
        os.makedirs(PARSED_DIR)

    if os.path.exists(DATA_DIR + "log.log"):
        os.remove(DATA_DIR + "log.log")

    # Setting up loggers
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)

    log_fh = logging.FileHandler(DATA_DIR + "log.log")
    log_fh.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    log_fh.setFormatter(formatter)
    logger.addHandler(log_fh)

    # Split contest to question and answer
    print("\tSpliting post")
    # split_post(raw_dir=RAW_DIR, data_dir=DATA_DIR)

    # Extract question-user, answer-user, and question-answer information
    # Generate Question and Answer/User map
    print("\tProcessing QA")
    # process_QA(data_dir=DATA_DIR)

    print("\tGenerating question statistics...")
    # question_stats(data_dir=DATA_DIR)

    print("\tExtracting question content ...")
    # extract_question_content(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)

    print("\tBuilding test sets")
    # build_test_set(data_dir=DATA_DIR, parsed_dir=PARSED_DIR,
    #                threshold=threshold, test_sample_size=sample_size,
    #                test_proportion=prop_test)

    print("\tExtracting Q, R, A relations ...")
    extract_question_user(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)

    # extract_question_answer_user(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)

    # extract_answer_score(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)
    # extract_question_best_answerer_2(data_dir=DATA_DIR, parsed_dir=PARSED_DIR)

    # write_part_users(parsed_dir=PARSED_DIR)

    print("Done!")

if __name__ == "__main__":
    if len(sys.argv) < 3 + 1:
        print("\t Usage: {} [name of dataset] [threshold] [prop of test] [test sample size]"
              .format(sys.argv[0]), file=sys.stderr)
        sys.exit(0)
    threshold = int(sys.argv[2])
    test_proportion = float(sys.argv[3])
    sample_size = int(sys.argv[4])
    preprocess_(sys.argv[1], threshold, test_proportion, sample_size)


# -*- coding: utf-8 -*-

import numpy as np
import pandas as pd

import argparse
import glob
import sys
import os
import sys
import random
import copy
import json
import time
from datetime import datetime as dt
import matplotlib.pyplot as plt

path = os.path.join(os.path.dirname(__file__), '../')
sys.path.append(path)
import ddim_change_symptom.gmm_generator as gg
import ddim_change_symptom.ts_clustering as tc
import ddim_change_symptom.dynamic_model_selection as dms
import ddim_change_symptom.descriptional_dimension as ddim
import ddim_change_symptom.symptom_evaluation as sym_eval
import ddim_change_symptom.fixed_share as fs

class simDdimRealData():
    '''Simulate Descriptional Dimension (Ddim)
    [Parameters]
        random_state: int, optional (default:0)
            The state of random element

        conf_file: string, optional (default:'ddim_paramset_single.json')
            The file of paramter set

        c_pattern: string, optional (default:'single')
            The gradual change point pattern

        temp_beta: float, optional (default:1)
            The temperature parameter 'beta' in Ddim

        out_flg: boolean, optional (default:False)
            Whether to output result or not

    '''

    def __init__(self, data_file, random_state=0, conf_file='ddim_paramset_realdata.json',
                 temp_beta=None, out_flg=False):

        now_time = dt.now()
        now_str = now_time.strftime("%Y%m%d-%H%M%S")
        date_str = now_time.strftime("%Y%m%d")

        ### params
        self.random_state = random_state
        self.conf_file = conf_file
        self.temp_beta = temp_beta
        self.out_flg = out_flg

        ### internal parameter
        self.data_column = 'amount_data'
        self.sim_title = 'ddim_realdata'
        self.date_str = date_str
        self.file_name = 'ddim_result'

        self.FILE_DIR = os.path.dirname(os.path.abspath(__file__))
        self.OUT_DIR = os.path.normpath(os.path.join(self.FILE_DIR, './data/out/%s_%s'%(self.sim_title,date_str)))
        self.CONF_DIR = os.path.normpath(os.path.join(self.FILE_DIR, './conf'))
        self.out_data_file = '%s/%s_rand%d_%s.json'%(self.OUT_DIR,self.file_name,random_state,now_str)

        self.data_file = os.path.normpath(os.path.join(self.FILE_DIR, './data/%s'%(data_file)))

        if not os.path.isdir(self.OUT_DIR):
            os.mkdir(self.OUT_DIR)

        ### args
        self._read_conf()

        ### input dataset
        self._read_dataset()

        ### output
        self.out = {}

    def _read_conf(self):
        '''Read conf file
        '''
        conf_file = self.CONF_DIR + '/' + self.conf_file

        ### read json
        f = open(conf_file, 'r')
        params = json.load(f)
        f.close()

        self.params = params

    def _read_dataset(self):
        '''Read given dataset
        '''
        ### read json
        f = open(self.data_file, 'r')
        j_data = json.load(f)
        f.close()

        amount_dataset = np.array(j_data[self.data_column])

        self.dataset = amount_dataset

    def simulate_algorithm_to_dataset(self):
        '''Simulate algorithm using given dataset
        '''

        ### parameter of simulation
        norm_file = self.params['norm_file']
        iter_count = self.params['simulation']['iter']

        ### parameter of clustering
        num_em = self.params['clustering_params']['num_em']
        max_clu = self.params['clustering_params']['max_clu']

        ### parameter of SDMS
        sdms_lam = self.params['sdms_params']['sdms_lam']
        sdms_beta = self.params['sdms_params']['sdms_beta']

        ### parameter of Ddim
        temp_beta_unit = self.params['sim_params']['temp_beta_unit']
        th_change_diff = self.params['sim_params']['th_change_diff']
        th_change_th = self.params['sim_params']['th_change_th']
        th_change_sdms = self.params['sim_params']['th_change_sdms']
        th_change_fs = self.params['sim_params']['th_change_fs']
        th_change_ws_fs = self.params['sim_params']['th_change_ws_fs']
        th_change_ws_fs_th = self.params['sim_params']['th_change_ws_fs_th']

        ### parameter of fixed share
        fs_alpha = self.params['sim_params']['fs_alpha']
        fs_temp_beta = self.params['sim_params']['fs_temp_beta']

        # num = self.dataset.shape[1]
        if self.temp_beta is None:
            # temp_beta = temp_beta_unit / np.sqrt(num)
            temp_beta = 0.05
        else:
            temp_beta = self.temp_beta

        st = time.time()
        res_opt_k = np.zeros((iter_count,self.dataset.shape[0]))
        res_ent = np.zeros((iter_count,self.dataset.shape[0]))
        res_ddim = np.zeros((iter_count,self.dataset.shape[0]))
        res_fs = np.zeros((iter_count,self.dataset.shape[0]))
        res_ws_fs = np.zeros((iter_count,self.dataset.shape[0]))
        for i in range(iter_count):
            tc_obj = tc.gmmSeqClustering(max_clu=max_clu,num_em=num_em,norm_file=norm_file,sdms_lam=sdms_lam,sdms_beta=sdms_beta)
            tc_obj.fit(self.dataset,progress=True)

            dms_obj = dms.dynamicModelSelection(sdms_lam=sdms_lam,sdms_beta=sdms_beta)
            dms_obj.calc_simple_sdms(tc_obj.ts_nml)

            ### calculate Descriptional Dimension
            ddim_obj = ddim.calcDdim(sdms_lam=sdms_lam,sdms_beta=sdms_beta,temp_beta=temp_beta)
            ddim_value = ddim_obj.calc_ts_of_Ddim(tc_obj.ts_nml)

            ### fixed share
            fs_obj = fs.calcFS(sdms_lam=sdms_lam, sdms_beta=sdms_beta, temp_beta=fs_temp_beta, fs_alpha=fs_alpha)
            fs_obj.calc_ts_of_FS(tc_obj.ts_nml)

            res_opt_k[i,:] = dms_obj.l_opt_k
            res_ddim[i,:] = ddim_value
            res_fs[i,:] = fs_obj.l_bx_fs
            res_ws_fs[i,:] = fs_obj.l_bx_fs_weightsum

            elapsed_time = time.time() - st
            print('=== optimal k (iter:%d,elapsed-time:%d[s]) ==='%(i,elapsed_time))
            print(dms_obj.l_opt_k)

        ### evaluation
        eval_obj = sym_eval.evalSymptom()

        self.tc_obj = tc_obj

        avg_opt_k = np.mean(res_opt_k,axis=0)
        avg_ddim = np.mean(res_ddim,axis=0)
        avg_fs = np.mean(res_fs,axis=0)
        avg_ws_fs = np.mean(res_ws_fs,axis=0)

        ### symptom detection: Diff
        t_criterion_diff = [np.nan]
        t_criterion_diff.extend( ( np.diff(avg_ddim) ).tolist())
        t_criterion_diff = np.array(t_criterion_diff)
        t_criterion_diff = np.abs(t_criterion_diff)
        sy_diff = eval_obj.symptom_detection(t_criterion_diff,th_change=th_change_diff)

        ### symptom detection: Threshold
        t_criterion_th = np.absolute( avg_opt_k - avg_ddim )
        sy_th = eval_obj.symptom_detection(t_criterion_th,th_change=th_change_th)

        ### SDMS
        t_criterion_sdms = [np.nan]
        t_criterion_sdms.extend( ( np.diff(avg_opt_k) ).tolist())
        t_criterion_sdms = np.array(t_criterion_sdms)
        sy_sdms = eval_obj.symptom_detection(t_criterion_sdms,th_change=th_change_sdms)

        ### fixed share
        t_criterion_fs = [np.nan]
        t_criterion_fs.extend( ( np.diff(avg_fs) ).tolist())
        t_criterion_fs = np.array(t_criterion_fs)
        sy_fs = eval_obj.symptom_detection(t_criterion_fs,th_change=th_change_fs)

        ### weighted average fixed share: Diff
        t_criterion_ws_fs = [np.nan]
        t_criterion_ws_fs.extend( ( np.diff(avg_ws_fs) ).tolist())
        t_criterion_ws_fs = np.array(t_criterion_ws_fs)
        sy_ws_fs = eval_obj.symptom_detection(t_criterion_ws_fs,th_change=th_change_ws_fs)

        ### weighted average fixed share: Threshold
        t_criterion_ws_fs_th = np.absolute( avg_opt_k - avg_ws_fs )
        sy_ws_fs_th = eval_obj.symptom_detection(t_criterion_ws_fs_th,th_change=th_change_ws_fs_th)

        self.out['params'] = self.params
        self.out['params']['file'] = self.conf_file
        self.out['params']['temp_beta'] = self.temp_beta
        self.out['result'] = {}
        self.out['result']['opt_k'] = res_opt_k.tolist()
        self.out['result']['ddim'] = res_ddim.tolist()
        self.out['result']['avg_opt_k'] = avg_opt_k.tolist()
        self.out['result']['avg_ddim'] = avg_ddim.tolist()
        self.out['result']['avg_fs'] = avg_fs.tolist()
        self.out['result']['avg_ws_fs'] = avg_ws_fs.tolist()
        self.out['result']['symptom_diff'] = sy_diff
        self.out['result']['symptom_th'] = sy_th
        self.out['result']['symptom_sdms'] = sy_sdms
        self.out['result']['symptom_fs'] = sy_fs
        self.out['result']['symptom_ws_fs'] = sy_ws_fs

        self.out['result']['evaluation'] = {}
        ### Diff
        self.out['result']['evaluation']['t_criterion_diff'] = t_criterion_diff.tolist()
        ### TH
        self.out['result']['evaluation']['t_criterion_th'] = t_criterion_th.tolist()
        ### SDMS
        self.out['result']['evaluation']['t_criterion_sdms'] = t_criterion_sdms.tolist()
        ### FS
        self.out['result']['evaluation']['t_criterion_fs'] = t_criterion_fs.tolist()
        ### Weighted average FS: Diff
        self.out['result']['evaluation']['t_criterion_ws_fs'] = t_criterion_ws_fs.tolist()
        ### Weighted average FS: TH
        self.out['result']['evaluation']['t_criterion_ws_fs_th'] = t_criterion_ws_fs_th.tolist()

        if self.out_flg:
            f = open(self.out_data_file, 'w')
            json.dump(self.out, f, indent=4)
            f.close()

    def create_graph(self,initial_period=0,ts_ind=None,c_ind_th=None,c_ind_diff=None,fs_flg=False,tofile=False,file_format='eps'):

        '''Create graph with result of Ddim
        [Parameters]
            tofile: boolean, optional (default:False)
                Whether to out a file or not

            file_format: string, optional (default:'eps')
                Output file type (Usually 'eps' or 'png')
        '''

        opt_k = np.array(self.out['result']['avg_opt_k'])
        l_ddim = np.array(self.out['result']['avg_ddim'])
        avg_ws_fs = np.array(self.out['result']['avg_ws_fs'])

        ### Result of Diff
        t_criterion_diff = self.out['result']['evaluation']['t_criterion_diff']
        sy_diff = self.out['result']['symptom_diff']

        ### Result of TH
        t_criterion_th = self.out['result']['evaluation']['t_criterion_th']
        sy_th = self.out['result']['symptom_th']

        ### SDMS
        sy_sdms = self.out['result']['symptom_sdms']

        sy_th_move = ( np.array(sy_th) + initial_period ).tolist()
        sy_diff_move = ( np.array(sy_diff) + initial_period ).tolist()
        sy_sdms_move = ( np.array(sy_sdms) + initial_period ).tolist()

        ### Define directory name and file name
        now_time = dt.now()
        now_str = now_time.strftime("%Y%m%d-%H%M%S")
        date_str = now_time.strftime("%Y%m%d")

        dir_name = self.OUT_DIR
        if not os.path.isdir(dir_name):
            os.mkdir(dir_name)

        file_name = '%s/gmm_clu_ddim_realdata'%(dir_name)
        out_fig = '%s_%s.%s'%(file_name,now_str,file_format)

        max_time = opt_k.shape[0]
        ts = np.array([i for i in range(max_time)])
        if ts_ind is None:
            ts_ind = ts
        if c_ind_th is None:
            c_ind_th = sy_th
        if c_ind_diff is None:
            c_ind_diff = sy_diff

        ts_sdms, opt_k_sdms, ts_ind_sdms = self._create_sequence(ts,ts_ind,opt_k)

        alpha_true = 1.
        l_width = 5.
        ddim_width = 12.

        fig = plt.figure(figsize=(20,12))
        plt.rcParams["font.size"] = 24
        ax = fig.add_subplot(111)

        title_str = 'Ddim for Gradual Change \n<Symptom> TH:%s, Diff:%s, SDMS:%s' \
                                %(str(sy_th_move), str(sy_diff_move), str(sy_sdms_move) )
        ax.plot(ts_sdms[ts_ind_sdms]+initial_period,opt_k_sdms[ts_ind_sdms],'b-',linewidth=l_width,label='SDMS')
        if fs_flg:
            ax.plot(ts[ts_ind]+initial_period,avg_ws_fs[ts_ind],'c-',linewidth=l_width,label='WA-FS')
        avg_ws_fs
        ax.plot(ts[ts_ind]+initial_period,l_ddim[ts_ind],'g-',linewidth=ddim_width, label='Ddim')
        plt.ylabel('SDMS / Ddim')
        plt.xlabel('Time')

        ax2 = ax.twinx()

        cri_min, cri_max = 0, 1.5
        ax2.plot(ts[ts_ind]+initial_period,np.array(t_criterion_th)[ts_ind],'r--',linewidth=l_width, label='TH')
        for c in c_ind_th:
            ax2.plot([c+initial_period,c+initial_period],[cri_min,cri_max],'r--',linewidth=l_width)
        ax2.plot(ts[ts_ind]+initial_period,np.array(t_criterion_diff)[ts_ind],'m-.',linewidth=l_width,label='Diff')
        for c in c_ind_diff:
            ax2.plot([c+initial_period,c+initial_period],[cri_min,cri_max],'m-.',linewidth=l_width)
        ax.legend(loc=2)
        ax2.legend(loc=4)
        plt.ylabel('TH / Diff')
        # plt.title(title_str)
        plt.xlabel('time')
        if tofile:
            plt.savefig(out_fig, format=file_format)
            plt.show()
        else:
            plt.show()

    def _create_sequence(self,ts,ts_ind,opt_k):
        ts_out = []
        opt_k_out = []
        ts_ind_out = []
        curr_o = None
        curr_lag = 0
        for i,(t,o) in enumerate(zip(ts,opt_k)):
            if i == 0:
                ts_out.append(t)
                opt_k_out.append(o)
                if t in ts_ind:
                    ts_ind_out.append(t + curr_lag)
                curr_o = o
                continue

            if curr_o == o:
                ts_out.append(t)
                opt_k_out.append(o)
                if t in ts_ind:
                    ts_ind_out.append(t + curr_lag)
                curr_o = o
            else:
                ts_out.append(t)
                ts_out.append(t)
                opt_k_out.append(curr_o)
                opt_k_out.append(o)
                if t in ts_ind:
                    ts_ind_out.append(t + curr_lag)
                    ts_ind_out.append(t + curr_lag + 1)
                curr_lag += 1
                curr_o = o
        ts_out = np.array(ts_out)
        opt_k_out = np.array(opt_k_out)

        return ts_out, opt_k_out, ts_ind_out


import json
from glob import glob
from os.path import isdir, join

import click
import semantic_version

from platformio import app, exception, util
from platformio.compat import glob_escape
from platformio.managers.package import BasePkgManager
from platformio.managers.platform import PlatformFactory, PlatformManager
from platformio.project.config import ProjectConfig

class LibraryManager(BasePkgManager):

    FILE_CACHE_VALID = "30d"  # 1 month

    def __init__(self, package_dir=None):
        self.config = ProjectConfig.get_instance()
        super(LibraryManager, self).__init__(
            package_dir or self.config.get_optional_dir("globallib")
        )

    @property
    def manifest_names(self):
        return [".library.json", "library.json", "library.properties", "module.json"]

    def get_manifest_path(self, pkg_dir):
        path = BasePkgManager.get_manifest_path(self, pkg_dir)
        if path:
            return path

        # if library without manifest, returns first source file
        src_dir = join(glob_escape(pkg_dir))
        if isdir(join(pkg_dir, "src")):
            src_dir = join(src_dir, "src")
        chs_files = glob(join(src_dir, "*.[chS]"))
        if chs_files:
            return chs_files[0]
        cpp_files = glob(join(src_dir, "*.cpp"))
        if cpp_files:
            return cpp_files[0]

        return None

    def max_satisfying_repo_version(self, versions, requirements=None):
        def _cmp_dates(datestr1, datestr2):
            date1 = util.parse_date(datestr1)
            date2 = util.parse_date(datestr2)
            if date1 == date2:
                return 0
            return -1 if date1 < date2 else 1

        semver_spec = None
        try:
            semver_spec = (
                semantic_version.SimpleSpec(requirements) if requirements else None
            )
        except ValueError:
            pass

        item = {}

        for v in versions:
            semver_new = self.parse_semver_version(v["name"])
            if semver_spec:
                if not semver_new or semver_new not in semver_spec:
                    continue
                if not item or self.parse_semver_version(item["name"]) < semver_new:
                    item = v
            elif requirements:
                if requirements == v["name"]:
                    return v

            else:
                if not item or _cmp_dates(item["released"], v["released"]) == -1:
                    item = v
        return item

    def get_latest_repo_version(self, name, requirements, silent=False):
        item = self.max_satisfying_repo_version(
            util.get_api_result(
                "/lib/info/%d"
                % self.search_lib_id(
                    {"name": name, "requirements": requirements}, silent=silent
                ),
                cache_valid="1h",
            )["versions"],
            requirements,
        )
        return item["name"] if item else None

    def _install_from_piorepo(self, name, requirements):
        assert name.startswith("id="), name
        version = self.get_latest_repo_version(name, requirements)
        if not version:
            raise exception.UndefinedPackageVersion(
                requirements or "latest", util.get_systype()
            )
        dl_data = util.get_api_result(
            "/lib/download/" + str(name[3:]), dict(version=version), cache_valid="30d"
        )
        assert dl_data

        return self._install_from_url(
            name,
            dl_data["url"].replace("http://", "https://")
            if app.get_setting("strict_ssl")
            else dl_data["url"],
            requirements,
        )

    def search_lib_id(  # pylint: disable=too-many-branches
        self, filters, silent=False, interactive=False
    ):
        assert isinstance(filters, dict)
        assert "name" in filters

        # try to find ID within installed packages
        lib_id = self._get_lib_id_from_installed(filters)
        if lib_id:
            return lib_id

        # looking in PIO Library Registry
        if not silent:
            click.echo(
                "Looking for %s library in registry"
                % click.style(filters["name"], fg="cyan")
            )
        query = []
        for key in filters:
            if key not in ("name", "authors", "frameworks", "platforms"):
                continue
            values = filters[key]
            if not isinstance(values, list):
                values = [v.strip() for v in values.split(",") if v]
            for value in values:
                query.append(
                    '%s:"%s"' % (key[:-1] if key.endswith("s") else key, value)
                )

        lib_info = None
        result = util.get_api_result(
            "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h"
        )
        if result["total"] == 1:
            lib_info = result["items"][0]
        elif result["total"] > 1:
            if silent and not interactive:
                lib_info = result["items"][0]
            else:
                click.secho(
                    "Conflict: More than one library has been found "
                    "by request %s:" % json.dumps(filters),
                    fg="yellow",
                    err=True,
                )
                # pylint: disable=import-outside-toplevel
                from platformio.commands.lib import print_lib_item

                for item in result["items"]:
                    print_lib_item(item)

                if not interactive:
                    click.secho(
                        "Automatically chose the first available library "
                        "(use `--interactive` option to make a choice)",
                        fg="yellow",
                        err=True,
                    )
                    lib_info = result["items"][0]
                else:
                    deplib_id = click.prompt(
                        "Please choose library ID",
                        type=click.Choice([str(i["id"]) for i in result["items"]]),
                    )
                    for item in result["items"]:
                        if item["id"] == int(deplib_id):
                            lib_info = item
                            break

        if not lib_info:
            if list(filters) == ["name"]:
                raise exception.LibNotFound(filters["name"])
            raise exception.LibNotFound(str(filters))
        if not silent:
            click.echo(
                "Found: %s"
                % click.style(
                    "https://platformio.org/lib/show/{id}/{name}".format(**lib_info),
                    fg="blue",
                )
            )
        return int(lib_info["id"])

    def _get_lib_id_from_installed(self, filters):
        if filters["name"].startswith("id="):
            return int(filters["name"][3:])
        package_dir = self.get_package_dir(
            filters["name"], filters.get("requirements", filters.get("version"))
        )
        if not package_dir:
            return None
        manifest = self.load_manifest(package_dir)
        if "id" not in manifest:
            return None

        for key in ("frameworks", "platforms"):
            if key not in filters:
                continue
            if key not in manifest:
                return None
            if not util.items_in_list(
                util.items_to_list(filters[key]), util.items_to_list(manifest[key])
            ):
                return None

        if "authors" in filters:
            if "authors" not in manifest:
                return None
            manifest_authors = manifest["authors"]
            if not isinstance(manifest_authors, list):
                manifest_authors = [manifest_authors]
            manifest_authors = [
                a["name"]
                for a in manifest_authors
                if isinstance(a, dict) and "name" in a
            ]
            filter_authors = filters["authors"]
            if not isinstance(filter_authors, list):
                filter_authors = [filter_authors]
            if not set(filter_authors) <= set(manifest_authors):
                return None

        return int(manifest["id"])

    def install(  # pylint: disable=arguments-differ
        self,
        name,
        requirements=None,
        silent=False,
        after_update=False,
        interactive=False,
        force=False,
    ):
        _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
        if not _url:
            name = "id=%d" % self.search_lib_id(
                {"name": _name, "requirements": _requirements},
                silent=silent,
                interactive=interactive,
            )
            requirements = _requirements
        pkg_dir = BasePkgManager.install(
            self,
            name,
            requirements,
            silent=silent,
            after_update=after_update,
            force=force,
        )

        if not pkg_dir:
            return None

        manifest = self.load_manifest(pkg_dir)
        if "dependencies" not in manifest:
            return pkg_dir

        if not silent:
            click.secho("Installing dependencies", fg="yellow")

        builtin_lib_storages = None
        for filters in manifest["dependencies"]:
            assert "name" in filters

            # avoid circle dependencies
            if not self.INSTALL_HISTORY:
                self.INSTALL_HISTORY = []
            history_key = str(filters)
            if history_key in self.INSTALL_HISTORY:
                continue
            self.INSTALL_HISTORY.append(history_key)

            if any(s in filters.get("version", "") for s in ("\\", "/")):
                self.install(
                    "{name}={version}".format(**filters),
                    silent=silent,
                    after_update=after_update,
                    interactive=interactive,
                    force=force,
                )
            else:
                try:
                    lib_id = self.search_lib_id(filters, silent, interactive)
                except exception.LibNotFound as e:
                    if builtin_lib_storages is None:
                        builtin_lib_storages = get_builtin_libs()
                    if not silent or is_builtin_lib(
                        builtin_lib_storages, filters["name"]
                    ):
                        click.secho("Warning! %s" % e, fg="yellow")
                    continue

                if filters.get("version"):
                    self.install(
                        lib_id,
                        filters.get("version"),
                        silent=silent,
                        after_update=after_update,
                        interactive=interactive,
                        force=force,
                    )
                else:
                    self.install(
                        lib_id,
                        silent=silent,
                        after_update=after_update,
                        interactive=interactive,
                        force=force,
                    )
        return pkg_dir

def get_builtin_libs(storage_names=None):
    items = []
    storage_names = storage_names or []
    pm = PlatformManager()
    for manifest in pm.get_installed():
        p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
        for storage in p.get_lib_storages():
            if storage_names and storage["name"] not in storage_names:
                continue
            lm = LibraryManager(storage["path"])
            items.append(
                {
                    "name": storage["name"],
                    "path": storage["path"],
                    "items": lm.get_installed(),
                }
            )
    return items

def is_builtin_lib(storages, name):
    for storage in storages or []:
        if any(l.get("name") == name for l in storage["items"]):
            return True
    return False


"""
Script for raw yield extraction
"""

import os
import sys
import argparse
import yaml
import pandas as pd
import numpy as np
from hist import Hist
import uproot
from flarefly.data_handler import DataHandler
from flarefly.fitter import F2MassFitter
from particle import Particle

def create_hist(pt_lims, contents, errors, label_pt=r"$p_\mathrm{T}~(\mathrm{GeV}/c)$"):
    """
    Helper method to create histogram

    Parameters
    ----------

    - pt_lims (list): list of pt limits
    - contents (list): histogram contents
    - errors (list): histogram errors
    - label_pt (str): label for x axis

    Returns
    ----------
    - histogram (hist.Hist)

    """

    pt_cent = [(pt_min+pt_max)/2 for pt_min, pt_max in zip(pt_lims[:-1], pt_lims[1:])]
    histo = Hist.new.Var(pt_lims, name="x", label=label_pt).Weight()
    histo.fill(pt_cent, weight=contents)
    histo.view(flow=False).variance = np.array(errors)**2

    return histo

def extract_rawyield(input_file, config, output_dir, suffix):
    """
    function for raw yield computation

    Parameters
    ----------

    - input_file (str): path of input file
    - config (str): path of config file
    - output_dir (str): output directory
    - suffix (str): suffix to append to output files
    """

    with open(config, "r") as yml_cfg:  # pylint: disable=bad-option-value
        cfg = yaml.load(yml_cfg, yaml.FullLoader)

    pdg_reso = cfg["pdg_reso"]

    # define titles and labels
    extra_info_loc = None
    if pdg_reso == 10433:
        name_reso = "Ds1plus"
        label_mass = r"M($\mathrm{D}^{*+}\mathrm{K_S^0}$)$-$M($\mathrm{D}^{*+}$)"
        pdg_v0 = 310
        pdg_d = 413
        extra_info_loc = ["lower right", "upper center"]
    elif pdg_reso == 435:
        name_reso = "Ds2starplus"
        label_mass = r"M($\mathrm{D}^+\mathrm{K_S^0}$)$-$M($\mathrm{D}^+$)"
        pdg_v0 = 310
        pdg_d = 411
        extra_info_loc = ["upper left", "lower right"]
    else:
        print(f"ERROR: pdg code {pdg_reso} not supported")
        sys.exit()

    trigger = ""
    if "MB" in input_file:
        trigger = "MB"
    elif "HM" in input_file:
        trigger = "HM"

    df_reso = pd.read_parquet(input_file)

    pt_mins_reso = cfg["fit_config"]["pt_mins"]
    pt_maxs_reso = cfg["fit_config"]["pt_maxs"]
    mass_mins_reso = cfg["fit_config"]["mass_mins"]
    mass_maxs_reso = cfg["fit_config"]["mass_maxs"]
    signal_pdf_reso = cfg["fit_config"]["signal_pdf"]
    bkg_pdf_reso = cfg["fit_config"]["bkg_pdf"]
    pt_lims_reso = pt_mins_reso.copy()
    pt_lims_reso.append(pt_maxs_reso[-1])

    # define output file
    outfile_name = os.path.join(
        output_dir, f"mass_{name_reso}_pt{pt_mins_reso[0]}-{pt_maxs_reso[-1]}_{trigger}{suffix}.root")
    file_root = uproot.recreate(outfile_name)
    file_root.close()

    # fit resonance
    signif, signif_unc = [], []
    s_over_b, s_over_b_unc = [], []
    raw_yields, raw_yields_unc = [], []
    bkgs, bkgs_unc = [], []
    means, means_unc = [], []
    sigmas, sigmas_unc = [], []
    for pt_min, pt_max, mass_min, mass_max, spdf, bpdf in zip(
        pt_mins_reso, pt_maxs_reso, mass_mins_reso, mass_maxs_reso, signal_pdf_reso, bkg_pdf_reso):

        mass_min = max([mass_min, Particle.from_pdgid(pdg_v0).mass*1e-3])
        df_sel_pt = df_reso.query(f"{pt_min} < pt_reso < {pt_max}")
        data_hdl = DataHandler(df_sel_pt, var_name="delta_inv_mass_reso", limits=[mass_min, mass_max], nbins=100)

        fitter_reso = F2MassFitter(data_hdl, [spdf], [bpdf], name=f"{name_reso}_pt{pt_min}_{pt_max}")
        delta_mass = Particle.from_pdgid(pdg_reso).mass*1e-3 - Particle.from_pdgid(pdg_d).mass*1e-3
        width = Particle.from_pdgid(pdg_reso).width*1.e-3
        fitter_reso.set_signal_initpar(0, "gamma", width, fix=True)
        fitter_reso.set_signal_initpar(0, "sigma", 0.0008, limits=[0.0004, 0.0020])
        fitter_reso.set_particle_mass(0, mass=delta_mass, limits=[delta_mass-0.1, delta_mass+0.1])
        fitter_reso.set_signal_initpar(0, "frac", 0.01, limits=[0., 1.])
        fitter_reso.set_background_initpar(0, "mass", Particle.from_pdgid(pdg_v0).mass*1e-3)
        fitter_reso.set_background_initpar(0, "c0", 100)
        fitter_reso.set_background_initpar(0, "c1", 1)
        fitter_reso.set_background_initpar(0, "c2", 1)
        fitter_reso.mass_zfit()
        fig_reso = fitter_reso.plot_mass_fit(style="ATLAS",
                                             figsize=(8, 8),
                                             axis_title=rf"{label_mass} (GeV/$c^2$)",
                                             show_extra_info=True,
                                             extra_info_massnhwhm=3.,
                                             extra_info_loc=extra_info_loc)
        fig_reso_res = fitter_reso.plot_raw_residuals(style="ATLAS", figsize=(8, 8), axis_title=rf"{label_mass} (GeV/$c^2$)")
        fig_reso.savefig(os.path.join(output_dir, f"mass_{name_reso}_pt{pt_min}-{pt_max}_{trigger}{suffix}.pdf"))
        fig_reso_res.savefig(os.path.join(output_dir,
                                          f"mass_residuals_{name_reso}_pt{pt_min}-{pt_max}_{trigger}{suffix}.pdf"))
        fitter_reso.dump_to_root(outfile_name, option="update")

        rawy, rawy_unc = fitter_reso.get_raw_yield(0)
        bkg, bkg_unc = fitter_reso.get_background(0, min=mass_min, max=mass_max)

        sign, sign_unc = fitter_reso.get_significance(0, nhwhm=3.)
        soverb, soverb_unc = fitter_reso.get_signal_over_background(0, nhwhm=3.)
        mean, mean_unc = fitter_reso.get_signal_parameter(0, "mu")
        sigma, sigma_unc = fitter_reso.get_signal_parameter(0, "sigma")

        raw_yields.append(rawy)
        raw_yields_unc.append(rawy_unc)
        bkgs.append(bkg)
        bkgs_unc.append(bkg_unc)
        signif.append(sign)
        signif_unc.append(sign_unc)
        s_over_b.append(soverb)
        s_over_b_unc.append(soverb_unc)
        means.append(mean)
        means_unc.append(mean_unc)
        sigmas.append(sigma)
        sigmas_unc.append(sigma_unc)

    file_root = uproot.update(outfile_name)
    file_root["h_rawyields"] = create_hist(pt_lims_reso, raw_yields, raw_yields_unc)
    file_root["h_bkg_allrange"] = create_hist(pt_lims_reso, bkgs, bkgs_unc)
    file_root["h_significance"] = create_hist(pt_lims_reso, signif, signif_unc)
    file_root["h_soverb"] = create_hist(pt_lims_reso, s_over_b, s_over_b_unc)
    file_root["h_means"] = create_hist(pt_lims_reso, means, means_unc)
    file_root["h_sigmas"] = create_hist(pt_lims_reso, sigmas, sigmas_unc)
    file_root.close()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Arguments")
    parser.add_argument("input_file", metavar="text",
                        default="projected_data.parquet.gzip",
                        help="input parquet file")
    parser.add_argument("config", metavar="text",
                        default="config_fit.yml", help="input config file")
    parser.add_argument("--output_dir", "-o", metavar="text",
                        default=".", help="output directory")
    parser.add_argument("--suffix", "-s", metavar="text",
                        default="", help="suffix for output files")
    args = parser.parse_args()

    extract_rawyield(args.input_file, args.config, args.output_dir, args.suffix)


from django.test import TestCase
from knesset.settings import PROJECT_ROOT
from knesset.mks.models import Member
from knesset.committees.models import Committee
from knesset.mmm.models import Document, parse_json, text_lookup, verify
from datetime import datetime
import simplejson
import re

FP = open(PROJECT_ROOT + "/mmm/test_matches.json")
JSON = simplejson.load(FP)
OK_CANDIDATES = re.sub(r"\s+", r" " , " ".join(JSON[0]['candidates']))
OK_DATE = datetime.strptime(JSON[0]['date'], '%d/%m/%Y')

class MmmTest(TestCase):
    """ Testing mmm functions """

    def test_parse_json(self):
        """
        Tests data modification
        """

        j = parse_json(FP)

        self.assertEqual(OK_CANDIDATES, j[0]['candidates'])
        self.assertEqual(OK_DATE, j[0]['date'])

    def test_text_lookup(self):
        """
        Tests in text look up for mks and committees names
        """
        mk_name = u'\u05e8\u05d5\u05e0\u05d9\u05ea \u05ea\u05d9\u05e8\u05d5\u05e9'
        mks = Member.objects.create(name=mk_name)
        c = Committee.objects.create(name='c1')

        self.assertIn(mk_name, OK_CANDIDATES)

        mk = text_lookup(Member, OK_CANDIDATES)
        self.assertEqual(mks.id, mk[0])

        self.assertNotIn('c1', OK_CANDIDATES)

        committees = text_lookup(Committee, OK_CANDIDATES)
        self.assertEqual([], committees)

    def test_verify(self):
        """
        Tests verify method
        """
        o = JSON[0]
        mk_name = u'\u05e8\u05d5\u05e0\u05d9\u05ea \u05ea\u05d9\u05e8\u05d5\u05e9'
        mks = Member.objects.create(name=mk_name)
        mkses = [1]
        committees = []
        d1 = Document.objects.create(url=JSON[0]['url'], title=JSON[0]['title'],
                                     publication_date=OK_DATE, author_names=JSON[0]['author'])
        d1.req_mks.add(mks.id)

        self.assertEqual(o['url'], d1.url)
        self.assertEqual(o['title'], d1.title)
        self.assertEqual(OK_DATE, d1.publication_date)
        self.assertEqual(o['author'], d1.author_names)
        self.assertEqual(1, d1.req_mks.count())
        self.assertEqual(0, d1.req_committee.count())

        self.assertTrue(verify(o, [d1], mkses, committees))


from thread import Thread
import os
import time
from .bluetoothctl import Bluetoothctl

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, GdkPixbuf

class Handler:
    def __init__(self, builder, controller=None):
        self.builder = builder
        self.bluetooth = None
        self.devices = {}
        self.paired_devices = {}
        self.what_to_do = 'check'
        Thread(self)

# ADDING CONTROLLER TO HANDLER
# ----------------------------------------------------------------------------------------------------------------------
    def add_controller(self, controller):
        self.controller = controller

# ----------------------------------------------------------------------------------------------------------------------
    def fulfill_devices(self):
        self.devices = self.bluetooth.get_discoverable_devices()
        self.paired_devices = self.bluetooth.get_paired_devices()
        if self.devices == [] or self.devices == None:
            return

        tree = self.builder.get_object('bluetooth_tree')

        columns_to_remove = tree.get_columns()         #REMOVING COLUMNS IF EXIST ALREADY
        if columns_to_remove is not None:
            for column in columns_to_remove:
                tree.remove_column(column)

        store = Gtk.ListStore(str, str)
        tree.set_model(store)

        for paired in self.paired_devices:
            store.append(["/opt/settings/bluetooth/img/bluetooth_row_paired.svg", paired['name']])

        for device in self.devices:
            store.append(["/opt/settings/bluetooth/img/bluetooth_row.svg", device['name']])

        px_column = Gtk.TreeViewColumn('Devices')
        px_renderer = Gtk.CellRendererPixbuf()

        px_column.pack_start(px_renderer, True)
        str_renderer = Gtk.CellRendererText()
        px_column.pack_start(str_renderer, True)

        px_column.set_cell_data_func(px_renderer, self.get_tree_cell_pixbuf)
        px_column.set_cell_data_func(str_renderer, self.get_tree_cell_text)
        tree.append_column(px_column)

# ----------------------------------------------------------------------------------------------------------------------
    def get_tree_cell_text(self, col, cell, model, iter, user_data):
        cell.set_property('text', model.get_value(iter, 1))

    def get_tree_cell_pixbuf(self, col, cell, model, iter, user_data):
        cell.set_property('pixbuf', GdkPixbuf.Pixbuf.new_from_file_at_scale(filename=model.get_value(iter, 0),width=32, height=32,
                                                             preserve_aspect_ratio=True))

# ----------------------------------------------------------------------------------------------------------------------
    def check_state(self):
        state = os.popen('rfkill list bluetooth | grep -oP "Soft blocked: [a-z]+"').read().rstrip().split(' ')[-1:]
        self.builder.get_object('bluetooth_switch').set_active(True if state[0]=='no' else False)
        self.builder.get_object('bluetooth_switch').connect('state-set', self.state_changed)
        return state[0]

# ----------------------------------------------------------------------------------------------------------------------
    def set_widgets_to(self, state):
        self.builder.get_object('refresh_button').set_sensitive(state)
        self.builder.get_object('scrolled_window').set_sensitive(state)
        self.builder.get_object('discoverable_button').set_sensitive(state)

# ----------------------------------------------------------------------------------------------------------------------
    def state_changed(self, widget, state):
        if not state:
            os.popen('rfkill block bluetooth')
            self.set_widgets_to(False)
        else:
            self.bluetooth = Bluetoothctl()
            self.set_widgets_to(True)
            self.bluetooth.start_scan()
            self.fulfill_devices()

# ----------------------------------------------------------------------------------------------------------------------
    def create_modal_of_device(self, widget=None, position=None, column=None):
        position = int(str(position))
        device = None
        try:
            device = self.paired_devices[position]
            dialog = self.builder.get_object('paired_device_modal')
            dialog.set_attached_to(self.builder.get_object('bluetooth'))
            dialog.show_all()

        except Exception:
            position = position - len(self.paired_devices)
            device = self.devices[position]
            if len(device['name'])> 17:
                   device['name'] = device['name'][17:] + '...'
            self.builder.get_object('device_name_label').set_text(device['name'])
            dialog = self.builder.get_object('device_modal')
            dialog.set_attached_to(self.builder.get_object('bluetooth'))
            dialog.show_all()

    def delete_modal_of_device(self, widget=None):
        dialog = self.builder.get_object('device_modal')
        dialog.hide()
        dialog = self.builder.get_object('paired_device_modal')
        dialog.hide()
        self.fulfill_devices()

# ----------------------------------------------------------------------------------------------------------------------
    def pair(self, widget):
        name_of_device = widget.get_text()
        for device in self.devices:
            if device['name'] == name_of_device:
                self.bluetooth.pair(device['mac_address'])
                self.delete_modal_of_device()

# ----------------------------------------------------------------------------------------------------------------------
    def refresh(self, widget):
        self.what_to_do = 'refresh'
        Thread(self)
# ----------------------------------------------------------------------------------------------------------------------
    def set_discoverable(self, widget):
        self.bluetooth.make_discoverable()
        widget.set_sensitive(False)
        self.what_to_do = 'disable discoverable'
        Thread(self)

# ----------------------------------------------------------------------------------------------------------------------
    def thread_function(self):
        if self.what_to_do == 'check':
            is_off = self.check_state()
            if is_off == 'yes':
                self.set_widgets_to(False)
            else:
                self.bluetooth = Bluetoothctl()
                self.bluetooth.start_scan()
                self.fulfill_devices()

        elif self.what_to_do == 'disable discoverable':
            time.sleep(20)
            self.bluetooth.stop_discoverable()
            self.builder.get_object('discoverable_button').set_sensitive(True)
        elif self.what_to_do == 'refresh':
            self.fulfill_devices()