import torch
import torch.nn as nn
import torch.nn.functional as F
import random
from torch.distributions import Beta

class Flatten(nn.Module):
    def __init__(self):
        super(Flatten, self).__init__()

    def forward(self, x):
        return x.view(x.size(0), -1)

class Conv_Standard(nn.Module):
    def __init__(self, args, x_dim, hid_dim, z_dim, final_layer_size):
        super(Conv_Standard, self).__init__()
        self.args = args
        self.net = nn.Sequential(self.conv_block(x_dim, hid_dim), self.conv_block(hid_dim, hid_dim),
                                 self.conv_block(hid_dim, hid_dim), self.conv_block(hid_dim, z_dim))
        self.dist = Beta(torch.FloatTensor([2]), torch.FloatTensor([2]))
        self.hid_dim = hid_dim

        self.logits = nn.Linear(final_layer_size, self.args.num_classes)

    def conv_block(self, in_channels, out_channels):
        return nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, padding=1),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(),
            nn.MaxPool2d(2)
        )

    def functional_conv_block(self, x, weights, biases,
                              bn_weights, bn_biases, is_training):
        x = F.conv2d(x, weights, biases, padding=1)
        x = F.batch_norm(x, running_mean=None, running_var=None, weight=bn_weights, bias=bn_biases,
                         training=is_training)
        x = F.relu(x)
        return x

    def mixup_data(self, xs, ys, xq, yq):
        query_size = xq.shape[0]

        shuffled_index = torch.randperm(query_size)

        xs = xs[shuffled_index]
        ys = ys[shuffled_index]
        lam = self.dist.sample().cuda()
        mixed_x = lam * xq + (1 - lam) * xs

        return mixed_x, yq, ys, lam

    def forward(self, x):
        x = self.net(x)

        x = torch.mean(x, dim=(2, 3))

        return self.logits(x)

    def functional_forward(self, x, weights, is_training=True):
        for block in range(4):
            x = self.functional_conv_block(x, weights[f'net.{block}.0.weight'], weights[f'net.{block}.0.bias'],
                                           weights.get(f'net.{block}.1.weight'), weights.get(f'net.{block}.1.bias'),
                                           is_training)

        x = torch.mean(x, dim=(2, 3))

        x = F.linear(x, weights['logits.weight'], weights['logits.bias'])

        return x

    def channel_shuffle(self, hidden, label, shuffle_dict, shuffle_channel_id):
        concept_idx = [0, 12, 22, 32, 44, 54, 64]

        new_data = []

        start = concept_idx[shuffle_channel_id]
        end = concept_idx[shuffle_channel_id + 1]

        for i in range(self.args.num_classes):
            cur_class_1 = hidden[label == i]
            cur_class_2 = hidden[label == shuffle_dict[i]]

            new_data.append(
                torch.cat((cur_class_1[:, :start], cur_class_2[:, start:end], cur_class_1[:, end:]), dim=1))

        new_data = torch.cat(new_data, dim=0)

        indexes = torch.randperm(new_data.shape[0])

        new_data = new_data[indexes]
        new_label = label[indexes]

        return new_data, new_label

    def forward_metamix(self, hidden_support, label_support, hidden_query, label_query, weights, is_training=True):

        sel_layer = random.randint(0, 3)
        flag = 0

        for layer in range(4):
            if layer == sel_layer:
                hidden_query, reweighted_query, reweighted_support, lam = self.mixup_data(hidden_support, label_support,
                                                                                hidden_query, label_query)
                flag = 1

            if not flag:
                hidden_support = self.functional_conv_block(hidden_support, weights[f'net.{layer}.0.weight'],
                                                            weights[f'net.{layer}.0.bias'],
                                                            weights.get(f'net.{layer}.1.weight'),
                                                            weights.get(f'net.{layer}.1.bias'),
                                                            is_training)
            hidden_query = self.functional_conv_block(hidden_query, weights[f'net.{layer}.0.weight'],
                                                      weights[f'net.{layer}.0.bias'],
                                                      weights.get(f'net.{layer}.1.weight'),
                                                      weights.get(f'net.{layer}.1.bias'),
                                                      is_training)

        hidden_query = torch.mean(hidden_query, dim=(2, 3))

        x = F.linear(hidden_query, weights['logits.weight'], weights['logits.bias'])

        return x, reweighted_query, reweighted_support, lam

    def functional_forward_cf(self, hidden, label, sel_layer, shuffle_list, shuffle_channel_id, weights,
                                           is_training=True):

        label_new = label

        for layer in range(4):
            if layer == sel_layer:
                hidden, label_new = self.channel_shuffle(hidden, label, shuffle_list, shuffle_channel_id)

            hidden = self.functional_conv_block(hidden, weights[f'net.{layer}.0.weight'],
                                                weights[f'net.{layer}.0.bias'],
                                                weights.get(f'net.{layer}.1.weight'),
                                                weights.get(f'net.{layer}.1.bias'),
                                                is_training)

        hidden = torch.mean(hidden, dim=(2, 3))

        x = F.linear(hidden, weights['logits.weight'], weights['logits.bias'])

        return x, label_new

    def mix_cf(self, hidden_support, label_new_support, hidden_query, label_new_query, shuffle_list,
               shuffle_channel_id):
        hidden_support, label_new_support = self.channel_shuffle(hidden_support, label_new_support, shuffle_list,
                                                                 shuffle_channel_id)
        hidden_query, label_new_query = self.channel_shuffle(hidden_query, label_new_query, shuffle_list,
                                                             shuffle_channel_id)

        hidden_query, label_new_query, label_new_support, lam = self.mixup_data(hidden_support, label_new_support, hidden_query,
                                                        label_new_query)

        return hidden_support, label_new_support, hidden_query, label_new_query, lam

    def functional_forward_cf_mix_query(self, hidden_support, label_support, hidden_query, label_query, sel_layer,
                                                 shuffle_list, shuffle_channel_id, weights,
                                                 is_training=True):

        flag = 0

        for layer in range(4):
            if layer == sel_layer:
                hidden_support, label_new_support, hidden_query, label_new_query, lam = self.mix_cf(hidden_support,
                                                                                label_support,
                                                                                hidden_query,
                                                                                label_query,
                                                                                shuffle_list,
                                                                                shuffle_channel_id)
                flag = 1

            if not flag:
                hidden_support = self.functional_conv_block(hidden_support, weights[f'net.{layer}.0.weight'],
                                                            weights[f'net.{layer}.0.bias'],
                                                            weights.get(f'net.{layer}.1.weight'),
                                                            weights.get(f'net.{layer}.1.bias'),
                                                            is_training)

            hidden_query = self.functional_conv_block(hidden_query, weights[f'net.{layer}.0.weight'],
                                                      weights[f'net.{layer}.0.bias'],
                                                      weights.get(f'net.{layer}.1.weight'),
                                                      weights.get(f'net.{layer}.1.bias'),
                                                      is_training)

        hidden_query = torch.mean(hidden_query, dim=(2, 3))

        x = F.linear(hidden_query, weights['logits.weight'], weights['logits.bias'])

        return x, label_new_query, label_new_support, lam


from django.contrib.auth.models import User, Group
from rest_framework import serializers
from reala_api.models import Parcel, Owner, Event
from reala_api.helpers import search_parcel, get_user_created_event
from django.utils import timezone

class UserSerializer(serializers.ModelSerializer):
    """
    Simple User Serializer for read/write API
    """

    class Meta:
        model = User
        fields = ('url', 'username', 'email', 'groups')

class GroupSerializer(serializers.ModelSerializer):
    """
    Simple Group Serializer for read/write API
    """

    class Meta:
        model = Group
        fields = ('url', 'name')

class ParcelSerializer(serializers.ModelSerializer):
    street_number = serializers.CharField(read_only=True)
    route = serializers.CharField(read_only=True)
    city = serializers.CharField(read_only=True)
    state = serializers.CharField(read_only=True)
    postal_code = serializers.CharField(read_only=True)
    lat = serializers.CharField(read_only=True)
    lng = serializers.CharField(read_only=True)

    class Meta:
        model = Parcel
        fields = (
            'id', 'contact', 'formatted_address', 'street_number', 'route', 'city', 'state', 'postal_code', 'lat',
            'lng', 'swis', 'assessment', 'frontage', 'depth', 'acres', 'year_built', 'beds', 'baths', 'fireplace',
            'description', 'school', 'notes')

    def create(self, validated_data):
        parcel = search_parcel(validated_data.pop('formatted_address'))
        for attr, val in validated_data.items():
            setattr(parcel, attr, val)
        parcel.save()

        return parcel

    def update(self, instance, validated_data):
        # We don't let Parcel objects update address data
        # Since every address given exists, we want a new
        # Parcel instance to be created instead.
        parcel = search_parcel(validated_data.pop('formatted_address'))
        if parcel is not None and parcel.formatted_address != instance.formatted_address:
            return parcel

        for attr, val in validated_data.items():
            setattr(instance, attr, val)
        instance.save()

        return instance

class ParcelCompactSerializer(serializers.ModelSerializer):
    formatted_address = serializers.CharField(allow_blank=True)

    class Meta:
        model = Parcel
        fields = ('id', 'formatted_address')

class EventCompactSerializer(serializers.ModelSerializer):
    class Meta:
        model = Event
        fields = ('id', 'occurred', 'user', 'type')

class OwnerSerializer(serializers.ModelSerializer):
    events = EventCompactSerializer(many=True, read_only=True)
    home = ParcelCompactSerializer()
    parcels = ParcelCompactSerializer(many=True, read_only=True)

    gender = serializers.CharField(allow_blank=True)
    phone_number = serializers.CharField(allow_blank=True)
    email_address = serializers.CharField(allow_blank=True)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        """
        user = self.context['request'].user
        owner_id = args[0].id
        e = get_user_created_event(user, owner_id)
        e.type = "Contact"
        e.occurred = timezone.now()
        e.save()
        """

    class Meta:
        model = Owner
        fields = ('id', 'first_name', 'last_name', 'parcels', 'events',
                  'phone_number', 'phone_verified', 'do_not_contact',
                  'email_address', 'age', 'gender', 'home')

    def create(self, validated_data):
        home_data = validated_data.pop('home')
        owner = Owner.objects.create(**validated_data)
        owner.save()
        owner.home = search_parcel(home_data['formatted_address'])
        if owner.home is not None:
            owner.home.contact = owner
            owner.home.save()
        owner.save()

        return owner

    def update(self, instance, validated_data):
        home_data = validated_data['home']
        parcel = search_parcel(home_data['formatted_address'])
        instance.home = parcel

        instance.first_name = validated_data.get('first_name', instance.first_name)
        instance.last_name = validated_data.get('last_name', instance.last_name)
        instance.phone_number = validated_data.get('phone_number', instance.phone_number)
        instance.phone_verified = validated_data.get('phone_verified', instance.phone_verified)
        instance.do_not_contact = validated_data.get('do_not_contact', instance.do_not_contact)
        instance.email_address = validated_data.get('email_address', instance.email_address)
        instance.age = validated_data.get('age', instance.age)
        instance.gender = validated_data.get('gender', instance.gender)

        parcel.contact = instance
        parcel.save()
        instance.save()

        return instance

class EventSerializer(serializers.ModelSerializer):
    user = UserSerializer(read_only=True)
    # owner = serializers.CharField(required=True)

    class Meta:
        model = Event
        fields = ('id', 'occurred', 'user', 'owner', 'type', 'details', 'notes',
                  'claimed', 'called', 'mailed', 'social', 'other')

    def create(self, validated_data):
        user = self.context['request'].user
        owner = validated_data['owner']

        occurred = timezone.now()
        etype = 'Contact'
        if 'occurred' in validated_data:
            occurred = validated_data['occurred']
        if 'type' in validated_data:
            etype = validated_data['type']

        e = get_user_created_event(user, owner.id)
        e.occurred = occurred
        e.type = etype

        if 'details' in validated_data:
            e.details = validated_data['details']
        if 'notes' in validated_data:
            e.notes = validated_data['notes']

        if 'claimed' in validated_data:
            e.claimed = validated_data['claimed']
        if 'called' in validated_data:
            e.called = validated_data['called']
        if 'mailed' in validated_data:
            e.mailed = validated_data['mailed']
        if 'social' in validated_data:
            e.social = validated_data['social']
        if 'other' in validated_data:
            e.other = validated_data['other']

        e.save()
        return e

class LeadSerializer(serializers.ModelSerializer):
    home = ParcelCompactSerializer()
    events = EventCompactSerializer(many=True, read_only=True)

    class Meta:
        model = Owner
        fields = ('id', 'first_name', 'last_name', 'home', 'events')

    def create(self, validated_data):
        home_data = validated_data.pop('home')
        owner = Owner.objects.create(**validated_data)
        owner.home = search_parcel(home_data['formatted_address'])
        owner.home.contact = owner
        owner.home.save()
        owner.save()

        return owner

    def update(self, instance, validated_data):
        home_data = validated_data['home']
        parcel = search_parcel(home_data['formatted_address'])
        instance.home = parcel
        parcel.contact = instance
        parcel.save()
        instance.save()

        return instance


"""
/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""

from builtins import str
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from .ui_disp_ini import Ui_Dialog
import webbrowser
import os

class Initialization_dialog(QtWidgets.QDialog):
    def __init__(self):
        QtWidgets.QDialog.__init__(self)
        self.ui = Ui_Dialog()
        self.ui.setupUi(self)
        self.center()

        self.ui.lineEdit.setText("")
        self.ui.lineEditDEM.setText("")
        self.ui.checkBox.setChecked(False)
        self.ui.lineEdit_2.setText("")

        openFile = self.ui.toolButton
        openFile.clicked.connect(self.showDialog)
        openFile = self.ui.toolButtonDEM
        openFile.clicked.connect(self.showDialogDEM)

        self.ui.buttonBox.helpRequested.connect(self.helpWindow)
        self.ui.checkBox.stateChanged[int].connect(self.orthoActivate)

        self.ui.lineEditDEM.dropped.connect(self.dropEvent)
        self.openOrtho = self.ui.toolButton_2
        self.openOrtho.clicked.connect(self.showDialogOrtho)
        self.ui.lineEdit_2.setReadOnly(True)
        self.currentPath = '/home'
        self.setAcceptDrops(True)

    def dragEnterEvent(self, event):
        event.accept()

    def dropEvent(self, event):
        lineEdit =  self.childAt(event.pos().x(),event.pos().y());
        fileURL = event.mimeData().urls()[0].toString()
        fileName = fileURL.split('file:///')[1]
        lineEdit.setText(fileName)

    def orthoActivate(self, state):
        if state == 2:
            self.openOrtho.setEnabled(True)
        if state == 0:
            self.openOrtho.setEnabled(False)
            self.ui.lineEdit_2.setText('')

    def helpWindow(self):
        name = os.path.dirname(os.path.abspath(__file__))
        url = name + "/help/build/html/index.html"
        url.replace("\\","/")
        webbrowser.open(url,new=2)

    def center(self):
        qr = self.frameGeometry()
        cp = QtWidgets.QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def showDialogOrtho(self):
        fname = QtWidgets.QFileDialog.getOpenFileName(self, 'Open file',self.currentPath)[0]
        self.ui.lineEdit_2.setText(fname)
        if fname:
             self.currentPath = fname.rsplit("/",1)[0]

    def showDialog(self):
        fname = QtWidgets.QFileDialog.getOpenFileName(self, 'Open file',self.currentPath)[0]
        self.ui.lineEdit.setText(fname)
        if fname:
             self.currentPath = fname.rsplit("/",1)[0]

    def showDialogDEM(self):
        fname = QtWidgets.QFileDialog.getOpenFileName(self, 'Open file',self.currentPath, "Images (*.tiff *.tif)")[0]
        self.ui.lineEditDEM.setText(fname)
        if fname:
             self.currentPath = fname.rsplit("/",1)[0]


"""create investment table

Revision ID: 3a27d37dedae
Revises: 37013825f347
Create Date: 2021-07-15 00:22:33.371345

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '3a27d37dedae'
down_revision = '37013825f347'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('chart_dummy',
    sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
    sa.Column('status', sa.String(length=200), nullable=True),
    sa.Column('value', sa.BigInteger(), nullable=True),
    sa.Column('created_date', sa.DateTime(), nullable=True),
    sa.Column('updated_date', sa.DateTime(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('chart_dummy')
    # ### end Alembic commands ###


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

class BinaryDiceLoss(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, predict, target):
        assert predict.shape[0] == target.shape[0]
        predict = predict.view(predict.shape[0], -1)
        target = target.view(target.shape[0], -1)

        num = 2 * torch.sum(torch.mul(predict, target), dim=1) + 1.
        den = torch.sum(predict * predict + target * target, dim=1) + 1.

        loss = 1 - num / den
        return loss.mean()

class DiceLoss(nn.Module):
    def __init__(self, weight):
        super().__init__()
        self.weight = weight
        self.dice = BinaryDiceLoss()

    def forward(self, logits, target):
        n_fgs = target.shape[1] - 1 # no bg
        weight = [1] * n_fgs if self.weight is None else self.weight

        total_loss = 0
        predict = F.softmax(logits, dim=1)

        for i in range(1, target.shape[1]): # no bg
            total_loss += weight[i - 1] * self.dice(predict[:, i], target[:, i])

        return total_loss / (sum(weight) + 1e-6)

class MultiHeadDiceLoss(nn.Module):
    def __init__(self, weights):
        super().__init__()
        self.dice = nn.ModuleList([DiceLoss(w) for w in weights])
        self.weight = [sum(w) for w in weights]

    def forward(self, logits, targets):
        total_loss = 0

        for i, dice in enumerate(self.dice):
            total_loss += self.weight[i] * dice(logits[i], targets[i])

        return total_loss / (sum(self.weight) + 1e-6)

class MultiHeadCELoss(nn.Module):
    def __init__(self, weights):
        super().__init__()
        self.ce = nn.ModuleList([nn.CrossEntropyLoss() for _ in weights])
        self.weight = [sum(w) for w in weights]

    def forward(self, logits, targets):
        total_loss = 0

        for i, ce in enumerate(self.ce):
            total_loss += self.weight[i] * ce(logits[i], torch.argmax(targets[i], 1))
        return total_loss / (sum(self.weight) + 1e-6)

class DCandCELoss(nn.Module):
    def __init__(self, weights):
        super().__init__()
        self.ce = nn.CrossEntropyLoss()
        self.dc = DiceLoss(weights)

    def forward(self, logits, target):
        dc_loss = self.dc(logits, target)
        ce_loss = self.ce(logits, torch.argmax(target, axis=1))

        result = (dc_loss + ce_loss) / 2
        return result

class MultiHeadDCandCELoss(nn.Module):
    def __init__(self, weights):
        super().__init__()
        self.dc_and_ce = nn.ModuleList([DCandCELoss(w) for w in weights])
        self.weight = [sum(w) for w in weights]

    def forward(self, logits, targets):
        total_loss = 0

        for i, dc_and_ce in enumerate(self.dc_and_ce):
            total_loss += self.weight[i] * dc_and_ce(logits[i], targets[i])
        return total_loss / (sum(self.weight) + 1e-6)


import logging
import os
import uuid
from pathlib import Path
from typing import List

import mlflow

from . import Experimentation

class MlflowExperimentation(Experimentation):
    def __init__(
        self,
        tracking_uri: str = None,
        log_package: bool = True,
        files_to_log: List[str] = None,
    ):
        """
        Wrapper for the MLFlow object
        :param tracking_uri: Where runs gets stored.
        Either a local folder, or the uri of the remote (or local) tracking server.
        See https://mlflow.org/docs/0.4.0/tracking.html#where-runs-get-recorded
        :param log_package Whether to save all code in this package into mlflow
        :param List of file paths to save into mlflow as artifacts.
        """
        super().__init__()

        self.log_package = log_package
        self.files_to_log = files_to_log

        mlflow.set_tracking_uri(tracking_uri)

    def set_experiment(self, name, artifact_location=None):
        mlflow.set_experiment(name)

    def start_run(self):
        if mlflow.active_run() is not None:
            mlflow.end_run()
        mlflow.start_run()

        if self.log_package:
            current_dir = os.path.dirname(os.path.realpath(__file__))
            package_dir = str(Path(current_dir, "../").resolve())
            print(f"Logging package in {package_dir}")
            mlflow.log_artifacts(local_dir=package_dir)

        if self.files_to_log:
            for file in self.files_to_log:
                mlflow.log_artifact(local_path=file)

    def end_run(self):
        mlflow.end_run()

    def log_param(self, key, value):
        mlflow.log_param(key, value)

    def log_params(self, params):
        mlflow.log_params(params)

    def log_metric(self, key, value, step=None):
        mlflow.log_metric(key, value, step)

    def log_metrics(self, metrics, step=None):
        mlflow.log_metrics(metrics, step)

    def log_artifact(self, local_path, artifact_path=None):
        mlflow.log_artifact(local_path, artifact_path)

    def log_artifacts(self, local_path, artifact_path=None):
        mlflow.log_artifacts(local_path, artifact_path)

    def log_image(self, title, fig):
        # Save figure
        try:
            fig.savefig(title + ".png")
            mlflow.log_artifact(title + ".png")
        except OSError:
            # Error saving file probably due to bad title string
            unique_filename = str(uuid.uuid4()) + ".png"
            fig.savefig(unique_filename)
            mlflow.log_artifact(unique_filename)

    @staticmethod
    def search_runs(
        experiment_ids=None,
        filter_string="",
        run_view_type=1,
        max_results=100000,
        order_by=None,
    ):
        """
        Get a pandas DataFrame of runs that fit the search criteria.
        See mlflow.search_runs
        :return:
        """
        mlflow.search_runs(
            experiment_ids=experiment_ids,
            filter_string=filter_string,
            run_view_type=run_view_type,
            max_results=max_results,
            order_by=order_by,
        )

    @staticmethod
    def get_local_tracking_uri():
        dir_path = os.path.dirname(os.path.realpath(__file__))
        tracking_uri = str(Path("file:", dir_path, "../../experiments"))

        logging.info(
            f"Tracking uri not passed, " "experiments will be logged in {tracking_uri}"
        )

        return tracking_uri


from collections import deque
import sys

sys.stdin = open("input.txt", "r")

dx = [-1, 1, 0, 0]
dy = [0, 0, -1, 1]

class Cell:
    def __init__(self, n, m, size, life):
        self.n = n
        self.m = m
        self.size = size
        self.life = life

    def __str__(self):
        return f'{self.n}, {self.m}, {self.size}, {self.life}'

for tc in range(1, int(input()) + 1):
    N, M, K = list(map(int, input().split()))
    cell_check = [[0] * 401 for _ in range(401)]
    non_active = deque()
    active = deque()
    for n in range(N):
        temp = list(map(int, input().split()))
        for m in range(M):
            if temp[m] != 0:
                non_active.append(Cell(175+n, 175+m, temp[m], temp[m]))
                cell_check[175 + n][175 + m] = 1

    breed = list()
    breed_map = [[0] * 401 for _ in range(401)]
    for k in range(K):
        na_len = len(non_active)
        a_len = len(active)

        breed = list(set(breed))

        while breed:
            n, m = breed.pop(0)
            non_active.append(Cell(n, m, breed_map[n][m], breed_map[n][m]))
            cell_check[n][m] = 1

        for i in range(na_len):
            na = non_active.popleft()
            na.life -= 1

            if na.life == 0:
                active.append(Cell(na.n, na.m, na.size, na.size))
                for j in range(4):
                    nn, nm = na.n + dx[j], na.m + dy[j]
                    if cell_check[nn][nm]:
                        continue
                    if breed_map[nn][nm] < na.size:
                        breed_map[nn][nm] = na.size
                        breed.append((nn, nm))
            else:
                non_active.append(na)

        for i in range(a_len):
            a = active.popleft()
            a.life -= 1
            if a.life != 0:
                active.append(a)
    print(f'#{tc} {len(active) + len(non_active)}')


"""Hangups - open client for the undocumented Hangouts chat API."""

import re
import random
import json
import requests
import time
import hashlib
import datetime

from hangups import javascript, longpoll

class HangupsClient(object):

    def __init__(self, cookies, origin_url):
        self._cookies = cookies
        self._origin_url = origin_url

        # discovered automatically:

        # the api key sent with every request
        self.api_key = None
        # fields sent in request headers
        self.header_date = None
        self.header_version = None
        self.header_id = None
        self.header_client = None
        # parameters related talkgadget channel requests
        self.channel_path = None
        self.gsessionid = None
        self.clid = None
        self.channel_ec_param = None
        self.channel_prop_param = None
        self.channel_session_id = None

        # make initialization requests
        self._init_talkgadget_1()
        self._init_talkgadget_2()

    def _init_talkgadget_1(self):
        """Make first talkgadget request and parse response.

        The response body is a HTML document containing a series of script tags
        containing JavaScript object. We need to parse the object to get at the
        data.
        """
        url = 'https://talkgadget.google.com/u/0/talkgadget/_/chat'
        params = {
            'prop': 'aChromeExtension',
            'fid': 'gtn-roster-iframe-id',
            'ec': '["ci:ec",true,true,false]',
        }
        headers = {
            # appears to require a browser user agent
            'user-agent': (
                'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
                '(KHTML, like Gecko) Chrome/34.0.1847.132 Safari/537.36'
            ),
        }
        res = requests.get(url, cookies=self._cookies, params=params,
                           headers=headers)
        if res.status_code != 200:
            raise ValueError("First talkgadget request failed")
        res = res.text

        # Parse the response by using a regex to find all the JS objects, and
        # parsing them.
        res = res.replace('\n', '')
        regex = re.compile(
            r"(?:<script>AF_initDataCallback\((.*?)\);</script>)"
        )
        data_dict = {}
        for data in regex.findall(res):
            try:
                data = javascript.loads(data)
                data_dict[data['key']] = data['data']
            except ValueError:
                pass # not everything will be parsable, but we don't care

        # TODO: handle errors here
        self.api_key = data_dict['ds:7'][0][2]
        self.header_date = data_dict['ds:2'][0][4]
        self.header_version = data_dict['ds:2'][0][6]
        self.header_id = data_dict['ds:4'][0][7]
        self.channel_path = data_dict['ds:4'][0][1]
        self.gsessionid = data_dict['ds:4'][0][3]
        self.clid = data_dict['ds:4'][0][7]
        self.channel_ec_param = data_dict['ds:4'][0][4]
        self.channel_prop_param = data_dict['ds:4'][0][5]

    def _init_talkgadget_2(self):
        """Make second talkgadget request and parse response."""
        url = 'https://talkgadget.google.com{}bind'.format(self.channel_path)
        params = {
            'VER': 8,
            'clid': self.clid,
            'prop': self.channel_prop_param,
            'ec': self.channel_ec_param,
            'gsessionid': self.gsessionid,
            'RID': 81187, # TODO
            'CVER': 1,
            't': 1, # trial
        }
        res = requests.post(url, cookies=self._cookies, params=params,
                            data='count=0', stream=True)
        if res.status_code != 200:
            raise ValueError("Second talkgadget request failed with {}: {}"
                             .format(res.status_code, res.raw.read()))
        res = list(longpoll.load(res.raw))[0]
        val = res[3][1][1][1][1] # ex. foo@bar.com/AChromeExtensionBEEFBEEF
        self.header_client = val.split('/')[1] # ex. AChromeExtensionwBEEFBEEF
        self.channel_session_id = res[0][1][1]

    def _receive_push_events(self):
        """Open channel to receive push events."""
        url = 'https://talkgadget.google.com/u/0/talkgadget/_/channel/bind'
        params = {
            'VER': 8,
            'clid': self.clid,
            'prop': self.channel_prop_param,
            'ec': self.channel_ec_param,
            'gsessionid': self.gsessionid,
            'RID': 'rpc',
            't': 1, # trial
            'SID': self.channel_session_id,
            'CI': 0,
        }
        res = requests.get(url, params=params, cookies=self._cookies,
                           stream=True)
        if res.status_code != 200:
            raise ValueError('Push channel request returned {}: {}'
                             .format(res.status_code, res.raw.read()))
        for message in longpoll.load(res.raw):
            yield message

    def _get_authorization_header(self):
        # technically, it doesn't matter what the url and time are
        time_msec = int(time.time() * 1000)
        auth_string = '{} {} {}'.format(time_msec, self._get_cookie("SAPISID"),
                                        self._origin_url)
        auth_hash = hashlib.sha1(auth_string.encode()).hexdigest()
        return 'SAPISIDHASH {}_{}'.format(time_msec, auth_hash)

    def _get_cookie(self, name):
        try:
            return self._cookies[name]
        except KeyError:
            raise KeyError("Cookie '{}' is required".format(name))

    def _get_request_header(self):
        return [
            [3, 3, self.header_version, self.header_date],
            [self.header_client, self.header_id],
            None,
            "en"
        ]

    def _request(self, endpoint, body_json):
        url = 'https://clients6.google.com/chat/v1/{}'.format(endpoint)
        headers = {
            'authorization': self._get_authorization_header(),
            'x-origin': self._origin_url,
            'x-goog-authuser': '0',
            'content-type': 'application/json+protobuf',
        }
        required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
        cookies = {cookie: self._get_cookie(cookie)
                   for cookie in required_cookies}
        params = {
            'key': self.api_key,
            'alt': 'json', # json or protojson
        }
        return requests.post(url, headers=headers, cookies=cookies,
                             params=params, data=json.dumps(body_json))

    def getselfinfo(self):
        """Return information about your account."""
        res = self._request('contacts/getselfinfo', [
            self._get_request_header(),
            [], []
        ])
        return json.loads(res.text)

    def setfocus(self, conversation_id):
        """Set focus (occurs whenever you give focus to a client)."""
        res = self._request('conversations/setfocus', [
            self._get_request_header(),
            [conversation_id],
            1,
            20
        ])
        return json.loads(res.text)

    def searchentities(self, search_string, max_results):
        """Search for people."""
        res = self._request('contacts/searchentities', [
            self._get_request_header(),
            [],
            search_string,
            max_results
        ])
        return json.loads(res.text)

    def querypresence(self, chat_id):
        """Check someone's presence status."""
        res = self._request('presence/querypresence', [
            self._get_request_header(),
            [
                [chat_id]
            ],
            [1, 2, 5, 7, 8]
        ])
        return json.loads(res.text)

    def getentitybyid(self, chat_id_list):
        """Return information about a list of contacts."""
        res = self._request('contacts/getentitybyid', [
            self._get_request_header(),
            None,
            [[str(chat_id)] for chat_id in chat_id_list],
        ])
        return json.loads(res.text)

    def getconversation(self, conversation_id, num_events,
                        storage_continuation_token, event_timestamp):
        """Return data about a conversation.

        Seems to require both a timestamp and a token from a previous event
        """
        res = self._request('conversations/getconversation', [
            self._get_request_header(),
            [
                [conversation_id], [], []
            ],
            True, True, None, num_events,
            [None, storage_continuation_token, event_timestamp]
        ])
        return json.loads(res.text)

    def syncallnewevents(self, after_timestamp):
        """List all events occuring at or after timestamp."""
        res = self._request('conversations/syncallnewevents', [
            self._get_request_header(),
            after_timestamp,
            [], None, [], False, [],
            1048576
        ])
        return json.loads(res.text)

    def sendchatmessage(self, conversation_id, message, is_bold=False,
                        is_italic=False, is_strikethrough=False,
                        is_underlined=False):
        """Send a chat message to a conversation."""
        client_generated_id = random.randint(0, 2**32)
        res = self._request('conversations/sendchatmessage', [
            self._get_request_header(),
            None, None, None, [],
            [
                [
                    [0, message, [is_bold, is_italic, is_strikethrough,
                                  is_underlined]]
                ],
                []
            ],
            None,
            [
                [conversation_id], client_generated_id, 2
            ],
            None, None, None, []
        ])
        return json.loads(res.text)

def load_cookies_txt():
    """Return cookies dictionary loaded from cookies.txt file.

    Expected format is the same as the body of an HTTP cookie header.
    """
    cookies_str = open('cookies.txt').read()
    cookies_list = [s.split('=', 1) for s in
                    cookies_str.strip('\n').split('; ')]
    return {cookie[0]: cookie[1] for cookie in cookies_list}

def main():
    """Make a chat request."""
    cookies = load_cookies_txt()
    hangups = HangupsClient(cookies, 'https://talkgadget.google.com')

    # Get all events in the past hour
    now = time.time() * 1000000
    one_hour = 60 * 60 * 1000000
    events = hangups.syncallnewevents(now - one_hour)

    conversations = {}
    for conversation in events['conversation_state']:
        id_ = conversation['conversation']['id']['id']
        participants = {
            (p['id']['chat_id'], p['id']['gaia_id']): p['fallback_name']
            for p in conversation['conversation']['participant_data']
        }
        conversations[id_] = {
            'participants': participants,
        }

    conversations_list = list(enumerate(conversations.items()))
    print('Activity has recently occurred in the conversations:')
    for n, (_, conversation) in conversations_list:
        print(' [{}] {}'
              .format(n,
                      ', '.join(sorted(conversation['participants'].values()))))
    conversation_index = int(input('Select a conversation to listen to: '))
    conversation_id = conversations_list[conversation_index][1][0]
    conversation = conversations_list[conversation_index][1][1]
    print('Now listening to conversation\n')

    for msg in hangups._receive_push_events():
        msg = longpoll.parse_message(msg)
        if 'payload_type' in msg and msg['payload_type'] == 'list':
            for submsg in longpoll.parse_list_payload(msg['payload']):
                if submsg['conversation_id'] == conversation_id:
                    print('({}) {}: {}'.format(
                        datetime.datetime.fromtimestamp(
                            submsg['timestamp'] / 1000000
                        ).strftime('%I:%M:%S %p'),
                        conversation['participants'][submsg['sender_ids']],
                        submsg['text']
                    ))

    print('Connection closed')

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# Copyright 2017 IBM Corporation
# Copyright 2017 The Johns Hopkins University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function
import os
import argparse

FILE_SIZE = 500
NUM_ITER = 5

class FsBenchmark(object):

    def __init__(self):
        self.parse_args = self._parse_args()
        for iter_num in self.parse_args.iter:
            self.parse_args.func(iter_num)

    def read_write(self, iter_num):
        """Run the read write benchmark"""
        file_des = os.open('{}/{}'.format(self.parse_args.dir_path, iter_num), os.O_RDWR)

    def _parse_args(self):
        """Parse arguments"""

        parser = argparse.ArgumentParser(description='ObjectFS benchmark utils')
        parser.add_argument('dir_path', type=str, help='Directory path to run the benchmark')
        parser.add_argument('-i', '--iter', type=int, default=NUM_ITER, help='Iterations to run for the test. Default:{}'.format(NUM_ITER))
        sub_parsers = parser.add_subparsers(help='commands')

        # read and write files
        rw_parser = sub_parsers.add_parser('rw', help='Run and read-write workload')
        rw_parser.add_argument('-s', '--size', type=int, default=FILE_SIZE, help='Size of files to read and write. Default:{}'.format(FILE_SIZE))
        rw_parser.set_defaults(func=self.read_write)

def main():

    fs_benchmark = FsBenchmark()

if __name__ == '__main__':
    main()


import parse
import sys

def run(file_):
    with open(file_, 'rb') as file:
        for line in file.readlines():
            parse.parse(line)

if __name__ == "__main__":
    if len(sys.argv) < 2:
        sys.exit("Usage: ./httplang <file.http>")
    inputFile = sys.argv[1]
    run(inputFile)


# -*- coding: utf-8 -*-
# =============================================================================
# MIT License
#
# Copyright (c) 2018 Andrés Bernardo
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
# =============================================================================

##=============================================================================
## """ locateBubbleOrigin.py """ :
##     Mark the origin of the experimental data
##         based on various techniques on a 2D plot,
##             where Z data is represented with a colormap;
##=============================================================================
from time import time

t0 = time()

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

#def locateBubbleOrigin(dicFileInput):
def locateBubbleOrigin(dicFileInput, plotFilePath2D, plotFilePath3D):
    ## Note: code for data point annotations have been commented out,
    ## but were left within code for future possible improvements

    ##=========================================================================
    ## utilizing np.loadtxt
    ## see "reduceFileFormatSizeDemo.py" file
    ##=========================================================================
    values = np.loadtxt(dicFileInput, skiprows = 3)

    ##=========================================================================
    ## removing data points where Z-displacement is 0mm
    ## see "identifySpecimenDemo.py" file
    ##=========================================================================
    values = values[ (values[:,2] != 0) | (values[:,5] != 0) ]

#    print ('\nThere are')
#    print (len(values))
#    print ('data entries when zero-displacement points are removed')

    X = values[:,0]
    Y = values[:,1]
    Z = values[:,2]
    dispX = values[:,3]
    dispY = values[:,4]
    dispZ = values[:,5]

    # the displaced nodes are equal to "X+dispX", "Y+dispY", "Z+dispZ"
    finalX = X+dispX
    finalY = Y+dispY
    finalZ = Z+dispZ

# =============================================================================
# =============================================================================
# =============================================================================
# # #     ##=========================================================================
# # #     ## solving for maximum & minimum values
# # #     ##=========================================================================
# # #
# # #     # find the minimum and maximum values in the x- and y-directions
# # #     xMin = min(finalX)
# # #     xMax = max(finalX)
# # #     yMin = min(finalY)
# # #     yMax = max(finalY)
# # #
# # #     # find the x & y values associated with each extreme
# # #     y_coordinate_of_xMax = finalY[ finalX.argmax() ]
# # #     y_coordinate_of_xMin = finalY[ finalX.argmin() ]
# # #     x_coordinate_of_yMax = finalX[ finalY.argmax() ]
# # #     x_coordinate_of_yMin = finalX[ finalY.argmin() ]
# # #
# # #     # include x & y values associated with maximum z
# # #     x_coordinate_of_zMax = finalX[ finalZ.argmax() ]
# # #     y_coordinate_of_zMax = finalY[ finalZ.argmax() ]
# # #
# # #
# # #     #==========================================================================
# # #     # 2D plot
# # #     #==========================================================================
# # #     fig2D = plt.figure()
# # #     ax2D = fig2D.add_subplot(111)
# # #     map2D = ax2D.scatter(finalX, finalY, c=finalZ, s=8, edgecolor='')
# # #
# # #     # include the colorbar to illustrate Z-values in 2D plot
# # #     plt.colorbar(map2D, ax=ax2D, format='%.0f mm')
# # #
# # #     # label axes
# # #     plt.xlabel('X+dispX (mm)')
# # #     plt.ylabel('Y+dispY (mm)')
# # #     plt.title('Z-displacement, 2D representation')
# # #
# # #     # include grid; low alpha value for good visibility
# # #     ax2D.grid(True, alpha = 0.2, ls='dotted')
# # #
# # #     # x-axis and y-axis
# # #     ax2D.axhline(0, color='k', lw=0.5, ls='dashed')
# # #     ax2D.axvline(0, color='k', lw=0.5, ls='dashed')
# # #
# # #     ## annotate origin [x = 0 mm, y = 0 mm]
# # # #    plt.scatter(0, 0, c='k', s=10, edgecolor='')
# # # #    ax2D.annotate('(x=0, y=0)',
# # # #                 size=7,
# # # #                 xy = [0, 0],
# # # #                 xytext=(5, -15),
# # # #                 textcoords = 'offset pixels')
# # #
# # #     # plot & annotate maximum Z value
# # #     ax2D.scatter(x_coordinate_of_zMax, y_coordinate_of_zMax, \
# # #                         s=100, marker='s', facecolors='none', edgecolors='k', \
# # #                         label='Z$_{raw Max}$ = (%.2f, %.2f) mm' \
# # #                         %(x_coordinate_of_zMax, y_coordinate_of_zMax ))
# # # #    ax2D.annotate('Z$_{max}$',
# # # #                 size=7,
# # # #                 xy = [x_coordinate_of_zMax, y_coordinate_of_zMax],
# # # #                 xytext=(-20, -15),
# # # #                 textcoords = 'offset pixels')
# # #
# # #     # plot the x & y extrema
# # #     ax2D.scatter(xMax, y_coordinate_of_xMax, c='orange', marker='x', s=50)
# # #     ax2D.scatter(xMin, y_coordinate_of_xMin, c='orange', marker='x', s=50)
# # #     ax2D.scatter(x_coordinate_of_yMax, yMax, c='orange', marker='x', s=50)
# # #     ax2D.scatter(x_coordinate_of_yMin, yMin, c='orange', marker='x', s=50)
# # #
# # #     # include perpendicular lines marking the extrema locations
# # #     ax2D.axvline(x_coordinate_of_yMax, color='orange', lw=0.75, ls='dashdot')
# # #     ax2D.axvline(x_coordinate_of_yMin, color='orange', lw=0.75, ls='dashdot')
# # #     ax2D.axhline(y_coordinate_of_xMax, color='orange', lw=0.75, ls='dashdot')
# # #     ax2D.axhline(y_coordinate_of_xMin, color='orange', lw=0.75, ls='dashdot')
# # #
# # #     # plot lines connecting max & min in the x & y directions
# # #     ax2D.plot([xMax,xMin],
# # #              [y_coordinate_of_xMax,y_coordinate_of_xMin],
# # #              c='r',
# # #              lw=.75,
# # #              ls='dashed')
# # #
# # #     ax2D.plot([x_coordinate_of_yMax,x_coordinate_of_yMin],
# # #              [yMax,yMin],
# # #              c='r',
# # #              lw=.75,
# # #              ls='dashed')
# # #
# # #     # calculate point of intersection
# # #     slopeH = (y_coordinate_of_xMin - y_coordinate_of_xMax)/(xMin - xMax)
# # #     slopeV = (yMin - yMax)/(x_coordinate_of_yMin - x_coordinate_of_yMax)
# # #
# # #     bV = yMax - slopeV*x_coordinate_of_yMax
# # #     bH = y_coordinate_of_xMax - slopeH*xMax
# # #
# # #     x_intersect = (bH - bV)/(slopeV - slopeH)
# # #     y_intersect = slopeV*x_intersect + bV
# # #
# # #     # plot & annotate the intersection
# # #     ax2D.scatter(x_intersect, y_intersect, \
# # #                  c='r', s=25, \
# # #                  label='Z$_{intersect}$ = (%.2f, %.2f) mm' \
# # #                      %(x_intersect, y_intersect))
# # # #    ax2D.annotate('(%.2f, %.2f)' %(x_intersect, y_intersect),
# # # #                 size=7,
# # # #                 color='k',
# # # #                 xy = [x_intersect, y_intersect],
# # # #                 xytext=(120, -75),
# # # #                 textcoords = 'offset pixels',
# # # #                 arrowprops=dict(facecolor='black', arrowstyle='->'))
# =============================================================================
# =============================================================================
# =============================================================================

    ##=========================================================================
    ## 3D plot (plot of the surfaces fits of Disp Z)
    ##=========================================================================
    from doubleVander import doubleVander
    from polySurfaceFit import polySurfaceFit
    from circleFit import circleFit

    x_center, y_center, radius = circleFit(finalX, finalY)

    cDispZ, rDispZ, fittedDispZ = \
        polySurfaceFit(finalX, finalY, finalZ, 4)

    X,Y = np.meshgrid(np.linspace(min(finalX), max(finalX), 50), \
                      np.linspace(min(finalY), max(finalY), 50))

    Axy = doubleVander(X.flatten(),Y.flatten(),4)

    ZZ = np.dot(Axy,cDispZ)
    Z = ZZ.reshape(X.shape)

    #   3D plot of the X, Y, disp Z
    fig3D = plt.figure()
    ax3D = fig3D.add_subplot(111, projection='3d')

    ax3D.plot_surface(X, Y, Z, rstride=1, cstride=1, alpha=0.2)

    ax3D.scatter(finalX, finalY, finalZ, zdir='z', \
                 s=.2,c='b',edgecolor='', depthshade=False)

    ax3D.set_aspect('equal')

    # suitable axes limits were determined to encompass all data for all
    # bubble tests; the hard-coded limits below are a result of inspecting
    # the resulting fitted graphs from matplotlib's default fitting behavior...
    # ...these limits were selected as a result of the above:
    ax3D.set_xlim3d(-111, 100)
    ax3D.set_ylim3d(-107, 115)
    ax3D.set_zlim3d(-122, 91)
    # (there may be some cut-offs from hyperbolic-shaped polynomial fit curves)

    # =========================================================================
    # After running this code and analyzing the results of each Bubble Test,
    # the following are the absolute extrema for all data from all 16 test:

    #The highest X value is: 99.648605
    #The highest Y value is: 114.224746
    #The highest Z value is: 90.3031988648
    #The lowest X value is: -110.493572
    #The lowest Y value is: -106.315826
    #The lowest Z value is: -121.446724849

    #The highest X value is: 99.648605
    #The highest Y value is: 114.224746
    #The highest Z value is: 3166.28546918
    #The lowest X value is: -110.493572
    #The lowest Y value is: -106.315826
    #The lowest Z value is: -1173.76981509
    #The total elapsed time is: 585.5147442817688 seconds
    # =========================================================================

    ax3D.set_xlabel('X (mm)')
    ax3D.set_ylabel('Y (mm)')
    ax3D.set_zlabel('Z (mm)')

    ax3D.set_title('3D plot of Disp Z over the XY bubble position' + \
                   '\n(with polynomial fit)')

# =============================================================================
# =============================================================================
# =============================================================================
# # #     #==========================================================================
# # #     # Marking the highest Z from the fitted polynomial on 2D plot
# # #     #
# # #     # (This method has been commented out; for middle/high pressure data,
# # #     # the max Z value of the polyfit was useful, because the curve of the
# # #     # polyfit was primarily "parabolic", so Z_polyfit would be at the
# # #     # "top of the dome". However, low-pressure data resulted in curves
# # #     # that were primarily "hyperbolic", which lead to Z_polyfit lying upon a
# # #     # "corner" of the resulting curve.)
# # #     #==========================================================================
# # # #    x_polyfit, y_polyfit = np.unravel_index(Z.argmax(), Z.shape)
# # # #
# # # #    ax3D.scatter(X[x_polyfit, y_polyfit], \
# # # #                 Y[x_polyfit, y_polyfit], \
# # # #                 Z[x_polyfit, y_polyfit], \
# # # #                 c='k', s=100, edgecolor='', marker='+')
# # # #
# # # #    ax2D.scatter(X[x_polyfit, y_polyfit], \
# # # #                 Y[x_polyfit, y_polyfit], \
# # # #                 c='k', s=100, edgecolor='', marker='+', \
# # # #                 label='Z$_{polyfit}$ = (%.2f, %.2f) mm' \
# # # #                     %(X[x_polyfit,y_polyfit], Y[x_polyfit,y_polyfit]))
# # # ##    ax2D.annotate('Z$_{polyfit}$ = (%.2f, %.2f)' \
# # # ##                      %(X[x_coord,y_coord], Y[x_coord,y_coord]), \
# # # ##                 size=7,
# # # ##                 xy = [X[x_coord,y_coord], Y[x_coord,y_coord]],
# # # ##                 xytext=(-20, 15),
# # # ##                 textcoords = 'offset pixels',
# # # ##                 arrowprops=dict(facecolor='black', arrowstyle='->'))
# # #
# # #     #==========================================================================
# # #     # Marking the median X & median Y from the fitted polynomial on 2D plot
# # #     #==========================================================================
# # #     x_median = np.median(X)
# # #     y_median = np.median(Y)
# # #
# # #     ax2D.scatter(x_median, \
# # #                  y_median, \
# # #                  c='k', s=200, edgecolor='', marker='+', \
# # #                  label='Z$_{poly XY median}$ = (%.2f, %.2f) mm' \
# # #                      %(x_median, y_median))
# # #
# # #     # perhaps we can look at the median from the raw data as well??
# # #     x_median = np.median(finalX)
# # #     y_median = np.median(finalY)
# # #     ax2D.scatter(x_median, \
# # #                  y_median, \
# # #                  s=200, edgecolors='k', marker='^', facecolors='none', \
# # #                  label='Z$_{raw XY median}$ = (%.2f, %.2f) mm' \
# # #                      %(x_median, y_median))
# # #
# # #     #==========================================================================
# # #     # Marking center derived from circleFit function
# # #     #==========================================================================
# # #     ax2D.scatter(x_center, y_center, \
# # #                  s=200, marker='o', facecolors='none', edgecolors='k', \
# # #                  label='Z$_{circlefit}$ = (%.2f, %.2f) mm' \
# # #                      %(x_center, y_center))
# # # #    ax2D.annotate('Z$_{circlefit}$ = (%.2f, %.2f)' %(x_center, y_center),
# # # #                 size=7,
# # # #                 xy = [x_center, y_center],
# # # #                 xytext=(45, 40),
# # # #                 textcoords = 'offset pixels',
# # # #                 arrowprops=dict(facecolor='white', arrowstyle='->'))
# # #
# # #     #==========================================================================
# # #     # Resize 2D figure, generate legend, and show and/or save plots
# # #     #==========================================================================
# # #     fig2D.set_size_inches(np.array([10,10]))
# # #     ax2D.set_aspect('equal')
# # #     ax2D.xaxis.set_ticks(np.arange(-100, 125, 25))
# # #     ax2D.yaxis.set_ticks(np.arange(-100, 125, 25))
# # #
# # #     ax2D.legend(fancybox=True, fontsize='small', loc='lower right', \
# # #                 markerscale=0.5)
# # #
# # # #    fig2D.show()
# # #     fig2D.savefig(plotFilePath2D, bbox_inches='tight', dpi=200)
# # #     plt.close(fig2D)
# =============================================================================
# =============================================================================
# =============================================================================

    fig3D.set_size_inches(np.array([6,6]))
#    fig3D.show()
    fig3D.savefig(plotFilePath3D, bbox_inches='tight', dpi=200)
    plt.close(fig3D)

##=============================================================================

# =============================================================================
#    return X[np.unravel_index(np.argmax(X),X.shape)], \
#           Y[np.unravel_index(np.argmax(Y),Y.shape)], \
#           Z[np.unravel_index(np.argmax(Z),Z.shape)], \
#           X[np.unravel_index(np.argmin(X),X.shape)], \
#           Y[np.unravel_index(np.argmin(Y),Y.shape)], \
#           Z[np.unravel_index(np.argmin(Z),Z.shape)], \
# =============================================================================

###############################################################################

# =============================================================================
# locatebubbleOrigin.py loop
# =============================================================================

plt.ioff()
## used when running locateBubbleOrigin.py in the following loop;
## this will help suppress figure output (see plt.close() commands above)

import os
import os.path as path
import glob

# =============================================================================
#maxX = 90
#maxY = 90
#maxZ = 90
#minX = -90
#minY = -90
#minZ = -90
# =============================================================================

datFolderNameList = \
    [ \
    'C:\\temp\Blue_PVC_Valmex_BubbleTest00',\
    'C:\\temp\Blue_PVC_Valmex_BubbleTest01',\
    'C:\\temp\Blue_PVC_Valmex_BubbleTest02',\
    'C:\\temp\Blue_PVC_Valmex_BubbleTest03',\
    'C:\\temp\Black_PVC_Cape_Coaters_BubbleTest01',\
    'C:\\temp\Black_PVC_Cape_Coaters_BubbleTest02',\
#    'C:\\temp\Black_PVC_Cape_Coaters_BubbleTest03',\
#    'C:\\temp\Black_PVC_Cape_Coaters_BubbleTest04' \
    ]

for line in datFolderNameList :

    baseName = path.basename(line)
    plotFolderName2D = baseName + '_origin2Dplots'
    plotFolderPath2D = path.normpath(path.join('C:\\temp', plotFolderName2D))
    plotFolderName3D = baseName + '_poly3Dplots'
    plotFolderPath3D = path.normpath(path.join('C:\\temp', plotFolderName3D))

# =============================================================================
# =============================================================================
# =============================================================================
# # #     os.makedirs(plotFolderPath2D)
# =============================================================================
# =============================================================================
# =============================================================================
    os.makedirs(plotFolderPath3D)

    line = path.join(line, '*.dat')

    datList = glob.glob(line)

    for line in datList:
        plotFilePath = path.splitext(path.basename(line))[0]
        plotFilePath = baseName + '_' + plotFilePath
        plotFilePath2D = plotFilePath + '_origin2Dplot.png'
        plotFilePath2D = path.join(plotFolderPath2D, plotFilePath2D)
        plotFilePath3D = plotFilePath + '_poly3Dplot.png'
        plotFilePath3D = path.join(plotFolderPath3D, plotFilePath3D)
        locateBubbleOrigin(line, plotFilePath2D, plotFilePath3D)
#        testXmax, testYmax, testZmax, testXmin, testYmin, testZmin = \
#            locateBubbleOrigin(line, plotFilePath2D, plotFilePath3D)
#            locateBubbleOrigin(line)

# =============================================================================
#        if maxX < testXmax: maxX = testXmax
#        if maxY < testYmax: maxY = testYmax
#        if maxZ < testZmax: maxZ = testZmax
#        if minX > testXmin: minX = testXmin
#        if minY > testYmin: minY = testYmin
#        if minZ > testZmin: minZ = testZmin
# =============================================================================

#        print(line)
#        print(plotFilePath2D)
#        print(plotFilePath3D)
#    print('\n\n')

# =============================================================================
#print('The highest X value is:', maxX)
#print('The highest Y value is:', maxY)
#print('The highest Z value is:', maxZ)
#print('The lowest X value is:', minX)
#print('The lowest Y value is:', minY)
#print('The lowest Z value is:', minZ)
# =============================================================================
# =============================================================================
# After running this code and analyzing the results of each Bubble Test,
# the following are the absolute extrema for all data from all 16 test:

#The highest X value is: 99.648605
#The highest Y value is: 114.224746
#The highest Z value is: 90.3031988648
#The lowest X value is: -110.493572
#The lowest Y value is: -106.315826
#The lowest Z value is: -121.446724849

# Note: Black Cape Coaters tests 3 & 4 require manual fixes;
# including those tests in the script results in a higher maxZ
# and a lower minZ:
#(result if Black Cape Coaters tests 3 & 4 remain in queue...)
####The highest X value is: 99.648605
####The highest Y value is: 114.224746
####The highest Z value is: 3166.28546918
####The lowest X value is: -110.493572
####The lowest Y value is: -106.315826
####The lowest Z value is: -1173.76981509
####The total elapsed time is: 585.5147442817688 seconds

# The previous limits were used pending fixes to Black Cape Coaters tests 3 & 4

# =============================================================================

t1 = time()

Ts = t1-t0
Tmin = Ts/60
print('The total elapsed time is:', Ts, 'seconds =', Tmin, 'minutes')


#!/usr/bin/env python
from sklearn.metrics import precision_recall_fscore_support
import numpy as numpy
import pandas as panda
import re
import nl_core_news_sm

from medstruct.classify.tnm.extractmeasurement import ExtractMeasurement

concept_tumor = "massa|tumor|nodu|densiteit|haarden|carcino|meningeno|laesie|letsel|weke delen|rip|maligniteit|afwijking" #afwijking
concept_kwab = "kwab"
unit_mm_pattern = r'mm|millimeter'
unit_cm_pattern = r'cm\b|centimeter\b'
unit_dm_pattern = r'dm\b|decimeter\b'
unit_pattern = unit_mm_pattern+'|'+unit_cm_pattern+'|'+unit_dm_pattern
number_attribute_pattern = r'(x|bij|maal|op|plus.*minus|ongeveer|om en nabij|a|circa|en)'
ima_attribute_pattern = r'(-|//)'
decrease_synonyms = 'afname|afgenomen|gekrompen'

trachea = "trachea|luchtpijp"
bronchus = "bronchus"
right_bronchus = "rechter bronchus"
left_bronchus = "left bronches"

lung = "lung|long"
lung_right = "pulmo dexter|recht.*long|long.*recht"
lung_left = "pulmo sinister|link.*long|long.*link"

superior_lobe = "bovenste longlob|bovenste kwab|bovenste lob"
superior_lobe_right = "recht.*boven.*(kwab|lob)|boven.*(lob|kwab).*recht|\bRBK\b|\bRBL\b"
# superior_lobe_left = "link.*boven.*(kwab|lob)|boven.*(lob|kwab).*link|\bLBK\b|\bLBL\b"
superior_lobe_left = "linker bovenkwab"

middle_lobe = "midde.*(lob|kwab)|\bMK\b|\bML\b"
inferior_lobe = "onderste longlob|onderste kwab|onderste lob"
inferior_lobe_right = "recht.*onder.*(kwab|lob)|onder.*(lob|kwab).*recht|\bROK\b|\bROL\b"
# inferior_lobe_left = "link.*onder.*(kwab|lob)|onder.*(lob|kwab).*link|\bLOK\b|\bLOL\b"
inferior_lobe_left = "linker onderkwab"

fissura_obliqua = ""
fissura_horizontalis = ""
arteria_pulmonalis = ""

multiple = "meerdere|verschillende|verscheidenen|tweetal|drietal|viertal|vijftal|zestal|zevental"
multiple_tumors = "noduli"

"apicaal"
"basaal"
"craniaal"
"caudaal"

#parietal pleural|PL3|borstwand|borstkaswand|thoraxwand|phrenic|parietal pericard|hartzak|longvlies|rib|spier|kraakbeen|diafragma

# t2_exceptions_manual = "inv.*pleur|pleur.*inv|hilu|hila|hilum pulmonis|(atelect|obstruct).*pneum|atelect"
t2_exceptions_atelect = "atelect"
t2_exceptions_main_bronchus = "(centrale|hoofd).*bronch"
t2_exceptions_visceral_pleura = "pleura\b"

t3_exceptions_borstwand = "borst.*wan|thorax.*wan"
t3_exceptions_parietale_pericard = "pericard"
t3_exceptions_nervus_phrenicus = "nervus.*phrenicus"
t3_exceptions_satellite_same_lobe = "satelliet"

t4_exceptions_mediastinum = "geen.*afgrenzing.*mediast|ingroei.*mediast"
t4_exceptions_diafragma = "diafragm|middenrif"
t4_exceptions_carina = "carina"

t4_exceptions_large_vessels = "grote vaten|aorta|vena cava|vat\b|arteri.*pul|ven.*pul"
t4_exceptions_trachea = "trachea|luchtpijp"
t4_exceptions_nervus_laryngeus_recurrens = "nervus.*laryngeus.*recurrens|strot|slijmvlies|stembanden"
t4_exceptions_oesophagus = "oesophagus|slok"
t4_exceptions_wervels = "wervel|bot|rib|kraak|cartilago|thoracicae|ruggengraat|columna|vertebra" #niet specifiek

# unit_pattern = r'(mm\b|millimeter\b|cm\b|centimeter\b|dm\b|decimeter\b)'

lympfe_filter = 'lymf|lymph'

#exclude lympfe?

class ReportT(object):

    t_column = 3

    def __init__(self):
        self.nlp = nl_core_news_sm.load()

    # Preprocess documents using spaCy's nlp pipeline
    def run_spacy(self, doc):
        return self.nlp(str(doc))

    # Create function that processes new dataframe in one go.
    def NLP_pipeline(self, df, report_column, t_column):
        """
        Input = dataframe where the first column contains all the raw text files and column 15 contains the label.

        Output = dataframe of 4 columns, namely:
                            - original raw text
                            - sentences with numbers
                            - sentences with sizes
                            - sizes

        Necessary packages: - numpy
                            - pandas
                            - re
                            - itertools
                            - spacy
                            - nl_core_news_sm
                            - precision_recall_fscore_support from sklearn.metrics

        """
        self.t_column = t_column
        self.report_column = report_column

        print("Preprocessing raw text...")
        result_df = df.iloc[:, self.report_column]
        result_df = result_df.apply(ExtractMeasurement.preprocess_raw_numbers_dictation_artifacts_correction)
        result_df = result_df.apply(ReportT.preprocess_raw_abbreviations)

        result_df = result_df.apply(self.run_spacy) # Preprocess raw text using spaCy's nlp function
        result_df = panda.concat([result_df, result_df.apply(get_num_sents)], axis=1) # Get sentences with numbers
        result_df.columns = ['document', 'num_sents'] # Rename columns
        result_df['size_sents'] = result_df['num_sents'].apply(get_size_sents) # Get sentences with sizes
        result_df['tumor_sents'] = result_df['document'].apply(get_tumor_sents) # Get sentences with 'tumor' synoniems

        result_df['tumor_sents_unit'] = result_df['num_sents'].apply(ExtractMeasurement.get_measurements_length)
        result_df['tumor_sents_cm'] = result_df['tumor_sents_unit'].apply(ExtractMeasurement.get_measurements_in_cm)
        result_df["class_size_cm"] = result_df["tumor_sents_cm"].apply(ExtractMeasurement.get_max_size)

        result_df["two_tumors"] = result_df.iloc[:, 0].apply(has_tumors_in_multiple_lobes_same_lung)

        print("Extracting T classification...")
        result_df["naive_T_class"] = result_df["class_size_cm"].apply(get_naive_t_class)
        result_df["naive_T_class_simple"] = result_df["naive_T_class"].apply(get_t_score_label)

        result_df["T2"] = result_df['tumor_sents'].apply(t2_classification)

        result_df["T3"] = result_df.iloc[:,0].apply(t3_classification)
        result_df["T3+naive_class"] = numpy.where(
            (result_df["T3"] == "T3") & (result_df["naive_T_class_simple"] != "T4"), "T3",
            result_df["naive_T_class_simple"])

        result_df["T4"] = result_df.iloc[:,0].apply(t4_classification)
        result_df["T4+naive_class"] = numpy.where(result_df["T4"] == "T4", "T4", result_df["naive_T_class_simple"])

        result_df["T4+T3+naive_class"] = result_df.apply(lambda x: get_class(x['naive_T_class_simple'], x['T2'], x['T3'], x['T4']), axis=1)
        result_df = panda.concat([result_df, df.iloc[:, self.t_column]], axis=1)
        result_df = result_df.rename(columns={result_df.columns[-1]:'label'})
        result_df["label_T"] = result_df["label"].apply(get_t_score_label)

        print("Computing evaluation metrics...")
        acc_df = result_df[result_df["label_T"] != "?"] # Filter dataframe on reports that have a valid label
        print("")
        print("ACCURACY RESULTS FROM VARIOUS METHODS:")
        print("-----------------------------------------------")
        acc_naive = acc_df[acc_df["label_T"] == acc_df["naive_T_class_simple"]].shape[0]/acc_df.shape[0] # Compute accuracy
        print("Naive classification (only tumor measurements):")
        print("Accuracy = " + str(round(acc_naive*100, 2)) + "%") # Print result in percentages
        print("-----------------------------------------------")
        acc_t4 = acc_df[acc_df["T4+naive_class"] == acc_df["label_T"]].shape[0]/acc_df.shape[0] # Compute accuracy
        print("Classification using additional T4 rules:")
        print("Accuracy = " + str(round(acc_t4*100, 2)) + "%") # Print result in percentages
        print("-----------------------------------------------")
        acc_t3 = acc_df[acc_df["T3+naive_class"] == acc_df["label_T"]].shape[0]/acc_df.shape[0] # Compute accuracy
        print("Classification using additional T3 rules:")
        print("Accuracy = " + str(round(acc_t3*100, 2)) + "%") # Print result in percentages
        print("-----------------------------------------------")
        acc_t4_t3 = acc_df[acc_df["T4+T3+naive_class"] == acc_df["label_T"]].shape[0]/acc_df.shape[0] # Compute accuracy
        print("Classification using additional T4 & T3 rules:")
        print("Accuracy = " + str(round(acc_t4_t3*100, 2)) + "%") # Print result in percentages
        print("-----------------------------------------------")
        print("")

        y_actu = panda.Series(acc_df['label_T'], name='Actual')
        y_pred = panda.Series(acc_df['naive_T_class_simple'], name='Predicted')
        df_confusion = panda.crosstab(y_actu, y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)
        print("CONFUSION MATRIX OF CLASSIFICATION USING ADDITIONAL naive_T_class_simple:")
        print("------------------------------------------------------------------")
        print("")
        print(df_confusion)

        y_actu = panda.Series(acc_df['label_T'], name='Actual')
        y_pred = panda.Series(acc_df['T4+T3+naive_class'], name='Predicted')
        df_confusion = panda.crosstab(y_actu, y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)
        print("CONFUSION MATRIX OF CLASSIFICATION USING ADDITIONAL T4 & T3 RULES:")
        print("------------------------------------------------------------------")
        print("")
        print(df_confusion)
        print("")
        prf = precision_recall_fscore_support(y_actu, y_pred, average=None, labels=['T1', 'T2', 'T3', 'T4'])
        evaldf = panda.DataFrame({"Precision": prf[0], "Recall": prf[1], "F-score": prf[2]}, index=["T1", "T2", "T3", "T4"])
        print(evaldf)
        print("-------------------------------------------------------------------")

        return result_df

    @staticmethod
    def preprocess_raw_abbreviations(text):
        """
        Map abbreviations targeting numbers to its full form
        :param text including abbrevations
        :return: text including full form of selected abbreviations
        """
        abbreviations = {"ca.": "circa"}
        for key in abbreviations:
            text = text.replace(key, abbreviations[key])
        return text

# Extract all sentences from a document that contain numerical tokens using spaCy's PoS tagger
def get_num_sents(doc):
    sentlist = list()
    for sent in list(doc.sents):
        if 'NUM' in [token.pos_ for token in sent]:
            sentlist.append(sent)
    return sentlist

# Extract all sentences that contain 'mm', 'cm', '...meter'.
# mm can't be followed by another letter to ensure we don't include sentences with mmol in result.
def get_size_sents(sentences):
    sizelist = list()
    for sent in filter_out_lymph(sentences):
        if len(re.findall(r'('+unit_pattern+')', sent.string)) > 0:
            sizelist.append(sent)
    return sizelist

def get_tumor_size_sents(sentences):
    tumorlist = list()
    for sent in sentences:
        if len(re.findall(r'('+concept_tumor+')', sent.string, re.IGNORECASE)) > 0:
            tumorlist.append(sent)
    return tumorlist

def get_tumor_sents(doc):
    tumorlist = list()
    for sent in list(doc.sents):
        if len(re.findall(r'('+concept_tumor+')', sent.string, re.IGNORECASE)) > 0:
            tumorlist.append(sent)
    return tumorlist

def has_multiple_tumors(doc):
    tumor_sents = remove_negex_sents(get_tumor_sents(doc))
    for sent in tumor_sents:
        if re.search(r'('+multiple_tumors+')', sent.string, re.IGNORECASE) \
                or re.search(r'('+multiple+')', sent.string, re.IGNORECASE):
            return True
    return False

def filter_out_lymph(sents):
    clean = list()
    for sent in sents:
        if not re.search(r'('+lympfe_filter+')', sent.string, re.IGNORECASE):
            clean.append(sent)
    return clean

def has_tumors_in_multiple_lobes_same_lung(doc):
    tumor_sent = remove_uncertain_sents(remove_negex_sents(get_tumor_sents(doc)))

    for sent in tumor_sent:
        slr_found = 0
        sll_found = 0
        ml_found = 0
        irl_found = 0
        ill_found = 0

        if re.search(r'('+superior_lobe_right+')', sent.string, re.IGNORECASE):
            slr_found = 1
        if re.search(r'('+superior_lobe_left+')', sent.string, re.IGNORECASE):
            sll_found = 1
        if re.search(r'('+middle_lobe+')', sent.string, re.IGNORECASE):
            ml_found = 1
        if re.search(r'('+inferior_lobe_right+')', sent.string, re.IGNORECASE):
            irl_found = 1
        if re.search(r'('+inferior_lobe_left+')', sent.string,  re.IGNORECASE):
            ill_found = 1

        multiple_lobes_right = slr_found + irl_found + ml_found
        multiple_lobes_left = sll_found + ill_found
        multiple_lobes_same_lung = (multiple_lobes_right > 1) or (multiple_lobes_left > 1)
        if multiple_lobes_same_lung:
            print("multiple_lobes", multiple_lobes_same_lung)
            return True

    return False

def has_multiple_tumors_in_same_lobe(doc):
    tumor_sent = remove_negex_sents(get_tumor_sents(doc))

    ull_found = 0
    url_found = 0
    ml_found = 0
    lrl_found = 0
    lll_found = 0

    for sent in tumor_sent:
        if re.search(r'('+superior_lobe_right+')', sent.string, re.IGNORECASE):
            url_found = 1
        if re.search(r'('+superior_lobe_left+')', sent.string, re.IGNORECASE):
            ull_found = 1
        if re.search(r'('+middle_lobe+')', sent.string, re.IGNORECASE):
            ml_found = 1
        if re.search(r'('+inferior_lobe_right+')', sent.string, re.IGNORECASE):
            lrl_found = 1
        if re.search(r'('+inferior_lobe_left+')', sent.string,  re.IGNORECASE):
            lll_found = 1

    multiple_lobes_right = ull_found + url_found + ml_found > 1
    multiple_lobes_left = lrl_found + lll_found > 1

    multiple_lobes_same_lung = multiple_lobes_right > 1 or multiple_lobes_left > 1
    if multiple_lobes_same_lung:
        print("multiple_lobes", multiple_lobes_same_lung)
    return multiple_lobes_same_lung

def get_class(naive, t2, t3, t4):
    if naive == "T4" or t4 == "T4":
        return "T4"
    if naive == "T3" or t3 == "T3":
        return "T3"
    if naive == "T2" or t2 == "T2":
        return "T2"
    return "T1"

def get_naive_t_class(max_size):
    if type(max_size) == float:
        if 7 < max_size <= 40: # Sanity check in case max_size is an illogical value
            return 'T4'
        elif 5 < max_size <= 7:
            return 'T3'
        elif 4 < max_size <= 5:
            return 'T2b'
        elif 3 < max_size <= 4:
            return 'T2a'
        elif 2 < max_size <= 3:
            return 'T1c'
        elif 1 < max_size <= 2:
            return 'T1b'
        elif 0 < max_size <= 1:
            return 'T1a'
        else:
            return '?'
    else:
        return '?'

def t2_classification(sents):
    t2list = t2_exceptions(sents)
    if len(t2list) > 0:
        tclass = "T2"
    else:
        tclass = "?"
    return tclass

def t3_classification(doc):
    t3list = t3_exceptions(doc)
    t3tumorlist = get_tumor_size_sents(t3list)
    nonneglist = remove_negex_sents(t3tumorlist)
    if len(nonneglist) > 0:
        tclass = "T3"
    else:
        tclass = "?"
    return tclass

def t4_classification(doc):
    t4list = t4_exceptions(doc)
    t4tumorlist = get_tumor_size_sents(t4list)
    nonneglist = remove_negex_sents(t4tumorlist)
    if len(nonneglist) > 0:
        tclass = "T4"
    elif has_tumors_in_multiple_lobes_same_lung(doc):
        tclass = "T4"
    else:
        tclass = "?"
    return tclass

def t2_exceptions(sents):
    t2list = list()
    regex = t2_exceptions_atelect \
            + "|" + t2_exceptions_main_bronchus \
            + "|" + t2_exceptions_visceral_pleura

    for sent in sents:
        if len(re.findall(r'('+regex+')', sent.string, re.IGNORECASE)) > 0:
            t2list.append(sent)
    return t2list

# Detect exception areas for T3 Classification
def t3_exceptions(doc):
    t3list = list()

    regex = t3_exceptions_borstwand \
            + "|" + t3_exceptions_nervus_phrenicus \
            + "|" + t3_exceptions_parietale_pericard \
            + "|" + t3_exceptions_satellite_same_lobe

    for sent in get_tumor_sents(doc):
        if len(re.findall(r'('+regex+')', sent.string, re.IGNORECASE)) > 0:
            t3list.append(sent)
    return t3list

# Detect exception areas for T4 Classification
def t4_exceptions(doc):
    t4list = list()

    regex = t4_exceptions_diafragma \
            + "|" + t4_exceptions_mediastinum

    for sent in get_tumor_sents(doc):
        if len(re.findall(r'('+regex+')', sent.string, re.IGNORECASE)) > 0:
            t4list.append(sent)
    return t4list

# Detect negations for T3/T4 exceptions
def remove_negex_sents(sentlist):
    resultlist = list()
    for sent in sentlist:
        if len(re.findall(r'(geen|niet|normaal|zonder|conform|ongewijzigd)', sent.string, re.IGNORECASE)) == 0:
            resultlist.append(sent)
        if len(re.findall(r'(geen.*afgrenzing)', sent.string, re.IGNORECASE)) > 0:
            resultlist.append(sent)
    return resultlist

def remove_uncertain_sents(sentlist):
    resultlist = list()
    for sent in sentlist:
        if len(re.findall(r'(verdenking|lijkt|mogelijk)', sent.string, re.IGNORECASE)) == 0:
            resultlist.append(sent)
    return resultlist

# Extract first two characters from TNM code
def get_t_score_label(label):
    return str(label)[:2] if (str(label)[0] == "T") and ((str(label)[1] == "1") or (str(label)[1] == "2") or (str(label)[1] == "3") or (str(label)[1] == "4")) else "?"


from config import Config
from common_utils import *
from query import Query
import json
from collections import defaultdict
import requests
import string
from category_vendor_details import CategoryVendorDetails, OmnitureDetails, PriceDetails

class BuildProductDetail(object):

    def __init__(self):
        self.product_info = dict()
        self.category_vendor = CategoryVendorDetails()
        self.omniture = OmnitureDetails()
        self.price = PriceDetails()

    def add_to_product_dict(self, product_details_list):
        for product in product_details_list:
            product_detail = self.product_info.get(product.get('entity_id'), {})
            product_detail.update(product)
            self.product_info.update({product.get('entity_id'): product_detail})

    def add_keywords(self):
        map(lambda product: product.update({'keywords': Config.KEYWORDS}), self.product_info.values())
        map(lambda product: product.update({'fabric': ''}), self.product_info.values())
        map(lambda product: product.update({'tags': ['']}), self.product_info.values())

    def get_page_title(self, product):
        return product.get('name', '') + ' ' + product.get('sku') + ' - buy ' + \
                product.get('category_name', ['', ''])[1] + ' online from ' + product.get('vendor_name', '') + \
                ' at CraftsVilla.com'

    def add_page_title(self):
        map(lambda product: product.update({'page_title': self.get_page_title(product)}), self.product_info.values())

    def prepare_product_detail_from_id_list(self, product_list_block):
        product_category_vendor_details = self.category_vendor.get_details(Query.product_category_vendor_info, product_list_block)
        self.add_to_product_dict(product_category_vendor_details)
        product_omniture_details = self.omniture.get_details(Query.omniture_info, product_list_block)
        self.add_to_product_dict(product_omniture_details)
        product_price_details = self.price.get_details(Query.price_details, product_list_block)
        self.add_to_product_dict(product_price_details)
        self.add_keywords()
        self.add_page_title()
        return self.product_info


"""
NSLS2 Visualization Lib

:Author: Lingyun Yang <lyyang@bnl.gov>
:Date: 2014-07-11 18:21:20

"""

from __future__ import print_function
import time
import numpy as np
import aphla as ap
import matplotlib.pylab as plt
from datetime import datetime
from matplotlib.dates import YEARLY, DateFormatter, rrulewrapper, RRuleLocator, drange
import numpy as np
import datetime

_figsize = (12, 4)

def pltOrbit(**kwargs):
    """
    figsize -
    """
    fig = plt.figure(figsize=kwargs.pop("figsize", _figsize))
    ax1 = fig.add_subplot(2,1,1)
    ax2 = fig.add_subplot(2,1,2)
    obt = ap.getOrbit(spos=True)
    ax1.plot(obt[:,-1], obt[:,0], label="std(X)= %.2f" % np.std(obt[:,0]))
    ax1.set_ylabel("X [mm]", fontdict={"size": 18})
    ax1.legend(loc="best")
    ax1.grid(True)
    ax2.plot(obt[:,-1], obt[:,1], label="std(Y)= %.2f" % np.std(obt[:,1]))
    ax2.set_ylabel("Y [mm]", fontdict={"size": 18})
    ax2.legend(loc="best")
    ax2.grid(True)
    return [ax1, ax2]

def pltFullOrbit(**kwargs):
    """
    figsize -
    """
    fig = plt.figure(figsize=kwargs.pop("figsize", _figsize))
    ax1 = fig.add_subplot(2,1,1)
    ax2 = fig.add_subplot(2,1,2)
    allbpms = ap.getGroupMembers(["BPM", "UBPM"], op="union")
    s = [b.sb for b in allbpms]
    x = ap.fget(allbpms, 'x', unitsys=None)
    y = ap.fget(allbpms, 'y', unitsys=None)
    ax1.plot(s, x, label="std(X)= %.2f" % np.std(x))
    ax1.set_ylabel("X [mm]")
    ax1.legend(loc="best")
    ax1.grid(True)
    ax2.plot(s, y, label="std(Y)= %.2f" % np.std(y))
    ax2.set_ylabel("Y [mm]")
    ax2.legend(loc="best")
    ax2.grid(True)
    return [ax1, ax2]

def pltOrbitRms(**kwargs):
    nsample = kwargs.get("nsample", 10)
    obt0 = ap.getOrbit(spos=True)
    nbpm = len(obt0)
    dat = np.zeros((nbpm, 2, nsample), 'd')
    for i in range(nsample):
        dat[:,:,i] = ap.getOrbit(spos=False)
        time.sleep(0.12)
    err = np.std(dat, axis=-1) * 1000.0
    fig = plt.figure(figsize=kwargs.get("figsize", _figsize))
    ax1 = plt.subplot(2,1,1)
    ax1.plot(obt0[:,-1], err[:,0])
    ax1.set_ylabel("X [um]", fontdict={"size": 18})
    ax1.grid(True)
    ax2 = plt.subplot(2,1,2)
    ax2.plot(obt0[:,-1], err[:,1])
    ax2.set_ylabel("Y [um]", fontdict={"size": 18})
    ax2.grid(True)
    return ax1, ax2

def pltAllBpmRms(**kwargs):
    nsample = kwargs.get("nsample", 10)
    ignore = kwargs.get("ignore", [])
    allbpms = [b for b in ap.getGroupMembers(["BPM", "UBPM"], op="union")
               if b.name not in ignore and b.devname not in ignore]
    s = [b.sb for b in allbpms]
    obtx0 = ap.fget(allbpms, "x0", unitsys=None)
    obty0 = ap.fget(allbpms, "y0", unitsys=None)
    nbpm = len(allbpms)
    dat = np.zeros((nbpm, 2, nsample), 'd')
    for i in range(nsample):
        dat[:,0,i] = ap.fget(allbpms, "x0", unitsys=None)
        dat[:,1,i] = ap.fget(allbpms, "y0", unitsys=None)
        time.sleep(0.12)
    err = np.std(dat, axis=-1) * 1000.0
    ix = np.argmax(err[:,0])
    iy = np.argmax(err[:,1])
    fig = plt.figure(figsize=kwargs.get("figsize", _figsize))
    ax1 = plt.subplot(2,1,1)
    b180 = [i for i,b in enumerate(allbpms) if "BPM" in b.group]
    uidx = [i for i,b in enumerate(allbpms) if "UBPM" in b.group]
    b60 = [i for i,b in enumerate(allbpms) if b.devname[-1] in ["3", "4"]]
    ax1.semilogy(np.take(s, b180), np.take(err[:,0], b180), '-b', lw=2)
    ax1.semilogy(np.take(s, uidx), np.take(err[:,0], uidx), 'og')
    ax1.semilogy(np.take(s, b60), np.take(err[:,0], b60), 'bv')
    ax1.set_ylabel("Xstd [um]", fontdict={"size": 18})
    ax1.grid(True)
    ax1.annotate("peak={0}".format(allbpms[ix].devname),
                 xy=(0.02, 0.85), xycoords="axes fraction", size=18)
    ax2 = plt.subplot(2,1,2)
    #ax2.plot(s, err[:,1])
    ax2.semilogy(np.take(s, b180), np.take(err[:,1], b180), '-b', lw=2)
    ax2.semilogy(np.take(s, uidx), np.take(err[:,1], uidx), 'og')
    ax2.semilogy(np.take(s, b60), np.take(err[:,1], b60), 'bv')
    ax2.set_ylabel("Ystd [um]", fontdict={"size": 18})
    ax2.grid(True)
    ax2.annotate("peak={0}".format(allbpms[iy].devname),
                 xy=(0.02, 0.85), xycoords="axes fraction", size=18)
    return ax1, ax2

def pltOrbitFft(x, y, **kwargs):
    fig = plt.figure(figsize=kwargs.get("figsize", (16,7)))
    nbpm, nturns = np.shape(x)
    obt = np.zeros((nbpm, nturns, 3), 'd')
    obt[:,:,0] = x
    obt[:,:,1] = y
    if "Isum" in kwargs:
        obt[:,:,2] =kwargs["Isum"]
    #
    Fs = np.zeros_like(obt)

    n = nturns // 2
    c = [('g-', "X"), ("b-", "Y")]
    obtc = np.zeros((nbpm, 2), 'd')
    for iplane in range(2):
        obtc[:,iplane] = np.average(obt[:,:,iplane], axis=1)
        #Fx = np.zeros((nbpm, nturns), 'd')

        for ibpm in range(180): #[1,2,3,4,5,6]:
            dobt = obt[ibpm,:,iplane]-obtc[ibpm,iplane]
            Fs[ibpm,:,iplane] = np.abs(np.fft.fft(dobt))

    # plotting
    vc = np.average(obtc, axis=-1)*1000.0
    vr = np.std(obtc, axis=-1) * 1000.0
    plt.subplot(3,1,1)
    plt.plot(obtc[:,0], 'r-', label="X %.1f+/-%.1f [um]" % (vc[0], vr[0]))
    plt.plot(obtc[:,1], 'b-', label="Y %.1f+/-%.1f [um]" % (vc[1], vr[1]))
    plt.grid(True)
    plt.legend(loc="best")

    freq, p = np.fft.fftfreq(nturns), np.sum(np.abs(Fs), axis=0)
    # print np.shape(Fs), np.shape(p)
    i0, i1 = 2+int(1e-4*nturns), nturns // 2
    if "tune_range" in kwargs:
        nuref = kwargs["tune_range"]
        # ref tune and +/- range
        #nu0 = [(0.2, 0.04), (0.18, 0.03), (0.01, 0.008)]
        nu = [0.0, 0.0, 0.0]
        for i in range(3):
            j0 = max(i0, int((nuref[i][0] - nuref[i][1])/(1.0/nturns)))
            j1 = min(i1, (nuref[i][0] + nuref[i][1])/(1.0/nturns))
            print(i, j0, j1, np.shape(p), freq[j0], freq[j1])
            if i < 2:
                im = j0 + np.argmax(p[j0:j1,i])
            elif i == 2:
                im = j0 + np.argmax(p[j0:j1,0] + p[j0:j1,1])
            nu[i] = freq[im]
    else:
        ff = 1.0/nturns
        psum = p[:,0]/np.max(p[:,0]) + p[:,1]/np.max(p[:,1])
        nu = [(i0+np.argmax(p[i0:i1,0]))*ff, (i0+np.argmax(p[i0:i1,1]))*ff,
              (i0+np.argmax(psum[i0:int(0.05*nturns)]))*ff]

    print(nu)
    plt.subplot(3,1,2)
    plt.plot(obt[3,:,0], 'r-')
    plt.plot(obt[5,:,0], 'r-')
    plt.plot(obt[3,:,1], 'b-')
    plt.plot(obt[5,:,1], 'b-')
    plt.grid(True)

    ax1 = plt.subplot(3,1,3)
    ax2 = ax1.twinx()
    p1, = ax1.semilogy(freq[i0:i1], p[i0:i1,0], 'g-', linewidth=2)
    p2, = ax2.semilogy(freq[i0:i1], p[i0:i1,1], 'b-', linewidth=2)
    #ax1.set_xlim([freq[i0], freq[i1]])
    ax1.grid(True)
    p1 = ax1.axvline(nu[0], color='g', linestyle='--')
    p2 = ax1.axvline(nu[1], color='b', linestyle='--')
    labels= ["X: %.4f" % nu[0], "Y: %.4f" % nu[1]]
    if nu[2]:
        p3 = ax1.axvline(nu[2], color='k', linestyle='--')
        labels.append("S: %.4f (%.2fkHz)" % (nu[2], nu[2]*500e3/1320.))
        plt.legend([p1, p2, p3], labels, loc="best")
    else:
        plt.legend([p1, p2], labels)
    #plt.savefig("tunes_1.png")

def plotArchivedData(data, **kwargs):
    fig = plt.figure(figsize=kwargs.pop("figsize", _figsize))
    formatter = DateFormatter('%m/%d/%y')
    t0 = kwargs.get("t0", datetime.now())
    delta = np.min([v[0][0] for k,v in data.items()])
    t1 = t0 + datetime.delta

def plotChromaticity(chrom, info, **kwargs):
    """
    plot with returns from measChromaticity.
    """
    fig = plt.figure(figsize=kwargs.pop("figsize", _figsize))
    #df = 1e3*(info["freq"] - info["freq0"])
    dpp = info["dpp"]
    w = dpp[-1] - dpp[0]
    t = np.linspace(dpp[0]-0.1*w, dpp[-1]+0.1*w, 100)
    # plt.clf()
    ax1 = plt.subplot(1,2,1)
    ax1.plot(dpp, info["tune"][:,0], '-rx')
    ax1.set_ylabel(r"$\nu_x$", fontdict={"size": 18})

    ax2 = plt.subplot(1,2,2)
    ax2.plot(dpp, info["tune"][:,1], '-go')
    ax2.set_ylabel(r"$\nu_y$", fontdict={"size": 18})

    if info["deg"] < 2:
        pass
    else:
        p2, p1, p0 = info["p"][:,0]
        nux = p2*t*t + p1*t + p0
        ax1.plot(t, nux, '--r', label=r"H: $%.1fx^2%+.2fx%+.2f$" % (p2, p1, p0))
        p2, p1, p0 = info["p"][:,1]
        nuy = p2*t*t + p1*t + p0
        ax2.plot(t, nuy, '--r', label=r"V: $%.1fx^2%+.2fx%+.2f$" % (p2, p1, p0))

    ax1.annotate(r"$C_x=%.2f$" %  (chrom[0], ),
                 xy=(0.02, 0.85), xycoords="axes fraction", size=18)
    ax2.annotate(r"$C_y=%.2f$" %  (chrom[1], ),
                 xy=(0.02, 0.85), xycoords="axes fraction", size=18)

    for ax in [ax1, ax2]:
        ax.legend(loc='lower right')
        ax.set_xlabel(r"$\delta p/p$", fontdict={"size": 18})
        ax.grid(True)
    plt.savefig('measchrom.png')


# coding=utf-8
import json
import time
import datetime
from django.db import models
from garblog.settings import DEBUG, PAGE_SIZE

__author__ = 'garfield'

def log(content, path='/tmp/default.log'):
    with open(path, 'w') as f:
        if DEBUG:
            content = str(content) + '\n'
        else:
            now = time.strftime("%Y-%m-%d %H %M %S", time.localtime())
            content = '%s---%s\n' % (now, content)
            pass
        f.write(str(content))

def get_range(page):
    start = (int(page)-1) * PAGE_SIZE
    end = start + PAGE_SIZE
    return start, end

def shift_paths(exclude, name):
    return tuple(item.split('.', 1)[1] for item in exclude
                 if item.startswith(('{0}.'.format(name), '*.')))

def deep_dump_instance(instance,
                       depth=1,
                       exclude=('password', 'is_admin', 'is_active'),
                       include=(),
                       order_by=(),
                       seen=None):
    if not seen:
        seen = set()
    if (instance.__class__, instance.pk) in seen:
        return '<recursive>'
    seen.add((instance.__class__, instance.pk))
    field_names = sorted(
        [field.name for field in instance._meta.fields] +
        [f.get_accessor_name() for f in instance._meta.get_all_related_objects()])

    dump = []
    exclude_all = '*' in exclude
    for name in field_names:
        if name in include or (not exclude_all and name not in exclude):
            try:
                value = getattr(instance, name)
            except models.ObjectDoesNotExist:
                value = None
            if value.__class__.__name__ == 'RelatedManager':
                if depth >= 1:
                    related_objects = value.all()
                    for ordering in order_by:
                        parts = ordering.split('.')
                        if len(parts) == 2 and parts[0] == name:
                            related_objects = related_objects.order_by(parts[1])
                    value = [deep_dump_instance(related,
                                                depth=depth - 1,
                                                exclude=shift_paths(exclude, name),
                                                include=shift_paths(include, name),
                                                order_by=shift_paths(order_by, name),
                                                seen=seen)
                             for related in related_objects]
                else:
                    continue
            elif isinstance(value, models.Model):
                if depth >= 1:
                    value = deep_dump_instance(value,
                                               depth=depth - 1,
                                               exclude=shift_paths(exclude, name),
                                               include=shift_paths(include, name),
                                               order_by=shift_paths(order_by, name),
                                               seen=seen)
                else:
                    continue
            dump.append((name, value))
    return dump

class DateTimeEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            encoded_object = list(obj.timetuple())[0:6]
        else:
            encoded_object =json.JSONEncoder.default(self, obj)
        return encoded_object

def check_object_validate(Klass, **kwargs):
    for key in kwargs:
        if not hasattr(Klass, key):
            return False
    return True