# -*- coding: utf-8 -*-
'''
陌生人识别模型和情感分析模型的结合的主程序

用法：
python checkingstrangersandfacialexpression.py
python checkingstrangersandfacialexpression.py --filename tests/room_01.mp4
'''

# 导入包
import argparse
from oldcare.utils.insertingassistant import inserting
from oldcare.utils.pathassistant import get_path
from oldcare.utils import communicationassistant
from oldcare.facial import FaceUtil
from PIL import Image, ImageDraw, ImageFont
from oldcare.utils import fileassistant
from keras.models import load_model
from keras.preprocessing.image import img_to_array
import cv2
import time
import numpy as np
import imutils
from oldcare.utils.streampushassistant import stream_pusher
import multiprocessing

if __name__ == '__main__':
    raw_q = multiprocessing.Queue()
    my_pusher = stream_pusher(rtmp_url=get_path('rtmp_room_output', 2), raw_frame_q=raw_q)
    my_pusher.run()

    stranger_type = 5
    old_smile_type = 1

    # 全局变量
    output_stranger_path = get_path('output_stranger_path', 1)
    output_smile_path = get_path('output_smile_path', 1)

    facial_recognition_model_path = get_path('facial_recognition_model_path')
    facial_expression_model_path = get_path('facial_expression_model_path')

    people_info_path = get_path('people_info_path')
    facial_expression_info_path = get_path('facial_expression_info_path')

    # 更新一下people_info.csv
    communicationassistant.get_people_info()

    # 得到当前时间
    current_time = time.strftime('%Y-%m-%d %H:%M:%S',
                                 time.localtime(time.time()))
    print('[INFO] %s 陌生人检测程序和表情检测程序启动了.' % (current_time))

    # 传入参数
    ap = argparse.ArgumentParser()
    ap.add_argument("-f", "--filename", required=False, default='',
                    help="")
    args = vars(ap.parse_args())
    input_video = args['filename']

    # 全局常量
    FACIAL_EXPRESSION_TARGET_WIDTH = 48
    FACIAL_EXPRESSION_TARGET_HEIGHT = 48

    VIDEO_WIDTH = 640
    VIDEO_HEIGHT = 480

    ANGLE = 20

    # 得到 ID->姓名的map 、 ID->职位类型的map、
    # 摄像头ID->摄像头名字的map、表情ID->表情名字的map
    id_card_to_name, id_card_to_type = fileassistant.get_people_info(
        people_info_path)
    facial_expression_id_to_name = fileassistant.get_facial_expression_info(
        facial_expression_info_path)

    # 控制陌生人检测
    strangers_timing = 0  # 计时开始
    strangers_start_time = 0  # 开始时间
    strangers_limit_time = 4  # if >= 2 seconds, then he/she is a stranger.

    # 控制情感分析
    facial_expression_timing = 0  # 计时开始
    facial_expression_start_time = 0  # 开始时间
    facial_expression_limit_time = 2  # if >= 2 seconds, he/she is smiling

    # 初始化摄像头
    if not input_video:
        vs = cv2.VideoCapture(get_path('rtmp_room_input', 2))
        time.sleep(2)
    else:
        vs = cv2.VideoCapture(input_video)

    # 初始化人脸识别模型
    faceutil = FaceUtil(facial_recognition_model_path)
    facial_expression_model = load_model(facial_expression_model_path)

    print('[INFO] 开始检测陌生人和表情...')
    # 不断循环
    counter = 0
    old_count = 0
    old_name = ''
    while True:
        counter += 1
        # grab the current frame
        (grabbed, frame) = vs.read()
        # for i in range(5):
        #     vs.grab()

        # if we are viewing a video and we did not grab a frame, then we
        # have reached the end of the video
        if input_video and not grabbed:
            break

        if not input_video:
            frame = cv2.flip(frame, 1)

        frame = imutils.resize(frame, width=VIDEO_WIDTH, height=VIDEO_HEIGHT)  # 压缩，加快识别速度
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)  # grayscale，情感分析

        face_location_list, names = faceutil.get_face_location_and_name(
            frame)

        # 得到画面的四分之一位置和四分之三位置，并垂直划线
        one_fourth_image_center = (int(VIDEO_WIDTH / 4),
                                   int(VIDEO_HEIGHT / 4))
        three_fourth_image_center = (int(VIDEO_WIDTH / 4 * 3),
                                     int(VIDEO_HEIGHT / 4 * 3))

        cv2.line(frame, (one_fourth_image_center[0], 0),
                 (one_fourth_image_center[0], VIDEO_HEIGHT),
                 (0, 255, 255), 1)
        cv2.line(frame, (three_fourth_image_center[0], 0),
                 (three_fourth_image_center[0], VIDEO_HEIGHT),
                 (0, 255, 255), 1)

        # 处理每一张识别到的人脸
        for ((left, top, right, bottom), name) in zip(face_location_list, names):
            # 人脸识别错误矫正
            if len(face_location_list) == 1 and id_card_to_type[name] == 'old_people':
                old_count = 15
                old_name = name
            if old_count != 0:
                name = old_name
                old_count -= 1
            # 将人脸框出来
            rectangle_color = (0, 0, 255)
            print(name)
            if id_card_to_type[name] == 'old_people':
                rectangle_color = (0, 0, 128)
            elif id_card_to_type[name] == 'employee':
                rectangle_color = (255, 0, 0)
            elif id_card_to_type[name] == 'volunteer':
                rectangle_color = (0, 255, 0)
            else:
                pass
            cv2.rectangle(frame, (left, top), (right, bottom),
                          rectangle_color, 2)

            # 陌生人检测逻辑
            if 'Unknown' in names:  # alert
                if strangers_timing == 0:  # just start timing
                    strangers_timing = 1
                    strangers_start_time = time.time()
                else:  # already started timing
                    strangers_end_time = time.time()
                    difference = strangers_end_time - strangers_start_time

                    current_time = time.strftime('%Y-%m-%d %H:%M:%S',
                                                 time.localtime(time.time()))

                    if difference < strangers_limit_time:
                        print('[INFO] %s, 房间, 陌生人仅出现 %.1f 秒. 忽略.'
                              % (current_time, difference))
                    else:  # strangers appear
                        event_desc = '陌生人出现!!!'
                        event_location = '房间'
                        print('[EVENT] %s, 房间, 陌生人出现!!!'
                              % (current_time))

                        # event_desc, event_type, event_location, old_people_id, output_path, frame
                        inserting(event_desc, stranger_type, event_location, None, output_stranger_path,
                                  frame)

                        # 开始陌生人追踪f
                        unknown_face_center = (int((right + left) / 2),
                                               int((top + bottom) / 2))

                        cv2.circle(frame, (unknown_face_center[0],
                                           unknown_face_center[1]), 4, (0, 255, 0), -1)

                        direction = ''
                        # face locates too left, servo need to turn right,
                        # so that face turn right as well
                        if unknown_face_center[0] < one_fourth_image_center[0]:
                            direction = 'right'
                        elif unknown_face_center[0] > three_fourth_image_center[0]:
                            direction = 'left'

                        # adjust to servo
                        if direction:
                            print('%d-摄像头需要 turn %s %d 度' % (counter,
                                                             direction, ANGLE))
            else:  # everything is ok
                strangers_timing = 0

            # 表情检测逻辑
            # 如果不是陌生人，且对象是老人
            if name != 'Unknown' and id_card_to_type[name] == 'old_people':
                # 表情检测逻辑
                roi = gray[top:bottom, left:right]
                roi = cv2.resize(roi, (FACIAL_EXPRESSION_TARGET_WIDTH,
                                       FACIAL_EXPRESSION_TARGET_HEIGHT))
                roi = roi.astype("float") / 255.0
                roi = img_to_array(roi)
                roi = np.expand_dims(roi, axis=0)

                # determine facial expression
                (smile, neural) = facial_expression_model.predict(roi)[0]
                facial_expression_label = 'Neural' if neural > smile else 'Smile'

                if facial_expression_label == 'Smile':  # alert
                    if facial_expression_timing == 0:  # just start timing
                        facial_expression_timing = 1
                        facial_expression_start_time = time.time()
                    else:  # already started timing
                        facial_expression_end_time = time.time()
                        difference = facial_expression_end_time - facial_expression_start_time

                        current_time = time.strftime('%Y-%m-%d %H:%M:%S',
                                                     time.localtime(time.time()))
                        if difference < facial_expression_limit_time:
                            print('[INFO] %s, 房间, %s仅笑了 %.1f 秒. 忽略'
                                  % (current_time,
                                     id_card_to_name[name], difference))
                        else:  # he/she is really smiling
                            event_desc = '%s正在笑' % (id_card_to_name[name])
                            event_location = '房间'
                            print('[EVENT] %s, 房间, %s正在笑.'
                                  % (current_time, id_card_to_name[name]))

                            # event_desc, event_type, event_location, old_people_id, output_path, frame
                            inserting(event_desc, old_smile_type, event_location, int(name),
                                      output_smile_path, frame)

                else:  # everything is ok
                    facial_expression_timing = 0

            else:  # 如果是其他人，则不检测表情
                facial_expression_label = ''

            # 人脸识别和情感分析都结束后，把表情和人名写上
            # (同时处理中文显示问题)
            img_PIL = Image.fromarray(cv2.cvtColor(frame,
                                                   cv2.COLOR_BGR2RGB))

            draw = ImageDraw.Draw(img_PIL)
            final_label = id_card_to_name[name] + ': ' + facial_expression_id_to_name[
                facial_expression_label] if facial_expression_label else id_card_to_name[name]
            draw.text((left, top - 30), final_label,
                      # font=ImageFont.truetype('NotoSansCJK-Black.ttc', 40),
                      font=ImageFont.truetype('simsun.ttc', 40),
                      fill=(255, 0, 0))  # linux

            # 转换回OpenCV格式
            frame = cv2.cvtColor(np.asarray(img_PIL), cv2.COLOR_RGB2BGR)
        # show our detected faces along with smiling/not smiling labels
        # cv2.imshow("Checking Strangers and Old People's Face Expression",
        #            frame)

        info = (frame, '2', '3', '4')
        if not raw_q.full():
            raw_q.put(info)
        cv2.waitKey(1)

        # Press 'ESC' for exiting video
        k = cv2.waitKey(1) & 0xff
        if k == 27:
            break

    # cleanup the camera and close any open windows
    vs.release()
    cv2.destroyAllWindows()


import torch
from typing import Dict, List, Optional, Tuple, Union
from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou
from detectron2.modeling.proposal_generator.proposal_utils import (
    add_ground_truth_to_proposals,
)
from detectron2.utils.events import get_event_storage
from detectron2.modeling.roi_heads.box_head import build_box_head
from detectron2.layers import ShapeSpec
from detectron2.modeling.roi_heads import (
    ROI_HEADS_REGISTRY,
    StandardROIHeads,
)
from detectron2.modeling.roi_heads.fast_rcnn import FastRCNNOutputLayers
from adapteacher.modeling.roi_heads.fast_rcnn import FastRCNNFocaltLossOutputLayers, FastRCNNExtendedOutputLayers

import numpy as np
from detectron2.modeling.poolers import ROIPooler

from detectron2.modeling.sampling import subsample_labels

@ROI_HEADS_REGISTRY.register()
class StandardROIHeadsPseudoLab(StandardROIHeads):
    @classmethod
    def _init_box_head(cls, cfg, input_shape):
        # fmt: off
        in_features       = cfg.MODEL.ROI_HEADS.IN_FEATURES
        pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION
        pooler_scales     = tuple(1.0 / input_shape[k].stride for k in in_features)
        sampling_ratio    = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO
        pooler_type       = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE
        # fmt: on

        in_channels = [input_shape[f].channels for f in in_features]
        # Check all channel counts are equal
        assert len(set(in_channels)) == 1, in_channels
        in_channels = in_channels[0]

        box_pooler = ROIPooler(
            output_size=pooler_resolution,
            scales=pooler_scales,
            sampling_ratio=sampling_ratio,
            pooler_type=pooler_type,
        )
        box_head = build_box_head(
            cfg,
            ShapeSpec(
                channels=in_channels, height=pooler_resolution, width=pooler_resolution
            ),
        )

        if cfg.MODEL.ROI_HEADS.LOSS == "CrossEntropy":

            ## @darkhan-s for continual learning (expanding network)
            classesAdded = []
            for element in cfg.MODEL.ROI_HEADS.NEW_CLASSES:
                if element not in cfg.MODEL.ROI_HEADS.OLD_CLASSES:
                    classesAdded.append(element)

            if len(classesAdded) > 0:
                print(f"New classes are requested to train: {classesAdded}. Adding extra classifier layer..")
                box_predictor = FastRCNNExtendedOutputLayers(cfg, box_head.output_shape)
            else:
                # regular prediction output
                box_predictor = FastRCNNOutputLayers(cfg, box_head.output_shape)

        elif cfg.MODEL.ROI_HEADS.LOSS == "FocalLoss":
            box_predictor = FastRCNNFocaltLossOutputLayers(cfg, box_head.output_shape)

            for element in cfg.MODEL.ROI_HEADS.NEW_CLASSES:
                if element not in cfg.MODEL.ROI_HEADS.OLD_CLASSES:
                    raise NotImplementedError("Extra classifier not implemented for this head loss.")
        else:
            raise ValueError("Unknown ROI head loss.")

        return {
            "box_in_features": in_features,
            "box_pooler": box_pooler,
            "box_head": box_head,
            "box_predictor": box_predictor,
        }

    def forward(
        self,
        images: ImageList,
        features: Dict[str, torch.Tensor],
        proposals: List[Instances],
        targets: Optional[List[Instances]] = None,
        compute_loss=True,
        branch="",
        compute_val_loss=False,
    ) -> Tuple[List[Instances], Dict[str, torch.Tensor]]:

        del images
        if self.training and compute_loss:  # apply if training loss
            assert targets
            # 1000 --> 512
            proposals = self.label_and_sample_proposals(
                proposals, targets, branch=branch
            )
        elif compute_val_loss:  # apply if val loss
            assert targets
            # 1000 --> 512
            temp_proposal_append_gt = self.proposal_append_gt
            self.proposal_append_gt = False
            proposals = self.label_and_sample_proposals(
                proposals, targets, branch=branch
            )  # do not apply target on proposals
            self.proposal_append_gt = temp_proposal_append_gt
        del targets

        if (self.training and compute_loss) or compute_val_loss:
            losses, _, box_features = self._forward_box(
                features, proposals, compute_loss, compute_val_loss, branch
            )
            return proposals, losses, box_features
        else:
            pred_instances, predictions, box_features = self._forward_box(
                features, proposals, compute_loss, compute_val_loss, branch
            )

            return pred_instances, predictions, box_features

    def _forward_box(
        self,
        features: Dict[str, torch.Tensor],
        proposals: List[Instances],
        compute_loss: bool = True,
        compute_val_loss: bool = False,
        branch: str = "",
    ) -> Union[Dict[str, torch.Tensor], List[Instances]]:
        features = [features[f] for f in self.box_in_features]
        box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
        box_features = self.box_head(box_features)
        predictions = self.box_predictor(box_features)
        #@ darkhan-s dont delete box features because they are needed for instance level alignment
        #del box_features
        if (self.training and compute_loss) or compute_val_loss:  # apply if training loss or val loss
            losses = self.box_predictor.losses(predictions, proposals)

            if self.train_on_pred_boxes:
                with torch.no_grad():
                    pred_boxes = self.box_predictor.predict_boxes_for_gt_classes(
                        predictions, proposals
                    )
                    for proposals_per_image, pred_boxes_per_image in zip(
                        proposals, pred_boxes
                    ):
                        proposals_per_image.proposal_boxes = Boxes(pred_boxes_per_image)
            return losses, predictions, box_features
        else:

            pred_instances, _ = self.box_predictor.inference(predictions, proposals)
            return pred_instances, predictions, box_features

    @torch.no_grad()
    def label_and_sample_proposals(
        self, proposals: List[Instances], targets: List[Instances], branch: str = ""
    ) -> List[Instances]:
        gt_boxes = [x.gt_boxes for x in targets]
        if self.proposal_append_gt:
            proposals = add_ground_truth_to_proposals(gt_boxes, proposals)

        proposals_with_gt = []

        num_fg_samples = []
        num_bg_samples = []
        for proposals_per_image, targets_per_image in zip(proposals, targets):
            has_gt = len(targets_per_image) > 0
            match_quality_matrix = pairwise_iou(
                targets_per_image.gt_boxes, proposals_per_image.proposal_boxes
            )
            matched_idxs, matched_labels = self.proposal_matcher(match_quality_matrix)
            sampled_idxs, gt_classes = self._sample_proposals(
                matched_idxs, matched_labels, targets_per_image.gt_classes
            )

            proposals_per_image = proposals_per_image[sampled_idxs]
            proposals_per_image.gt_classes = gt_classes

            if has_gt:
                sampled_targets = matched_idxs[sampled_idxs]
                for (trg_name, trg_value) in targets_per_image.get_fields().items():
                    if trg_name.startswith("gt_") and not proposals_per_image.has(
                        trg_name
                    ):
                        proposals_per_image.set(trg_name, trg_value[sampled_targets])
            else:
                gt_boxes = Boxes(
                    targets_per_image.gt_boxes.tensor.new_zeros((len(sampled_idxs), 4))
                )
                proposals_per_image.gt_boxes = gt_boxes

            # check if  extra classes are added
            try:
                num_bg_samples.append((gt_classes == (self.num_classes + self.box_predictor.extra_num_classes)).sum().item())
            except AttributeError:
                num_bg_samples.append((gt_classes == self.num_classes).sum().item())

            num_fg_samples.append(gt_classes.numel() - num_bg_samples[-1])
            proposals_with_gt.append(proposals_per_image)

        storage = get_event_storage()

        if branch != "val_loss":
            storage.put_scalar(
                "roi_head/num_target_fg_samples_" + branch, np.mean(num_fg_samples)
            )
            storage.put_scalar(
                "roi_head/num_target_bg_samples_" + branch, np.mean(num_bg_samples)
            )

        return proposals_with_gt

    def _sample_proposals(
        self, matched_idxs: torch.Tensor, matched_labels: torch.Tensor, gt_classes: torch.Tensor
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        """
        Based on the matching between N proposals and M groundtruth,
        sample the proposals and set their classification labels.

        Args:
            matched_idxs (Tensor): a vector of length N, each is the best-matched
                gt index in [0, M) for each proposal.
            matched_labels (Tensor): a vector of length N, the matcher's label
                (one of cfg.MODEL.ROI_HEADS.IOU_LABELS) for each proposal.
            gt_classes (Tensor): a vector of length M.

        Returns:
            Tensor: a vector of indices of sampled proposals. Each is in [0, N).
            Tensor: a vector of the same length, the classification label for
                each sampled proposal. Each sample is labeled as either a category in
                [0, num_classes) or the background (num_classes).
        """
        has_gt = gt_classes.numel() > 0
        # Get the corresponding GT for each proposal
        if has_gt:
            gt_classes = gt_classes[matched_idxs]
            # Label unmatched proposals (0 label from matcher) as background (label=num_classes)
            try:
                gt_classes[matched_labels == 0] = (self.num_classes + self.box_predictor.extra_num_classes)
            except AttributeError:
                gt_classes[matched_labels == 0] = self.num_classes

            # Label ignore proposals (-1 label)
            gt_classes[matched_labels == -1] = -1
        else:
            try:
                gt_classes = torch.zeros_like(matched_idxs) + (self.num_classes + self.box_predictor.extra_num_classes)
            except AttributeError:
                gt_classes = torch.zeros_like(matched_idxs) + self.num_classes

        try:
            sampled_fg_idxs, sampled_bg_idxs = subsample_labels(
                gt_classes, self.batch_size_per_image, self.positive_fraction, (self.num_classes + self.box_predictor.extra_num_classes)
            )
        except AttributeError:
            sampled_fg_idxs, sampled_bg_idxs = subsample_labels(
                gt_classes, self.batch_size_per_image, self.positive_fraction, self.num_classes
            )

        sampled_idxs = torch.cat([sampled_fg_idxs, sampled_bg_idxs], dim=0)
        return sampled_idxs, gt_classes[sampled_idxs]


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('variety_trials_data', '0001_initial'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='location_year_plantingmethods_survey_answer',
            name='location',
        ),
        migrations.RemoveField(
            model_name='trial_entry_history',
            name='trial_entry',
        ),
        migrations.DeleteModel(
            name='Location_Year_PlantingMethods_Survey_Answer',
        ),
        migrations.DeleteModel(
            name='Trial_Entry_History',
        ),
    ]


'''
Scrapes hockey reference for game data and team-specific game data
    -Use scrape_game_data() for game data
    -Use scrape_team_data() for team-specific game data
'''

import pandas as pd
import requests
from bs4 import BeautifulSoup
import time
from tqdm import tqdm_notebook

team_name = ['San Jose Sharks',
             'Toronto Maple Leafs',
             'Vancouver Canucks',
             'Washington Capitals',
             'Buffalo Sabres',
             'Carolina Hurricanes',
             'Colorado Avalanche',
             'Dallas Stars',
             'Detroit Red Wings',
             'New York Rangers',
             'Ottawa Senators',
             'Pittsburgh Penguins',
             'St. Louis Blues',
             'Vegas Golden Knights',
             'Columbus Blue Jackets',
             'Los Angeles Kings',
             'Arizona Coyotes',
             'Calgary Flames',
             'Minnesota Wild',
             'New Jersey Devils',
             'New York Islanders',
             'Tampa Bay Lightning',
             'Chicago Blackhawks',
             'Anaheim Ducks',
             'Boston Bruins',
             'Nashville Predators',
             'Philadelphia Flyers',
             'Winnipeg Jets',
             'Florida Panthers',
             'Montreal Canadiens',
             'Edmonton Oilers']

team_code = ['SJS',
            'TOR',
            'VAN',
            'WSH',
            'BUF',
            'CAR',
            'COL',
            'DAL',
            'DET',
            'NYR',
            'OTT',
            'PIT',
            'STL',
            'VEG',
            'CBJ',
            'LAK',
            'ARI',
            'CGY',
            'MIN',
            'NJD',
            'NYI',
            'TBL',
            'CHI',
            'ANA',
            'BOS',
            'NSH',
            'PHI',
            'WPG',
            'FLA',
            'MTL',
            'EDM']

name_to_code = dict(zip(team_name, team_code))

def scrape_team_data(years):
    '''
    Scrape all team stats for specified years from https://www.hockey-reference.com/

    Inputs:

    years -> An iterable of years to scrape

    Outputs:

    Saves a pickle for each year containing:

        -list of tuples
        -tuple[0] is team 3-letter code
        -tuple[1] is a pandas dataframe with team stats
    '''
    for year in years:
        teams = ['ANA',
                'ARI',
                'BOS',
                'BUF',
                'CAR',
                'CGY',
                'CHI',
                'CBJ',
                'COL',
                'DAL',
                'DET',
                'EDM',
                'FLA',
                'LAK',
                'MIN',
                'MTL',
                'NSH',
                'NJD',
                'NYI',
                'NYR',
                'OTT',
                'PHI',
                'ARI',
                'PIT',
                'SJS',
                'STL',
                'TBL',
                'VAN',
                'TOR',
                'VEG',
                'WPG',
                'WSH']
        # Vegas Golden Knight Entered NHL in 2017-2018 Season
        if year < 2017:
            teams.remove('VEG')

        team_data = []
        for team in tqdm_notebook(teams):
            url = f'https://www.hockey-reference.com/teams/{team}/{year}_gamelog.html'
            headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'}
            resp2 = requests.get(url=url, headers=headers)
            soup = BeautifulSoup(resp2.text, features='lxml')
            table = soup.find('table')
            table_body = table.find('tbody')
            rows = table_body.find_all('tr')
            data= []
            for row in rows:
                cols = row.find_all('td')
                games_played = row.find('th')
                cols = [ele.text.strip() for ele in cols]
                cols.append(games_played.text)
                data.append(cols)
            team_game_data_df = pd.DataFrame(data)
            team_game_data_df.columns = ['date',
                                    '?',
                                    'opponent',
                                    'goals_for',
                                    'goals_against',
                                    'win/loss',
                                    'ot/so',
                                    '?',
                                    'team_sog',
                                    'team_pim',
                                    'team_pp_goal',
                                    'team_pp_oppurtunities',
                                    'team_shorthanded_goals',
                                    '?',
                                    'opponent_sog',
                                    'opponent_pim',
                                    'opponent_pp_goal',
                                    'opponent_pp_oppurtunities',
                                    'opponent_shorthanded_goals',
                                    '?',
                                    'corsi_for',
                                    'corsi_against',
                                    'corsi_for%',
                                    'fenwick_for',
                                    'fenwick_against',
                                    'fenwick_for%',
                                    'faceoff_wins',
                                    'faceoff_loses',
                                    'faceoff%',
                                    'off_zone_start%',
                                    'pdo',
                                    'games_played'
                                    ]
            team_game_data_df.drop(columns=['?'], inplace=True)
            team_game_data_df.dropna(inplace=True)
            team_game_data_df['date'] = team_game_data_df['date'].map(lambda x: pd.to_datetime(x))
            team_game_data_df['opponent'] = team_game_data_df['opponent'].map(lambda x: name_to_code[x])
            team_data.append((team, team_game_data_df))
            time.sleep(2)
        pd.to_pickle(team_data, f'./data/team_game_data_{year}')

def scrape_game_data(years):
    '''
    Scrape all game stats for specified years from https://www.hockey-reference.com/

    Inputs:

    years -> An iterable of years to scrape

    Outputs:

    Pickles a dataframe for each year
    '''
    for year in years:
        url = f'https://www.hockey-reference.com/leagues/NHL_{year}_games.html'
        headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'}
        resp = requests.get(url=url, headers=headers)
        soup = BeautifulSoup(resp.text, features='lxml')
        table = soup.find('table')
        table_body = table.find('tbody')
        rows = table_body.find_all('tr')

        data= []
        for row in rows:
            cols = row.find_all('td')
            date = row.find('th')
            cols = [ele.text.strip() for ele in cols]
            cols.append(date.text)
            data.append(cols)

        game_data_df = pd.DataFrame(data)
        game_data_df.columns = ['visitor', 'visitor_goals', 'home',
                                'home_goals', 'ot/so', 'attendance',
                                'len_game', 'notes', 'date']
        game_data_df.drop(columns=['notes'], inplace=True)
        game_data_df['visitor'] = game_data_df['visitor'].map(lambda x: name_to_code[x])
        game_data_df['home'] = game_data_df['home'].map(lambda x: name_to_code[x])
        game_data_df['date'] = game_data_df['date'].map(lambda x: pd.to_datetime(x))

        pd.to_pickle(game_data_df, f'./data/all_game_data_{year}')


# -*- coding: utf-8 -*-
from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes
from plone import api
from plone.app.dexterity.behaviors import constrains
from plone.app.textfield.value import RichTextValue
from zope.dottedname.resolve import resolve
from Products.Five.utilities.marker import mark
from Products.CMFPlone.interfaces import INonInstallable
from zope.interface import implementer
from plone.namedfile.file import NamedImage

from logging import getLogger
logger = getLogger(__name__)

@implementer(INonInstallable)
class HiddenProfiles(object):

    def getNonInstallableProfiles(self):
        """Hide uninstall profile from site-creation and quickinstaller"""
        return [
            'qyxycjh.policy:uninstall',
        ]

# for image field data

def _load_image(slider):
    from plone.namedfile.file import NamedImage
    import os
    filename = os.path.join(
        os.path.dirname(__file__),
        'browser',
        'static',
        'slide_{0}.jpg'.format(slider),
    )
    return NamedImage(
        data=open(filename, 'r').read(),
        filename=u'slide_{0}.jpg'.format(slider)
    )
image1 = _load_image(1)
image2 = _load_image(2)
image3 = _load_image(3)

default = { 'i': 'portal_type',
           'o': 'plone.app.querystring.operation.string.is',
            'v': 'Document'}
query = []
defaultpath = {
                    'i': 'path',
                    'o': 'plone.app.querystring.operation.string.path',
                    'v': '/',
                }
import copy

defaultpath.update({'v':'/guanyuqixie'})
guanyuqixie = [default,defaultpath]

xiehuidongtaipath = copy.copy(defaultpath)
xiehuidongtaipath.update({'v':'/xiehuidongtai'})
xiehuidongtai = [default,xiehuidongtaipath]

hangyezixunpath = copy.copy(defaultpath)
hangyezixunpath.update({'v':'/hangyezixun'})
hangyezixun = [default,hangyezixunpath]

zhengcefaguipath = copy.copy(defaultpath)
zhengcefaguipath.update({'v':'/zhengcefagui'})
zhengcefagui = [default,zhengcefaguipath]

meitibaodaopath = copy.copy(defaultpath)
meitibaodaopath.update({'v':'/meitibaodao'})
meitibaodao = [default,meitibaodaopath]

huiyuandanweipath = copy.copy(defaultpath)
huiyuandanweipath.update({'v':'/huiyuandanwei'})
huiyuandanwei = [default,huiyuandanweipath]

STRUCTURE = [
    {
        'type': 'Folder',
        'title': u'关于企协',
        'id': 'guanyuqixie',
        'description': u'关于企协',
        'layout': 'tableview'
    },
    {
        'type': 'Folder',
        'title': u'协会动态',
        'id': 'xiehuidongtai',
        'description': u'协会动态',
        'layout': 'tableview',
        'children': [
                     {
            'type': 'my315ok.products.productfolder',
            'title': u'图片新闻',
            'id': 'tupianxinwen',
            'description': u'图片新闻',
            'children': [
                     {
            'type': 'my315ok.products.product',
            'title': u'图片新闻',
            'id': 'prdt1',
            'image':image1,
            'description': u'图片新闻'
                        } ,
                         {
            'type': 'my315ok.products.product',
            'title': u'图片新闻2',
            'id': 'prdt2',
            'image':image2,
            'description': u'图片新闻2'
                        } ,
                         {
            'type': 'my315ok.products.product',
            'title': u'图片新闻3',
            'id': 'prdt3',
            'image':image3,
            'description': u'图片新闻3'
                        }
                         ]
                      }
                     ]
    },
    {
        'type': 'Folder',
        'title': u'行业资讯',
        'id': 'hangyezixun',
        'description': u'行业资讯',
        'layout': 'tableview'
    },
    {
        'type': 'Folder',
        'title': u'政策法规',
        'id': 'zhengcefagui',
        'description': u'政策法规',
        'layout': 'tableview'

    },
    {
        'type': 'Folder',
        'title': u'会员单位',
        'id': 'huiyuandanwei',
        'description': u'会员单位',
        'layout': 'tableview'

    },
    {
        'type': 'Folder',
        'title': u'媒体报道',
        'id': 'meitibaodao',
        'description': u'媒体报道',
        'layout': 'tableview'
    },
    {
        'type': 'Folder',
        'title': u'查询集',
        'id': 'sqls',
        'description': u'查询集',
        'children': [
                     {
                     'type':'Collection',
                     'title':u'协会动态',
                     'description': u'协会动态查询集',
                     'id':'xiehuidongtai',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':xiehuidongtai,
                     },
                     {
                     'type':'Collection',
                     'title':u'行业资讯',
                     'description': u'行业资讯查询集',
                     'id':'hangyezixun',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':hangyezixun,
                     },
                     {
                     'type':'Collection',
                     'title':u'会员单位',
                     'description': u'会员单位查询集',
                     'id':'huiyuandanwei',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':huiyuandanwei,
                     },
                    {
                     'type':'Collection',
                     'title':u'政策法规',
                     'description': u'最新政策法规',
                     'id':'zhengcefagui',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':zhengcefagui,
                     },
                    {
                     'type':'Collection',
                     'title':u'媒体报道',
                     'description': u'媒体报道',
                     'id':'meitibaodao',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':meitibaodao,
                     }

                     ]
    },
    {
        'type': 'Folder',
        'title': u'帮助',
        'id': 'help',
        'description': u'帮助',
        'layout': 'tableview',
    }

]

def isNotCurrentProfile(context):
    return context.readDataFile('policy_marker.txt') is None

def post_install(context):
    """Setuphandler for the profile 'default'
    """
#     if isNotCurrentProfile(context):
#         return
    # Do something during the installation of this package
#     return
    portal = api.portal.get()
    members = portal.get('events', None)
    if members is not None:
        api.content.delete(members)
    members = portal.get('news', None)
    if members is not None:
        api.content.delete(members)
    members = portal.get('Members', None)
    if members is not None:
       members.exclude_from_nav = True
       members.reindexObject()

    for item in STRUCTURE:
        _create_content(item, portal)

    import_article(portal)
    members = portal.get('sqls', None)
    if members is not None:
       members.exclude_from_nav = True
       members.reindexObject()
    members = portal.get('help', None)
    if members is not None:
       members.exclude_from_nav = True
       members.reindexObject()

def uninstall(context):
    """Uninstall script"""
    # Do something at the end of the uninstallation of this package.

def _create_content(item, container):
    new = container.get(item['id'], None)
    if not new:

        new = api.content.create(
            type=item['type'],
            container=container,
            title=item['title'],
            description=item['description'],
            id=item['id'],
            safe_id=False)
        logger.info('Created item {}'.format(new.absolute_url()))

    if item.get('layout', False):
        new.setLayout(item['layout'])
    if item.get('query', False):
        new.query = item['query']
    if item.get('sort_on', False):
        new.sort_on = item['sort_on']
    if item.get('sort_reversed', False):
        new.sort_reversed = item['sort_reversed']
    if item.get('image', False):
        new.image = item['image']
    if item.get('markif', False):
        try:
            ifobj = resolve(item['markif'])
            mark(new,ifobj)
        except:
            pass
    if item.get('default-page', False):
        new.setDefaultPage(item['default-page'])
    if item.get('allowed_types', False):
        _constrain(new, item['allowed_types'])
    if item.get('local_roles', False):
        for local_role in item['local_roles']:
            api.group.grant_roles(
                groupname=local_role['group'],
                roles=local_role['roles'],
                obj=new)
    if item.get('publish', False):
        api.content.transition(new, to_state=item.get('state', 'published'))
    new.reindexObject()
    # call recursively for children
    for subitem in item.get('children', []):
        _create_content(subitem, new)

def _constrain(context, allowed_types):
    behavior = ISelectableConstrainTypes(context)
    behavior.setConstrainTypesMode(constrains.ENABLED)
    behavior.setLocallyAllowedTypes(allowed_types)
    behavior.setImmediatelyAddableTypes(allowed_types)

article = {"id":'example','title':u'测试文档','content':u'
这是一个测试文档
'}

def _create_article(article, container):
    id = str(article['id'])

    new = container.get(id, None)
    if not new:
        new = api.content.create(
            type='Document',
            container=container,
            title=article['title'],
            text = RichTextValue(article['content']),
            id=id,
            safe_id=False)
        new.reindexObject()

def import_article(portal):
    "migrate articles to document"

    containers = list(item['id'] for item in STRUCTURE if item['id'] not in ["sqls","help"])

    for con in containers:
        container =  portal[con]
        try:
            _create_article(article,container)
        except:
            continue


# -*- coding: utf-8 -*-
"""Module vcf.py from yapp

This module exports functions to parse VCF files into various input
for other modules in yapp or other packages. VCF files are read using cyvcf2.
"""
import sys
import logging
from collections import defaultdict
import numpy as np
import zarr
from cyvcf2 import VCF

logger = logging.getLogger(__name__)

modes_avail = ["genotype", "inbred", "phased", "likelihood"]
default_mode = modes_avail[0]

def geno2int(a1, a2):
    if a1 == -1 or a2 == -1:
        return -1
    else:
        return a1 + a2

def geno2hap(a1, a2):
    if a1 == -1 or a2 == -1 or (a1 != a2):
        return -1
    else:
        return a1

def phred2ln(x):
    """Transforms a PHRED score into a natural logarithm"""
    return -0.1 * np.log(10) * x

def chunk_region(reg, chunksize=-1):
    """Decompose reg into smaller regions
    Parameters
    ----------
    reg : str
       region in bed format ("chr:begin-end").
       begin and end may contain commas.
    chunksize : float
       size of chunks in basepairs

    Returns
    -------
    list of str
        list of sub-regions in bed format.
        If chunksize is <0, returns a list with the original region
    """
    if chunksize < 0:
        return [reg]
    chrom, coords = reg.split(":")
    beg, end = coords.split("-")
    beg = int(beg.replace(",", ""))
    end = int(end.replace(",", ""))
    starts = np.arange(beg, end, step=chunksize, dtype=int)
    ends = [int(x + chunksize - 1) for x in starts]
    ends[-1] = end
    reglist = []
    for i, s in enumerate(starts):
        newreg = chrom + ":" + str(s) + "-" + str(ends[i])
        reglist.append(newreg)
    return reglist

def vcf_chunk_regions(fname, chunksize=-1):
    """Creates regions of chunksize in a VCF file from contig header information
    Parameters
    ----------

    fname : str
        The name of the vcf(.gz) file
    chunksize : int
        Size of chunks

    Returns
    -------
    list of str
        List of regions in bed format. If chunksize is <0, regions are contigs.
    """
    v = VCF(fname, lazy=True)
    regions = [name + ":" + "0-" + str(l) for name, l in zip(v.seqnames, v.seqlens)]
    if chunksize > 0:
        new_regions = []
        for r in regions:
            new_regions += chunk_region(r, chunksize)
        regions = new_regions
    return regions

def vcf2fph(
    fname,
    mode=default_mode,
    samples=None,
    reg=None,
    maf=0.01,
    varids=None,
    genotypes=True,
):
    """Parses a VCF file and returns data arrays usable with
    the fastphase package

    Parameters
    ----------
    fname : str
        The name of the vcf(.gz) file
    mode : str
        One of 'genotype', 'inbred', 'phased' or 'likelihood'.

        In 'genotype' mode, the returning 1D arrays are genotypes
        (0,1,2 and -1 for missing data)

        In 'inbred' mode, the resulting 1D arrays are haplotypes (0,1 and -1).
        Heterozygote calls are treated as missing data.

        In 'phased' mode, two haplotype 1D arrays are returned per
        sample, using phased information in the VCF (eg. 0|1). Any non
        phased genotype is considered missing.

        In 'likelihood' mode, the resulting 2D arrays contain the
        genotype log-likelihoods extracted from the 'PL' field.

    samples : list
        List of samples to extract from full set in file
    reg : str
        query variants in region reg only in bed format ("chr:begin-end").
        VCF file must be indexed.
    maf : double
        Minimum allele frequency to include variants
    varids : list
        List of variants IDs to extract. Only variants found in vcf are
        returned, irrespective of MAF.
    genotypes : bool
        Return Genotype data. For large files this can be quite big.
    Returns
    -------
    dict
        {
         regions : list of regions (str)
         samples : list of samples (str)
         variants : dict {region : list of variants (ID, CHROM, POS, REF, ALT)}
         data : dict {region : dict {
                                  sample ID: data (numpy array)
                                  data type depends on mode. see Parameters.)}
        }

    """

    v = VCF(fname, gts012=True, strict_gt=True, samples=samples, lazy=True)
    smp = v.samples  # might differ if some requested are not found

    if reg is None:
        regions = v.seqnames
    else:
        regions = [reg]

    if varids is None:
        keepvar = defaultdict(lambda: True)
    else:
        maf = 0
        keepvar = defaultdict(lambda: False)
        for s in varids:
            keepvar[s] = True
    variants = {}
    trueregions = []
    variants_summary = {}
    fphdata = {}
    for r in regions:
        snps = []
        for s in v(r):
            if len(s.ALT) > 1 or s.aaf < maf or not keepvar[s.ID]:
                continue
            snps.append(s)
        if len(snps) == 0:
            continue
        variants[r] = sorted(snps, key=lambda x: x.POS)
        variants_summary[r] = [
            (s.ID, s.CHROM, s.POS, s.REF, s.ALT[0]) for s in variants[r]
        ]
        fphdata[r] = {}
        trueregions.append(r)
        if genotypes:
            for i, sid in enumerate(smp):
                if mode == "likelihood":
                    fphdata[r][sid] = phred2ln(
                        np.array(
                            [
                                [
                                    s.gt_phred_ll_homref[i],
                                    s.gt_phred_ll_het[i],
                                    s.gt_phred_ll_homalt[i],
                                ]
                                for s in variants[r]
                            ],
                            dtype=np.float,
                        )
                    )
                else:
                    geno = [s.genotypes[i] for s in variants[r]]
                    if mode == "genotype":
                        fphdata[r][sid] = np.array(
                            [geno2int(*g[:2]) for g in geno], dtype=np.int
                        )
                    elif mode == "inbred":
                        fphdata[r][sid] = np.array(
                            [geno2hap(*g[:2]) for g in geno], dtype=np.int
                        )
                    elif mode == "phased":
                        h1 = []
                        h2 = []
                        for g in geno:
                            if g[2]:
                                h1.append(g[0])
                                h2.append(g[1])
                            else:
                                h1.append(-1)
                                h2.append(-1)
                        fphdata[r][sid + ".h1"] = np.array(h1, dtype=np.int)
                        fphdata[r][sid + ".h2"] = np.array(h2, dtype=np.int)
    return {
        "regions": trueregions,
        "samples": smp,
        "variants": variants_summary,
        "data": fphdata,
    }

def vcf2zarr(
    fname,
    output_prefix=None,
    mode=default_mode,
    samples=None,
    reg=None,
    maf=0.01,
    varids=None,
):
    """Parses a VCF file and creates a ZARR file with its contents

    Parameters
    ----------
    fname : str
        The name of the vcf(.gz) file
    output_prefix: str
        The prefix for zarr file name (prefix.zarr)
    mode : str
        One of 'genotype', 'inbred', 'phased' or 'likelihood'.

        In 'genotype' mode, the returning 1D arrays are genotypes
        (0,1,2 and -1 for missing data)

        In 'inbred' mode, the resulting 1D arrays are haplotypes (0,1 and -1).
        Heterozygote calls are treated as missing data.

        In 'phased' mode, two haplotype 1D arrays are returned per
        sample, using phased information in the VCF (eg. 0|1). Any non
        phased genotype is considered missing.

        In 'likelihood' mode, the resulting 2D arrays contain the
        genotype log-likelihoods extracted from the 'PL' field.

    samples : list
        List of samples to extract from full set in file
    reg : str
        query variants in region reg only in bed format
        ("chr:begin-end"). VCF file must be indexed.
    maf : double
        Minimum allele frequency to include variants
    varids : list
        List of variants IDs to extract. Only variants found in vcf
        are returned, irrespective of MAF.
    genotypes : bool
        Return Genotype data. For large files this can be quite big.
    Returns
    -------
    str
       handler to the zarr root group

    """

    v = VCF(fname, gts012=True, strict_gt=True, samples=samples, lazy=True)
    if not output_prefix:
        output_prefix = fname.split(".")[0]
    fzname = output_prefix + ".zarr"
    fz = zarr.open(fzname, "w")
    fz.attrs["archive"] = fzname
    smp = v.samples  # might differ if some requested are not found
    if reg is None:
        _regions = v.seqnames
    else:
        _regions = [reg]

    regions = []
    for r in _regions:
        ns = len([s for s in v(r)])
        if ns > 0:
            regions.append(r)
    if len(regions) < 1:
        raise IOError(f"No SNP found in {fname}")

    logger.info("Creating Archive structure")
    fz["samples"] = zarr.array(smp, dtype="U50")
    fz["regions"] = zarr.array(regions, dtype="U50")
    for i, reg in enumerate(regions):
        fz.create_group(f"variants/{reg}")
    fz.create_group("genotypes")

    if varids is None:
        keepvar = defaultdict(lambda: True)
    else:
        maf = 0
        keepvar = defaultdict(lambda: False)
        for s in varids:
            keepvar[s] = True

    for r in regions:
        logger.info(f"Getting data for region {r}")
        snps = []
        for s in v(r):
            if len(s.ALT) > 1 or s.aaf < maf or not keepvar[s.ID]:
                continue
            snps.append(s)
        if len(snps) == 0:
            continue
        variants = sorted(snps, key=lambda x: x.POS)
        fz[f"variants/{r}/ID"] = np.array([s.ID for s in variants], dtype="U50")
        fz[f"variants/{r}/CHROM"] = np.array([s.CHROM for s in variants], dtype="U50")
        fz[f"variants/{r}/POS"] = np.array([s.POS for s in variants], dtype=np.uint32)
        fz[f"variants/{r}/REF"] = np.array([s.REF for s in variants], dtype="S20")
        fz[f"variants/{r}/ALT"] = np.array([s.ALT[0] for s in variants], dtype="S20")

        data = []
        for i, sid in enumerate(smp):
            sys.stdout.write(f"{sid:<20} {100*(i+1)/len(smp):.2f} %\r")
            sys.stdout.flush()
            if mode == "likelihood":
                gind = phred2ln(
                    np.array(
                        [
                            [
                                s.gt_phred_ll_homref[i],
                                s.gt_phred_ll_het[i],
                                s.gt_phred_ll_homalt[i],
                            ]
                            for s in variants[r]
                        ],
                        dtype=np.float,
                    )
                )
            else:
                geno = [s.genotypes[i] for s in variants]
                if mode == "genotype":
                    gind = np.array([geno2int(*g[:2]) for g in geno], dtype=np.int8)
                elif mode == "inbred":
                    gind = np.array([geno2hap(*g[:2]) for g in geno], dtype=np.int8)
                elif mode == "phased":
                    gind = np.array(
                        [
                            [g[0] if g[2] else -1 for g in geno],
                            [g[1] if g[2] else -1 for g in geno],
                        ],
                        dtype=np.int8,
                    )
            data.append(gind)
        fz[f"genotypes/{r}"] = zarr.array(data, chunks=False)
    return fz


#!/usr/bin/python3
"""
Fabric script based on the file 2-do_deploy_web_static.py that creates and
distributes an archive to the web servers
"""

import os.path
from fabric.api import *
from fabric.operations import run, put

env.user = 'ubuntu'
env.hosts = ['xx.xx.xx.xx', 'xx.xx.xx.xx']

def do_pack():
    """Create a .tgz archive from the contents of the web_static folder"""
    try:
        if not os.path.exists("versions"):
            local("mkdir versions")
        created_at = datetime.now().strftime("%Y%m%d%H%M%S")
        file_path = "versions/web_static_{}.tgz".format(created_at)
        local("tar -cvzf {} web_static".format(file_path))
        return file_path
    except:
        return None

def do_deploy(archive_path):
    """Distributes an archive to your web servers"""
    if not os.path.exists(archive_path):
        return False
    try:
        file_name = archive_path.split("/")[-1]
        no_ext = file_name.split(".")[0]
        path_release = "/data/web_static/releases/{}/".format(no_ext)
        put(archive_path, "/tmp/")
        run("mkdir -p {}".format(path_release))
        run("tar -xzf /tmp/{} -C {}".
            format(file_name, path_release))
        run("rm /tmp/{}".format(file_name))
        run("mv {}web_static/* {}".format(path_release, path_release))
        run("rm -rf {}web_static".format(path_release))
        run("rm -rf /data/web_static/current")
        run("ln -s {} /data/web_static/current".
            format(path_release))
        return True
    except:
        return False

def deploy():
    """Call the do_pack() function and store the path of the created archive.
    Call the do_deploy(archive_path) function, using the new path of the new archive"""
    archive_path = do_pack()
    if not archive_path:
        return False
    return do_deploy(archive_path)


import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as VM
from Models import basic
from Utils import util, tool

eps = 0.0000001

def l2_loss(y_input, y_target):
    return F.mse_loss(y_input, y_target)

def weightedL2_loss(y_input, y_target, weight_map):
    N,C,H,W = y_input.shape
    _,C1,_,_ = weight_map.shape
    interval = C//C1
    diff_map = y_input-y_target
    diff_map = torch.cat([torch.mean(torch.abs(diff_map[:,interval*i:interval*(i+1),:,:]), dim=1, keepdim=True) for i in range(0,C1)], dim=1)
    return torch.sum(diff_map*diff_map*weight_map) / (eps+torch.sum(weight_map))

def l1_loss(y_input, y_target):
    return F.l1_loss(y_input, y_target)

def weightedL1_loss(y_input, y_target, weight_map):
    N,C,H,W = y_input.shape
    _,C1,_,_ = weight_map.shape
    interval = C//C1
    diff_map = y_input-y_target
    diff_map = torch.cat([torch.mean(torch.abs(diff_map[:,interval*i:interval*(i+1),:,:]), dim=1, keepdim=True) for i in range(0,C1)], dim=1)
    return torch.sum(diff_map*weight_map) / (eps+torch.sum(weight_map))

def ssimLoss(yInput, yTarget):
    # data range is [-1,1]
    ssim = tool.ssim(yInput / 2. + 0.5, yTarget / 2. + 0.5, window_size=11)
    return 1. - ssim

def wrapLightFieldVGGLoss(yInput, yTarget, vggLossFunc):
    # data range is [-1,1]
    N,C,H,W = yInput.shape
    batch_input = torch.cat([yInput[:, i*3:(i+1)*3, :, :] for i in range(C//3)], dim=0)
    batch_target = torch.cat([yTarget[:, i*3:(i+1)*3, :, :] for i in range(C//3)], dim=0)
    vggLoss = vggLossFunc(batch_input/2.+0.5, batch_target/2.+0.5)
    return vggLoss

def depth_warp_loss(ray_depths, input_lightfields, c_views):
    N, frame_num, H, W = ray_depths.shape
    warping_loss, no = 0, 0
    c_depth = ray_depths[:,frame_num//2,:,:].unsqueeze(1)
    #! warping loss: central view
    for t in [-2, 2]:       # s is along the x axis of image
        for s in [-2, 2]:   # t is along the y axis of image
            index = (t+3)*7+(s+3)
            st_view = input_lightfields[:,3*index:3*(index+1),:,:]
            st2c_disp = torch.cat((s*c_depth, t*c_depth), dim=1)
            c_warp, _ = basic.flowWarp(st_view, st2c_disp)
            c_loss = l2_loss(c_warp, c_views)
            warping_loss += c_loss
            no += 1
            #! output test
            #meta_imgs = basic.tensor2array(c_warp)
            #util.save_images_from_batch(meta_imgs, './caca/', no*1)
    #! warping loss: surround subviews
    occlusion_maps = torch.ones(N, 1*frame_num, H, W, device=c_views.device)
    warped_frames = torch.ones(N, 3*frame_num, H, W, device=c_views.device)
    for i in range(frame_num):
        t = (i // 7) - 3       # t is along the y axis of image
        s = (i % 7) - 3        # s is along the x axis of image
        if s == 0 and t == 0:
            c_errormap = torch.zeros_like(c_depth, device=c_views.device)
            occlusion_maps[:,i:i+1,:,:] = c_errormap
            warped_frames[:,3*i:3*(i+1),:,:] = c_views
        else:
            i_depth = ray_depths[:,i:i+1,:,:]
            c2i_disp = torch.cat((-s*i_depth, -t*i_depth), dim=1)
            i_warp, _ = basic.flowWarp(c_views, c2i_disp)
            i_loss = l2_loss(i_warp, input_lightfields[:,3*i:3*(i+1),:,:])
            warping_loss += i_loss
            no += 1
            i_errormap = basic.checkLRDepthErrorMap(i_depth, c_depth, -s, -t)
            occlusion_maps[:,i:i+1,:,:] = i_errormap
            warped_frames[:,3*i:3*(i+1),:,:] = i_warp
            #! output test
            #meta_imgs = basic.tensor2array(i_warp)
            #util.save_images_from_batch(meta_imgs, './caca/', no*1)
    return warping_loss / no, occlusion_maps, warped_frames

def depth_consistency_loss(ray_depths):
    _, frame_num, _, _ = ray_depths.shape
    consistency_loss, no = 0, 0
    for t in range(-3,3):      # t is along the y axis of image：-3~2
        for s in range(-3,3):  # s is along the x axis of image：-3~2
            index = (t+3)*7+(s+3)
            i_depth = ray_depths[:,index,:,:].unsqueeze(1)
            # d(s,t) of half its neighborhood views
            d10_depth = ray_depths[:,index+1,:,:].unsqueeze(1)
            d01_depth = ray_depths[:,index+7,:,:].unsqueeze(1)
            d11_depth = ray_depths[:,index+8,:,:].unsqueeze(1)
            d10_loss = basic.checkLRDepthErrorMap(i_depth, d10_depth, 1, 0)
            d01_loss = basic.checkLRDepthErrorMap(i_depth, d01_depth, 0, 1)
            d11_loss = basic.checkLRDepthErrorMap(i_depth, d11_depth, 1, 1)
            consistency_loss += (d01_loss.mean() + d10_loss.mean() + d11_loss.mean()) / 3.0
            no += 1
    return consistency_loss / no

def computeDispWarpLoss(disparity_maps, input_lightfields, c_view):
    N, dframe_num, H, W = disparity_maps.shape
    frame_num = dframe_num//2   # 49
    cview_no = frame_num//2     # 24
    c_disp = disparity_maps[:,2*cview_no:2*(cview_no+1),:,:]
    warping_loss, no = 0, 0
    #! warping loss: central view
    for t in [-2, 2]:       # s is along the x axis of image
        for s in [-2, 2]:   # t is along the y axis of image
            index = (t+3)*7+(s+3)
            st_view = input_lightfields[:,3*index:3*(index+1),:,:]
            st2c_dispmap = torch.cat((s*c_disp[:,0:1,:,:], t*c_disp[:,1:2,:,:]), dim=1)
            c_warp, _ = basic.flowWarp(st_view, st2c_dispmap)
            c_loss = l2_loss(c_warp, c_view)
            warping_loss += c_loss
            no += 1
            #! output test
            #meta_imgs = basic.tensor2array(c_warp)
            #util.save_images_from_batch(meta_imgs, './caca/', no*1)
    #! warping loss: surround subviews
    for i in range(frame_num):
        t = (i // 7) - 3       # t is along the y axis of image
        s = (i % 7) - 3        # s is along the x axis of image
        if s == 0 and t == 0:
            # c_errormap = torch.zeros((N,1,H,W), device=c_views.device)
            continue
        else:
            i_disp = disparity_maps[:,2*i:2*(i+1),:,:]
            c2i_dispmap = torch.cat((-s*i_disp[:,0:1,:,:], -t*i_disp[:,1:2,:,:]), dim=1)
            i_warp, _ = basic.flowWarp(c_view, c2i_dispmap)
            i_loss = l2_loss(i_warp, input_lightfields[:,3*i:3*(i+1),:,:])
            warping_loss += i_loss
            no += 1
    return warping_loss / no

def computeDispWarpLossNew(disparity_maps, input_lightfields, c_view, frameNos):
    N, dframe_num, H, W = disparity_maps.shape
    frame_num = dframe_num//2   # 49
    c_disp = disparity_maps[:,0:2,:,:]
    cview_loss = 0
    #! warping loss: central view
    for t in [-2, 2]:       # s is along the x axis of image
        for s in [-2, 2]:   # t is along the y axis of image
            index = (t+3)*7+(s+3)
            st_view = input_lightfields[:,3*index:3*(index+1),:,:]
            st2c_dispmap = torch.cat((s*c_disp[:,0:1,:,:], t*c_disp[:,1:2,:,:]), dim=1)
            c_warp, _ = basic.flowWarp(st_view, st2c_dispmap)
            c_loss = l2_loss(c_warp, c_view)
            cview_loss += c_loss
            #! output test
            #meta_imgs = basic.tensor2array(c_warp)
            #util.save_images_from_batch(meta_imgs, './caca/', no*1)
    #! warping loss: surround subviews
    warping_loss = cview_loss / 4.0
    no = 1
    for i in frameNos:
        t = (i // 7) - 3       # t is along the y axis of image
        s = (i % 7) - 3        # s is along the x axis of image
        if s == 0 and t == 0:
            # c_errormap = torch.zeros((N,1,H,W), device=c_views.device)
            continue
        else:
            i_disp = disparity_maps[:,2*no:2*(no+1),:,:]
            c2i_dispmap = torch.cat((-s*i_disp[:,0:1,:,:], -t*i_disp[:,1:2,:,:]), dim=1)
            i_warp, _ = basic.flowWarp(c_view, c2i_dispmap)
            i_loss = l2_loss(i_warp, input_lightfields[:,3*i:3*(i+1),:,:])
            warping_loss += i_loss
            no += 1
    return warping_loss / no

def computeDispConsistLoss(disparity_maps):
    _, dframe_num, _, _ = disparity_maps.shape
    frame_num = dframe_num//2   # 49
    consistency_loss, no = 0, 0
    for t in range(-3,3):      # t is along the y axis of image：-3~2
        for s in range(-3,3):  # s is along the x axis of image：-3~2
            index = (t+3)*7+(s+3)
            i_disp = disparity_maps[:,2*index:2*(index+1),:,:]
            # d(s,t) of half its neighborhood views
            no10, no01, no11 = 2*(index+1), 2*(index+7), 2*(index+8)
            d10_disp = disparity_maps[:,no10:no10+2,:,:]
            d01_disp = disparity_maps[:,no01:no01+2,:,:]
            d11_disp = disparity_maps[:,no11:no11+2,:,:]
            d10_errormap = basic.checkLRDispErrorMap(i_disp, d10_disp, 1, 0)
            d01_errormap = basic.checkLRDispErrorMap(i_disp, d01_disp, 0, 1)
            d11_errormap = basic.checkLRDispErrorMap(i_disp, d11_disp, 1, 1)
            consistency_loss += (d01_errormap.mean() + d10_errormap.mean() + d11_errormap.mean()) / 3.0
            no += 1
    return consistency_loss / no

def computeSmoothLoss(disparity_a2b, image_b):
    """
    image_batch: [B, 3, H, W], value of [-1,1]
    flow_batch: [B, 2, H, W], value of [-7,7]
    """
    guide_map = 10.0*(torch.mean(image_b, dim=1, keepdim=True)+1.0)
    gy_disp = torch.norm(disparity_a2b[:,:,:-1,:]-disparity_a2b[:,:,1:,:], dim=1)
    gx_disp = torch.norm(disparity_a2b[:,:,:,:-1]-disparity_a2b[:,:,:,1:], dim=1)
    y_weight = torch.exp(-torch.abs(guide_map[:,:,:-1,:]-guide_map[:,:,1:,:]))
    x_weight = torch.exp(-torch.abs(guide_map[:,:,:,:-1]-guide_map[:,:,:,:1]))
    smoothLoss = torch.mean(gy_disp*y_weight) + torch.mean(gx_disp*x_weight)
    return smoothLoss

def computeTotalVariation(flow_batch):
    """
    flow_batch: [B, C, H, W] flow or depth
    """
    y_tv = torch.pow((flow_batch[:,:,1:,:]-flow_batch[:,:,:-1,:]),2).mean()
    x_tv = torch.pow((flow_batch[:,:,:,1:]-flow_batch[:,:,:,:-1]),2).mean()
    totalVar = (x_tv + y_tv) / 2.0
    return totalVar

def computeMaskPriorLoss(mask_batch, guidance_batch):
    """
    image_batch: [B, 1, H, W], value of [-1,1]
    guidance_batch: [B, 1, H, W], value of [0,2]  threshold_value: 0.2
    """
    weight_map = torch.exp(-10.0*guidance_batch)
    return torch.sum(mask_batch*weight_map) / (eps+torch.sum(weight_map))

class Vgg19Loss:
    def __init__(self, multiGpu=True):
        # data in RGB format, [0,1] range
        self.mean = [0.485, 0.456, 0.406]
        self.std = [0.229, 0.224, 0.225]
        vgg19 = VM.vgg19(pretrained=True)
        # maxpoll after conv4_4
        self.featureExactor = nn.Sequential(*list(vgg19.features)[:28])
        for param in self.featureExactor.parameters():
            param.requires_grad = False
        if multiGpu:
            self.featureExactor = torch.nn.DataParallel(self.featureExactor).cuda()
        #! evaluation mode
        self.featureExactor.eval()
        print('[*] Vgg19Loss init!')

    def normalize(self, tensor):
        tensor = tensor.clone()
        mean = torch.as_tensor(self.mean, dtype=torch.float32, device=tensor.device)
        std = torch.as_tensor(self.std, dtype=torch.float32, device=tensor.device)
        tensor.sub_(mean[None, :, None, None]).div_(std[None, :, None, None])
        return tensor

    def __call__(self, yInput, yTarget):
        inFeature = self.featureExactor(self.normalize(yInput))
        targetFeature = self.featureExactor(self.normalize(yTarget))
        return l2_loss(inFeature, targetFeature)


from __future__ import print_function
import cplex
import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
from itertools import product
from cplex.exceptions import CplexError, CplexSolverError
import time
from scipy.sparse import find
import numpy as np
import networkx as nx
from multiplex import Multiplex

def solveMultiplexConductance(mx,local_genes,seed,fdr_bound,time_limit,relative_gap):
    fdr_data = mx.get_fdr()
    nodes = [x + '#' + y for x in local_genes for y in mx.get_layers()]
    local_index = [mx.get_state_node().index(x) for x in nodes]
    layers = mx.get_layers()
    adjacency_connection = mx.get_adjacency()[local_index,:][:,local_index]
    transition_multiplied = mx.get_trans_multiply_stat()[local_index,:][:,local_index]
    stationary_distribution = mx.get_stationary()[local_index]
    [conn_indi,conn_indj,conn_val] = find(adjacency_connection)
    [tran_indi,tran_indj,tran_val] = find(transition_multiplied)
    n_transition = len(tran_val);
    nNodes = len(nodes)
    nEdges = len(conn_val)
    u = 1.001 # upper bound of Z
    l = -0.001 # lower bound of Z
    tmp1 = [(x,y) for (x,y) in zip(tran_indi,tran_indj) if x<=y]
    tmp2 = [(y,x) for (x,y) in zip(tran_indi,tran_indj) if x>y]
    xx_entrys = list(set(tmp1 + tmp2))
    xx_entrys_value = [transition_multiplied[x,y] + transition_multiplied[y,x] for (x,y) in xx_entrys]

    ######## Variables Start
    # Variable Name
    variable_x = [x for x in nodes] # x_i, indicator variable
    variable_z = "obj" # Z
    variable_zx = ["".join(("z", x)) for x in nodes] # zx_i
    variable_xx = ["".join((np.array(nodes)[x],"*",np.array(nodes)[y])) for (x,y) in xx_entrys] # xx_ij #transition nonzero for conductance calculation
    variable_y = ["".join((x, "->", y)) for x, y in zip(np.array(nodes)[conn_indi],np.array(nodes)[conn_indj])] #y_ij #egdes for flow
    variable_source = "source" # source flow
    variable_s2x = ["".join((variable_source, "_", seed + "_" + layers[0]))] # source to seed
    my_colnames = variable_x + [variable_z] + variable_zx + variable_xx +  variable_y + [variable_source] + variable_s2x
    variable_dict = dict(zip(my_colnames,range(len(my_colnames)))) # variable map, speed up problem building of cplex
    # Variable Type: x_i binary, other continous
    my_ctype = "I" * nNodes + "C" * (nEdges + nNodes + len(xx_entrys) + 3)
    # Variable Bound: x {0,1}, obj 0-1, zx: 0-1, xx: 0-1, y: 0-nodes, source: 0-nodes, source2seed: 0-nodes
    my_ub = [1.0] * nNodes + [1.0] * (1 + nNodes + len(xx_entrys)) + [float(nNodes)] * (nEdges + 2)
    my_lb = [0.0] * nNodes + [0.0] * (1 + nNodes + len(xx_entrys) + nEdges + 2)
    ######### Variables End

    ######## Objective
    my_obj = [0.0] * nNodes + [1.0] + [0.0] * (nEdges + nNodes + len(xx_entrys) + 2) # objective: min Z
    ######## Objective End

    ######## Constraints Start
    my_row = []

    ### seed constrain  x_seed = 1
    tmp_con = [[variable_dict[seed + '#' + layers[0]]], [1.0]]
    my_row.append([[variable_dict[seed + '#' + layers[0]]], [1.0]])
    my_rhs_seed = [1.0]
    my_sense_seed = "E"
    my_rownames_seed = ["seed"]

    #### Budget constraint fdr: sum(lfdr-B) <= 0
    tmp_con = [ [variable_dict[x + '#' + layers[0]] for x in local_genes], [ float(fdr_data[x] - fdr_bound) for x in local_genes if x in fdr_data.keys()] ]
    my_row.append([ [variable_dict[x + '#' + layers[0]] for x in local_genes], [ float(fdr_data[x] - fdr_bound) for x in local_genes if x in fdr_data.keys()] ] )
    my_sense_budget_fdr = "L"
    my_rhs_budget_fdr = [0.0]
    my_rownames_budget_fdr = ["Budget_fdr"]

    #### Connectivity constraints
    # residual flow + flow injected into network = total flow  z0 + sum(y_zv) = N
    my_row.append([[variable_dict[variable_source], variable_dict[variable_s2x[0]]], [1.0] * (1 + 1)])
    my_sense_total = "E"
    my_rhs_total = [float(nNodes)]
    my_rownames_total = ["total"]

    # positive flow
    for i in range(nEdges):
        my_row.append([[variable_dict[variable_y[i]], variable_dict[variable_y[i].split("->")[-1]]], [1.0, -float(nNodes)]])
    my_rhs_positive = [0.0] * nEdges
    my_sense_positive = "L" * nEdges
    my_rownames_positive = ["".join((variable_y[i], "flow")) for i in range(nEdges)]

    # consuming flow
    my_rhs_consuming = []
    my_sense_consuming = "E" * nNodes
    my_rownames_consuming = []
    for i in range(nNodes):
        edge_incoming = [variable_dict["".join((nodes[y],"->",nodes[i]))] for y in list(adjacency_connection[i,:].nonzero()[1])]
        edge_ongoing = [variable_dict["".join((nodes[i],"->",nodes[y]))] for y in list(adjacency_connection[:,i].nonzero()[0])]
        if nodes[i] == (seed + '#' + layers[0]):
            edge_incoming.append(variable_dict[variable_s2x[0]])
        edge_ongoing.append(variable_dict[nodes[i]])
        my_row.append([edge_incoming + edge_ongoing, [1.0] * len(edge_incoming) + [-1.0] * (len(edge_ongoing))])
        my_rhs_consuming = my_rhs_consuming + [0.0]
        my_rownames_consuming = my_rownames_consuming + ["".join(("consuming_", nodes[i]))]

    # injected = consumed
    my_rhs_equal = [0.0]
    my_sense_equal = "E"
    my_rownames_equal = ["Equal"]
    tmp = [variable_dict[x] for x in variable_x]
    tmp.append(variable_dict[variable_s2x[0]])
    my_row.append([tmp, [1.0] * nNodes + [-1.0]])

    #### Helper constraints
    # Multiplex conductance: -sum(zx_i * D_ii) + sum(p*x_ii) - sum(P*p*x_ij) <= 0 and x_ii == x_i
    my_rhs_obj = [0.0]
    my_sense_obj = "L"
    my_rownames_obj = ["obj_main"]
    var_zx = [variable_dict[x] for x in variable_zx]
    var_x = [variable_dict[x] for x in variable_x]
    var_xx = [variable_dict[x] for x in variable_xx]
    tmpvar = []
    tmpvar.extend(var_zx + var_x + var_xx)

    value_zx = [-1.0*val for val in stationary_distribution.squeeze()] # stationary
    value_x = [1.0*val for val in stationary_distribution.squeeze()] # stationary
    value_xx = [-1.0*val for val in xx_entrys_value] # transition*stationary
    tmpvalue = []
    tmpvalue.extend(value_zx + value_x + value_xx)

    my_row.append([tmpvar,tmpvalue])

    # Constraints for zx:
    # 1. zx_i >= z - u(1-x_i) ===>   z +   x_i - zx_i <= u
    # 2. zx_i >= l*x_i        ===>       l*x_i - zx_i <= 0
    # 3. zx_i <= z - l(1-x_i) ===>  -z - l*x_i + zx_i <= -l
    # 4. zx_i <= u*x_i        ===>     - u*x_i + zx_i <= 0
    my_rhs_zx = [u] * nNodes + [0.0] * nNodes + [-1.0*l] * nNodes + [0.0] * nNodes
    my_sense_zx = "L" * 4 * nNodes
    my_rownames_zx = ["".join(("obj_helper_zx1_",nodes[i])) for i in range(nNodes)] + ["".join(("obj_helper_zx2_",nodes[i])) for i in range(nNodes)] + ["".join(("obj_helper_zx3_",nodes[i])) for i in range(nNodes)] + ["".join(("obj_helper_zx4_",nodes[i])) for i in range(nNodes)]

    constrain_zx_1 = [[[variable_dict[variable_z],variable_dict[variable_x[i]],variable_dict[variable_zx[i]]],[1.0,u,-1.0]] for i in range(nNodes)]
    constrain_zx_2 = [[[variable_dict[variable_x[i]],variable_dict[variable_zx[i]]],[l,-1.0]] for i in range(nNodes)]
    constrain_zx_3 = [[[variable_dict[variable_z],variable_dict[variable_x[i]],variable_dict[variable_zx[i]]],[-1.0,-l,1.0]] for i in range(nNodes)]
    constrain_zx_4 = [[[variable_dict[variable_x[i]],variable_dict[variable_zx[i]]],[-u,1.0]] for i in range(nNodes)]
    my_row.extend(constrain_zx_1 + constrain_zx_2 + constrain_zx_3 + constrain_zx_4)

    # Constraints for xx
    # 1. xx_ij <= x_i                ===>  xx_ij - x_i        <= 0
    # 2 xx_ij <= x_j               ===>  xx_ij - x_j        <= 0
    # 3 xx_ij >= x_i + x_j - 1     ===> -xx_ij + x_i + x_j  <= 1

    my_rhs_xx = [0.0] * (2 * len(variable_xx)) + [1.0] * len(variable_xx)
    my_sense_xx = "L" * (3 * len(variable_xx))
    my_rownames_xx = ["".join(("obj_helper_xx1_",x.split('*')[0],x.split('*')[1])) for x in variable_xx] + \
                    ["".join(("obj_helper_xx2_",x.split('*')[0],x.split('*')[1])) for x in variable_xx] + \
                    ["".join(("obj_helper_xx3_",x.split('*')[0],x.split('*')[1])) for x in variable_xx]

    constrain_1 = [[[variable_dict[x],variable_dict[x.split('*')[0]]],[1.0,-1.0]] for x in variable_xx]
    constrain_2 = [[[variable_dict[x],variable_dict[x.split('*')[1]]],[1.0,-1.0]] for x in variable_xx]
    constrain_3 = [[[variable_dict[x],variable_dict[x.split('*')[0]],variable_dict[x.split('*')[1]]],[-1.0,1.0,1.0]] for x in variable_xx]
    my_row.extend(constrain_1 + constrain_2 + constrain_3)

    # Adherent constraint
    # gene_test1 = gene_test2 = gene_test3
    my_rhs_adh = [0.0] * len(local_genes) * (len(layers)-1)
    my_sense_adh = "E" * len(local_genes) * (len(layers)-1)
    my_rownames_adh = ["".join(("adh_",x,"")) for x in local_genes for i in range(1,len(layers))]
    constrain_adh = [[[variable_dict[x + '#'+layers[0]],variable_dict[x + '#' + layers[i]]],[1.0,-1.0]] for x in local_genes for i in range(1,len(layers))]
    my_row.extend(constrain_adh)

    #### Merge Constraints
    my_sense = my_sense_seed + my_sense_budget_fdr + my_sense_total + my_sense_positive + my_sense_consuming + my_sense_equal + \
                my_sense_obj + my_sense_zx + my_sense_xx + my_sense_adh
    my_rhs = my_rhs_seed + my_rhs_budget_fdr + my_rhs_total + my_rhs_positive + my_rhs_consuming + my_rhs_equal + \
                my_rhs_obj + my_rhs_zx + my_rhs_xx + my_rhs_adh
    my_rownames = my_rownames_seed + my_rownames_budget_fdr + my_rownames_total + \
                    my_rownames_positive + my_rownames_consuming + my_rownames_equal + my_rownames_obj + my_rownames_zx + my_rownames_xx + my_rownames_adh
    ######## Constraints End
    #print(len(my_row),len(my_sense),len(my_rhs),len(my_rownames))
    ######## Problem Build Start
    try:
        build_start = time.time()
        subnetwork = cplex.Cplex()
        subnetwork.parameters.timelimit.set(time_limit)
        subnetwork.parameters.mip.tolerances.mipgap.set(relative_gap)
        subnetwork.set_log_stream(None)
        subnetwork.set_warning_stream(None)
        subnetwork.set_results_stream(None)
        #subnetwork.parameters.simplex.tolerances.markowitz.set(0.99999)
        #subnetwork.parameters.read.scale.set(1)
        subnetwork.objective.set_sense(subnetwork.objective.sense.minimize)
        subnetwork.variables.add(obj=my_obj, lb=my_lb, ub=my_ub, types=my_ctype, names=my_colnames)
        subnetwork.linear_constraints.add(lin_expr=my_row, senses=my_sense,rhs=my_rhs, names=my_rownames)
        #build_end = time.time()
        #print("build time: ", build_end - build_start)
        #solve_start = time.time()
        subnetwork.solve()
        #solve_end = time.time()
        #print("Solve time: ", solve_end - solve_start)
    except CplexError as exc:
        print(exc)
        subnetwork_result = [];
        status = -1;
        return
    ######## Problem Build End

    ######## Solution Start
    solution = subnetwork.solution
    #solution.get_status() returns  an integer code
    #print("Solution status = ", solution.get_status(), ":", end=' ')
# # # the following line prints the corresponding string
# #   print(solution.status[solution.get_status()])
    status = solution.status[solution.get_status()]
    try:
        #print("Objective value = ", solution.get_objective_value())
        x = solution.get_values(0, subnetwork.variables.get_num() - 1)
        result = dict(zip(my_colnames, x))
    except CplexSolverError as exc:
#     #print(exc)
        subnetwork_result = []
        return subnetwork_result,status
# #print(result)
    subnetwork_result = [x for x in variable_x if result[x] > 0.9]
    return subnetwork_result, status

if __name__ == '__main__':
    database_list = ['test1','test2','test3']
    data_path = 'toy_data/'
    index_file_list = ['test1_index_gene','test2_index_gene','test3_index_gene']
    index_file_list_full = [data_path + x for x in index_file_list]
    edge_file_list = ['test1_edge_list','test2_edge_list','test3_edge_list']
    edge_file_list_full = [data_path + x for x in edge_file_list]
    score_file = 'test_fdr.txt'
    score_file_full = data_path + score_file
    seed = 'TCGA'
    fdr_bound = 0.1


from celery import shared_task
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
from .models import Counter

@shared_task(bind=True)
def add_first(self):
    counter = Counter.objects.first()
    counter.number = counter.number + 20
    counter.save()
    logger.info(str(counter.number))
    # logger.info("first task celery bright")
    return 'done'


import math
import random
from time import sleep
import sys
import subprocess
import os
#avoiding the prints of pygame library
os.environ['PYGAME_HIDE_SUPPORT_PROMPT'] = "hide"

#downloading the required packages(numpy and pygame) if they aren't installed
try:
    import numpy as np
except ImportError:
    subprocess.check_call([sys.executable, "-m", "pip", "install", 'numpy'], stdout=subprocess.DEVNULL)
    print("Installed numpy")
    import numpy as np

try:
    import pygame as pg
except ImportError:
    subprocess.check_call([sys.executable, "-m", "pip", "install", 'pygame'], stdout=subprocess.DEVNULL)
    print("Installed pygame")
    import pygame as pg

#this class represents an organism in the simulation
class Organism:
    # ctor
    def __init__(self, state, speed, X):
        self.state = state
        self.speed = speed
        self.sick_time = X

    # setting the state of the organism
    def set_state(self, new_state):
        self.state = new_state

    # returns true if the organism is healthy, else - false
    def is_healthy(self):
        if self.state == 1:
            return True
        return False

    # returns true if the organism is sick, else - false
    def is_sick(self):
        if self.state == 2:
            return True
        return False

    #checking if the organism still has time to be sick
    def is_still_sick(self):
        if self.sick_time > 0:
            return True
        return False

    #updating the number of generations that the organism has to be sick
    def update_sick_time(self):
        self.sick_time -= 1
        return self.sick_time

#the board size - size*size
size = 200

#moving an organism randomly - setting the selected new location in "update_positions"
#parameters: the board, a list with the selected locations for all the organisms,
# the row and column of the organism's current location, the speed of the organism, the id of the organism
def move(board, update_positions, row, col, steps, id):
    new_position = []
    options_list = [1,2,3,4,5,6,7,8,9]
    valid_position = False

    #untill the selected new location doesn't collide with other organism's location - selecting a new location
    while valid_position == False:
        #selecting the movement of the organism randomly
        x = random.choice(options_list)

        # Stay in the same place
        if x == 1:
            new_position = [row, col]
        # Up + Left
        elif x == 2:
            new_position = [((row - steps) + size) % size, ((col - steps) + size) % size]
        # Up
        elif x == 3:
            new_position = [((row - steps) + size) % size, col]
        # Up + Right
        elif x == 4:
            new_position = [((row - steps) + size) % size, ((col + steps) + size) % size]
        # Left
        elif x == 5:
            new_position = [row, ((col - steps) + size) % size]
        # Right
        elif x == 6:
            new_position = [row, ((col + steps) + size) % size]
        # Down + Left
        elif x == 7:
            new_position = [((row + steps) + size) % size, ((col - steps) + size) % size]
        # Down
        elif x == 8:
            new_position = [((row + steps) + size) % size, col]
        # Down + Right
        elif x == 9:
            new_position = [((row + steps) + size) % size, ((col + steps) + size) % size]

        #checking if the position is valid - not colliding with other organism's position
        if board[new_position[0], new_position[1]] == 0 or board[new_position[0], new_position[1]] == id:
            valid_position = True
        #if invalid - removing the selected movement from the possible movements array
        else:
            options_list.remove(x)

    #checks if the selected new location collides with a new location of a previous organism
    #if there is a collision - the current organism will stay in the same place
    for i in range(id - 1):
        if update_positions[i + 1][0] == new_position[0] and update_positions[i + 1][1] == new_position[1]:
            new_position = [row, col]

    #updates the new position of the current organism
    update_positions[id] = new_position

#initializing the board of the simulation
def initialize_board(N, init_positions):
    # Create an empty 2D numpy array by the size(0 - an empty position)
    an_array = np.full([size, size], 0)

    #adding the organisms to the board
    for i in range(N):
        #finds a position that is empty
        row = random.randint(0, size - 1)
        col = random.randint(0, size - 1)

        while an_array[row, col] != 0:
            row = random.randint(0, size - 1)
            col = random.randint(0, size - 1)

        #setting the current organism in the empty position
        an_array[row, col] = i + 1
        init_positions[i] = [row , col]

    return an_array

#initializing dictionary that holds information about every organism
def initialize_dict(N, init_positions, D, R, X):
    org_dict = {}
    #the initial states of the organisms
    states = np.full(N, 1)
    #selecting randomly by D the sick organisms
    sick_indexes = random.sample(range(0, N), D)
    #setting the state of the sick organisms(2)
    for index in sick_indexes:
        states[index] = 2

    #the initial speeds of the organisms
    speeds = np.full(N, 1)
    # selecting randomly by R the fast organisms
    fast_indexes = random.sample(range(0, N), R)
    # setting the speed of the sick organisms(10)
    for index in fast_indexes:
        speeds[index] = 10

    #adding the organisms to the dictionary
    for i in range(N):
        org_dict[i + 1] = (init_positions[i], Organism(states[i], speeds[i], X))

    return org_dict

#checking if there is a neighbor near the input position. if there is - returning it's id
def get_neighbors_ids(array, row, col):
    ids = []
    if array[((row - 1) + size) % size, ((col - 1) + size) % size] != 0:
        ids.append(array[((row - 1) + size) % size, ((col - 1) + size) % size])
    if array[((row - 1) + size) % size, col] != 0:
        ids.append(array[((row - 1) + size) % size, col])
    if array[((row - 1) + size) % size, ((col + 1) + size) % size] != 0:
        ids.append(array[((row - 1) + size) % size, ((col + 1) + size) % size])
    if array[row, ((col - 1) + size) % size] != 0:
        ids.append(array[row, ((col - 1) + size) % size])
    if array[row, ((col + 1) + size) % size] != 0:
        ids.append(array[row, ((col + 1) + size) % size])
    if array[((row + 1) + size) % size, ((col - 1) + size) % size] != 0:
        ids.append(array[((row + 1) + size) % size, ((col - 1) + size) % size])
    if array[((row + 1) + size) % size, col] != 0:
        ids.append(array[((row + 1) + size) % size, col])
    if array[((row + 1) + size) % size, ((col + 1) + size) % size] != 0:
        ids.append(array[((row + 1) + size) % size, ((col + 1) + size) % size])

    return ids

#randomly selecting sick or healthy by P
def random_sick(P):
    state = ["sick", "healthy"]
    if np.random.choice(state, p=[P, 1 - P]) == "sick":
        return True
    return False

#setting P by the sick organisms rate and T
def initialize_p(T, P_low, P_high, sick_rate):
    if sick_rate <= T:
        return P_high
    elif sick_rate > T:
        return P_low

#printing the information about the organisms
def print_organisms(organisms_list, N):
    for i in range(N):
        print('id:', i + 1)
        print('position:', organisms_list[i + 1][0])
        print('data:', organisms_list[i + 1][1].state, organisms_list[i + 1][1].sick_time, organisms_list[i + 1][1].speed)

#checks if the input parameters are valid
def are_parameters_valid(N, D, R, X, P_low, P_high, T):
    valid_parameters = True
    if N >= size*size:
        print("Too Many Organisms - N Needs To Be Smaller Than The Size Of The Board.")
        valid_parameters = False
    if N < 1:
        print("N Needs To Be At Least 1.")
        valid_parameters = False
    if not isinstance(N, int):
        print("N Needs To Be An Integer.")
        valid_parameters = False
    if D > 1 or D < 0:
        print("D Needs To Be Between 0 And 1.")
        valid_parameters = False
    if R > 1 or R < 0:
        print("R Needs To Be Between 0 And 1.")
        valid_parameters = False
    if X < 0:
        print("X Needs To Be At Least 0.")
        valid_parameters = False
    if not isinstance(X, int):
        print("X Needs To Be An Integer.")
        valid_parameters = False
    if P_low > 1 or P_low < 0:
        print("P_low Needs To Be Between 0 And 1.")
        valid_parameters = False
    if P_high > 1 or P_high < 0:
        print("P_high Needs To Be Between 0 And 1.")
        valid_parameters = False
    if P_low > P_high:
        print("P_low Needs To Be Lower Or Equal To P_high.")
        valid_parameters = False
    if T > 1 or T < 0:
        print("T Needs To Be Between 0 And 1.")
        valid_parameters = False

    return valid_parameters

#main function
if __name__ == '__main__':
    if len(sys.argv) < 8:
        print("Not Enough Parameters!")
    else:
        # initializing the parameters by the input of the user
        N = int(sys.argv[1])
        D = float(sys.argv[2])
        R = float(sys.argv[3])
        X = int(sys.argv[4])
        P_low = float(sys.argv[5])
        P_high = float(sys.argv[6])
        T = float(sys.argv[7])

        #if the parameters are valid - proceeding
        if are_parameters_valid(N, D, R, X, P_low, P_high, T):
            num_of_fast_organisms = math.ceil(N*R)
            sick_organisms = math.ceil(N*D)
            print("Number Of Organisms:", N, ", Initial Number Of Sick Organisms:", sick_organisms,
                  " ,Number Of Fast Organisms: ", num_of_fast_organisms)
            #initializing the board and the starting positions of the organisms
            init_positions = [None] * N
            arr = initialize_board(N, init_positions)

            #initializing the properties for every organism
            organisms_dict = initialize_dict(N, init_positions, sick_organisms, num_of_fast_organisms, X)

            #the colors on the board
            colors = np.array([[255, 255, 255], [120, 250, 90], [250, 90, 120], [80, 128, 255]])

            isLastGeneration = False

            #generations loop
            generation = 0
            while not isLastGeneration:
                #if there are not anymore sick organisms - the simulation will stop
                if sick_organisms == 0:
                    isLastGeneration = True
                sick_rate = float(sick_organisms/N)
                # check for updating p
                P = initialize_p(T, P_low, P_high, sick_rate)

                print("Generation: ", generation, " Sick Rate: ", sick_rate)

                #creating a board with the organism's states instead of the organism's ids
                states_board = np.empty((size, size))
                for i in range(size):
                    for j in range(size):
                        if arr[i][j] != 0:
                            states_board[i][j] = organisms_dict[arr[i][j]][1].state
                        else:
                            states_board[i][j] = 0

                #presenting the visual board
                states_board = states_board.astype(int)
                pg.init()
                screen = pg.display.set_mode((700, 700))

                print_array = np.transpose(states_board)
                colors_arr = colors[print_array]
                surface = pg.surfarray.make_surface(colors_arr)
                surface = pg.transform.scale(surface, (600, 600))  # Scaled a bit.

                pg.font.init()
                my_font = pg.font.SysFont('Comic Sans MS', 30)
                text_surface = my_font.render(
                    'Generation: ' + str(generation) + ", Sick Organisms Rate: " + str(sick_rate*100) + "%",
                    False, (255, 255, 255))

                screen.fill((30, 30, 30))
                screen.blit(surface, (60, 60))
                screen.blit(text_surface, (10, 10))
                pg.display.flip()
                sleep(3)

                #scans the array for the sick organisms
                update_states = {}
                for i in range(N):
                    #if the current organism is still sick
                    if organisms_dict[i + 1][1].is_sick():
                        #if the current organism has no time left to be sick - updating it's state and the sick counter
                        if not organisms_dict[i + 1][1].is_still_sick():
                            update_states[i + 1] = 3
                            sick_organisms -= 1

                        #if the current organism if still sick
                        else:
                            #getting the organism's neighbors and randomizing by P if the healthy neighbors will be sick
                            for neighbor_id in get_neighbors_ids(arr, organisms_dict[i + 1][0][0], organisms_dict[i + 1][0][1]):
                                #if the neighbor is healthy
                                if organisms_dict[neighbor_id][1].is_healthy():
                                    #randomize if sick
                                    if random_sick(P):
                                        #means that no other organism already got the current neighbor sick
                                        if neighbor_id not in update_states:
                                            #updating the neighbor's state and the sick counter
                                            update_states[neighbor_id] = 2
                                            sick_organisms += 1

                            organisms_dict[i + 1][1].update_sick_time()

                #updating the states for the next generation
                for update_index in update_states:
                    organisms_dict[update_index][1].set_state(update_states[update_index])

                #scans the array and for every organism's randomizing it's new position
                update_positions = {}
                for i in range(N):
                    move(arr, update_positions, organisms_dict[i + 1][0][0], organisms_dict[i + 1][0][1], organisms_dict[i + 1][1].speed, i + 1)

                #update the positions on the dictionary and on the board
                for i in range(N):
                    # updates the new position for the organism on the board
                    arr[update_positions[i + 1][0], update_positions[i + 1][1]] = i + 1
                    # setting the old position as empty if no previous organism moves to this location
                    if arr[organisms_dict[i + 1][0][0], organisms_dict[i + 1][0][1]] == i + 1 and organisms_dict[i + 1][0] != update_positions[i + 1]:
                        arr[organisms_dict[i + 1][0][0], organisms_dict[i + 1][0][1]] = 0

                    # updates the new position for the organism in the dictionary
                    organisms_dict[i + 1][0][0] = update_positions[i + 1][0]
                    organisms_dict[i + 1][0][1] = update_positions[i + 1][1]

                #updates the generation number
                generation += 1