# coding: utf-8

# In[1]:

import numpy as np

# In[2]:

import json

# In[3]:

with open("yelp_academic_dataset_review.json", encoding="utf8") as f:
    data = f.readlines()

# In[4]:

jsondat = []
for f in data:
    jsondat.append(json.loads(f))

# In[5]:

userset = set()
restset = set()
for data in jsondat:
    userset.add(data['user_id'])
    restset.add(data['business_id'])

# In[6]:

mapuser={}
i = 1
mapbackuser = {}
for data in userset:
    mapuser[data] = i
    mapbackuser[i] = data
    i+=1
i=1
maprest = {}
mapbackrest = {}
for data in restset:
    maprest[data] = i
    mapbackrest[i] = data
    i+=1

# In[55]:

actualdata=[]

# In[56]:

print (len(jsondat))

# In[57]:

import csv

# In[58]:

with open('train.dat', "w", encoding = "latin-1") as f:
    writer = csv.writer(f, delimiter = ",")
    i=0
    thresh = len(jsondat)*0.8
    for data in jsondat:
        if (mapuser[data['user_id']]<=100000 and maprest[data['business_id']]<=10000):
            actualdata.append([str(mapuser[data['user_id']]), str(maprest[data['business_id']]), str(data['stars']), str(99999)])
#             writer.writerow([str(mapuser[data['user_id']]), str(maprest[data['business_id']]), str(data['stars'])])
#         if (i>thresh):
#             break
# with open('test.dat', "w", encoding = "latin-1") as f:
#     writer = csv.writer(f, delimiter = ",")
#     i=0
#     thresh = len(jsondat)*0.8
#     for data in jsondat:
#         i+=1
#         if (maprest[data['business_id']]>=):
#             writer.writerow([str(mapuser[data['user_id']]), str(maprest[data['business_id']]), str(data['stars'])])

# In[43]:

with open('train.dat', "w", encoding = "latin-1") as f:
    writer = csv.writer(f, delimiter = ",")
    i=0
    thresh = len(actualdata)*0.8
    for data in actualdata:
        i+=1
        if (i<thresh):
#             actualdata.append([str(mapuser[data['user_id']]), str(maprest[data['business_id']]), str(data['stars'])])
            writer.writerow(data)

# In[44]:

with open('test.dat', "w", encoding = "latin-1") as f:
    writer = csv.writer(f, delimiter = ",")
    i=0
    thresh = len(actualdata)*0.8

    for data in actualdata:
        i+=1
        if (i>=thresh):
#             actualdata.append([str(mapuser[data['user_id']]), str(maprest[data['business_id']]), str(data['stars'])])
            writer.writerow(data)

# In[59]:

actualdata = [list(map(int,i)) for i in actualdata]
actualdata = np.array(actualdata)

# In[60]:

np.shape(actualdata)

# In[62]:

actualdata[:,0]
test = actualdata[(int)(len(actualdata)*0.8):]
actualdata = actualdata[:(int)(len(actualdata)*0.8)]

# In[48]:

import pandas as pd
import json
import numpy as np
import keras
from IPython.display import SVG
from keras.optimizers import Adam
from keras.utils.vis_utils import model_to_dot
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from math import sqrt
from keras.models import Model
from keras.layers import dot
import warnings
warnings.filterwarnings('ignore')

# In[49]:

n_movies = 9999
n_users = 99999
n_latent_factors = 5

# In[50]:

movie_input = keras.layers.Input(shape=[1],name='Item')
movie_embedding = keras.layers.Embedding(n_movies + 1, n_latent_factors, name='Movie-Embedding')(movie_input)
movie_vec = keras.layers.Flatten(name='FlattenMovies')(movie_embedding)

user_input = keras.layers.Input(shape=[1],name='User')
user_vec = keras.layers.Flatten(name='FlattenUsers')(keras.layers.Embedding(n_users + 1, n_latent_factors,name='User-Embedding')(user_input))

prod = dot([movie_vec, user_vec], axes = 1)

model = Model([user_input, movie_input], prod)
model.compile('adam', 'mean_squared_error')
model.summary()

# In[51]:

history = model.fit([actualdata[:,0], actualdata[:,1]], actualdata[:,2], epochs=100, verbose=0)

# In[53]:

from sklearn.metrics import mean_squared_error
from math import sqrt

y_hat = np.round(model.predict([test[:,0], test[:,1]]),0)
y_true = test[:,2]
sqrt(mean_squared_error(y_true, y_hat))

# In[21]:

len(mapuser)

# In[22]:

len(maprest)

# In[63]:

len(test)


import sys
sys.path.append("..")  # 包含上级目录
import pandas as pd
import json
import math
import numpy as np
from scipy.ndimage.filters import gaussian_filter
import tqdm
import cv2
import tensorflow as tf
from keras.utils.multi_gpu_utils import multi_gpu_model

from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
import util

from config_reader import config_reader
from config import COCOSourceConfig, GetConfig

from keras import backend as K

import os

# os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = "0"
K.set_learning_phase(1)

# the middle joints heatmap correpondence
limbSeq = [[1, 0], [1, 14], [1, 15], [1, 16], [1, 17], [0, 14], [0, 15], [14, 16], [15, 17],
           [1, 2], [2, 3], [3, 4], [1, 5], [5, 6], [6, 7], [1, 8], [8, 9],
           [9, 10], [1, 11], [11, 12], [12, 13], [8, 11], [2, 16], [5, 17]]

mapIdx = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13], [14, 15], [16, 17], [18, 19], [20, 21], [22, 23],
          [24, 25], [26, 27], [28, 29], [30, 31], [32, 33], [34, 35], [36, 37], [38, 39], [40, 41], [42, 43], [44, 45],
          [46, 47]]

# visualize
colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0],
          [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255],
          [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85], [193, 193, 255], [106, 106, 255], [20, 147, 255],
          [128, 114, 250], [130, 238, 238], [48, 167, 238], [180, 105, 255]]

#  TODO: when evaluation, change the keypoint id mapping!! Because we use different ids compared with MSCOCO dataset!
dt_gt_mapping = {0: 0, 1: None, 2: 6, 3: 8, 4: 10, 5: 5, 6: 7, 7: 9, 8: 12, 9: 14, 10: 16, 11: 11, 12: 13, 13: 15,
                 14: 2, 15: 1, 16: 4, 17: 3, 18: None}

def predict(image, params, model, model_params, heat_layers, paf_layers):
    # print (image.shape)
    heatmap_avg = np.zeros((image.shape[0], image.shape[1], heat_layers))
    paf_avg = np.zeros((image.shape[0], image.shape[1], paf_layers))
    multiplier = [x * model_params['boxsize'] / image.shape[0] for x in params['scale_search']]
    for m in range(len(multiplier)):
        scale = multiplier[m]

        imageToTest = cv2.resize(image, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
        imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model_params['stride'],
                                                          model_params['padValue'])

        input_img = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]),
                                 (3, 0, 1, 2))  # required shape (1, width, height, channels)

        output_blobs = model.predict(input_img)

        # extract outputs, resize, and remove padding
        heatmap = np.squeeze(output_blobs[1])  # output 1 is heatmaps
        heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'],
                             interpolation=cv2.INTER_CUBIC)
        heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
        heatmap = cv2.resize(heatmap, (image.shape[1], image.shape[0]), interpolation=cv2.INTER_CUBIC)

        paf = np.squeeze(output_blobs[0])  # output 0 is PAFs
        paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'],
                         interpolation=cv2.INTER_CUBIC)
        paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
        paf = cv2.resize(paf, (image.shape[1], image.shape[0]), interpolation=cv2.INTER_CUBIC)
        #
        heatmap_avg = heatmap_avg + heatmap / len(multiplier)
        paf_avg = paf_avg + paf / len(multiplier)

    return heatmap_avg, paf_avg

def find_peaks(heatmap_avg, params):
    all_peaks = []
    peak_counter = 0

    for part in range(18):
        map_ori = heatmap_avg[:, :, part]
        map = gaussian_filter(map_ori, sigma=3)  # TODO: fintune the sigma

        map_up = np.zeros(map.shape)
        map_up[1:, :] = map[:-1, :]
        map_down = np.zeros(map.shape)  # todo： NMS with a sliding window of 3*3
        map_down[:-1, :] = map[1:, :]
        map_left = np.zeros(map.shape)
        map_left[:, 1:] = map[:, :-1]
        map_right = np.zeros(map.shape)
        map_right[:, :-1] = map[:, 1:]

        map_left_up = np.zeros(map.shape)
        map_left_up[1:, :] = map_left[:-1, :]
        map_right_up = np.zeros(map.shape)
        map_right_up[1:, :] = map_right[:-1, :]
        map_left_down = np.zeros(map.shape)
        map_left_down[:-1, :] = map_left[1:, :]
        map_right_down = np.zeros(map.shape)
        map_right_down[:-1, :] = map_right[1:, :]

        peaks_binary = np.logical_and.reduce((map >= map_left, map >= map_right,
                                              map >= map_up, map >= map_down, map >= map_right_up,
                                              map >= map_right_down,
                                              map >= map_left_up, map >= map_left_down,
                                              map > params['thre1']))  # fixme: finetue it

        peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]))  # note reverse
        peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks]
        id = range(peak_counter, peak_counter + len(peaks))
        peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]

        all_peaks.append(peaks_with_score_and_id)
        peak_counter += len(peaks)

    return all_peaks

def find_connections(all_peaks, paf_avg, image_width, params):
    connection_all = []
    special_k = []

    for k in range(len(mapIdx)):
        score_mid = paf_avg[:, :, mapIdx[k][0] // 2]
        candA = all_peaks[limbSeq[k][0]]
        candB = all_peaks[limbSeq[k][1]]
        nA = len(candA)
        nB = len(candB)
        indexA, indexB = limbSeq[k]
        if (nA != 0 and nB != 0):
            connection_candidate = []
            for i in range(nA):
                for j in range(nB):
                    vec = np.subtract(candB[j][:2], candA[i][:2])
                    norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
                    mid_num = max(int(norm), 10)  # todo: tune it?
                    # failure case when 2 body parts overlaps
                    if norm == 0:
                        # https://github.com/ZheC/Realtime_Multi-Person_Pose_Estimation/issues/54
                        continue

                    startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num),
                                        np.linspace(candA[i][1], candB[j][1], num=mid_num)))

                    limb_response = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0]))] \
                                              for I in range(len(startend))])
                    score_midpts = limb_response

                    score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min(0.5 * image_width / norm - 1, 0)
                    # https://github.com/michalfaber/keras_Realtime_Multi-Person_Pose_Estimation/issues/48

                    criterion1 = len(np.nonzero(score_midpts > params['thre2'])[0]) >= params['connect_ration'] * len(
                        score_midpts)  # todo　改成了0.8
                    # fixme: tune 手动调整, 0.7 or 0.8
                    criterion2 = score_with_dist_prior > 0
                    if criterion1 and criterion2:
                        connection_candidate.append([i, j, score_with_dist_prior, norm,
                                                     0.5 * score_with_dist_prior + 0.25 * candA[i][2] + 0.25 * candB[j][
                                                         2]])

            connection_candidate = sorted(connection_candidate, key=lambda x: x[4], reverse=True)  # todo: sort by what

            connection = np.zeros((0, 6))
            for c in range(len(connection_candidate)):
                i, j, s, limb_len = connection_candidate[c][0:4]
                if (i not in connection[:, 3] and j not in connection[:, 4]):
                    connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j, limb_len]])
                    if (len(connection) >= min(nA, nB)):
                        break
            connection_all.append(connection)
        else:
            special_k.append(k)
            connection_all.append([])

    return connection_all, special_k

def find_people(connection_all, special_k, all_peaks, params):
    # last number in each row is the total parts number of that person
    # the second last number in each row is the score of the overall configuration
    subset = -1 * np.ones((0, 20, 2))
    candidate = np.array([item for sublist in all_peaks for item in sublist])
    for k in range(len(mapIdx)):

        if k not in special_k:
            partAs = connection_all[k][:, 0]
            partBs = connection_all[k][:, 1]
            indexA, indexB = np.array(limbSeq[k])

            for i in range(len(connection_all[k])):

                found = 0
                subset_idx = [-1, -1]
                for j in range(len(subset)):
                    if subset[j][indexA][0].astype(int) == (partAs[i]).astype(int) or subset[j][indexB][0].astype(
                            int) == partBs[i].astype(int):

                        if found >= 2:
                            print('************ error occurs! 3 joints sharing have been found  *******************')
                            continue
                        subset_idx[found] = j
                        found += 1

                if found == 1:
                    j = subset_idx[0]

                    if subset[j][indexB][0].astype(int) == -1 and \
                            params['len_rate'] * subset[j][-1][1] > connection_all[k][i][-1]:

                        subset[j][indexB][0] = partBs[i]
                        subset[j][indexB][1] = connection_all[k][i][2]
                        subset[j][-1][0] += 1
                        # last number in each row is the total parts number of that person

                        subset[j][-2][0] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
                        subset[j][-1][1] = max(connection_all[k][i][-1], subset[j][-1][1])

                    elif subset[j][indexB][0].astype(int) != partBs[i].astype(int):
                        if subset[j][indexB][1] >= connection_all[k][i][2]:
                            pass

                        else:
                            if params['len_rate'] * subset[j][-1][1] <= connection_all[k][i][-1]:
                                continue
                            subset[j][-2][0] -= candidate[subset[j][indexB][0].astype(int), 2] + subset[j][indexB][1]

                            subset[j][indexB][0] = partBs[i]
                            subset[j][indexB][1] = connection_all[k][i][2]
                            subset[j][-2][0] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]

                            subset[j][-1][1] = max(connection_all[k][i][-1], subset[j][-1][1])
                    else:
                        pass

                elif found == 2:  # if found 2 and disjoint, merge them
                    j1, j2 = subset_idx

                    membership1 = ((subset[j1][..., 0] >= 0).astype(int))[:-2]
                    membership2 = ((subset[j2][..., 0] >= 0).astype(int))[:-2]
                    membership = membership1 + membership2
                    if len(np.nonzero(membership == 2)[0]) == 0:  # if found 2 and disjoint, merge them

                        min_limb1 = np.min(subset[j1, :-2, 1][membership1 == 1])
                        min_limb2 = np.min(subset[j2, :-2, 1][membership2 == 1])
                        min_tolerance = min(min_limb1, min_limb2)

                        if connection_all[k][i][2] < params['connection_tole'] * min_tolerance or params['len_rate'] * \
                                subset[j1][-1][1] <= connection_all[k][i][-1]:
                            # todo: finetune the tolerance of connection
                            continue  #

                        subset[j1][:-2][...] += (subset[j2][:-2][...] + 1)
                        subset[j1][-2:][:, 0] += subset[j2][-2:][:, 0]

                        subset[j1][-2][0] += connection_all[k][i][2]
                        subset[j1][-1][1] = max(connection_all[k][i][-1], subset[j1][-1][1])
                        subset = np.delete(subset, j2, 0)

                    else:
                        if connection_all[k][i][0] in subset[j1, :-2, 0]:
                            c1 = np.where(subset[j1, :-2, 0] == connection_all[k][i][0])
                            c2 = np.where(subset[j2, :-2, 0] == connection_all[k][i][1])
                        else:
                            c1 = np.where(subset[j1, :-2, 0] == connection_all[k][i][1])
                            c2 = np.where(subset[j2, :-2, 0] == connection_all[k][i][0])

                        c1 = int(c1[0])
                        c2 = int(c2[0])
                        assert c1 != c2, "an candidate keypoint is used twice, shared by two people"

                        if connection_all[k][i][2] < subset[j1][c1][1] and connection_all[k][i][2] < subset[j2][c2][1]:
                            continue  # the trick here is useful

                        small_j = j1
                        big_j = j2
                        remove_c = c1

                        if subset[j1][c1][1] > subset[j2][c2][1]:
                            small_j = j2
                            big_j = j1
                            remove_c = c2

                        subset[small_j][-2][0] -= candidate[subset[small_j][remove_c][0].astype(int), 2] + \
                                                  subset[small_j][remove_c][1]
                        subset[small_j][remove_c][0] = -1  # todo
                        subset[small_j][remove_c][1] = -1
                        subset[small_j][-1][0] -= 1

                # if find no partA in the subset, create a new subset
                elif not found and k < 24:
                    # Fixme: 原始的时候是18,因为我加了limb，所以是24,因为真正的limb是0~16，最后两个17,18是额外的不是limb
                    # FIXME: 但是后面画limb的时候没有把鼻子和眼睛耳朵的连线画上，要改进
                    row = -1 * np.ones((20, 2))
                    row[indexA][0] = partAs[i]
                    row[indexA][1] = connection_all[k][i][2]
                    row[indexB][0] = partBs[i]
                    row[indexB][1] = connection_all[k][i][2]
                    row[-1][0] = 2
                    row[-1][1] = connection_all[k][i][-1]
                    row[-2][0] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2]
                    row = row[np.newaxis, :, :]
                    subset = np.concatenate((subset, row), axis=0)
        # todo: solve the unmathced keypoint?
    # delete some rows of subset which has few parts occur
    deleteIdx = []
    for i in range(len(subset)):
        if subset[i][-1][0] < 2 or subset[i][-2][0] / subset[i][-1][0] < 0.45:
            deleteIdx.append(i)
    subset = np.delete(subset, deleteIdx, axis=0)

    return subset, candidate

def process(input_image, params, model, model_params, heat_layers, paf_layers):
    oriImg = cv2.imread(input_image)  # B,G,R order
    # print(input_image)
    heatmap_avg, paf_avg = predict(oriImg, params, model, model_params, heat_layers, paf_layers)

    all_peaks = find_peaks(heatmap_avg, params)
    connection_all, special_k = find_connections(all_peaks, paf_avg, oriImg.shape[0], params)
    subset, candidate = find_people(connection_all, special_k, all_peaks, params)

    keypoints = []
    for s in subset[..., 0]:
        keypoint_indexes = s[:18]
        person_keypoint_coordinates = []
        for index in keypoint_indexes:
            if index == -1:
                X, Y = 0, 0
            else:
                X, Y = candidate[index.astype(int)][:2]
            person_keypoint_coordinates.append((X, Y))
        person_keypoint_coordinates_coco = [None] * 17

        for dt_index, gt_index in dt_gt_mapping.items():
            if gt_index is None:
                continue
            person_keypoint_coordinates_coco[gt_index] = person_keypoint_coordinates[dt_index]

        keypoints.append((person_keypoint_coordinates_coco, 1 - 1.0 / s[18]))
        # s[18] is the score, s[19] is the number of keypoint
    return keypoints

def get_image_name(coco, image_id):
    return coco.imgs[image_id]['file_name']

def predict_many(coco, images_directory, validation_ids, params, model, model_params, heat_layers, paf_layers):
    assert (not set(validation_ids).difference(set(coco.getImgIds())))

    keypoints = {}
    for image_id in tqdm.tqdm(validation_ids):
        image_name = get_image_name(coco, image_id)
        image_name = os.path.join(images_directory, image_name)
        keypoints[image_id] = process(image_name, dict(params), model, dict(model_params), heat_layers+1, paf_layers)
        # fixme: heat_layers + 1 if you use background keypoint  !!!
    return keypoints

def format_results(keypoints, resFile):
    format_keypoints = []
    # todo: do we need to sort the detections by scores before evaluation ?
    for image_id, people in keypoints.items():
        for keypoint_list, score in people:
            format_keypoint_list = []
            for x, y in keypoint_list:
                for v in [int(x), int(y), 1 if x > 0 or y > 0 else 0]:
                    format_keypoint_list.append(v)

            format_keypoints.append({
                "image_id": image_id,
                "category_id": 1,
                "keypoints": format_keypoint_list,
                "score": score,
            })

    json.dump(format_keypoints, open(resFile, 'w'))

def validation(model, dump_name, validation_ids=None, dataset='val2017'):
    annType = 'keypoints'
    prefix = 'person_keypoints'

    dataDir = 'dataset/coco/link2coco2017'

    # # #############################################################################
    # 在验证集上测试代码
    annFile = '%s/annotations/%s_%s.json' % (dataDir, prefix, dataset)
    print(annFile)
    cocoGt = COCO(annFile)

    if validation_ids == None:   # todo: we can set the validataion image ids here  !!!!!!
        validation_ids = cocoGt.getImgIds()
    # # #############################################################################

    resFile = '%s/results/%s_%s_%s100_results.json'
    resFile = resFile % (dataDir, prefix, dataset, dump_name)
    print('the path of detected keypoint file is: ', resFile)
    os.makedirs(os.path.dirname(resFile), exist_ok=True)

    keypoints = predict_many(cocoGt, os.path.join(dataDir, dataset), validation_ids, params, model, model_params,
                             config.heat_layers, config.paf_layers)
    format_results(keypoints, resFile)
    cocoDt = cocoGt.loadRes(resFile)
    cocoEval = COCOeval(cocoGt, cocoDt, annType)
    cocoEval.params.imgIds = validation_ids
    cocoEval.evaluate()
    cocoEval.accumulate()
    cocoEval.summarize()
    return cocoEval

def per_image_scores(eval_result):
    def convert_match_to_score(match):
        matches = match['gtMatches'][:, np.array(match['gtIgnore']) == 0]
        scores = {
            'image_id': match['image_id'],
            'gt_person_count': matches.shape[1],
        }

        for i in range(matches.shape[0]):
            okp_threshold = eval_result.params.iouThrs[i]
            scores['matched_%.2f' % okp_threshold] = sum(matches[i, :] != 0)
        scores['average'] = np.mean(np.sum(matches != 0, axis=1)) / scores['gt_person_count']

        return scores

    evalImgs = eval_result.evalImgs
    scores = [convert_match_to_score(image_match) for image_match in evalImgs if image_match is not None]

    return pd.DataFrame(scores)

if __name__ == "__main__":
    config = GetConfig('Canonical')
    params, model_params = config_reader(figstr='testing/config')
    from posenet.mymodel3 import get_testing_model

    # with tf.device("/cpu:0"):
    #     model_single = get_testing_model(np_branch1=config.paf_layers, np_branch2=config.heat_layers, stages=1)
    # model = multi_gpu_model(model_single, gpus=2)
    model = get_testing_model(np_branch1=config.paf_layers, np_branch2=config.heat_layers+1, stages=3)  # fixme: background + 1
    training_dir = './training/'
    trained_models = [
        'weights'
        # 'weights-cpp-lr'
        # 'weights-python-last',
    ]

    optimal_epoch_loss = 'val_weight_stage6_L1_loss'
    weights_path = 'training/cpu_weights/cpu_model.h5'
    # '../model/keras/model.h5'    # orginal weights converted from caffe
    model.load_weights(weights_path)
    eval_result_original = validation(model, dump_name='pose_focal_4scale', dataset='val2017')  # val2017

    print('over!')


import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

"""
If you want to continue to explore additional color channels,
I have seen that the L channel from LUV with lower and upper thresholds around 225 & 255
respectively works very well to pick out the white lines, even in the parts of the video
with heavy shadows.You can also try out the b channel from Lab which does a great job with
the yellow lines (you can play around with thresholds around 155 & 200).
"""

def get_binary_img(img, s_thresh=(120, 255), sx_thresh=(20, 255),l_thresh=(40,255)):
    img = np.copy(img)
    # Convert to HLS color space and separate the V channel
    hls = cv2.cvtColor(img, cv2.COLOR_BGR2HLS).astype(np.float)
    l_channel = hls[:,:,1]
    s_channel = hls[:,:,2]
    # Sobel x
    sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0) # Take the derivative in x
    abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal
    scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))

    # Threshold x gradient
    sxbinary = np.zeros_like(scaled_sobel)
    sxbinary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1])] = 1

    # Threshold color channel
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1

     # Threshold lightness
    l_binary = np.zeros_like(l_channel)
    l_binary[(l_channel >= l_thresh[0]) & (l_channel <= l_thresh[1])] = 1

    # Stack each channel
    channels = 255*np.dstack(( l_binary, sxbinary, s_binary)).astype('uint8')

    # Combine the three binary thresholds
    binary = np.zeros_like(sxbinary)
    binary[((l_binary == 1) & (s_binary == 1) | (sxbinary==1))] = 1
    binary = 255*np.dstack((binary,binary,binary)).astype('uint8')

    return binary, channels

if __name__ == '__main__':
    num_test_images = 6
    for test_image in range(1, num_test_images+1):
        print('processing image test_images/test{}.jpg'.format(test_image))
        image = cv2.imread('test_images/test{}.jpg'.format(test_image))
        combined_binary, color_binary = get_binary_img(image)
        cv2.imwrite('output_images/test{}_binary.png'.format(test_image), combined_binary)
        cv2.imwrite('output_images/test{}_color_binary.png'.format(test_image), color_binary)


from typing import TYPE_CHECKING, List, Optional

from pydantic import root_validator

from vkbottle.tools.mini_types.base import BaseMessageMin

if TYPE_CHECKING:
    from vkbottle.api import ABCAPI

from .foreign_message import ForeignMessageMin  # noqa: TCH001

class MessageMin(BaseMessageMin):
    user_id: Optional[int] = None
    reply_message: Optional["ForeignMessageMin"] = None
    fwd_messages: Optional[List["ForeignMessageMin"]] = []

    @property
    def is_mentioned(self) -> bool:
        return self.mention.id == self.user_id if self.mention else False

    @root_validator(pre=True)
    def __foreign_messages(cls, values):
        foreign_messages = []
        if values.get("fwd_messages"):
            foreign_messages.extend(values["fwd_messages"])
        if values.get("reply_message"):
            foreign_messages.append(values["reply_message"])
        for foreign_message in foreign_messages:
            foreign_message["unprepared_ctx_api"] = values["unprepared_ctx_api"]
            foreign_message["replace_mention"] = values["replace_mention"]
            foreign_message["user_id"] = values.get("user_id")
        return values

MessageMin.update_forward_refs()

async def message_min(
    message_id: int,
    ctx_api: "ABCAPI",
    replace_mention: bool = True,
) -> "MessageMin":
    response = await ctx_api.messages.get_by_id(message_ids=[message_id])
    if not response.items:
        raise ValueError(f"Message with id {message_id} not found, perhaps it was deleted")
    message_object = response.items[0].dict()
    return MessageMin(
        **message_object,
        unprepared_ctx_api=ctx_api,
        replace_mention=replace_mention,
    )


import sys, random, json
from gensim.models import Word2Vec
from argparse import ArgumentParser
from collections import Counter
from pathlib import Path

__dir__= Path(__file__).absolute().parent.parent

class DirSentences(object):
    def __init__(self, *args, **kwargs):
        self.dirs = args
        self.len = None
        self.sep = kwargs.get("sep", " ")
        self.shuffle = kwargs.get("shuffle", False)
        self.ext = kwargs.get("ext", "csv")

    def __len__(self):
        if self.len is not None:
            return self.len
        ret = 0
        for dirname in self.dirs:
            for p in Path(dirname).glob("*."+self.ext):
                with p.open("rb") as f:
                    content = f.read().decode("utf-8", "ignore")
                    ret += len(content.split("\n"))
        self.len = ret
        return ret

    def __iter__(self):
        total_len = 0
        for dirname in self.dirs:
            for p in Path(dirname).glob("*."+self.ext):
                with p.open("rb") as f:
                    content = f.read().decode("utf-8", "ignore")
                    for line in content.split("\n"):
                        ret = line.strip(' \r\n"\'\t\xa0`').split(self.sep)
                        if self.shuffle:
                            random.shuffle(ret)
                        yield ret
                        total_len += 1
        self.len = total_len

    def build_vocab(self, output=None):
        vocab = Counter()
        for dirname in self.dirs:
            for p in Path(dirname).glob("*."+self.ext):
                with p.open("rb") as f:
                    content = f.read().decode("utf-8", "ignore")
                    for line in content.split("\n"):
                        vocab+=Counter(line.strip().split(self.sep))
        vocab = dict(vocab)
        if output is not None:
            with open(output, "w") as f:
                json.dump(vocab, f)
        return vocab

def main(params):
    train_sentences = DirSentences(*params.input.split(','),
    sep=params.sep, shuffle=params.shuffle,ext=params.input_file_extension)
    if Path(params.model).exists():
        print ("Loading previous model " + params.model)
        model = Word2Vec.load(params.model)
    else:
        print("Creating new model and building vocab")
        model = Word2Vec(epochs=1, min_count=params.minTF, workers=params.workers, vector_size=params.size, window=params.window)
        if Path(params.vocab).exists():
            with open(params.vocab, 'r') as f:
                vocab_freq = json.load(f)
        else:
            print("Vocab file not found, building vocab")
            vocab_freq = train_sentences.build_vocab(output=params.vocab)
        model.build_vocab_from_freq(vocab_freq)
        print("Vocab generation done {c} words".format(c=len(model.wv.index_to_key)))
    print("Training model")
    model.train(train_sentences, total_examples=len(train_sentences), epochs=1)
    print("Saving model to " + params.model)
    model.save(params.model)
    return 0

if __name__ == "__main__":
    argparse = ArgumentParser()
    argparse.add_argument('--window', default=5, type=int, help='window size')
    argparse.add_argument('--size', default=50, type=int, help='vector size')
    argparse.add_argument('--shuffle', default=True, type=bool, help='shuffle order of tokens')
    argparse.add_argument('--sep', default=',', type=str, help='token seperator')
    argparse.add_argument('--input_file_extension', default='csv', type=str, help='input files to process')
    argparse.add_argument('--workers', default=4, type=int, help='number of processes')
    argparse.add_argument('--minTF', default=4, type=int, help='minimum term frequency')
    argparse.add_argument('--model', default=str(__dir__/'w2v.pickle'), type=str, help='model file')
    argparse.add_argument('--input', default=str(__dir__/'data'), type=str, help='dirs to learn from, comma separated')
    argparse.add_argument('--vocab', default=str(__dir__ / 'vocab.json'), type=str, help='vocab frequency file, in json format')
    params = argparse.parse_args()
    sys.exit(main(params))


import os
import json
from configparser import ConfigParser

class Configurations:

    def get_config(self):
        directory_path = os.getcwd()
        constants = ConfigParser()
        constants.read(directory_path + "/constants.ini")
        config = dict()

        config['aws_key_pub'] = constants.get("Aws_keys", "PUB")
        config['aws_key_review'] = constants.get("Aws_keys", "REVIEW")
        config['aws_key_station'] = constants.get("Aws_keys", "STATION")

        config['aws_prefix_pub'] = constants.get("Aws_prefixs", "PUB")
        config['aws_prefix_review'] = constants.get("Aws_prefixs", "REVIEW")
        config['aws_prefix_station'] = constants.get("Aws_prefixs", "STATION")

        config['model_pub'] = constants.get("Models", "PUB")
        config['model_review'] = constants.get("Models", "REVIEW")
        config['model_station'] = constants.get("Models", "STATION")

        config['column_photo'] = constants.get("Columns", "PHOTO")
        config['column_all'] = constants.get("Columns", "ALL")
        config['column_pub'] = constants.get("Columns", "PUB")
        config['column_review'] = constants.get("Columns", "REVIEW")
        config['column_score'] = constants.get("Columns", "SCORE")
        config['column_required'] = constants.get("Columns", "REQUIRED")

        config['visible_pub'] = constants.get("Visibles", "PUB")
        config['visible_review'] = constants.get("Visibles", "REVIEW")

        config['control_input'] = constants.get("Controls", "INPUT")
        config['control_dropdown'] = constants.get("Controls", "DROPDOWN")
        config['control_slider'] = constants.get("Controls", "SLIDER")

        config['colour_new'] = constants.get("Colours", "NEW")
        config['colour_review'] = constants.get("Colours", "REVIEW")

        # with open(os.getcwd() + '/config.json') as file:  # Opening JSON file
        #     config_file = json.load(file)  # returns JSON object as a dictionary
        #     config['google_key'] = config_file['configs']['local_key']
        #     config['access_id'] = config_file['configs']['access_id']
        #     config['access_key'] = config_file['configs']['access_key']
        #     config['bucket_name'] = config_file['configs']['bucket_name']
        # else:
        config['google_key'] = os.getenv("HEROKU_GOOGLE_API")
        config['access_id'] = os.environ.get("ACCESS_ID")
        config['access_key'] = os.environ.get("ACCESS_KEY")
        config['bucket_name'] = os.environ.get("BUCKET_NAME")
        return config


# IMPORTS #

import numpy as np
import matplotlib.pyplot as plt
from functools import reduce
from scipy.interpolate import interp1d
import SimpleITK as sitk
#from sklearn.linear_model import LinearRegression # for unused function

# MODULE IMPORTS #

from image import Image
from image3d import Image3d
from jsonReader import JsonReader

# INPUT FILE PARAMETER + other parameters #

json_file = "../json/default2.json"
display_images = True
display_graph = True
display_formulas = True
display_fingerprint_debug = True
save_images = False

# FUNCTIONS #

def MLRcoeffs_numpy():
    # calculates the multi-linear regression coefficients (r, g and b in the Dose equation)
    return np.linalg.lstsq(np.array([[nPVr[i], nPVg[i], nPVb[i]] for i in range(8)]), doses, rcond=None)[0]

def func_dose_to_cx_cal(val):
    arr = np.array([inter(val) for inter in i_funcs])
    return np.stack(arr, axis=-1)

def saveToTiff(srcImg, doseimg, filename):
    img = np.stack([doseimg, doseimg, doseimg], axis=-1)
    imagetif = sitk.GetImageFromArray(img.astype('uint16'), isVector=True)
    imagetif.CopyInformation(srcImg)
    writer = sitk.ImageFileWriter()
    writer.SetFileName(filename)
    writer.Execute(imagetif)

# UNUSED FUNCTIONS #

def MLRcoeffs_sklearn(): # /!\ uncomment the import to use the function
    reg = LinearRegression(fit_intercept=False)
    reg.fit(np.array([[nPVr[i], nPVg[i], nPVb[i]] for i in range(8)]), doses)
    return reg.coef_

# PROGRAM #

j = JsonReader()

params = j.getInfo(json_file)
variables = j.getVariables(json_file)
doses = params['doses'][::-1] # on inverse l'array de doses pour qu'il soit dans l'ordre croissant

img3d = Image3d(params['files'], params['path'])
img2d = img3d.get_2dImage()
arr = img2d.get_array()

# Initialisation des zones d'interet
zoi = [] # all of the strips in increasing dose value order(the first one being the unirradiated one)
rois = variables["ROIs"] # les régions d'intérêt ont été "calculées" à la main et sont indiquées dans le champ "ROIs" du fichier default2.json
for e in rois:
    zoi.append(arr[e[0]:e[1], e[2]:e[3], :])

print('\n---| Dose calculation part |---\n')

# Calcul de la valeur moyenne des pixels par bande
strips = []
for i in range(len(zoi)):
    strips.append((np.mean(zoi[i][:,:,0]), np.mean(zoi[i][:,:,1]), np.mean(zoi[i][:,:,2])))

# Valeurs de la bande de controle
base = (np.mean(zoi[0][:,:,0]), np.mean(zoi[0][:,:,1]), np.mean(zoi[0][:,:,2]))

# Calcul des nPVr, nPVg et nPVb
nPVr = []
nPVg = []
nPVb = []
for i in range(len(zoi)):
    nPVr.append(base[0]/strips[i][0] - 1)
    nPVg.append(base[1]/strips[i][1] - 1)
    nPVb.append(base[2]/strips[i][2] - 1)

# Calcul des nPVrgb
nPVrgb = []
coef = MLRcoeffs_numpy()
for i in range(8):
    nPVrgb.append(coef[0] * nPVr[i] + coef[1] * nPVg[i] + coef[2] * nPVb[i])
    if display_formulas:
        print(i, ') nPVrgb = r *', nPVr[i], '+ g *', nPVg[i], '+ b *', nPVb[i], '=', nPVrgb[i])

# Calcul de la dose avec le decay factor
d_lin = variables["d_lin"]
Dose = [(d_lin * e) for e in nPVrgb]
print('\nCalculated dose values :', Dose)
print('\nr, g and b coefficients :', coef)

arr[np.all(arr>0.75, axis=2)] = np.array(base) # making sure that the white background translates to a dose of zero

# Création de l'image de dose (voir équation (4) de la publication)
dose_arr = reduce(np.add, [coef[0] * (base[0]/arr[:,:,0] - 1), coef[1] * (base[1]/arr[:,:,1] - 1), coef[2] * (base[2]/arr[:,:,2] - 1)])
#dose_arr[np.all(dose_arr==0, axis=1)] =-1 # changes all the backgroud values to -1
#dose_arr[dose_arr>1250] = 0 # lowers the maximum dose value of the image (only present on the marker written notes)

if display_images:
    plt.figure("Dose map")
    plt.imshow(dose_arr, origin='lower')

if save_images:
    saveToTiff(img2d.get_image(), dose_arr, '../../DoseMap.tiff')

normal_denom = reduce(np.add, [arr[:, :, 0], arr[:, :, 1], arr[:, :, 2]]) # corresponds to (PVr_meas + PVg_meas + PVb_meas)
legend_list = [(arr[:, :, 0], 'r', 'red channel'), (arr[:, :, 1], 'g', 'green channel'), (arr[:, :, 2], 'b', 'blue channel')]

# cX_meas calculations
cR_meas = legend_list[0][0]/normal_denom
cG_meas = legend_list[1][0]/normal_denom
cB_meas = legend_list[2][0]/normal_denom
cX_meas = np.stack((cR_meas, cG_meas, cB_meas), axis=-1)

if display_graph:
    fig, ax = plt.subplots(figsize=(10, 10))

strips = np.array(strips)
normal_denom2 = reduce(np.add, [strips[:, 0], strips[:, 1], strips[:, 2]])
legend_list2 = [(strips[:, 0], 'r', 'red channel'), (strips[:, 1], 'g', 'green channel'), (strips[:, 2], 'b', 'blue channel')]

# Création des fonctions d'interpolation pour déterminer les cX à partir d'une valeur de dose
i_funcs = []
for e in legend_list2:
    x = e[0]/normal_denom2

    # Création des fonctions d'interpolation
    inter_func = interp1d(Dose, x, fill_value="extrapolate")
    i_funcs.append(inter_func)

    # Plot
    #xb = np.arange(0, 850, 50)
    xb = doses
    if display_graph:
        #ax.plot(doses, x, e[1]+'*')
        ax.plot(xb, inter_func(xb), e[1], label=e[2]+'-')

if display_graph:
    ax.set_xlim(left=0)
    xliml, xlimr = ax.get_xlim()
    ax.set_xticks(np.arange(xliml, xlimr, 100))
    ax.set_yticks(np.arange(0, 1.1, 0.1))
    ax.legend()
    ax.set_xlabel('Absorbed Dose (cGy)')
    ax.set_ylabel('Color (16 bits / channel)')

if display_images or display_graph:
    plt.show()

print('\n---| Fingerprint correction part |---\n')

corrected_dose_arr = dose_arr

# Initialisation de tous les cX à 1
old_cX = np.ones(arr.shape)

# Iterations of the fingerprint correction
for i in range(1, 6):
    cX_cal = func_dose_to_cx_cal(corrected_dose_arr)

    cX_tot = cX_meas / cX_cal # équation (12)
    cX_tot = 1 / cX_tot # essai avec (cX_cal / cX_meas) à cause de résultats étranges

    corrected_dose_arr = reduce(np.add, [cX_tot[:,:,0] * coef[0] * (base[0]/arr[:,:,0] - 1), cX_tot[:,:,1] * coef[1] * (base[1]/arr[:,:,1] - 1), cX_tot[:,:,2] * coef[2] * (base[2]/arr[:,:,2] - 1)])
    if display_images:
        plt.figure(f"Dose map w/ fingerprint v{i}")
        plt.imshow(corrected_dose_arr, origin='lower')
    if save_images:
        saveToTiff(img2d.get_image(), corrected_dose_arr, f"../../FingerprintDose_{i}th_iteration.tiff")

    # Calcul de la valeur moyenne des pixels par bande et autres informations sur les cX
    zoi2 = []
    for e in rois:
        zoi2.append(corrected_dose_arr[e[0]:e[1], e[2]:e[3]])
    strips2 = []
    for j in range(len(zoi2)):
        strips2.append(np.mean(zoi2[j]))

    if display_fingerprint_debug:
        print('\n', i, '-----------------\nstrips average values :', strips2, '\naverage cX amongst array :', np.mean(cX_tot), '\nminimum cX :', np.min(cX_tot), '\nmaximum cX :', np.max(cX_tot), '\nmean cX diff :', np.mean(cX_tot/old_cX))

    old_cX = cX_tot

if display_images:
    plt.show()


'''
Created on 28.12.2009

@author: KMihajlov
'''

from google.appengine.ext import db
import Models.BaseModels as bm
from datetime import datetime as dt
class Language(db.Model):
    DateCreated = db.DateTimeProperty(required=True);
    Creator = db.ReferenceProperty(bm.Person, required=True, collection_name='creator_languages')
    LanguageName = db.StringProperty(required=True)
    LanguageCode = db.StringProperty(required=True)

    ## Static Methods
    @classmethod
    def Get(cls, code):
        return cls.gql("WHERE LanguageCode =:c", c=code).get()
    @classmethod
    def GetAll(cls):
        return cls.all().fetch(100)
    @classmethod
    def CreateNew(cls, creator, lang_name, lang_code, _autoInsert=False):
        result = cls(
                        DateCreated=dt.now(),
                        Creator=creator,
                        LanguageName=lang_name,
                        LanguageCode=lang_code
                        )
        if _autoInsert:
            result.put()
        return result

class PrimaryLanguage(db.Model):
    '''
    Primary Language used for translation
    This language must be most complete of all
    Possibly to use some web service in order to retrieve it's items instead of storing them in datastore
    '''
    DateModified = db.DateTimeProperty(required=True)
    Language = db.ReferenceProperty(Language, required=True, collection_name='language_primary_language')
    @classmethod
    def Change(cls, new_language):
        if new_language:
            current = cls.all().get()
            if current:
                current.Language = new_language
                current.DateModified = dt.now()
            else:
                current = cls(DateModified=dt.now(), Language=new_language)
            current.put()
            return current
        else:
            raise('None returned as a parameter for the new language')
    @classmethod
    def GetCurrent(cls):
        return cls.all().get()
class VirtualLanguageItem(db.Model):
    DateCreated = db.DateTimeProperty(required=True);
    Creator = db.ReferenceProperty(bm.Person, required=True, collection_name='creator_virtual_language_items')
    Code = db.StringProperty(required=True)
    @classmethod
    def CreateNew(cls, creator, code, _autoInsert=True):
        result = cls(
                    DateCreated=dt.now(),
                    Creator=creator,
                    Code = code
                    )
        if _autoInsert:
            result.put()
        return result

    @classmethod
    def Get(cls, code):
        return cls.gql("WHERE Code =:c", c=code).get()

class DictionaryItem(db.Model):
    DateCreated = db.DateTimeProperty(required=True);
    Creator = db.ReferenceProperty(bm.Person, collection_name='creator_dictionary_items')
    Language = db.ReferenceProperty(Language, required=True, collection_name='language_dictionary_items')
    VItem = db.ReferenceProperty(VirtualLanguageItem, required=True, collection_name='vitem_dictionary_items')
    Value = db.StringProperty(required=True)
    Description = db.StringProperty(required=True)

    ## Static methods
    @classmethod
    def GetList(cls, language, word, limit=100, offset=0):
        '''List of all words with same value but different meaning'''
        return cls.gql("WHERE Language =:l AND Value =:v", l=language, v=word).fetch(limit, offset)
    @classmethod
    def GetAllByVItemLang(cls, vItem, language, limit=5,  offset=0):
        '''Get the list of synonyms that has the meaning of the virtual item given'''
        return vItem.vitem_dictionary_items.all().filter('Language =', language).fetch(limit=limit, offset=offset)
    @classmethod
    def GetFirstByVItemLang(cls, vItem, language):
        '''Get the first word that has the meaning of the virtual item given'''
        return vItem.vitem_dictionary_items.all().filter('Language =', language).get()
    @classmethod
    def GetByVItemByLangByValue(cls, vItem, language, value):
        '''Get the first word that has the meaning of the virtual item given'''
        return vItem.vitem_dictionary_items.all().filter('Language=', language).filter('Value=', value).get()
    @classmethod
    def CreateNew(cls, creator, language, vItem, value, description, _autoInsert=False):
        if not cls.DictionaryItem.GetByVItemByLangByValue(vItem, language, value):
            result = cls(
                            DateCreated=dt.now(),
                            Creator=creator,
                            Language=language,
                            VItem=vItem,
                            Value=value,
                            Description=description
                            )
            if _autoInsert:
                result.put()
            return result
        else:
            return None

    #Methods
    def TranslateList(self, language, limit=10, offset=0):
        '''
        Translate the word to another language if possible
        Return the list of all synonyms.
        '''
        return DictionaryItem.gql("WHERE VItem =:v AND Language =:l", v=self.VItem, l=language).fetch(limit=limit, offset=offset)


# -*- coding: utf-8 -*-
from django.contrib import admin

from .models import Pedido, ProductosPedido, OrdenTrabajo

class ProductoInline(admin.TabularInline):
    """Tabular Inline View for Producto"""

    model = ProductosPedido
    min_num = 1
    max_num = 20
    extra = 1
    exclude = ("is_removed",)

@admin.register(Pedido)
class PedidoAdmin(admin.ModelAdmin):
    def get_queryset(self, request):
        return Pedido.all_objects.all()

    list_display = (
        "id",
        "usuario",
        "cliente",
        "precio_total",
        "detalles",
        "estado",
        "pagado",
        "fecha_entrega",
        "created",
        "modified",
        "is_removed",
    )
    list_filter = ("cliente", "fecha_entrega")
    autocomplete_fields = ["productos_pedido"]

    inlines = [ProductoInline]

@admin.register(ProductosPedido)
class ProductosPedidoAdmin(admin.ModelAdmin):
    def get_queryset(self, request):
        return ProductosPedido.all_objects.all()

    list_display = (
        "id",
        "created",
        "modified",
        "is_removed",
        "pedido",
        "producto",
        "cantidad",
    )
    list_filter = ("created", "modified", "is_removed", "pedido", "producto")

@admin.register(OrdenTrabajo)
class OrdenTrabajoAdmin(admin.ModelAdmin):
    def get_queryset(self, request):
        return OrdenTrabajo.all_objects.all()

    list_display = (
        "id",
        "pedido",
        "estado",
        "detalles",
        "created",
        "modified",
        "is_removed",
    )
    list_filter = (
        "created",
        "modified",
        "is_removed",
        "pedido",
    )


from typing import FrozenSet, Optional, Set

from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name

from pipenv.patched.pip._internal.exceptions import CommandError

class FormatControl:
    """Helper for managing formats from which a package can be installed."""

    __slots__ = ["no_binary", "only_binary"]

    def __init__(
        self,
        no_binary: Optional[Set[str]] = None,
        only_binary: Optional[Set[str]] = None,
    ) -> None:
        if no_binary is None:
            no_binary = set()
        if only_binary is None:
            only_binary = set()

        self.no_binary = no_binary
        self.only_binary = only_binary

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, self.__class__):
            return NotImplemented

        if self.__slots__ != other.__slots__:
            return False

        return all(getattr(self, k) == getattr(other, k) for k in self.__slots__)

    def __repr__(self) -> str:
        return "{}({}, {})".format(
            self.__class__.__name__, self.no_binary, self.only_binary
        )

    @staticmethod
    def handle_mutual_excludes(value: str, target: Set[str], other: Set[str]) -> None:
        if value.startswith("-"):
            raise CommandError(
                "--no-binary / --only-binary option requires 1 argument."
            )
        new = value.split(",")
        while ":all:" in new:
            other.clear()
            target.clear()
            target.add(":all:")
            del new[: new.index(":all:") + 1]
            # Without a none, we want to discard everything as :all: covers it
            if ":none:" not in new:
                return
        for name in new:
            if name == ":none:":
                target.clear()
                continue
            name = canonicalize_name(name)
            other.discard(name)
            target.add(name)

    def get_allowed_formats(self, canonical_name: str) -> FrozenSet[str]:
        result = {"binary", "source"}
        if canonical_name in self.only_binary:
            result.discard("source")
        elif canonical_name in self.no_binary:
            result.discard("binary")
        elif ":all:" in self.only_binary:
            result.discard("source")
        elif ":all:" in self.no_binary:
            result.discard("binary")
        return frozenset(result)

    def disallow_binaries(self) -> None:
        self.handle_mutual_excludes(
            ":all:",
            self.no_binary,
            self.only_binary,
        )


import typing
import itertools

import numpy as np
import torch
from rdkit import Chem

from graph_neural_networks.sparse_pattern import graph_as_adj_list
from graph_neural_networks.core import nd_ten_ops
from graph_neural_networks.core import utils

from .rdkit_ops import rdkit_reaction_ops
from .rdkit_ops import rdkit_general_ops
from .rdkit_ops import rdkit_featurization_ops
from .rdkit_ops import chem_details
from . import lef_consistency_errors
from . import action_list_funcs
from . import mask_creators
from . import graph_ds
from ..misc import data_types

"""
 ==== Datastructures from the different transforms ====
"""

class BrokenDownParts(typing.NamedTuple):
    reactants: str  # AM-SMILES
    reagents: str  # AM-SMILES
    products: str  # AM-SMILES
    ordered_am_path: typing.List[int]  # should include self-bond break at start if applicable by repeated action.

class RdKitIntermediates(typing.NamedTuple):
    intermediates: typing.List[Chem.Mol]
    reagents: Chem.Mol
    ordered_indx_path: typing.List[int]
    am_to_idx_map: typing.Mapping[int, int]

class ElectroTrainInput:
    def __init__(self,
            graphs: graph_as_adj_list.GraphAsAdjList,
            reagents: graph_as_adj_list.GraphAsAdjList,

            initial_input: graph_ds.ActionSelectorGraphIds,
            remove_input: graph_ds.ActionSelectorGraphIds,
            add_input: graph_ds.ActionSelectorGraphIds,

            initial_target: graph_ds.ActionSelectorTarget,
            remove_target: graph_ds.ActionSelectorTarget,
            add_target: graph_ds.ActionSelectorTarget,

            stop_label: nd_ten_ops.Nd_Ten,
            stop_mask: nd_ten_ops.Nd_Ten,

            num_reactions:int):
        self.graphs = graphs
        self.reagents = reagents

        self.initial_input = initial_input
        self.remove_input = remove_input
        self.add_input = add_input

        self.initial_target = initial_target
        self.remove_target = remove_target
        self.add_target = add_target

        self.stop_label = stop_label
        self.stop_mask = stop_mask
        self.num_reactions = num_reactions

    def to_torch(self, cuda_details: utils.CudaDetails):
        self.graphs = self.graphs.to_torch(cuda_details)
        self.reagents = self.reagents.to_torch(cuda_details)
        def func(op_arr: typing.Optional[np.ndarray]):
            if op_arr is not None:
                if op_arr.dtype == np.bool_:
                    op_arr = op_arr.astype(np.uint8)  # torch has no bool_ type.
                return torch.from_numpy(op_arr).to(cuda_details.device_str)
            else:
                return None

        self._over_nd_ten_fields(func)
        return self

    def _over_nd_ten_fields(self, func):
        for inp_ in [self.initial_input, self.remove_input, self.add_input]:
            inp_.graphs_ids = func(inp_.graphs_ids)
            inp_.reagent_context_ids = func(inp_.reagent_context_ids)
            inp_.prev_action_per_graph = func(inp_.prev_action_per_graph)

        for inp_ in [self.initial_target, self.remove_target, self.add_target]:
            inp_.action_mask = func(inp_.action_mask)
            inp_.label_per_graph = func(inp_.label_per_graph)

        self.stop_label = func(self.stop_label)
        self.stop_mask = func(self.stop_mask)

"""
 ==== Transforms between the different datastructures ====
"""

class TransformStrToBrokenDownParts:
    """
    (A) Creates path and assigns order see Fig 3 of the paper.
    (B) Breaks out the reagents,

    It runs the following consistency checks:
    C1. The atoms can be lined up end to end on the path
    """
    def __call__(self, input_: typing.Tuple[str, str, str]) -> BrokenDownParts:
        reactants, products, bond_changes = input_

        # A.2 We first create the path (checks for C1.)
        change_list = bond_changes.split(';')
        atom_pairs = np.array([c.split('-') for c in change_list]).astype(int)
        unordered_electron_path_am = action_list_funcs.actions_am_from_pairs(atom_pairs, consistency_check=True)

        # B Using these actions we can now work out what is a reagents and split these out.
        reactants, reagents, products = rdkit_reaction_ops.split_reagents_out_from_reactants_and_products(
            reactants, products, unordered_electron_path_am)

        # A.3 We can now order the electron path.
        reactant_mol = rdkit_general_ops.get_molecule(reactants, kekulize=False)
        product_mol = rdkit_general_ops.get_molecule(products, kekulize=False)
        ordered_electon_path_am = action_list_funcs.order_actions_am(unordered_electron_path_am, reactant_mol, product_mol)

        # A.3b Work out whether to add add a self-bond remove at the start
        # (we need to start with a remove action to pick up a pair of electrons)
        first_bond_in_reactants = rdkit_general_ops.get_bond_double_between_atom_mapped_atoms(reactant_mol, ordered_electon_path_am[0],
                                                                                ordered_electon_path_am[1])
        first_bond_in_products = rdkit_general_ops.get_bond_double_between_atom_mapped_atoms(product_mol, ordered_electon_path_am[0],
                                                                                ordered_electon_path_am[1])
        starts_already_with_remove_bond = first_bond_in_reactants - first_bond_in_products > 0
        if not starts_already_with_remove_bond:
            ordered_electon_path_am = [ordered_electon_path_am[0]] + ordered_electon_path_am

        # We can now create the o/p data-structure
        op = BrokenDownParts(reactants, reagents, products, ordered_electon_path_am)
        return op

class TransformToRdKitIntermediates:
    """
    This creates a series of intermediate molecules showing the molecule state at each stage of the reaction path.
    Effectively this is panel 4 of Fig3 of the the paper.

    It runs the following consistency checks:
    C2. The bond differences between the reactants and products along the path vary by +- one
    C3. The final molecule created by editing the reactants should be consistent with reported product.
         However, it can contain more information, eg minor products.
        we therefore check consistency of "super molecule" formed by changing actions versus the "sub molecule",
         which is the product given in the dataset. The super molecule should have disconnected extra molecules but
         the major product should show up in it.
    """
    def __call__(self, reaction_parts: BrokenDownParts) -> RdKitIntermediates:
        # 1. Work out the number of add/remove steps.
        # We always start on a remove bond (can be remove self bond though) and go in intermediate add/remove steps)
        # Therefore the add remove steps is as follows:
        action_types = [bond_change for bond_change, _ in
                 zip(
                     itertools.cycle([chem_details.ElectronMode.REMOVE, chem_details.ElectronMode.ADD]),
                     range(len(reaction_parts.ordered_am_path) -1)
                 )]

        # 2. Form the intermediate states
        reactant_mol = rdkit_general_ops.get_molecule(reaction_parts.reactants, kekulize=True)
        reactant_atom_map_to_idx_map = rdkit_general_ops.create_atom_map_indcs_map(reactant_mol)
        intermediates = [reactant_mol, reactant_mol]  # twice as after picked up half a pair nothing has happened.
        action_pairs = zip(reaction_parts.ordered_am_path[:-1], reaction_parts.ordered_am_path[1:])
        for step_mode, (start_atom_am, next_atom_am) in zip(action_types, action_pairs):
            # Nb note we do not change any Hydrogen numbers on the first and last step -- these may change in practice
            # eg gaining H from water in solution, however, we do not represent Hydrogens in our graph structure,
            # apart from in the features.
            prev_mol = intermediates[-1]
            start_atom_idx = reactant_atom_map_to_idx_map[start_atom_am]
            next_atom_idx = reactant_atom_map_to_idx_map[next_atom_am]
            if start_atom_idx == next_atom_idx:
                # then a self bond removal:
                intermed = rdkit_reaction_ops.change_mol_atom(prev_mol, step_mode, start_atom_idx)
            else:
                intermed = rdkit_reaction_ops.change_mol_bond(prev_mol, step_mode, (start_atom_idx, next_atom_idx))
            intermediates.append(intermed)

        # 3. Form the reagent molecule
        reagent_mol = rdkit_general_ops.get_molecule(reaction_parts.reagents, kekulize=True)

        # 4. Consistency checks.
        product_mol = rdkit_general_ops.get_molecule(reaction_parts.products)
        # we first check C2:
        if not rdkit_reaction_ops.is_it_alternating_add_and_remove_steps(reactant_mol, product_mol,
                                                                         reaction_parts.ordered_am_path):
            raise lef_consistency_errors.NotAddingAndRemovingError

        if not rdkit_reaction_ops.is_sub_mol_consistent_with_super(product_mol, intermediates[-1]):
            raise lef_consistency_errors.InconsistentActionError(f"Inconsistent action error for molecule:"
                                                                 f" {[str(part) for part in reaction_parts]}")

        # 5. Switch ordered path to refer to atoms indices rather than atom mapped number.
        ordered_index_path = [reactant_atom_map_to_idx_map[am] for am in reaction_parts.ordered_am_path]

        # 6. Create the final output
        op = RdKitIntermediates(intermediates, reagent_mol, ordered_index_path, reactant_atom_map_to_idx_map)
        return op

class TransformRdKitIntermediatesToElectroTrainInput:
    """
    Takes the intermediate molecules and produces the inputs that we need for training ELECTRO.

    This consists of:
    1. Converting all intermediates to graphs that are easy to run through a GNN.
    2. Converting the reagents to a separate graph (or setting it as None if we choose to bin them)
    3. Get the graphids of initial, remove and add actions
    4. Compute the masks for stop and the atom actions.
    5. Create the targets for stop and atom actions.
    6. Split these different actions up (putting into datastructures)
    """
    def __init__(self, exclude_reagents_from_initial_flag):
        self.exclude_reagents_from_initial_flag = exclude_reagents_from_initial_flag
        self.atom_feat_params = rdkit_featurization_ops.AtomFeatParams()

        # mask creators
        self.initial_mask = mask_creators.InitialActionMaskCreator('cpu')
        self.add_mask = mask_creators.AddMaskCreator('cpu')
        self.remove_mask = mask_creators.RemoveMaskCreator('cpu')
        self.stop_mask = mask_creators.StopMaskCreator('cpu')

    def __call__(self, reaction_rdkit_mols: RdKitIntermediates) -> ElectroTrainInput:
        # 1. Convert all the intermediates to graphs
        mol_list = [mol for mol in reaction_rdkit_mols.intermediates]
        graphs_uncat = [rdkit_featurization_ops.mol_to_atom_feats_and_adjacency_list(mol, reaction_rdkit_mols.am_to_idx_map,
                                                                         params=self.atom_feat_params)
            for mol in reaction_rdkit_mols.intermediates]
        graphs = graphs_uncat[0].concatenate(graphs_uncat)

        # 2. Convert reagent to graph
        if rdkit_general_ops.get_num_atoms(reaction_rdkit_mols.reagents) == 0:
            reagent_graph = rdkit_featurization_ops.create_empty_graph(self.atom_feat_params)
        else:
            reagent_graph = rdkit_featurization_ops.mol_to_atom_feats_and_adjacency_list(reaction_rdkit_mols.reagents,
                                                                     params=self.atom_feat_params)
        reagent_graph_id = np.array([0])
        init_reagent_info = None if self.exclude_reagents_from_initial_flag else reagent_graph_id

        # 3.a Get the graphids of initial, remove and add actions
        initial_graphid = np.array([0])
        assert graphs.max_num_graphs == len(reaction_rdkit_mols.ordered_indx_path) + 1
        number_of_graphs_excluding_one_we_should_stop_on = graphs.max_num_graphs-1
        remove_graphids = np.arange(1, number_of_graphs_excluding_one_we_should_stop_on, 2)
        add_graphids = np.arange(2, number_of_graphs_excluding_one_we_should_stop_on, 2)

        # 3.b Get the previous atom for remove and add
        prev_action_idx_remove = np.array([reaction_rdkit_mols.ordered_indx_path[i-1] for i in remove_graphids], dtype=np.int64)
        prev_action_idx_add = np.array([reaction_rdkit_mols.ordered_indx_path[i-1] for i in add_graphids], dtype=np.int64)

        # 4. Compute the masks
        initial_mask = np.concatenate([self.initial_mask(mol_list[i]) for i in initial_graphid])
        remove_mask = np.concatenate([self.remove_mask(mol_list[i], int(reaction_rdkit_mols.ordered_indx_path[i-1]),
                                                   first_step_flag=i==1) for i in remove_graphids])
        add_mask = np.concatenate([self.add_mask(mol_list[i], reaction_rdkit_mols.ordered_indx_path[i - 1])
                              for i in add_graphids]) if len(add_graphids) else np.array([], dtype=np.bool_)
        stop_mask = self.stop_mask.all_time_steps_mask(action_list_ordered=reaction_rdkit_mols.ordered_indx_path)

        # 5. Compute the targets
        initial_target = np.array([reaction_rdkit_mols.ordered_indx_path[0]], dtype=data_types.INT)
        remove_target = np.array([reaction_rdkit_mols.ordered_indx_path[i] for i in remove_graphids], dtype=data_types.INT)
        add_target = np.array([reaction_rdkit_mols.ordered_indx_path[i] for i in add_graphids], dtype=data_types.INT)
        stop_target = np.zeros_like(stop_mask, dtype=data_types.INT)
        stop_target[-1] = 1  # we want to stop at the end!

        # 6. Put into the required datastructures.
        op = ElectroTrainInput(graphs, reagent_graph,
                                   initial_input=graph_ds.ActionSelectorGraphIds(initial_graphid, None, init_reagent_info),
                                   remove_input=graph_ds.ActionSelectorGraphIds(remove_graphids, prev_action_idx_remove, None),
                                   add_input=graph_ds.ActionSelectorGraphIds(add_graphids, prev_action_idx_add, None),
                                   initial_target=graph_ds.ActionSelectorTarget(initial_target, initial_mask),
                                   remove_target=graph_ds.ActionSelectorTarget(remove_target, remove_mask),
                                   add_target=graph_ds.ActionSelectorTarget(add_target, add_mask),
                                   stop_label=stop_target,
                                   stop_mask=stop_mask,
                                   num_reactions=1
                               )
        return op

class ElectroBatchCollate:
    """
    Takes the individual training items spat out by the datasets and collates them together eg collate_fn arg of
     dataloader.
    """
    def __call__(self, inp: typing.Iterable[ElectroTrainInput]) -> ElectroTrainInput:
        # Graphs
        all_graphs = [elem.graphs for elem in inp]
        num_graphs_per_inp = [elem.max_num_graphs for elem in all_graphs]
        graph_offset = [0] + np.cumsum(num_graphs_per_inp[:-1]).tolist()
        new_graphs = all_graphs[0].concatenate(all_graphs)

        # Reagents
        all_reagents = [elem.reagents for elem in inp]
        reagent_sizes = set([elem.max_num_graphs if elem is not None else 0 for elem in all_reagents])
        set_elem = reagent_sizes.pop()
        assert len(reagent_sizes) == 0 and set_elem == 1, "not all graphs have one reagent"
        new_reagents = all_reagents[0].concatenate(all_reagents)

        # New action inputs
        new_initial_input = self.concatenate_action_selectors([elem.initial_input for elem in inp], graph_offset)
        new_remove_input = self.concatenate_action_selectors([elem.remove_input for elem in inp], graph_offset)
        new_add_input = self.concatenate_action_selectors([elem.add_input for elem in inp], graph_offset)

        # New action targets
        new_initial_targets = self.concatenate_action_targets([elem.initial_target for elem in inp])
        new_remove_targets = self.concatenate_action_targets([elem.remove_target for elem in inp])
        new_add_targets = self.concatenate_action_targets([elem.add_target for elem in inp])

        # Stop targets
        new_stop_labels = np.concatenate([elem.stop_label for elem in inp])
        new_stop_mask = np.concatenate([elem.stop_mask for elem in inp])

        # num reactions
        num_reactions = sum([elem.num_reactions for elem in inp])

        # Pack back together
        op = ElectroTrainInput(graphs=new_graphs, reagents=new_reagents, initial_input=new_initial_input, remove_input=new_remove_input,
                          add_input=new_add_input, initial_target=new_initial_targets, remove_target=new_remove_targets,
                          add_target=new_add_targets, stop_label=new_stop_labels, stop_mask=new_stop_mask,
                               num_reactions=num_reactions)
        return op

    @staticmethod
    def concatenate_action_targets(targets: typing.List[graph_ds.ActionSelectorTarget]):
        # Labels
        new_labels = np.concatenate([elem.label_per_graph for elem in targets])

        # Masks
        new_masks = np.concatenate([elem.action_mask for elem in targets])

        # Put back together
        op = graph_ds.ActionSelectorTarget(new_labels, new_masks)
        return op

    @staticmethod
    def concatenate_action_selectors(action_selectors: typing.List[graph_ds.ActionSelectorGraphIds],
                                     graph_new_starting_locations: typing.List[int]):
        """
        Takes a list of  graph_ds.ActionSelectorGraphIds and creates a new one, taking into account that the
        indexing of the graphs and reagents has changed as we have concatenated them all together.
        """

        # Graph ids
        new_graphs_ids = np.concatenate([old_graph_id + offset for old_graph_id, offset
                          in zip([elem.graphs_ids for elem in action_selectors], graph_new_starting_locations)])

        # Actions
        old_actions = [elem.prev_action_per_graph for elem in action_selectors]
        if all([elem is None for elem in old_actions]):
            new_actions = None
        else:
            assert not any([elem is None for elem in old_actions])
            new_actions = np.concatenate(old_actions)

        # Reagents
        old_reagent_ids = [elem.reagent_context_ids for elem in action_selectors]
        if all([elem is None for elem in old_reagent_ids]):
            new_reagents = None
        elif all([elem == 0 for elem in old_reagent_ids]):
            new_reagents = np.arange(len(old_reagent_ids))
        else:
            raise RuntimeError("should only be one or None reagent per reaction with index 0")

        return graph_ds.ActionSelectorGraphIds(new_graphs_ids, new_actions, new_reagents)


####### Use /default/matplotlibrc and /default/header-matplotlib.tex
import os
import pathlib
os.environ['MATPLOTLIBRC'] = (pathlib.Path(__file__).absolute().parent.parent.parent / 'default' / 'matplotlibrc').__str__()
os.environ['TEXINPUTS'] =  (pathlib.Path(__file__).absolute().parent.parent.parent / 'default').__str__() + ':'
#######
import numpy as np
from uncertainties import (ufloat,ufloat_fromstr,unumpy as unp)
from uncertainties.unumpy import (nominal_values as noms, std_devs as stds)
from uncertainties.umath import *
import json
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

# Messdaten einlesen
t,N_mess = np.genfromtxt('data/Rhodium.dat',delimiter=',',unpack=True)

# Untergrundrate einlesen
Ergebnisse = json.load(open('data/Ergebnisse.json','r'))
N_U = ufloat_fromstr(Ergebnisse['Untergrundrate[Imp/15s]'])

# Messkonstanten
dt = 15 #s Integrationszeit

# Umrechnungen
N = unp.uarray(N_mess,np.sqrt(N_mess))
N = N - N_U # Imp/15s Untergrund abgezogen
lnN = unp.log(N)

###################################
## Langlebig
# Ausgleichsrechnung langlebig
def lnN_fit(t,a,b):
    return a*t+b
i_l = np.where(t == 300)[0][0]
i_l = 17
params_l,pcov_l = curve_fit(lnN_fit,t[i_l:],noms(lnN[i_l:]),sigma=stds(lnN[i_l:]))
a_l = ufloat(params_l[0],np.absolute(pcov_l[0][0])**0.5) # a=-lambda
b_l = ufloat(params_l[1],np.absolute(pcov_l[1][1])**0.5)

# Halbwertszeit langlebig berechnen
T_l = unp.log(2)/(-1*a_l)
# Zeit in Minuten und Sekunden Umrechnen
T_l_min = np.floor(T_l.n/60)
T_l_sec = (T_l/60 - T_l_min)*60
####################################

############################
## Kurzlebig
i_k = np.where(t == 210)[0][0]+1
i_k = 14
print(i_k)
# Messdaten korrigieren
lnN_k = unp.log(N[:i_k] - unp.exp(lnN_fit(t[:i_k],a_l,b_l)))

# Ausgleichsgerade
params_k,pcov_k = curve_fit(lnN_fit,t[:i_k],noms(lnN_k),sigma=stds(lnN_k))
a_k = ufloat(params_k[0],np.absolute(pcov_k[0][0])**0.5) # a=-lambda
b_k = ufloat(params_k[1],np.absolute(pcov_k[1][1])**0.5)

# Halbwertszeit kurzlebig berechnen
T_k = unp.log(2)/(-1*a_k)
# Zeit in Minuten und Sekunden Umrechnen
T_k_min = np.floor(T_k.n/60)
T_k_sec = (T_k/60 - T_k_min)*60
############################

# Ergebnisse speichern JSON
if not 'Rhodium' in Ergebnisse:
    Ergebnisse['Rhodium'] = {}
Ergebnisse['Rhodium']['a_l'] = '{}'.format(a_l)
Ergebnisse['Rhodium']['b_l'] = '{}'.format(b_l)
Ergebnisse['Rhodium']['T_l[s]'] = '{:3.1f}'.format(T_l)
Ergebnisse['Rhodium']['T_l[min]'] = '{} min + '.format(T_l_min) + '({:2.0f}) sec'.format(T_l_sec)
Ergebnisse['Rhodium']['a_k'] = '{}'.format(a_k)
Ergebnisse['Rhodium']['b_k'] = '{}'.format(b_k)
Ergebnisse['Rhodium']['T_k[s]'] = '{}'.format(T_k)
Ergebnisse['Rhodium']['T_k[min]'] = '{} min + '.format(T_k_min) + '({:2.0f}) sec'.format(T_k_sec)
json.dump(Ergebnisse,open('data/Ergebnisse.json','w'),indent=4)

# Messdaten mit Fehler in Tabelle speichern
data = list(zip(t,N_mess,np.sqrt(N_mess)))
np.savetxt('data/Rhodium_mit_Fehler.csv', data, header='t[s],N[Imp/15s]', fmt='%i,%i+-%i')

################
## PLOTS
################

# Plot der Ausgleichsgerade langlebig
t_l_linspace = np.linspace(t[i_l],np.max(t))
plt.plot(t_l_linspace,lnN_fit(t_l_linspace,*params_l),'k-', label='Ausgleichsgerade (langlebig)')

# Plot der Ausgleichsgerade kurzlebig
t_k_linspace = np.linspace(np.min(t),t[i_k-1])
plt.plot(t_k_linspace,lnN_fit(t_k_linspace,*params_k),'k--', label='Ausgleichsgerade (kurzlebig)')

# Plot der korrigierten Messwerte
plt.errorbar(t[:i_k], noms(lnN_k), yerr=stds(lnN_k), fmt='bo', ecolor='cornflowerblue', elinewidth=1.5, label='korrigierte Messpunkte')

# Plot der Messwerte
plt.errorbar(t, noms(lnN), yerr=stds(lnN), fmt='ro', ecolor='lightcoral', elinewidth=1.5, label='Messpunkte')

# Achsenbeschriftung
plt.xlabel(r'$t \:/\: \si{\second}$')
plt.ylabel(r'$\ln(N-N_0)$')

# in matplotlibrc leider (noch) nicht möglich
plt.legend()
plt.tight_layout(pad=0, h_pad=1.08, w_pad=1.08)

# Speicherort
plt.savefig('build/plot_Rhodium.pdf')


import scrapy
import re

article_count_to_load = 5000
num_loads = 100 # 500k articles

base_url = "http://www.wikihow.com/index.php?title=Special:PopularPages&limit={}&offset={}"

class WikihowLinksSpider(scrapy.Spider):
	name = 'wikihow_links'
	start_urls = [base_url.format(article_count_to_load, i*article_count_to_load) for i in range(num_loads)]
	DOWNLOAD_DELAY = 10

	def clean_views(self, view_str):
		view_str = re.search('\(([0-9,]+) views\)', view_str)
		return int(view_str.group(1).replace(',', ''))

	def parse(self, response):
		for link in response.css('ol li'):
			yield {'title': link.css('a::text').extract_first(), 'href': "http://www.wikihow.com"+link.css('a::attr("href")').extract_first(), 'views': self.clean_views(link.xpath('string()').extract_first())}


#defaultdict: another take on missing keys
#collections.defaultdict
'''
if 'new-key' is not in dd(defaultdict) then the expression dd['new-key'] does the following steps:
1. calls list() to create a new list
2. inserts the list into dd using 'new-key' as key
3. returns s reference to that list
'''
import sys
import re
import collections

WORD_RE = re.compile('\w+')

index = collections.defaultdict(list)
with open('test', encoding='utf-8') as fp:
    for line_no, line in enumerate(fp, 1):
        for match in WORD_RE.finditer(line):
            word = match.group()
            column_no = match.start() + 1
            location = (line_no, column_no)
            index[word].append(location)

for word in sorted(index, key=str.upper):
    print(word, index[word])


"""The alarm device class."""
# -*- coding: utf-8 -*-

import logging

from .fritzhomedevicebase import FritzhomeDeviceBase
from .fritzhomedevicefeatures import FritzhomeDeviceFeatures

_LOGGER = logging.getLogger(__name__)

class FritzhomeDeviceAlarm(FritzhomeDeviceBase):
    """The Fritzhome Device class."""

    alert_state = None

    def _update_from_node(self, node):
        super()._update_from_node(node)
        if self.present is False:
            return

        if self.has_alarm:
            self._update_alarm_from_node(node)

    # Alarm
    @property
    def has_alarm(self):
        """Check if the device has alarm function."""
        return self._has_feature(FritzhomeDeviceFeatures.ALARM)

    def _update_alarm_from_node(self, node):
        val = node.find("alert")
        try:
            self.alert_state = self.get_node_value_as_int_as_bool(val, "state")
        except (Exception, ValueError):
            self.alert_state = None


"""
Source File Name: slotMachine.py
Author's Name: Craig Bonin
Last Modified: 6/1/2013
Last Modified By: Craig Bonin
*************************************
Program Description-
This program simulates a slot machine. It provides the user with a gui to interact with the slot machine

*****************************************
Version History
*****************************************
v0.1
-Started Assignment

"""

#Import and initialize pygame
import pygame

def main():
    pygame.init()

    #Setting up Graphics
    screen = pygame.display.set_mode((800, 600))
    pygame.display.set_caption("Craig's Slot Machine")
    background = pygame.surface(screen.getsize())
    background = background.convert()
    betFont = pygame.font.SysFont("arial", 12)

    #Variables used throughout game
    clock = pygame.time.Clock()
    keepGoing = True
    userMoney = 1000
    userBet = 10

    while keepGoing:
        clock.tick(30)

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                keepgoing = false

            screen.blit(background, (0,0))
            pygame.display.flip()


import matplotlib.pyplot as plt
import matplotlib.colors as mcolors

color_list = ["Green", "Blue"]
color_map = mcolors.ListedColormap(["Green", "Blue"])

import seaborn as sns
sns.set()
sns.set_palette("tab10")

import random as rnd
import numpy as np
rnd.seed(0)
np.random.seed(0)

import scipy.stats as stats
import bayes_logistic
from pypolyagamma import PyPolyaGamma

#
from config import parse_args
from utils import print_config

def run_program(config=None):

    # get params
    args = parse_args()
    if config:
        args.update(config)
    print_config(args)

    # generate data_x
    data_x = generate_data_x(args)
    # print(data_x.shape)
    # stop

    # generate prior and posterior samples
    samples_a_weights_prior, samples_b_weights_prior, samples_a_weights_posterior = \
        generate_prior_and_posterior_samples(data_x, args)
    # print(samples_a_weights_prior.shape)
    # print(samples_a_weights_prior[:5])
    # print(samples_b_weights_prior[:5])
    # print(samples_a_weights_posterior[:5])
    # stop

    # Visualize the generated prior and posterior samples
    nrows = 2
    fig, axes = plt.subplots(nrows=nrows, ncols=1, sharex=True, sharey=True, figsize=(10,10))
    axes = axes.flatten()

    for i in range(nrows):
        sns.kdeplot(samples_a_weights_prior[:,0], fill=False, color="blue", label="sample_a_prior", ax=axes[i])
        sns.kdeplot(samples_b_weights_prior[:,0], fill=False, color="green", label="sample_b_prior", ax=axes[i])
        sns.kdeplot(samples_a_weights_posterior[:,0], fill=False, color="orange", label="sample_a_posterior", ax=axes[i])
        axes[i].legend()
    plt.savefig(f"./figures/{args['test_name']}_indi")
    plt.show()

    # Visualize the generated prior and posterior samples
    fig, axes = plt.subplots(
        nrows=3, ncols=1, sharex=True, sharey=True, figsize=(10,10))
    # axes = axes.flatten()

    sns.kdeplot(x=samples_a_weights_prior[:,0], y=samples_a_weights_prior[:,1],
                n_levels=20, cmap="inferno", fill=False, cbar=True, ax=axes[0])

    sns.kdeplot(x=samples_b_weights_prior[:,0], y=samples_b_weights_prior[:,1],
                n_levels=20, cmap="inferno", fill=False, cbar=True, ax=axes[1])

    sns.kdeplot(x=samples_a_weights_posterior[:,0], y=samples_a_weights_posterior[:,1],
                n_levels=20, cmap="inferno", fill=False, cbar=True, ax=axes[2])
    axes[0].set_title("sample_a_prior")
    axes[1].set_title("sample_b_prior")
    axes[2].set_title("sample_a_posterior")
    plt.savefig(f"./figures/{args['test_name']}_pair")
    plt.show()

def generate_data_x(args):
    # Generate data_x
    num_data = args["num_data"]
    num_feats = args["num_feats"]
    data_x_type = args["data_x_type"]

    if data_x_type == "twopoints":
        data_x = np.array([0, 1]*num_data//2)
    elif data_x_type == "uniform":
        data_x = np.random.randn(num_data, num_feats)
    elif data_x_type == "multigauss":
        data_x = np.random.rand(num_data, num_feats)
        mu = args["data_x_marginal_params"][0][0]
        sigma = args["data_x_marginal_params"][0][1]
        data_x = stats.multivariate_normal.rvs(mu, sigma, seed=12345)
    elif data_x_type == "mixgauss":
        data_x_marginal_dists = [
            stats.multivariate_normal(mu, sigma, seed=12345) \
            for mu, sigma in args["data_x_marginal_params"]]  # fixed random seed

        c0_x = data_x_marginal_dists[0].rvs(size=(num_data))
        c1_x = data_x_marginal_dists[1].rvs(size=(num_data))
        data_x = np.vstack((c0_x, c1_x))
    else:
        stop

    print(data_x.shape)

    return data_x

def generate_prior_and_posterior_samples(data_x, args):

    # np.random.seed(0)

    # params
    num_feats = args["num_feats"]
    num_samples = args["num_samples"]
    pglogistic_burnin_steps = args["pglogistic_burnin_steps"]

    # weights' prior distribution
    weights_prior_params = args["weights_prior_params"]

    weights_prior_dist_a = stats.multivariate_normal(
        weights_prior_params[0], weights_prior_params[1], seed=args['seed'])

    weights_prior_dist_b = stats.multivariate_normal(
        weights_prior_params[0], weights_prior_params[1], seed=args['seed']+10)

    samples_a_weights_prior = []
    samples_b_weights_prior = []
    samples_a_weights_posterior = []

    pg_dist = PyPolyaGamma(seed=0)
    for i in range(num_samples):
        # sample two set of weights' priors
        sample_a_weights_prior = weights_prior_dist_a.rvs(1)[None,:]
        sample_b_weights_prior = weights_prior_dist_b.rvs(1)[None,:]
        samples_a_weights_prior.append(sample_a_weights_prior)
        samples_b_weights_prior.append(sample_b_weights_prior)

        # generate sample y_i from theta_i in A
        sample_a_logit = 1.0 / (1 + np.exp(
            -np.matmul(data_x, sample_a_weights_prior.T)))
        sample_a_y = stats.bernoulli.rvs(sample_a_logit)
        # print(sample_a_y.shape)

        # sample weights' posterior from polyagamma inference
        sample_a_weights_posterior = pg_inference(
            data_x, sample_a_y.squeeze(-1), weights_prior_params, pg_dist,
            burnin_steps=pglogistic_burnin_steps)
        samples_a_weights_posterior.append(sample_a_weights_posterior)

    samples_a_weights_prior = np.vstack(samples_a_weights_prior)
    samples_b_weights_prior = np.vstack(samples_b_weights_prior)
    samples_a_weights_posterior = np.vstack(samples_a_weights_posterior)
    # print(samples_a_weights_prior.shape)
    # print(samples_b_weights_prior)
    # print(samples_a_weights_posterior)
    return samples_a_weights_prior, samples_b_weights_prior, \
        samples_a_weights_posterior

def pg_inference(X, y, weights_prior_params, pg_dist, burnin_steps=200):
    #
    num_data, num_feats = X.shape

    # Gibbs sampling with PG augmentation for burnin_steps
    # init states
    beta_mu = np.array(weights_prior_params[0])
    beta_cov = np.array(weights_prior_params[1])
    # beta_cov = np.diag(np.ones(num_feats))
    beta_hat = np.random.multivariate_normal(beta_mu, beta_cov)
    k = y - 1/2

    # pg = PyPolyaGamma(seed=0)
    # perform Gibbs sampling
    for bid in range(burnin_steps+1):
        # ω ~ PG(b, c) = PG(1, x*β).
        omega_b = np.ones(num_data)
        omega_c = X @ beta_hat
        omega_diag = np.array(
            [pg_dist.pgdraw(b, c) for b, c in zip(omega_b, omega_c)])

        # β ~ N(m, V).
        V = np.linalg.inv(X.T @ np.diag(omega_diag) @ X + np.linalg.inv(beta_cov))
        m = np.dot(V, X.T @ k + np.linalg.inv(beta_cov) @ beta_mu)
        beta_hat = np.random.multivariate_normal(m, V)

    return beta_hat

if __name__ == '__main__':
    run_program()


'''百度搜素'''
import requests
url = "https://www.baidu.com/s"
try:
    kw = {
        'wd':'比亚迪'
    }
    r = requests.get(url,params=kw)
    ss = r.raise_for_status()
    print(r.request.url)
    print(r.text)
except:
    print('爬取失败')

print('分割线'+'*'*50)
url2 = "https://www.so.com/s"
try:
    kw = {
        'q':'比亚迪'
    }
    r = requests.get(url2,params=kw)
    ss = r.raise_for_status()
    print(r.request.url)
    print(r.text)
except:
    print('爬取失败')


import os
import random
from enum import Enum

from .dictionary import Dictionary
from .wordentry import WordEntry, DefinitionEntry

class Category(Enum):
    """Enum class representing the categories for HSK3.0.

    Attributes:
        title: The string name of the category.
        index: The integer representing the associated index, used by the HSK3 class.
    """

    ENTRY = ("普及化 - Entry Level", 0)
    INTERMEDIATE = ("中级 - Intermediate Level", 1)
    ADVANCED = ("高级 - Advanced Level", 2)
    SUPPLEMENTAL = ("附录 - Supplemental Vocab", 3)

    def __init__(self, title, index):
        self.title = title
        self.index = index

    @classmethod
    def get_by_index(cls, index):
        for c in cls:
            if c.index == index:
                return c

class HSK3:
    """HSK3.0 utility for post-2020, categorizing Chinese words by difficulty.

    Parses HSK2.0 file on initialization. Contains 4 categories: entry, intermediate, advanced, supplemental.

    Attributes:
        dictionary: Optional Dictionary instance. If not provided, a new Dictionary instance will be instantiated.
    """

    def __init__(self, dictionary: Dictionary = None):
        if dictionary is None:
            dictionary = Dictionary()
        self.category_list = []  # contains categories to lists of WordEntry
        self.word_to_category_map = {}  # for retrieving category of given word
        self.dictionary = dictionary
        self.parse()

    def parse(self):
        """Parses the HSK3.0 file.

        Called on initialization of HSK3 instance. For each category in HSK3.0, a new entry is added to `category_list`; each category contains a list of corresponding words.
        """

        dirname = os.path.dirname(__file__)
        hsk_file = os.path.join(dirname, "hsk/HSK3.0 wordlist [post-2020].txt")

        category_index = -1
        category_words = None

        with open(hsk_file) as file:
            for line in file:

                # read new category for words that follow
                if line.startswith("//"):
                    category_words = []
                    self.category_list.append(category_words)
                    category_index += 1
                    continue

                word = line.strip()
                word_entry = self.dictionary.lookup(word)
                if word_entry is not None:
                    category_words.append(word_entry)
                    self.word_to_category_map[word] = category_index

    def get_category_for_word(self, word):
        """Returns the corresponding HSK category for the given word.

        Args:
            word: The string of the Chinese word to look up (can be multiple characters).

        Returns:
            The string name of the HSK category associated to the word, if found. Otherwise, returns None.
        """
        word = word.strip()
        if word in self.word_to_category_map:
            category_index = self.word_to_category_map.get(word)
            return Category.get_by_index(category_index).title
        return None

    def get_words_for_category(self, category=Category):
        """Returns the words corresponding to the given HSK category.

        Args:
            category: The Category to get words for.

        Returns:
            A list containing the words for the given HSK category.
        """
        return self.category_list[category.index]

    def get_entry(self):
        """Returns the words corresponding to HSK Entry level."""
        return self.get_words_for_category(Category.ENTRY)

    def get_intermediate(self):
        """Returns the words corresponding to HSK Intemediate level."""
        return self.get_words_for_category(Category.INTERMEDIATE)

    def get_advanced(self):
        """Returns the words corresponding to HSK Advanced level."""
        return self.get_words_for_category(Category.ADVANCED)

    def get_supplemental(self):
        """Returns the words corresponding to HSK Supplemental level."""
        return self.get_words_for_category(Category.SUPPLEMENTAL)


from flask import Flask, request
from pyzipcode import ZipCodeDatabase
import requests

app = Flask(__name__)
zcdb = ZipCodeDatabase()

# @app.route("/zipcode", methods=["GET"])
@app.route("/zipcode", methods=["GET"])
def get_zip_code():

    city = request.args.get("city")
    zip_code = zcdb.find_zip(city=city)
    # return {"zip_code": zip_code[0].zip }

    if zip_code:
        for zc in zip_code:
            zip_code = zc.zip
            break
        # return {"Zip code": zip_code}
        weather_response = requests.get(f"http://zipcode:3001/check_weather?zip_code={zip_code}")
        if weather_response.status_code == 200:
            return weather_response.text

        # instead of returning zipcode he is also calling out servis 2 url

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=3000, debug=True)


import click
import numpy as np

class cmd_depth(object):
    @classmethod
    def decorate(cls, cmd):
        cmd = click.option(
            '--depth', type=int, help="Normalize depth to positive (1) or negative(-1)"
        )(cmd)
        return cmd

    @classmethod
    def transform(cls, data, depth, **kw):
        for section in data:
            if "data" in section:
                for col in ("depth", "start_depth", "end_depth"):
                    if col in section["data"].columns:
                        section["data"][col] = np.abs(section["data"][col]) * depth
        return data


import tweepy
import time

auth = tweepy.OAuthHandler('X24kkVr17Tb1XhEETMXpdCrFR', '94i7VjUZqEF8QbHGjo1pVZGtfFOlLW2OAlqKNkDmXCXH5gGCGo')
auth.set_access_token('1238177705604112384-1Zq1xgii5DiNu61g3XjkTdDZVEhIc6', '7MoLXXTYmKwfTwOdvrFVNRWkBR6RrbTgzm65nr6s7XN2V')

api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)

user = api.me()

search = 'SoftServe'
nrTweets = 15

for tweet in tweepy.Cursor(api.search, search).items(nrTweets):
    try:
        print('Tweet Liked')
        tweet.favorite()
        time.sleep(2)
    except tweepy.TweepError as e:
        print(e.reason)
    except StopIteration:
        break

for tweet in tweepy.Cursor(api.search, search).items(nrTweets):
    try:
        print('Tweet Retweeted')
        tweet.retweet()
        time.sleep(2)
    except tweepy.TweepError as e:
        print(e.reason)
    except StopIteration:
        break


import stat
import shutil

from pathlib import Path
from typing import Any, Dict, List

from snapfs import transform

def make_dirs(path: Path):
    path.mkdir(0o774, True, True)

def store_file(file_path: Path, content: str, override: bool = False) -> None:
    if file_path.is_file() and override:
        # make file writeable
        file_path.chmod(stat.S_IWRITE | stat.S_IWGRP | stat.S_IROTH)

    if not file_path.is_file() or override:
        make_dirs(file_path.parent)

        with open(file_path, "w") as f:
            f.write(content)

        # make file read only
        file_path.chmod(stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)

def store_dict_as_file(
    file_path: Path, data: Dict[str, Any], override: bool = False
) -> None:
    store_file(file_path, transform.dict_as_json(data), override)

def store_dict_as_blob(directory: Path, data: Dict[str, Any]) -> str:
    contents = transform.dict_as_json(data)

    hashid = transform.string_as_hashid(contents)

    hashid_path = directory.joinpath(transform.hashid_as_path(hashid))

    store_file(hashid_path, contents)

    return hashid

def load_file(file_path: Path) -> str:
    data: str = ""

    with open(file_path, "r") as f:
        data = f.read()

    return data

def load_file_as_dict(file_path: Path) -> Dict[str, Any]:
    return transform.json_as_dict(load_file(file_path))

def load_blob_as_dict(directory: Path, hashid: str) -> Dict[str, Any]:
    hashid_path = directory.joinpath(transform.hashid_as_path(hashid))

    return load_file_as_dict(hashid_path)

def copy_file(source: Path, target: Path) -> None:
    make_dirs(target.parent)

    shutil.copyfile(source, target)

def copy_file_as_blob(directory: Path, source: Path) -> str:
    hashid = transform.file_as_hashid(source)

    hashid_path = directory.joinpath(transform.hashid_as_path(hashid))

    if not hashid_path.is_file():
        make_dirs(hashid_path.parent)

        copy_file(source, hashid_path)

        hashid_path.chmod(stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)

    return hashid

def load_ignore_file_as_patterns(directory: Path) -> List[str]:
    patterns = []

    ignore_file_path = directory.joinpath(".ignore")

    if ignore_file_path.is_file():
        contents = load_file(ignore_file_path)

        patterns = list(
            filter(
                bool,
                [
                    x.strip()
                    for x in contents.split("\n")
                    if not x.startswith("#")
                ],
            )
        )

    return patterns


from pyfcm import FCMNotification

APIKEY = "AAAAsLhtlp8:APA91bF7A7f_Rr0tBhpZmgBAzZ5wAIGifzAyMtLJgerJyR-uX6t6bdpjyZ6dNE1IDOk22wm3vItH9xx4mZwzzI_4lcbGBJ-ZBi8kHh9Cxwf_2Kb14SUAnDPkCFPZhHGEP-pdYEeucLfS"

# 파이어베이스 콘솔에서 얻어 온 서버 키를 넣어 줌
push_service = FCMNotification(APIKEY)

def sendMessage(body, title):
    # 메시지 (data 타입)
    data_message = {
        "body": body,
        "title": title
    }

    # topic을 이용해 다수의 구독자에게 푸시알림을 전송함
    result = push_service.notify_topic_subscribers(topic_name="generalTopic", data_message=data_message)

    # 전송 결과 출력
    print(result)

sendMessage("TEST Message", "데이터 업데이트 알림")


#!/usr/bin/python

import praw
import pdb
import re
import os

class redditReply(object):

    reddit = praw.Reddit('defbotboot')
    subreddit = reddit.subreddit("rsab")

    def reply_comment(self, limit):

        if not os.path.isfile("reply_text.txt"):
            reply_post = []

        else:

            # use with so we don't have to try, catch to debug errors
            with open("reply_text.txt", "r") as f:
                # read the file and split on newline
                reply_post = f.read()
                reply_post = reply_post.split("\n")
                # if file contains empty elements, remove them
                reply_post = list(filter(None, reply_post))

        for submission in self.subreddit.comments(limit = limit):
            if submission.id not in reply_post:

                print("submission id: ", submission.id, " // reply_post match: ", reply_post)
                if re.search("!defbot rate comments", submission.body, re.IGNORECASE):
                    # reply to the post
                    # submission.reply("Insert Sentiment Analysis here")
                    print("Bot replying to : ", submission.body)
                    reply_post.append(submission.id)

        with open("reply_post.txt", "w") as f:
            for post_id in reply_post:
                f.write(post_id + "\n")

def main():
    reddit = redditReply()
    reddit.reply_comment(limit = 25)

if __name__ == "__main__":
    main()