import sqlite3
import time

import httplib2
import os
import sys

from apiclient.discovery import build
from apiclient.errors import HttpError
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import argparser, run_flow

################################################################################

today = time.strftime("%x") + " " + time.strftime("%X")

ytDB = 'YouTube'
connection = sqlite3.connect(ytDB)
c = connection.cursor()

# ====================================

# The CLIENT_SECRETS_FILE variable specifies the name of a file that contains
# the OAuth 2.0 information for this application, including its client_id and
# client_secret. You can acquire an OAuth 2.0 client ID and client secret from
# the Google Developers Console at https://console.developers.google.com/.
CLIENT_SECRETS_FILE = "client_secrets.json"

# This OAuth 2.0 access scope allows for full read/write access to the
# authenticated user's account.
YOUTUBE_READ_WRITE_SCOPE = "https://www.googleapis.com/auth/youtube"
YOUTUBE_API_SERVICE_NAME = "youtube"
YOUTUBE_API_VERSION = "v3"

# This variable defines a message to display if the CLIENT_SECRETS_FILE is
# missing.
MISSING_CLIENT_SECRETS_MESSAGE = """
To make this sample run you will need to populate the client_secrets.json file
found at:
   %s
with information from the APIs Console
https://console.developers.google.com
""" % os.path.abspath(os.path.join(os.path.dirname(__file__),
                                   CLIENT_SECRETS_FILE))

# ===========================Define Methods====================================

# Authorize the request and store authorization credentials.
def get_authenticated_service(args):
  flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE, scope=YOUTUBE_READ_WRITE_SCOPE,
    message=MISSING_CLIENT_SECRETS_MESSAGE)

  storage = Storage("%s-oauth2.json" % sys.argv[0])
  credentials = storage.get()

  if credentials is None or credentials.invalid:
    credentials = run_flow(flow, storage, args)

  return build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
    http=credentials.authorize(httplib2.Http()))

# ====================================

# Insert video record to YT_Video table
def video_to_sql(video_id, name, published_date, channel_id):

    try:
        c.execute("INSERT INTO YT_Video VALUES (?, ?, ?, ?, ?, ?)",
                  (video_id, name, published_date, today, today, channel_id))
    except sqlite3.IntegrityError:
        print('you done goofed: sqlite3.IntegrityError')

# ====================================

# Insert user record into YT_User table
def user_to_sql(user_id, name, create_date):

    try:
        c.execute("INSERT INTO YT_User VALUES (?, ?, ?, ?, ?)",
                  (user_id, name, create_date, today, today))
    except sqlite3.IntegrityError:
        print('you done goofed')

# ====================================

# Insert playlist record into YT_Playlist table
def playlist_to_sql(playlist_id, name, create_date, channel_id):

    try:
        c.execute("INSERT INTO YT_Playlist VALUES (?, ?, ?, ?)",
                  (playlist_id, name, create_date, today, today, channel_id))
    except sqlite3.IntegrityError:
        print('you done goofed')

# ====================================

# Insert channel record into YT_Channel table
def channel_to_sql(channel_id, name, create_date, owner_id):

    try:
        c.execute("INSERT INTO YT_Channel VALUES (?, ?, ?, ?, ?, ?",
                  (channel_id, name, create_date, today, today, owner_id))
    except sqlite3.IntegrityError:
        print('you done goofed')

# ====================================

# Get details for specific YouTube Video from YouTube API
def get_video_details(youtube, video_id):
    video_results = youtube.videos().list(
        part="snippet, contentDetails",
        id=video_id,
        maxResults=50
    )

    video_results_response = video_results.execute()

    video_details_dict = {
        'id':
            video_results_response["items"][0]["id"],
        'title':
            video_results_response["items"][0]["snippet"]["title"],
        'published_at':
            video_results_response["items"][0]["snippet"]["publishedAt"],
        'channel_id':
            video_results_response["items"][0]["snippet"]["channelId"],
        'description':
            video_results_response["items"][0]["snippet"]["description"],
        'channel_title':
            video_results_response["items"][0]["snippet"]["channelTitle"],
        'duration':
            video_results_response["items"][0]["contentDetails"]["duration"],
        'definition':
            video_results_response["items"][0]["contentDetails"]["definition"],
        'licensedContent':
            video_results_response["items"][0]["contentDetails"][
                "licensedContent"]}

    return video_details_dict

# ====================================

# Get details for specific YouTube Playlist from YouTube API
def get_playlist_details(youtube, playlist_id):
    list_results = youtube.playlists().list(
        part="snippet, contentDetails",
        id=playlist_id,
        maxResults=50
    )

    list_results_response = list_results.execute()

    list_details_dict = {
        'playlist_id':
            list_results_response["items"][0]["id"],
        'playlist_title':
            list_results_response["items"][0]["snippet"]["title"],
        'playlist_published_at':
            list_results_response["items"][0]["snippet"]["publishedAt"],
        'playlist_channel_id':
            list_results_response["items"][0]["snippet"]["channelId"],
        'playlist_description':
            list_results_response["items"][0]["snippet"]["description"],
        'playlist_channel_title':
            list_results_response["items"][0]["snippet"]["channelTitle"],
        'playlist_item_count':
            list_results_response["items"][0]["contentDetails"]["itemCount"]
            }

    return list_details_dict

# ====================================

# Get details for specific YouTube Channel from YouTube API
def get_channel_details(youtube, channel_id):
    channel_results = youtube.videos().list(
        part="snippet, contentDetails",
        id=channel_id,
        maxResults=50
    )

    channel_results_response = channel_results.execute()

    channel_details_dict = {
        'id':
            channel_results_response["items"][0]["id"],
        'title':
            channel_results_response["items"][0]["snippet"]["title"],
        'published_at':
            channel_results_response["items"][0]["snippet"]["publishedAt"],
        'description':
            channel_results_response["items"][0]["snippet"]["description"],
        'googlePlusUserId':
            channel_results_response[
                "items"][0]["contentDetails"]["googlePlusUserId"]
    }

    return channel_details_dict

# ====================================

# Retrieve playlist info and relevant channel info
def get_channel_playlists(youtube, channel_id):

    channel_dict= {channel_id:get_channel_details(youtube, channel_id)}

    # Set up API call for Channel details
    channel_results = youtube.channels().list(
        part='snippet, contentDetails, contentOwnerDetails',
        id=channel_id
    )

    # Execute API Call for channel details
    channel_results_response = channel_results.execute()

    # Set up Channel Details
    yt_details = channel_results_response["items"][0]

    yt_channel_id = yt_details["id"]
    yt_channel_name = yt_details["snippet"]["title"]
    yt_channel_owner_id = yt_details["contentDetails"]["googlePlusUserId"]
    yt_channel_create_date = yt_details["snippet"]["publishedAt"]

    # Add channel to channel dictionary
    channel_dict = {yt_channel_id:yt_channel_name}

    # Print
    # 'Channel ID - Channel Name - Channel Owner ID - Channel Create Date'

    print("Channel: {1} ({0})\nOwner: {2}\nCreated On: {3}\n".format(
        yt_channel_id,
        yt_channel_name,
        yt_channel_owner_id,
        yt_channel_create_date))

    # Set up API call for Playlist details

    channel_results = youtube.playlists().list(
        part="snippet, contentDetails",
        channelId=channel_id,
        maxResults=50
    )

    # Execute API call for Playlist details
    channel_results_response = channel_results.execute()
    loop_number = 0

    # Loop through API call results for playlist details
    for playlist in channel_results_response["items"]:

        # Loop counter
        loop_number += 1

        # Grab playlist title
        playlist_title = playlist["snippet"]["title"]

        # Grab playlist ID
        playlist_id = playlist["id"]

        #
        playlist_dict[playlist_id] = playlist_title
        print("%s -- %s (%s)" % (loop_number, playlist_title, playlist_id))

# ====================================

# Retrieve and combine multiple playlists into a single list
def combine_channel_playlist(youtube, channel_id):

    # Set up API call
    channel_playlist_results = youtube.playlists().list(
        part="snippet, contentDetails",
        channelId=channel_id,
        maxResults=50
    )

    # Execute API call
    channel_playlist_results_response = channel_playlist_results.execute()

    # Set up method loop counter
    loop_number = 0

    # Set up array to record # of playlists in channel.
    # This is used later for print and playlist selection purposes.
    returned_playlists = []

    for playlist in channel_playlist_results_response["items"]:

        # Increment loop counter
        loop_number += 1

        # Add current loop counter value to returned playlist dictionary
        # This will act as a 'hacky' record id for the playlist
        returned_playlists.append(loop_number)

        # Set up playlist title and playlist id variables from API call return
        playlist_title = playlist["snippet"]["title"]
        playlist_id = playlist["id"]

        # List playlist on screen
        print("%s -- %s (%s)" % (loop_number, playlist_title, playlist_id))

    # Ask user for comma delimited input showing which playlists to pick based
    # on playlist record number (i.e. returned_playlists dictionary.
    picked_playlist = input("\nWhich playlist would you like to combine?\n"
                            "Enter numbers separated by commas\n")
    # Formatting line break
    print("\n")

    # Formats picked_playlist values; does work to convert string to list of
    # ints
    picked_playlist = picked_playlist.split(',')
    picked_playlist = filter(None, picked_playlist)
    picked_playlist = list(map(int, picked_playlist))

    # Validating user input to ensure picked values are within bounds of
    # available returned_playlist dict values
    if min(picked_playlist) < min(returned_playlists) or \
                    max(picked_playlist) > max(returned_playlists):
        exit('You specified a playlist that wasn\'t on the list!\n')

    # Reset loop counter
    loop_number = 0

    # Loops through returned playlist from API call and calls
    # get_playlist_videos method to return items in those playlists
    for playlist in channel_playlist_results_response["items"]:

        # Increment loop counter
        loop_number += 1

        # If current loop matches a number in the picked_playlist list, then
        # it runs get_playlist_videos method to return items in that playlist
        if loop_number in picked_playlist:
            get_playlist_videos(youtube, playlist["id"])

# ====================================

# Get list of videos in specific YouTube Playlist from YouTube API
# Record YouTube API response and return it
def get_playlist_videos(youtube, playlist_id):

    # Grab overall playlist details (e.g. title, channel, published date, ect)
    playlist_details = get_playlist_details(youtube, playlist_id)

    # Stick playlist details in dict.  We'll stick video info here late as well.
    playlist_dict = {'playlist_details':playlist_details}

    playlist_results = youtube.playlistItems().list(
        part="snippet, contentDetails",
        playlistId=playlist_id,
        maxResults=50
    )

    playlist_results_response = playlist_results.execute()

    # Record information about each video.  Continue looping through each page
    # of playlist repsonse until out of pages, then break loop.
    while True:

        for playlist_item in playlist_results_response["items"]:

            video_dict = {
                'video_title':
                    playlist_item["snippet"]["title"],
                'video_id':
                    playlist_item["snippet"]["resourceId"]["videoId"],
                'video_published_date':
                    playlist_item["snippet"]["publishedAt"],
                'video_description':
                    playlist_item["snippet"]["description"],
                'video_playlist_position':
                    playlist_item["snippet"]["position"]
            }

            playlist_dict[video_dict['video_playlist_position']] = video_dict

        # If reached end of response list, exit loop
        if "nextPageToken" not in playlist_results_response:
            break

        # Otherwise set things up to move to the next page in playlist response
        next_page = playlist_results_response["nextPageToken"]

        playlist_results = youtube.playlistItems().list(
            part="snippet, contentDetails",
            playlistId=playlist_id,
            maxResults=50,
            pageToken=next_page
            )
        playlist_results_response = playlist_results.execute()

    return playlist_dict

# ================================__main__=====================================

# if __name__ == "__main__":


import yaml
import logging

settings = dict()

log_levels = {
    'DEBUG': logging.DEBUG,
    'INFO': logging.INFO,
    'WARNING': logging.WARNING,
    'ERROR': logging.ERROR,
    'CRITICAL': logging.CRITICAL,
}

with open("../config.yaml", 'r') as stream:
    try:
        settings = yaml.safe_load(stream)
        settings['log_level'] = log_levels[settings['log_level']]
    except yaml.YAMLError as exc:
        print(exc)


# !/usr/bin/env python
# ! _*_ coding:utf-8 _*_
# @TIME   : 2018/12/14  19:26
# @Author : Noob
# @File   : basepage.py

"""
Project:基础类BasePage，封装所有页面都公用的方法；
定义open函数，重定义find_element,switch_frame,send_keys等函数；
在初始化方法中定义驱动driver,基本url，title；
WebDriverWait提供了显示等待方式；
"""

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import os
import time
import yaml
import csv
import xlrd

class BasePage(object):

    """
    1. 初始化driver、url、pagetitle等；
    2. 实例化BasePage类时，最先执行的就是__init__方法，该方法的入参，其实就是BasePage类的入参；
    3. __init__方法不能有返回值，只能返回None；
    4. self指实例本身，相较于类BasePage而言
    """

    def __init__(self, browser_driver, base_url, pagetitle):
        self.driver = browser_driver
        self.base_url = base_url
        self.pagetitle = pagetitle

    """
    1. 通过title断言进入的页面是否正确；
    2. 使用title获取当前窗口title，检查输入的title是否在当前title中，返回比较结果（True or False）
    """
    def on_page(self, pagetitle):
        return pagetitle in self.driver.title

    """
    1. 打开页面，并校验页面链接是否加载正确
    2. 以单下划线_开头的方法，在使用import *时，该方法不会被导入，保证该方法为类私有的；
    """
    def _open(self, url, pagetitle):
        # 使用get打开访问链接地址
        self.driver.get(url)
        self.driver.maximize_window()
        # 使用assert进行校验，打开的窗口title是否与配置的title一致。调用on_page方法。
        assert self.on_page(pagetitle), u'打开页面失败%s' % url

    # 定义open方法，掉用_open()进行打开链接
    def open(self):
        self._open(self.base_url, self.pagetitle)

    # 重写元素定位方法
    def find_element(self, *loc):
        # return self.driver.find_element(*loc)
        try:
            # 确保元素是可见的；
            # 注意：以下入参为元祖的元素，需要加*。Python存在这种特性，就是讲入参放在元祖里；
            # WebDriverWait(self.driver, 10).untli(lamba driver: driver.find_element(*loc).is_displayed())
            # 注意：以下入参本身是元祖，不需要加*
            WebDriverWait(self.driver, 10).until(EC.visibility_of_element_located(loc))
            return self.driver.find_element(*loc)
        except:
            print(u'%s 页面中未能找到%s元素' % (self, loc))

    # 重写switch_frame方法
    def switch_frame(self, loc):
        loc = self.find_element(loc)
        return self.driver.switch_to.frame(loc)

    # 定义scrip方法，用于执行js脚本，范围执行结果
    def scirpt(self, src):
        self.driver.execute_script(src)

    # 重写定义send_keys方法
    def send_keys(self, loc, value, clear_first=True, click_first=True):
        try:
            # getattr相当于实现self.loc
            loc = getattr(self, '_%s' % loc)
            if click_first:
                self.find_element(*loc).click()
            if clear_first:
                self.find_element(*loc).clear()
                self.find_element(*loc).send_keys(value)
        except AttributeError:
            print(u'%s页面中未能找到%s元素' % (self, loc))

    # 重写截图
    def screenshot(self, loc):
        picture_time = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time()))
        try:
            # 获取当前脚本的路径
            cwd_path = os.path.dirname(os.path.realpath(__file__))
            # self.__class__.__name__：当前运行文件的类名
            screenshot_path = os.path.join(cwd_path, 'screenshot_' + self.__class__.__name__)
            # 判断放置运行测试脚本的截图文件夹是否存在，如果不存在则创建
            if not os.path.exists(screenshot_path):
                os.makedirs(screenshot_path)
                print('截图目录新建成功：%s' % screenshot_path)
            else:
                print('该目录已存在！')
            error_text = self.find_element(*loc).text
            if 'Incorrect' in error_text:
                self.driver.get_screenshot_as_file(screenshot_path + '\\' + picture_time + '.png')
        except BaseException as msg:
            print('新建截图目录失败：%s' % msg)

    # 外部数据
    def get_data(self, filename):
        cwd_path = os.path.dirname(os.path.realpath(__file__))
        file_path = os.path.join(cwd_path, 'data\\' + filename)
        try:
            assert os.path.isfile(file_path), filename
        except AssertionError as msg:
            if filename == '':
                list_data = (['000', '000'], ['111', '111'])
                return list_data
            else:
                print('该文件不存在：%s' % msg)
        if file_path.endswith('.txt'):
            f = open(file_path, 'r', encoding='utf-8')
            cfg = f.readlines()
            cfg.remove(cfg[0])
            rows = []
            for i in cfg:
                i = i.split(',')
                i[1] = i[1].rstrip('\n')
                rows.append(i)
            return rows
        elif file_path.endswith('.csv'):
            cfg = csv.reader(open(file_path, 'r', encoding='utf-8'))
            next(cfg, None)
            rows = []
            for i in cfg:
                rows.append(i)
            return rows
        elif file_path.endswith('.xlsx'):
            book = xlrd.open_workbook(file_path)
            sheet = book.sheet_by_index(0)
            rows = []
            for idx in range(1, sheet.nrows):
                i = sheet.row_values(idx, 0, sheet.ncols)
                rows.append(i)
            rows[1][0] = int(rows[1][0])
            rows[1][1] = int(rows[1][1])
            return rows
        elif file_path.endswith('.yaml'):
            f = open(file_path, 'r', encoding='utf-8')
            cfg = yaml.load(f)
            f.close()
            cfg.remove(cfg[0])
            return cfg
        else:
            return '不能使用该类型的文件！'

if __name__ == '__main__':
    bp = BasePage(None, None, None)
    i = bp.get_data('data')
    print(i)


from cgitb import html
from keras.models import load_model
from flask import *
from PIL import Image
from sqlalchemy import false, true
from werkzeug.utils import secure_filename
import cv2
import numpy as np
import easyocr
import os

model = load_model('icrecognition.h5')

classes = {
    0 : 'No Pain',
    1 : 'Pain'
}

def textprocessing(img):

    print (img)
    img = cv2.imread(img)

    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    gray = cv2.GaussianBlur(src=gray, ksize=(5, 5), sigmaX=0, sigmaY=0) #Bluring image in a small amount
    filter = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]]) # Creating our sharpening filter

    sharpen_img = cv2.filter2D(gray,-1,filter) # Applying cv2.filter2D sharpening function on image

    reader = easyocr.Reader(['en'])
    result = reader.readtext(sharpen_img)

    result_val = " "
    for x in result:
        result_val += (x[1]+",")

    result_val.strip()

    return (result_val)

# main -----------------------------------------------------------------------------------------------------------------------
app = Flask(__name__)
print ("Hello")
@app.route("/", methods=['GET', 'POST'])
def summary():
    uploads_dir = os.path.join('')
    ic = request.files['ic']

    if ic != '':
        image = Image.open(ic)
        image = image.convert("RGB")
        image.save(os.path.join(uploads_dir, secure_filename("uploadedimage.jpg")))
        resizeimage = image.resize((200,200))
        resizeimage = np.expand_dims(resizeimage, axis = 0)
        resizeimage = np.array(resizeimage)
        pred = model.predict([resizeimage])[0]
        icpath = 'uploadedimage.jpg'
        icpath = str(icpath)
        dataic = textprocessing(icpath)

        if (pred == 0):
            data = {
                "status" : bool(true),
                "extraction" : dataic
            }
        else:
            data = {
                "tatus" : bool(false)
            }

        os.remove(icpath)
        return jsonify(data)

    else:
        return ('error')

if __name__ == '__main__':
    app.run(host = '0.0.0.0',port=8001,debug=True)

# main ----------------------------------------------------------------------------------------------------------------------------


import base58
import json
import time

from Crypto.Hash import SHA3_256, SHA3_512, RIPEMD160
from Crypto.Signature import PKCS1_v1_5

from transaction import Transaction, TransactionMerkleTree
from wallet import Wallet

class Block:
    def __init__(self, index, transactions, timestamp, previous_hash, nonce=0):
        self.index = index
        self.transactions = None
        if len(transactions) > 0:
            TransactionMerkleTree(transactions)
        self.timestamp = timestamp
        self.previous_hash = previous_hash
        self.nonce = nonce

    def __str__(self):
        return str(self.__dict__)

    def compute_hash(self):
        block_string = json.dumps(self, sort_keys=True, cls=Block.MyJsonEncoder)
        return SHA3_512.new(block_string.encode()).hexdigest()

    class MyJsonEncoder(json.JSONEncoder):
        def default(self, o):
            return o.__dict__

class Blockchain:
    def __init__(self, difficulty=3):
        self.unconfirmed_transactions = []
        self.chain = []
        self.create_genesis_block()
        self.difficulty = difficulty

    def __str__(self):
        return str([str(x) for x in self.chain])

    def create_genesis_block(self):
        genesis_block = Block(0, [], time.time(), "0")
        genesis_block.hash = genesis_block.compute_hash()
        self.chain.append(genesis_block)

    @property
    def last_block(self):
        return self.chain[-1]

    def proof_of_work(self, block):
        block.nonce = 0
        computed_hash = block.compute_hash()
        while not computed_hash.startswith('0' * self.difficulty):
            block.nonce += 1
            computed_hash = block.compute_hash()
        return computed_hash

    def is_valid_proof(self, block, block_hash):
        return (block_hash.startswith('0' * self.difficulty) and
                block_hash == block.compute_hash())

    def add_block(self, block, proof):
        previous_hash = self.last_block.hash
        if previous_hash != block.previous_hash:
            return False
        if not self.is_valid_proof(block, proof):
            return False
        block.hash = proof
        self.chain.append(block)
        return True

    def add_new_transaction(self, public_key, signed_transaction):
        try:
            Blockchain.validate_signature(public_key, signed_transaction)
        except AddressKeyMismatch:
            print("Wrong key")
        except ValueError:
            print("Invalid Signature")
        else:
            self.unconfirmed_transactions.append(signed_transaction)

    def mine(self):
        if not self.unconfirmed_transactions:
            return False
        last_block = self.last_block
        new_block = Block(index=last_block.index + 1,
                          transactions=self.unconfirmed_transactions,
                          timestamp=time.time(),
                          previous_hash=last_block.hash)
        proof = self.proof_of_work(new_block)
        self.add_block(new_block, proof)
        self.unconfirmed_transactions = []
        return new_block.index

    @staticmethod
    def validate_signature(public_key, signed_transaction):
        address = public_key
        h = RIPEMD160.new(SHA3_256.new(public_key.export_key()).hexdigest().encode())
        address = base58.b58encode(h.hexdigest()).decode("utf-8")
        if address != signed_transaction.transaction.sender:
            raise AddressKeyMismatch
        PKCS1_v1_5.new(public_key).verify(signed_transaction.transaction.compute_hash(), signed_transaction.signature)

class AddressKeyMismatch(Exception):
    pass

if __name__ == '__main__':
    wallet1 = Wallet()
    wallet2 = Wallet()

    wallet1.generate_new_wallet()
    wallet1.save_wallet()
    wallet2.generate_new_wallet()
    myBlockchain = Blockchain(difficulty=4)

    myTransaction01 = Transaction(wallet1.address, wallet2.address, 10.05)
    mySignedTransaction01 = myTransaction01.sign(wallet1.private_key)
    myBlockchain.add_new_transaction(wallet1.public_key, mySignedTransaction01)

    myTransaction02 = Transaction(wallet1.address, wallet2.address, 22.40)
    mySignedTransaction02 = myTransaction02.sign(wallet1.private_key)
    myBlockchain.add_new_transaction(wallet1.public_key, mySignedTransaction02)

    myTransaction03 = Transaction(wallet1.address, wallet2.address, 11.55)
    mySignedTransaction03 = myTransaction03.sign(wallet1.private_key)
    myBlockchain.add_new_transaction(wallet1.public_key, mySignedTransaction03)

    myTransaction04 = Transaction(wallet1.address, wallet2.address, 100.05)
    mySignedTransaction04 = myTransaction04.sign(wallet1.private_key)
    myBlockchain.add_new_transaction(wallet1.public_key, mySignedTransaction04)

    myBlockchain.mine()
    print(myBlockchain)


from cgi import test
from basicsr.losses.losses import L_color
import torch
from collections import OrderedDict
from os import path as osp
from tqdm import tqdm

from basicsr.archs import build_network
from basicsr.losses import build_loss
from basicsr.metrics import calculate_metric
from basicsr.utils import get_root_logger, imwrite, tensor2img
from basicsr.utils.registry import MODEL_REGISTRY
from .base_model import BaseModel

@MODEL_REGISTRY.register()
class LearnablePirorModel(BaseModel):
    """Base SR model for single image super-resolution."""

    def __init__(self, opt):
        super(LearnablePirorModel, self).__init__(opt)

        # define network
        self.net_g = build_network(opt['network_g'])
        self.net_g = self.model_to_device(self.net_g)
        self.print_network(self.net_g)

        self.net_noisePrior = build_network(opt['network_noisePrior'])
        self.net_noisePrior = self.model_to_device(self.net_noisePrior)

        # load pretrained models
        load_path = self.opt['path'].get('pretrain_network_g', None)
        if load_path is not None:
            param_key = self.opt['path'].get('param_key_g', 'params')
            self.load_network(self.net_g, load_path, self.opt['path'].get('strict_load_g', True), param_key)

        load_path_noisePrior = self.opt['path'].get('pretrain_network_noisePrior', None)
        if load_path_noisePrior is not None:
            param_key = self.opt['path'].get('param_key_decom', 'params')
            self.load_network(self.net_noisePrior, load_path_noisePrior, self.opt['path'].get('strict_load_noisePrior', True), param_key)

        if self.is_train:
            self.init_training_settings()

    def init_training_settings(self):
        self.net_g.train()
        train_opt = self.opt['train']

        self.ema_decay = train_opt.get('ema_decay', 0)
        if self.ema_decay > 0:
            logger = get_root_logger()
            logger.info(f'Use Exponential Moving Average with decay: {self.ema_decay}')
            # define network net_g with Exponential Moving Average (EMA)
            # net_g_ema is used only for testing on one GPU and saving
            # There is no need to wrap with DistributedDataParallel
            self.net_g_ema = build_network(self.opt['network_g']).to(self.device)
            # load pretrained model
            load_path = self.opt['path'].get('pretrain_network_g', None)
            if load_path is not None:
                self.load_network(self.net_g_ema, load_path, self.opt['path'].get('strict_load_g', True), 'params_ema')
            else:
                self.model_ema(0)  # copy net_g weight
            self.net_g_ema.eval()

        if train_opt.get('gtRecon_opt'):
            self.cri_gtRecon = build_loss(train_opt['gtRecon_opt']).to(self.device)
        else:
            self.cri_gtRecon = None

        if train_opt.get('lowRecon_opt'):
            self.cri_lowRecon = build_loss(train_opt['lowRecon_opt']).to(self.device)
        else:
            self.cri_lowRecon = None

        if train_opt.get('refl_opt'):
            self.cri_refl = build_loss(train_opt['refl_opt']).to(self.device)
        else:
            self.cri_refl = None

        if train_opt.get('illuMutualInput_opt'):
            self.cri_illuMutualInput = build_loss(train_opt['illuMutualInput_opt']).to(self.device)
        else:
            self.cri_illuMutualInput = None

        if train_opt.get('illuMutual_opt'):
            self.cri_illuMutual = build_loss(train_opt['illuMutual_opt']).to(self.device)
        else:
            self.cri_illuMutual = None

        if train_opt.get('pixel_opt'):
            self.cri_pix = build_loss(train_opt['pixel_opt']).to(self.device)
        else:
            self.cri_pix = None

        if train_opt.get('enhancedIllu_opt'):
            self.cri_enhancedIllu = build_loss(train_opt['enhancedIllu_opt']).to(self.device)
        else:
            self.cri_enhancedIllu = None

        if train_opt.get('enhancedIlluTV_opt'):
            self.cri_enhancedIlluTV = build_loss(train_opt['enhancedIlluTV_opt']).to(self.device)
        else:
            self.cri_enhancedIlluTV = None

        if train_opt.get('reflRestore_opt'):
            self.cri_reflRestore = build_loss(train_opt['reflRestore_opt']).to(self.device)
        else:
            self.cri_reflRestore = None

        if train_opt.get('noisePrior_opt'):
            self.cri_noisePrior = build_loss(train_opt['noisePrior_opt']).to(self.device)
        else:
            self.cri_noisePrior = None

        # set up optimizers and schedulers
        self.setup_optimizers()
        self.setup_schedulers()

    def setup_optimizers(self):
        train_opt = self.opt['train']
        optim_params = []
        for k, v in self.net_g.named_parameters():
            if v.requires_grad:
                optim_params.append(v)
            else:
                logger = get_root_logger()
                logger.warning(f'Params {k} will not be optimized.')

        optim_type = train_opt['optim_g'].pop('type')
        self.optimizer_g = self.get_optimizer(optim_type, optim_params, **train_opt['optim_g'])
        self.optimizers.append(self.optimizer_g)

    def feed_data(self, data):
        self.lq = data['lq'].to(self.device)
        if 'gt' in data:
            self.gt = data['gt'].to(self.device)

    def optimize_parameters(self, current_iter):
        self.optimizer_g.zero_grad()
        self.output, self.enhanced_L, self.L, self.restored_R, self.R, self.noise, self.L_hat = self.net_g(self.lq)

        _, _, self.gt_L, _, self.gt_R, self.gt_noise, _ = self.net_g(self.gt)

        _, _, self.output_noisePrior = self.net_noisePrior(self.output)
        _, _, self.gt_noisePrior = self.net_noisePrior(self.gt)

        l_total = 0
        loss_dict = OrderedDict()

        if self.cri_lowRecon:
            l_lowRecon = self.cri_lowRecon(self.L * self.R + self.noise, self.lq)
            l_total += l_lowRecon
            loss_dict['l_lowRecon'] = l_lowRecon

        if self.cri_gtRecon:
            l_gtRecon = self.cri_gtRecon(self.gt_L * self.gt_R, self.gt)
            l_total += l_gtRecon
            loss_dict['l_gtRecon'] = l_gtRecon

        if self.cri_refl:
            l_refl = self.cri_refl(self.R, self.gt_R)
            l_total += l_refl
            loss_dict['l_refl'] = l_refl

        if self.cri_illuMutualInput:
            l_illuMutualInputLQ = self.cri_illuMutualInput(self.L, self.lq)
            l_total += l_illuMutualInputLQ
            loss_dict['l_illuMutualInputLQ'] = l_illuMutualInputLQ

            l_illuMutualInputGT = self.cri_illuMutualInput(self.gt_L, self.gt)
            l_total += l_illuMutualInputGT
            loss_dict['l_illuMutualInputGT'] = l_illuMutualInputGT

        if self.cri_illuMutual:
            l_illuMutual = self.cri_illuMutual(self.L, self.gt_L)
            l_total += l_illuMutual
            loss_dict['l_illuMutual'] = l_illuMutual

            l_illuMutualEnhanced = self.cri_illuMutual(self.enhanced_L, self.gt_L)
            l_total += l_illuMutualEnhanced
            loss_dict['l_illuMutualEnhanced'] = l_illuMutualEnhanced

        if self.cri_pix:
            l_pix = self.cri_pix(self.output, self.gt)
            l_total += l_pix
            loss_dict['l_pix'] = l_pix

        if self.cri_enhancedIllu:
            l_EnhancedIllu = self.cri_enhancedIllu(torch.mean(self.enhanced_L, 1).unsqueeze(1), self.gt_L)
            l_total += l_EnhancedIllu
            loss_dict['l_enhancedIllu'] = l_EnhancedIllu

        if self.cri_enhancedIlluTV:
            l_EnhancedIlluTV = self.cri_enhancedIlluTV(torch.mean(self.enhanced_L, 1).unsqueeze(1), self.gt_L)
            l_total += l_EnhancedIlluTV
            loss_dict['l_EnhancedIlluTV'] = l_EnhancedIlluTV

        if self.cri_reflRestore:
            l_reflRestore = self.cri_reflRestore(self.restored_R, self.gt_R)
            l_total += l_reflRestore
            loss_dict['l_reflRestore'] = l_reflRestore

        if self.cri_noisePrior:
            l_noisePrior = self.cri_noisePrior(self.output_noisePrior, self.gt_noisePrior)
            l_total += l_noisePrior
            loss_dict['l_noisePrior'] = l_noisePrior

        l_total.backward()
        self.optimizer_g.step()

        self.log_dict = self.reduce_loss_dict(loss_dict)

        if self.ema_decay > 0:
            self.model_ema(decay=self.ema_decay)

    def test(self):
        if hasattr(self, 'net_g_ema'):
            self.net_g_ema.eval()
            with torch.no_grad():
                self.output_test, self.enhanced_L_test, self.L_test, self.restored_R_test, self.R_test, self.noise_test, self.L_prior_cond_test = self.net_g_ema(self.lq)
                _, _, self.gt_L_test, _, self.gt_R_test, self.gt_noise_test, _ = self.net_g_ema(self.gt)
                # self.gt_R_test = self.gt / (torch.max(self.gt, 1)[0].unsqueeze(1) + 1e-8)
        else:
            self.net_g.eval()
            with torch.no_grad():
                self.output_test, self.enhanced_L_test, self.L_test, self.restored_R_test, self.R_test, self.noise_test, self.L_prior_cond_test = self.net_g(self.lq)
                _, _, self.gt_L_test, _, self.gt_R_test, self.gt_noise_test, _ = self.net_g(self.gt)
                # self.gt_R_test = self.gt / (torch.max(self.gt, 1)[0].unsqueeze(1) + 1e-8)
            self.net_g.train()

    def dist_validation(self, dataloader, current_iter, tb_logger, save_img):
        if self.opt['rank'] == 0:
            self.nondist_validation(dataloader, current_iter, tb_logger, save_img)

    def nondist_validation(self, dataloader, current_iter, tb_logger, save_img):
        dataset_name = dataloader.dataset.opt['name']
        with_metrics = self.opt['val'].get('metrics') is not None
        if with_metrics:
            self.metric_results = {metric: 0 for metric in self.opt['val']['metrics'].keys()}
            metric_data = dict()
        pbar = tqdm(total=len(dataloader), unit='image')

        for idx, val_data in enumerate(dataloader):
            img_name = osp.splitext(osp.basename(val_data['lq_path'][0]))[0]
            self.feed_data(val_data)
            self.test()

            visuals = self.get_current_visuals()
            # [0-255]

            enhanced_img = tensor2img([visuals['enhanced']])
            metric_data['img'] = enhanced_img
            if 'gt' in visuals:
                # gt_img = tensor2img([visuals['gt']])
                gt_img = tensor2img([visuals['gt']])
                metric_data['img2'] = gt_img
                del self.gt

            # tentative for out of GPU memory
            del self.lq
            torch.cuda.empty_cache()

            if save_img:
                if self.opt['is_train']:
                    save_img_path_enhanced = osp.join(self.opt['path']['visualization'], img_name,
                                             f'{img_name}_{current_iter}_enhanced.png')
                else:
                    if self.opt['val']['suffix']:
                        save_img_path_enhanced = osp.join(self.opt['path']['visualization'], img_name,
                                                f'{img_name}_{current_iter}_enhanced.png')
                    else:
                        save_img_path_enhanced = osp.join(self.opt['path']['visualization'], img_name,
                                                f'{img_name}_{current_iter}_enhanced.png')
                imwrite(enhanced_img, save_img_path_enhanced)

            if with_metrics:
                # calculate metrics
                for name, opt_ in self.opt['val']['metrics'].items():
                    self.metric_results[name] += calculate_metric(metric_data, opt_)
            pbar.update(1)
            pbar.set_description(f'Test {img_name}')
        pbar.close()

        if with_metrics:
            for metric in self.metric_results.keys():
                self.metric_results[metric] /= (idx + 1)

            self._log_validation_metric_values(current_iter, dataset_name, tb_logger)

    def _log_validation_metric_values(self, current_iter, dataset_name, tb_logger):
        log_str = f'Validation {dataset_name}\n'
        for metric, value in self.metric_results.items():
            log_str += f'\t # {metric}: {value:.4f}\n'
        logger = get_root_logger()
        logger.info(log_str)
        if tb_logger:
            for metric, value in self.metric_results.items():
                tb_logger.add_scalar(f'metrics/{metric}', value, current_iter)

    def get_current_visuals(self):
        out_dict = OrderedDict()
        out_dict['enhanced'] = self.output_test.detach().cpu()
        if hasattr(self, 'gt'):
            out_dict['gt'] = self.gt.detach().cpu()
        return out_dict

    def save(self, epoch, current_iter):
        if hasattr(self, 'net_g_ema'):
            self.save_network([self.net_g, self.net_g_ema], 'net_g', current_iter, param_key=['params', 'params_ema'])
        else:
            self.save_network(self.net_g, 'net_g', current_iter)
        self.save_training_state(epoch, current_iter)


import json
import cv2

# PDI
from pipeline.PipelinePDI import PipelinePDI
pipelinePDI = PipelinePDI()
date, url_json, image = pipelinePDI.set_input('pdi_mocked.json')
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
cv2.imwrite('noe.jpg', image_rgb)
results = pipelinePDI.detect_offers(url_json, image, date)
with open('final_results_pdi.json', 'w') as json_file:
    json.dump(results, json_file, indent=2)

# NLP
#from pipeline.PipelineNLP import PipelineNLP
#pipelineNLP = PipelineNLP(ocr_threshold=0.6, similarity_threshold=0.2)
#pipelineNLP.set_input('final_results_pdi.json')

#pipelineNLP.run('intermediate_result_nlp.json', 'final_result_nlp.json')


# -*- coding: utf-8 -*-
import scrapy
import re
import json
from Scrapy_Cnluqiao_V1.items import ScrapyCnluqiaoV1Item
import time
from scrapy.utils import request
from Scrapy_Cnluqiao_V1.start_urls import url1

class CnluqiaoV1Spider(scrapy.Spider):
    name = 'cnluqiao_V1'
    # allowed_domains = ['123']
    # start_urls = ['http://123/']
    base_url = 'http://www.cnluqiao.com/'

    def start_requests(self):
        for i in range(643):
            url = f'http://www.cnluqiao.com/portal.php?mod=list&catid=1&page={i + 1}'
            req = scrapy.Request(url=url, callback=self.parse)
            yield req

    def parse(self, response):
        config_list = response.xpath('//div[@class="nex_artice"]/ul/li')
        # print(len(config_list))
        if config_list:
            for con in config_list:
                item = ScrapyCnluqiaoV1Item()

                nex_articer = con.xpath('./div[@class="nex_articer"]')
                nex_articer_tl = con.xpath('./div[@class="nex_articer_tl"]')

                if nex_articer:

                    title = con.xpath('./div[@class="nex_articer"]/h5/a/text()').extract_first()
                    link = con.xpath('./div[@class="nex_articer"]/h5/a/@href').extract_first()
                    title_images = con.xpath('.//div[@class="nex_articel"]//img/@src').extract_first()

                    # print(title, link, title_images)

                elif nex_articer_tl:
                    title = con.xpath('.//div[@class="nex_attltop"]/a/text()').extract_first()
                    link = con.xpath('.//div[@class="nex_attltop"]/a/@href').extract_first()
                    title_images = None

                    # print(title, link)

                author = con.xpath('.//div[@class="nex_atctl"]/span/text()').extract_first()
                issue_time = con.xpath('.//div[@class="nex_atmoresd"]/span[1]/text()').extract_first()

                req = scrapy.Request(url=link, callback=self.parse_detail, meta={'item': item})

                item['id'] = request.request_fingerprint(req)
                item['title'] = title
                item['title_images'] = title_images
                item['tags'] = None
                item['author'] = author
                item['industry_categories'] = 'E'
                item['industry_Lcategories'] = '48'
                item['industry_Mcategories'] = '481'
                item['industry_Scategories'] = None
                item['information_categories'] = '行业资讯'
                item['content_url'] = link
                item['issue_time'] = issue_time[3:12] if issue_time else None

                yield req

    def parse_detail(self, response):
        item = response.meta['item']
        content = response.xpath('//td[@id="article_content"]').extract_first()
        images = response.xpath('//td[@id="article_content"]//img/@src').extract()
        if images:
            images_url = []
            for img in images:
                if 'http' in img:
                    images_url.append(img)
                else:
                    image = f'{self.base_url}{img}'
                    images_url.append(image)
            images_urls = '; '.join(images_url)
            item['images'] = images_urls if images_urls else None
        else:
            item['images'] = None

        item['source'] = '路桥技术网'
        item['information_source'] = '路桥技术网'
        item['content'] = content
        item['attachments'] = None
        item['area'] = None
        item['address'] = None
        item['sign'] = '19'
        item['update_time'] = str(int(time.time() * 1000))
        # print(item)
        if content:
            yield item
            self.logger.info("title({}), issue_time({})".format(item['title'], item['issue_time']))

if __name__ == '__main__':
    from scrapy import cmdline
    cmdline.execute(['scrapy', 'crawl', 'cnluqiao_V1'])


"""
Run a nongaussian Kalman Filter twin experiment with Lorenz-63
"""

# Load modules
import numpy as np
import pickle
import mod_KalmanDA as da
import LorenzModels as LM
import matplotlib.pyplot as plt
import tikzplotlib as tpl

# Options for DA run
n_wind = 25                                # Number of DA windows for each run
obs_vars = 'xyz'                            # Observed variables, either 'xy', or 'xyz'
ml_method = 'array'                         # Nongaussian decision function, either
                                            #   'array',    selection based on observations before DA run
                                            #   'function', selection from decision function every DA window
seed = 3405                                 # Seed for random number generator, can be set to None
SV_init = [-5.0,-6.0,22.0]                  # Initial values of the Lorenz-63 run

###########################################################################################
###########################################################################################
###########################################################################################

def one_run():
    global n_wind
    global period_obs
    global obs_vars
    global var_obs
    global ml_method
    global ml_file
    global seed
    global SV_init

    # Load the machine learning model
    with open(ml_file,'rb') as f:
        clf = pickle.load(f)
        scaler = pickle.load(f)
        info = pickle.load(f)
        p = info['p']               # Parameters of L63, typically (10,28,8/3)
        dt = info['dt']             # Time step of L63
    rng = np.random.default_rng(seed)
    SV_init += rng.standard_normal(3)

    # Nature run
    t_max = n_wind*period_obs*dt        # End time of the Lorenz-63 run
    t = np.arange(0.0, t_max, dt)  # Evaluation time
    SV = LM.sol_L63(t, SV_init, p)
    xi_SV = np.nanmax(SV[2,:])+5.0      # Parameter of the reverse lognormal distribution

    ###########################################################################################
    ###########################################################################################
    ###########################################################################################

    # Create observation operator
    n_SV = 3
    if obs_vars == 'xyz':
        n_obs = 3
        obs_h = np.eye(n_SV)
    if obs_vars == 'xy':
        n_obs = 2
        obs_h = np.zeros((n_obs,n_SV))
        obs_h[0,0] = 1.0
        obs_h[1,1] = 1.0

    # Predict distribution from which to sample z-variable
    if obs_vars == 'xyz':
        # Predict distribution from nature run
        X_data = scaler.transform(np.transpose(SV[:2,:]))
        z_pred = clf.predict(X_data)
        # Define observation times when lognormal or reverse lognormal noise should be added
        ln_vars_ML = []
        rl_vars_ML = []
        for ii in range(t.size):
            if ii % period_obs == 0:
                if z_pred[ii] > 0.5: # Lognormal distribution predicted
                    ln_vars_ML.append([2])
                    rl_vars_ML.append([])
                elif z_pred[ii] < -0.5: # Reverse lognormal distribution predicted
                    ln_vars_ML.append([])
                    rl_vars_ML.append([2])
                else: # Gaussian distribution predicted
                    ln_vars_ML.append([])
                    rl_vars_ML.append([])
    if obs_vars == 'xy':
        # If z is not observed, ML model not necessary
        ln_vars_ML = []
        rl_vars_ML = []

    # Generate observations
    t_obs, y, R = da.gen_obs(t, SV, period_obs, obs_h, var_obs, \
        ln_vars = ln_vars_ML, rl_vars = rl_vars_ML, xi_obs = xi_SV, seed = seed, sample='mode')

    ###########################################################################################
    ###########################################################################################
    ###########################################################################################

    # DA initial guess
    init_guess = SV_init + rng.standard_normal(3)

    # Model function, of the form x = model(t,x_0)
    def model_l63(t, x0):
        y = LM.sol_L63(t, x0, p)
        return y

    # Decision function for lognormal state variables (for a single prediction)
    def f_ln_vars(SV):
        X = scaler.transform(SV[:2].reshape(1,-1))
        z_pred = clf.predict(X)
        if z_pred > 0.5:
            return [2]
        else:
            return []
    # Decision function for reverse-lognormal state variables (for a single prediction)
    def f_rl_vars(SV):
        X = scaler.transform(SV[:2].reshape(1,-1))
        z_pred = clf.predict(X)
        if z_pred < -0.5:
            return [2]
        else:
            return []

    # Create decision functions/arrays
    if ml_method == 'array':
        ln_vars = []
        rl_vars = []
        for ii in range(t_obs.size):
            ln_vars.append(f_ln_vars(y[:2,ii]))
            rl_vars.append(f_rl_vars(y[:2,ii]))
        ln_B0 = ln_vars[0]
        rl_B0 = rl_vars[0]
    if ml_method == 'function':
        ln_vars = f_ln_vars
        rl_vars = f_rl_vars
        ln_B0 = f_ln_vars(init_guess)
        rl_B0 = f_rl_vars(init_guess)

    # Store different decision functions for different (non)gaussian methods
    ln_vars_SV_OBJ = np.array([ \
        [],      # Gaussian for all t
        [2],     # Lognormal for all t
        [],      # Reverse Lognormal for all t
        ln_vars, # Gaussian - Lognormal ML
        [],      # Gaussian - Reverse Lognormal ML
        ln_vars, # Gaussian - Lognormal - Reverse Lognormal ML
        [],      # No DA (background run)
    ], dtype = object)
    rl_vars_SV_OBJ = np.array([ \
        [],      # Gaussian for all t
        [],      # Lognormal for all t
        [2],     # Reverse Lognormal for all t
        [],      # Gaussian - Lognormal ML
        rl_vars, # Gaussian - Reverse Lognormal ML
        rl_vars, # Gaussian - Lognormal - Reverse Lognormal ML
        [],      # No DA (background run)
    ], dtype = object)
    if obs_vars == 'xyz':
        ln_vars_obs_OBJ = ln_vars_SV_OBJ
        rl_vars_obs_OBJ = rl_vars_SV_OBJ
    elif obs_vars == 'xy':
        ln_vars_obs_OBJ = np.array([[],[],[],[],[],[],[-1.0]], dtype = object)
        rl_vars_obs_OBJ = np.array([[],[],[],[],[],[],[-1.0]], dtype = object)

    # Initialize analysis and background state variables
    n_meths = 7
    n_t_obs = t_obs.size           # number of observations
    n_t = n_t_obs * period_obs + 1 # number of total time steps
    X_A = np.empty((n_meths, n_SV, n_t_obs))
    X_B = np.empty((n_meths, n_SV, n_t))

    # Model error covariance matrix
    Q = np.array([ \
        [0.1491, 0.1505, 0.0007], \
        [0.1505, 0.9048, 0.0014], \
        [0.0007, 0.0014, 0.9180] \
    ])

    ln_vars_B_OBJ = np.array([ [], [2], [], ln_B0, [], ln_B0, []], dtype = object)
    rl_vars_B_OBJ = np.array([ [], [], [2], [], rl_B0, rl_B0, []], dtype = object)

    for iM in range(n_meths-1):
        # Create initial background error covariance matrix
        B_case = 'nmc'
        P_a = da.create_B_init(B_case, \
            SV_init = SV_init, \
            p = p, \
            seed = seed, \
            ln_vars=ln_vars_B_OBJ[iM], rl_vars=rl_vars_B_OBJ[iM])

        X_A[iM,:,:], X_B[iM,:,:], _, _ = da.kalman_filter( \
            init_guess, t_obs, period_obs, y, obs_h, P_a, R, Q, \
            model_l63, \
            ln_vars_SV = ln_vars_SV_OBJ[iM], ln_vars_obs = ln_vars_obs_OBJ[iM], \
            rl_vars_SV = rl_vars_SV_OBJ[iM], rl_vars_obs = rl_vars_obs_OBJ[iM], \
            xi_SV = xi_SV, xi_obs = xi_SV \
        )

    # Background run
    iM += 1
    dt_obs = t_obs[1] - t_obs[0] # Assuming evenly spaced observations
    t_true = np.linspace(t_obs[0],t_obs[-1] + dt_obs, n_t) # Assuming one prediction window
    X_B[iM,:,:] = model_l63(t_true, init_guess)
    X_A[iM,:,:] = X_B[iM,:,:-1:period_obs]

    MAE_A = np.empty(n_meths)
    MAE_B = np.empty(n_meths)
    for iM in range(n_meths):
        MAE_A[iM] = np.nanmean(np.abs((SV[:,::period_obs] - X_A[iM,:,:])))
        MAE_B[iM] = np.nanmean(np.abs((SV - X_B[iM,:,:-1])))

    return MAE_A, MAE_B, t_true, t_obs, X_A, X_B, y, SV, ln_vars_ML, rl_vars_ML, ln_vars, rl_vars

###########################################################################################
###########################################################################################
###########################################################################################
###########################################################################################
###########################################################################################
###########################################################################################
meths = [ \
    'Gaussian ', \
    'Lognormal', \
    'Rev logn ', \
    'G-LogNorm', \
    'G-RevLog ', \
    'All mixed', \
    'noDA     ' \
]
n_meths = len(meths)
ml_file = './kNN_l63_w14_s1.0.pkl'    # Location of the trained ML model

period_obs = 40
var_obs = 3.0

MAE_A, MAE_B, t, t_obs, X_A, X_B, y, SV, ln_vars_ML, rl_vars_ML, ln_vars, rl_vars = one_run()

print("Finished p = "+str(period_obs)+", s = "+str(var_obs))
for iM in range(n_meths):
    print(meths[iM]+ \
        ': mean(r_a) = '+format(np.round(MAE_A[iM],3),".3f")+ \
        ', mean(r_b) = '+format(np.round(MAE_B[iM],3),".3f"))

y_gs = np.full(y.shape,np.nan)
y_ln = np.full(y.shape,np.nan)
y_rl = np.full(y.shape,np.nan)
for jj in range(t_obs.size):
    if ln_vars_ML[jj]:
        y_ln[:,jj] = y[:,jj]
    elif rl_vars_ML[jj]:
        y_rl[:,jj] = y[:,jj]
    else:
        y_gs[:,jj] = y[:,jj]
y_gs_DA = np.full(y.shape,np.nan)
y_ln_DA = np.full(y.shape,np.nan)
y_rl_DA = np.full(y.shape,np.nan)
for jj in range(t_obs.size):
    if ln_vars[jj]:
        y_ln_DA[:,jj] = y[:,jj]
    elif rl_vars[jj]:
        y_rl_DA[:,jj] = y[:,jj]
    else:
        y_gs_DA[:,jj] = y[:,jj]

# n_plot = n_meths-1
n_plot = 2
fig, ax = plt.subplots(n_plot,3,figsize=(15, 5))
vars = ['$x$', '$y$', '$z$']
meth_select = [0,5]
for iMeth in range(n_plot):
    for jj in range(3):
        # True state
        ax[iMeth,jj].plot(t[:-1],SV[jj,:],label = 'Nature')

        # Background state
        ax[iMeth,jj].plot(t[:-1],X_B[meth_select[iMeth],jj,:-1],'--',label = 'Background')

        # Observations
        if not (jj==2 and obs_vars == 'xy'):
            ax[iMeth,jj].plot(t_obs,y_gs[jj,:],'.',label = 'Gs Observations')
            ax[iMeth,jj].plot(t_obs,y_ln[jj,:],'.',label = 'LN Observations')
            ax[iMeth,jj].plot(t_obs,y_rl[jj,:],'.',label = 'RL Observations')
        if not (jj==2 and obs_vars == 'xy'):
            ax[iMeth,jj].scatter(t_obs,y_gs_DA[jj,:],s=50, facecolors='none',edgecolors='#2ca02c',label = 'Gs ML')
            ax[iMeth,jj].scatter(t_obs,y_ln_DA[jj,:],s=50, facecolors='none',edgecolors='#d62728',label = 'LN ML')
            ax[iMeth,jj].scatter(t_obs,y_rl_DA[jj,:],s=50, facecolors='none',edgecolors='#9467bd',label = 'RL ML')

        # # Analysis
        # ax[iMeth,jj].plot(t_obs,X_A[meth_select[iMeth],jj,:],'.',label = 'Analysis')

        ax[iMeth,jj].set_xlabel('$t$')
        ax[iMeth,jj].set_ylabel(vars[jj])
    ax[iMeth,0].set_title(meths[meth_select[iMeth]])
    ax[iMeth,0].set_ylim(-25,25)
    ax[iMeth,1].set_ylim(-30,30)
    ax[iMeth,2].set_ylim(0,50)
ax[0,0].legend()
# tpl.clean_figure()
# tpl.save("KF_ML_example.tex")
plt.show()

# Write to files
vars = ['x', 'y', 'z']
for var in range(3):
    with open("./values/"+vars[var]+"_true.dat",'w') as f:
        for ii in range(t[:-1].size):
            f.write(str(t[ii])+" "+str(SV[var,ii])+"\n")
    with open("./values/"+vars[var]+"_b_gs.dat",'w') as f:
        for ii in range(t[:-1].size):
            f.write(str(t[ii])+" "+str(X_B[0,var,ii])+"\n")
    with open("./values/"+vars[var]+"_b_ml.dat",'w') as f:
        for ii in range(t[:-1].size):
            f.write(str(t[ii])+" "+str(X_B[5,var,ii])+"\n")

    with open("./values/"+vars[var]+"_obs_gs.dat",'w') as f:
        for ii in range(1,t_obs.size):
            f.write(str(t_obs[ii])+" "+str(y_gs[var,ii])+"\n")
    with open("./values/"+vars[var]+"_obs_ln.dat",'w') as f:
        for ii in range(1,t_obs.size):
            f.write(str(t_obs[ii])+" "+str(y_ln[var,ii])+"\n")
    with open("./values/"+vars[var]+"_obs_rl.dat",'w') as f:
        for ii in range(1,t_obs.size):
            f.write(str(t_obs[ii])+" "+str(y_rl[var,ii])+"\n")

    with open("./values/"+vars[var]+"_mlp_gs.dat",'w') as f:
        for ii in range(1,t_obs.size):
            f.write(str(t_obs[ii])+" "+str(y_gs_DA[var,ii])+"\n")
    with open("./values/"+vars[var]+"_mlp_ln.dat",'w') as f:
        for ii in range(1,t_obs.size):
            f.write(str(t_obs[ii])+" "+str(y_ln_DA[var,ii])+"\n")
    with open("./values/"+vars[var]+"_mlp_rl.dat",'w') as f:
        for ii in range(1,t_obs.size):
            f.write(str(t_obs[ii])+" "+str(y_rl_DA[var,ii])+"\n")


from django.shortcuts import render, redirect, get_object_or_404
from django.contrib.auth import get_user_model
from django.contrib.auth.decorators import login_required
from account.models import User
from .models import Task
import sweetify

from twilio.rest import Client
import os
import environ

@login_required(login_url='login')
def add_task(request, user_id):
    if request.user.is_superuser:
        employee = get_object_or_404(User, pk=user_id)
        if request.method == "POST":

            user_id = request.POST['employee_id']
            task = request.POST['task']
            priority = request.POST['priority']

            task = Task.objects.create(
                user_id=user_id,
                task=task,
                priority=priority,
                )

            task.save()
            env = environ.Env()
            environ.Env().read_env()
            TWILIO_NUMBER = env("TWILIO_NUMBER")

        # client credentials are read from TWILIO_ACCOUNT_SID and AUTH_TOKEN
            twilio  = Client()

            twilio.messages.create(
                from_=TWILIO_NUMBER,
                to='whatsapp:+2347030102959',
                body=f'New Task Added. {task}. The task is marked as {priority}')

            sweetify.info(request, f"Task added to {employee.username} Successfully", button='Ok', timer=3000)
            return redirect("all_employees")
        context = {
           "employee" : employee,
           'page_title':f'Add Task to {employee.username}',
       }

        return render(request, 'admin-dashboard/tasks.html', context)

    if not request.user.is_superuser:
        return render(request, 'employee-dashboard/index.html')

@login_required(login_url='login')
def update_task(request, task_id):

    # status = request.POST['status']
    employee_id = request.POST['employee_id']

    employee_task = Task.objects.filter(pk=task_id)
    for task in employee_task:
        this_status = task.status

    if this_status == 0:
        employee_task.update(status=1)
        sweetify.info(request, f"Task marked as complete ", button='Ok', timer=3000)
        return redirect("view_employee", employee_id)

    if this_status == 1:
        employee_task.update(status=0)
        sweetify.info(request, f"Task marked as incomplete ", button='Ok', timer=3000)
        return redirect("view_employee", employee_id)

@login_required(login_url='login')
def employee_update_task(request, task_id):

    # status = request.POST['status']
    employee_id = request.POST['employee_id']

    employee_task = Task.objects.filter(pk=task_id)
    for task in employee_task:
        this_status = task.status

    if this_status == 0:
        employee_task.update(status=1)
        sweetify.info(request, f"Task marked as complete ", button='Ok', timer=3000)
        return redirect('employee_dashboard')

    if this_status == 1:
        employee_task.update(status=0)
        sweetify.info(request, f"Task marked as incomplete ", button='Ok', timer=3000)
        return redirect('employee_dashboard')

@login_required(login_url='login')
def delete_task(request, task_id):
    employee_task = get_object_or_404(Task, pk=task_id)
    if request.method == "POST":
        employee_task.delete()
        sweetify.info(request, "Task Deleted Successfully", button='Ok', timer=3000)
        return redirect("all_employees")
    # context = {
    #     "employee" : employee
    # }

    # return render(request, 'admin-dashboard/delete-employee.html', context)