from ospark.nn.loss_function import LossFunction
from typing import Type
import importlib
import os

LOSS_REGISTRY_CLASS_NAME = set()
LOSS_REGISTRY = {}

class RegisterMeta(type):

    def __new__(mcls, name: str, base: tuple, attr: dict) -> Type[LossFunction]:
        cls = super().__new__(mcls, name, base, attr)
        if name not in LOSS_REGISTRY_CLASS_NAME:
            LOSS_REGISTRY_CLASS_NAME.add(name)
        else:
            raise KeyError(f"The class {name} has been registered.")
        registered_name = attr["register_name"]
        if registered_name is not None and registered_name not in LOSS_REGISTRY:
            LOSS_REGISTRY[registered_name] = cls

        elif registered_name in register_type:
            raise KeyError(f"The register name {registered_name} is duplicated.")

        else:
            raise KeyError("register_name is not named")
        return cls

# Automatically import python files under BAT/loss.
for filename in os.listdir(os.path.dirname(__file__)):
    if "__" not in filename and ".py" in filename:
        module_name = filename.replace(".py", "")
        importlib.import_module("BAT.loss." + module_name)


import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from chinaz.items import ChinazItem

class KexueSpider(CrawlSpider):
    name = "kexue"
    allowed_domains = ["chinaz.com"]
    start_urls = ["https://sc.chinaz.com/tupian/taikongkexuetupian.html"]

    rules = (
        Rule(LinkExtractor(restrict_xpaths=('//div[@class="tupian-list com-img-txt-list"]/div')),
             callback="parse_item", follow=False),
        Rule(LinkExtractor(restrict_xpaths=('//div[@class="new-two-page-box container"]/a')),
             callback="parse_item", follow=True),
    )

    def parse_item(self, response):
        img_url = response.xpath(
            '//div[@class="img-box"]/img/@src').get()

        img_name = response.xpath(
            './/h1[@class="name"]/text()').get()

        # 实例化item对象
        item = ChinazItem()
        item['img_name'] = img_name
        item['img_url'] = response.urljoin(img_url)

        yield item


import torch, re, yaml, os
import numpy as np
import collections.abc as container_abcs
from ot.backend import get_backend

class AlignmentDataset(torch.utils.data.Dataset):
    def __init__(self, s1_sents, s2_sents, input_s2t, alignment_list, subword_to_word_map, s2_start_idxes):
        self.s1_sents = [' '.join(sent) for sent in s1_sents]
        self.s2_sents = [' '.join(sent) for sent in s2_sents]
        self.input_s2t = input_s2t
        self.align_strs = alignment_list
        self.align_tuples = []
        for aligns in alignment_list:
            tuples = []
            for pair in aligns.split():
                i_j = pair.split('-')
                tuples.append((int(i_j[0]), int(i_j[1])))
            self.align_tuples.append(tuples)
        self.subword_to_word_maps = subword_to_word_map
        self.s2_start_idxes = s2_start_idxes

    def __getitem__(self, idx):
        item = {'s2t_' + key: val[idx].clone().detach() for key, val in self.input_s2t.items()}
        item['s1_sents'] = self.s1_sents[idx]
        item['s2_sents'] = self.s2_sents[idx]
        item['alignments'] = self.align_strs[idx]
        item['atuples'] = self.align_tuples[idx]
        item['subword_to_word_map'] = self.subword_to_word_maps[idx]
        item['s2_start_idx'] = self.s2_start_idxes[idx]
        return item

    def __len__(self):
        return len(self.align_strs)

def alignment_collate(batch):
    r"""Puts each data field into a tensor with outer dimension batch size"""

    np_str_obj_array_pattern = re.compile(r'[SaUO]')
    default_collate_err_msg_format = (
        "alignment_collate: batch must contain tensors, numpy arrays, numbers, "
        "dicts or lists; found {}")

    elem = batch[0]
    elem_type = type(elem)
    if isinstance(elem, torch.Tensor):
        out = None
        if torch.utils.data.get_worker_info() is not None:
            # If we're in a background process, concatenate directly into a
            # shared memory tensor to avoid an extra copy
            numel = sum([x.numel() for x in batch])
            storage = elem.storage()._new_shared(numel)
            out = elem.new(storage)
        return torch.stack(batch, 0, out=out)
    elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
            and elem_type.__name__ != 'string_':
        if elem_type.__name__ == 'ndarray' or elem_type.__name__ == 'memmap':
            # array of string classes and object
            if np_str_obj_array_pattern.search(elem.dtype.str) is not None:
                raise TypeError(default_collate_err_msg_format.format(elem.dtype))

            return alignment_collate([torch.as_tensor(b) for b in batch])
        elif elem.shape == ():  # scalars
            return torch.as_tensor(batch)
    elif isinstance(elem, float):
        return torch.tensor(batch, dtype=torch.float64)
    elif isinstance(elem, int):
        return torch.tensor(batch)
    elif isinstance(elem, str):
        return batch
    elif isinstance(elem, container_abcs.Mapping):
        return {key: alignment_collate([d[key] for d in batch]) for key in elem}
    elif isinstance(elem, tuple) and hasattr(elem, '_fields'):  # namedtuple
        return elem_type(*(alignment_collate(samples) for samples in zip(*batch)))
    elif isinstance(elem, container_abcs.Sequence):
        # Allow arbitrary length lists
        return batch
    raise TypeError(default_collate_err_msg_format.format(elem_type))

def send_batch_to_gpu(elem, device):
    if isinstance(elem, torch.Tensor):
        return elem.to(device)
    elif isinstance(elem, dict):
        return {key: send_batch_to_gpu(val, device) for key, val in elem.items()}
    elif isinstance(elem, list):
        return [send_batch_to_gpu(val, device) for val in elem]
    else:
        return elem

def read_Word_Alignment_Dataset(filename, sure_and_possible=False, transpose=False):
    data = []
    for line in open(filename):
        ID, sent1, _, sent2, _, _, _, sure_align, poss_align, *useless = line.strip('\n').split('\t')
        my_dict = {}
        my_dict['source'] = sent1
        my_dict['target'] = sent2
        my_dict['sureAlign'] = sure_align
        my_dict['possibleAlign'] = poss_align
        data.append(my_dict)
    sent1_list = []
    sent2_list = []
    alignment_list = []
    for i in range(len(data)):
        if transpose:
            source = data[i]['target']
            target = data[i]['source']
        else:
            source = data[i]['source']
            target = data[i]['target']
        alignment = data[i]['sureAlign']
        if sure_and_possible:
            alignment += ' ' + data[i]['possibleAlign']
        my_label = []
        for item in alignment.split():  # reverse the alignment
            i, j = item.split('-')
            if transpose:
                my_label.append(str(j) + '-' + str(i))
            else:
                my_label.append(str(i) + '-' + str(j))
        alignment = ' '.join(my_label)
        sent1_list.append(source.lower().split())
        sent2_list.append(target.lower().split())
        alignment_list.append(alignment)
    return (sent1_list, sent2_list, alignment_list)

def read_Preprocessed_Word_Alignment_Dataset(file_path):
    sent1_list = []
    sent2_list = []
    alignment_list = []

    with open(file_path) as f:
        for line in f:
            array = [item.strip() for item in line.split('\t')]
            sent1_list.append(array[0].lower().split())
            sent2_list.append(array[1].lower().split())
            alignment_list.append(array[2])

    return (sent1_list, sent2_list, alignment_list)

def load_WA_corpus(data, type, sure_and_possible):
    if data in ['mtref', 'wiki', 'msr', 'edinburgh']:
        load_data = data
    else:
        if type == 'test':
            load_data = data
        else:  # Use mtref train & dev sets for newsela and arxiv
            load_data = 'mtref'

    if load_data == 'mtref':
        examples = read_Word_Alignment_Dataset('../data/MultiMWA-data/MultiMWA-MTRef/mtref-' + type + '.tsv',
                                               sure_and_possible, True)
    elif load_data == 'wiki':
        examples = read_Word_Alignment_Dataset('../data/MultiMWA-data/MultiMWA-Wiki/wiki-' + type + '.tsv',
                                               sure_and_possible, False)
    elif load_data == 'newsela':
        examples = read_Word_Alignment_Dataset('../data/MultiMWA-data/MultiMWA-Newsela/newsela-' + type + '.tsv',
                                               sure_and_possible, False)
    elif load_data == 'arxiv':
        examples = read_Word_Alignment_Dataset('../data/MultiMWA-data/MultiMWA-arXiv/arxiv-' + type + '.tsv',
                                               sure_and_possible, False)
    elif load_data == 'msr':
        data_path = '../data/RTE-2006-Aligned/'
        if sure_and_possible:
            data_path += 'sure-and-possible_' + type + '.txt'
        else:
            data_path += 'sure_' + type + '.txt'
        examples = read_Preprocessed_Word_Alignment_Dataset(data_path)
    elif load_data == 'edinburgh':
        data_path = '../data/edinburgh/'
        if sure_and_possible:
            data_path += 'sure-and-possible_' + type + '.txt'
        else:
            data_path += 'sure_' + type + '.txt'
        examples = read_Preprocessed_Word_Alignment_Dataset(data_path)

    return examples

def load_lr_setting(file_path, patience, sure_and_possible, ot_type):
    with open(file_path, 'r') as yml:
        config = yaml.safe_load(yml)
    sp_key = 'sure_and_possible' if sure_and_possible else 'sure'
    lr = float(config['patience_{0}'.format(patience)][sp_key][ot_type]['lr'])
    return lr

def load_distortion_setting(file_path, data, sure_and_possible):
    with open(file_path, 'r') as yml:
        config = yaml.safe_load(yml)

    sure_type = 'sure_and_possible' if sure_and_possible else 'sure'
    distortion = config[sure_type][data]['distortion']

    return distortion

def load_ot_setting(dir_path, ot_type):
    if ot_type == 'ot':
        return 0.0
    elif ot_type == 'pot':
        files = os.listdir(dir_path)
        files_dir = [f for f in files if os.path.isdir(os.path.join(dir_path, f))]
        assert len(files_dir) == 2
        # Load setting without Sinkhorn
        with open(os.path.join(dir_path, files_dir[0], 'dev_log.txt')) as f:
            dev_logs = f.readlines()
        settings = dev_logs[-1].strip().split('\t')
        hyp = float(settings[1])
        return hyp
    else:
        files = os.listdir(dir_path)
        files_dir = [f for f in files if os.path.isdir(os.path.join(dir_path, f))]
        assert len(files_dir) == 1
        with open(os.path.join(dir_path, files_dir[0], 'dev_log.txt')) as f:
            dev_logs = f.readlines()
        settings = dev_logs[-1].strip().split('\t')
        hyp = float(settings[1])
        thresh = float(settings[2])

        return hyp

    raise Exception('OT setting load error!')

def get_token_word_mapping(all_input_s2t, tokenizer):
    all_subword_to_word_map, all_s2_start_idx = [], []
    for token_ids, offset_mapping in zip(all_input_s2t['input_ids'], all_input_s2t['offset_mapping']):
        word_idx = -1
        subword_to_word_conv = np.full((len(token_ids)), -1)
        # Bug in hugging face tokenizer? Sometimes Metaspace is inserted
        metaspace = getattr(tokenizer.decoder, "replacement", None)
        metaspace = tokenizer.decoder.prefix if metaspace is None else metaspace
        tokenizer_bug_idxes = [i for i, x in enumerate(tokenizer.convert_ids_to_tokens(token_ids)) if x == metaspace]

        for subw_idx, offset in enumerate(offset_mapping):
            if subw_idx in tokenizer_bug_idxes:
                continue
            elif offset[0] == offset[1]:  # Special token
                continue
            elif offset[0] == 0:
                word_idx += 1
                subword_to_word_conv[subw_idx] = word_idx
            else:
                subword_to_word_conv[subw_idx] = word_idx

        sep_tok_indices = [i for i, x in enumerate(token_ids) if x == tokenizer.sep_token_id]
        s2_start_idx = subword_to_word_conv[
            sep_tok_indices[0] + np.argmax(subword_to_word_conv[sep_tok_indices[0]:] > -1)]

        all_subword_to_word_map.append(subword_to_word_conv)
        all_s2_start_idx.append(s2_start_idx)

    return all_subword_to_word_map, all_s2_start_idx

def get_aligned_indices(alignments):
    aligned_s1_indices, aligned_s2_indices = set(), set()
    for a in alignments:
        i_j = a.split('-')
        aligned_s1_indices.add(int(i_j[0]))
        aligned_s2_indices.add(int(i_j[1]))
    return aligned_s1_indices, aligned_s2_indices

def grow_diag_final(e2f, f2e, len_e, len_f):
    alignments = grow_diag(e2f, f2e, len_e, len_f)
    alignments = final(len_e, len_f, e2f | f2e, alignments)
    return alignments

def get_aligned_points(alignments):
    aligned_e = set()
    aligned_f = set()
    for a in alignments:
        a = a.split('-')
        aligned_e.add(int(a[0]))
        aligned_f.add(int(a[1]))
    return aligned_e, aligned_f

def get_neighbors(e_idx, f_idx, e_len, f_len):
    neighboring = [(-1, 0), (0, -1), (1, 0), (0, 1), (-1, -1), (-1, 1), (1, -1), (1, 1)]
    new_points = [(e_idx + e, f_idx + f) for e, f in neighboring if
                  e_idx + e >= 0 and e_idx + e < e_len and f_idx + f >= 0 and f_idx + f < f_len]

    return new_points

def grow_diag(e2f, f2e, len_e, len_f):
    alignments = e2f & f2e
    union = e2f | f2e
    aligned_e, aligned_f = get_aligned_points(alignments)
    new_points_added = True

    while new_points_added:
        new_points_added = False
        for e_idx in range(len_e):
            for f_idx in range(len_f):
                if '{0}-{1}'.format(e_idx, f_idx) in alignments:
                    for e_new, f_new in get_neighbors(e_idx, f_idx, len_e, len_f):
                        new_point = '{0}-{1}'.format(e_new, f_new)
                        if (e_new not in aligned_e or f_new not in aligned_f) and new_point in union:
                            alignments.add(new_point)
                            aligned_e.add(e_new)
                            aligned_f.add(f_new)
                            new_points_added = True

    return alignments

def final(len_e, len_f, union, alignments):
    aligned_e, aligned_f = get_aligned_points(alignments)
    for e_new in range(len_e):
        for f_new in range(len_f):
            new_point = '{0}-{1}'.format(e_new, f_new)
            if (e_new not in aligned_e or f_new not in aligned_f) and new_point in union:
                alignments.add(new_point)
                aligned_e.add(e_new)
                aligned_f.add(f_new)

    return alignments

def atoi(text):
    return int(text) if text.isdigit() else text

def natural_keys(text):
    return [atoi(c) for c in re.split(r'(\d+)', text)]

def cosine_dist_matrix(matX, matY, eps=1e-08):
    num = torch.matmul(matX, matY.t())
    denom = torch.maximum(torch.full_like(num, eps),
                          torch.matmul(torch.norm(matX, dim=1, keepdim=True),
                                       torch.norm(matY, dim=1, keepdim=True).t()))
    cos_dist = 1 - num / denom
    return cos_dist

def convert_to_word_embeddings(offset_mapping, token_ids, hidden_tensors, tokenizer, pair):
    word_idx = -1
    subword_to_word_conv = np.full((hidden_tensors.shape[0]), -1)
    # Bug in hugging face tokenizer? Sometimes Metaspace is inserted
    metaspace = getattr(tokenizer.decoder, "replacement", None)
    metaspace = tokenizer.decoder.prefix if metaspace is None else metaspace
    tokenizer_bug_idxes = [i for i, x in enumerate(tokenizer.convert_ids_to_tokens(token_ids)) if
                           x == metaspace]

    for subw_idx, offset in enumerate(offset_mapping):
        if subw_idx in tokenizer_bug_idxes:
            continue
        elif offset[0] == offset[1]:  # Special token
            continue
        elif offset[0] == 0:
            word_idx += 1
            subword_to_word_conv[subw_idx] = word_idx
        else:
            subword_to_word_conv[subw_idx] = word_idx

    word_embeddings = torch.vstack(
        ([torch.mean(hidden_tensors[subword_to_word_conv == word_idx], dim=0) for word_idx in range(word_idx + 1)]))

    if pair:
        sep_tok_indices = [i for i, x in enumerate(token_ids) if x == tokenizer.sep_token_id]
        s2_start_idx = subword_to_word_conv[
            sep_tok_indices[0] + np.argmax(subword_to_word_conv[sep_tok_indices[0]:] > -1)]

        s1_word_embeddigs = word_embeddings[0:s2_start_idx, :]
        s2_word_embeddigs = word_embeddings[s2_start_idx:, :]

        return s1_word_embeddigs, s2_word_embeddigs
    else:
        return word_embeddings

def split_list_with_window(l, slide, window_size):
    if len(l) <= window_size - slide:
        return [l]
    else:
        return [l[i:i + window_size] for i in range(0, len(l) - window_size + slide, slide)]

def min_max_scaling(C):
    eps = 1e-10
    # Min-max scaling for stabilization
    nx = get_backend(C)
    C_min = nx.min(C)
    C_max = nx.max(C)
    C = (C - C_min + eps) / (C_max - C_min + eps)
    return C

def patch_attention(m):
    forward_orig = m.forward

    def wrap(*args, **kwargs):
        kwargs['need_weights'] = True
        # kwargs['average_attn_weights'] = False

        return forward_orig(*args, **kwargs)

    m.forward = wrap


from abc import ABC
from datetime import datetime

DATE_FORMAT = '%Y-%m-%d %H:%M:%S'

ERROR = -2
NOT_DELIVERED = -1
RECEIVED = 0
OPERATOR = 1
DELIVERED = 2
POSTPONED = 3

MESSAGE_STATUSES = (
    (ERROR, 'Ошибка', 'Ошибка, неправильные параметры запроса'),
    (NOT_DELIVERED, 'Не доставлено',
     'Сообщение не доставлено (не в сети, заблокирован, не взял трубку), PING - не в сети, HLR - не обслуживается (заблокирован)'),
    (RECEIVED, 'Новое', 'Новое сообщение/запрос, ожидает обработки у нас на сервере'),
    (OPERATOR, 'В очереди', 'Сообщение или запрос ожидают отправки на сервере оператора'),
    (DELIVERED, 'Доставлено', 'Доставлено, звонок совершен, PING - в сети, HLR - обслуживается'),
    (POSTPONED, 'Отложено', 'Отложенная отправка, отправка сообщения/запроса запланирована на другое время'),
)

def get_status_dict(short=True) -> dict:
    if short:
        return dict([(code, short) for code, short, description in MESSAGE_STATUSES])
    return dict([(code, description) for code, short, description in MESSAGE_STATUSES])

class MessageBase(ABC):
    def is_delivered(self) -> bool:
        return self.status == DELIVERED

    def is_error(self) -> bool:
        return self.status == ERROR

    def get_status_verbose(self) -> str:
        return get_status_dict().get(self.status)

class MessageResponse(MessageBase):

    def __init__(self, server_response: dict):
        msg_obj = server_response.get('send', [])[-1]
        self.server_id = int(msg_obj.get('server_id', 0))
        self.phone = int(msg_obj.get('phone', 0))
        self.price = float(msg_obj.get('price', 0))
        self.status = int(msg_obj.get('status', 0))
        self.sender = msg_obj.get('sender')
        self.sender_orig = msg_obj.get('sender_orig')
        try:
            self.created = datetime.strptime(msg_obj.get('created'), DATE_FORMAT)
        except (ValueError, TypeError):
            self.created = None
        try:
            self.modified = datetime.strptime(msg_obj.get('modified'), DATE_FORMAT)
        except (ValueError, TypeError):
            self.modified = None
        self.parts = int(msg_obj.get('parts', 0))
        self.country = msg_obj.get('country')
        self.operator = msg_obj.get('operator')
        self.error = int(msg_obj.get('error', 0))
        self.error_en = msg_obj.get('error_en')
        self.error_ru = msg_obj.get('error_ru')
        self.message = msg_obj.get('message')
        self.balance = float(server_response.get('balance'))
        self.cost = float(server_response.get('cost', 0))

    def is_success(self):
        return self.error == 0

    def __str__(self):
        return 'MessageResponse: %s (%s)' % (self.phone, self.error_en)

class Message(MessageBase):
    def __init__(self, **data):
        self.id = int(data.get('id'))
        self.server_id = int(data.get('server_id'))
        self.from_ = data.get('from')
        self.to = int(data.get('to'))
        self.text = data.get('text')
        self.parts = int(data.get('parts'))
        self.price = float(data.get('price'))
        self.status = int(data.get('status'))
        self.error = data.get('error')

        try:
            self.send_datetime = datetime.strptime(data.get('send_datetime'), '%Y-%m-%d %H:%M:%S')
        except ValueError:
            self.send_datetime = None

        self.country = data.get('country')
        self.operator = data.get('operator')

    def __str__(self):
        return 'Message(%s): %s %s' % (self.id, self.to, self.get_status_verbose())

    def __repr__(self):
        return str(self)

class MessagesResponse:

    def __init__(self, server_response: dict):
        self.send = [Message(**msg) for msg in server_response.get('send', [])]
        self.cost = float(server_response.get('cost', 0))
        self.balance = float(server_response.get('balance', 0))
        self.server_packet_id = int(server_response.get('server_packet_id', 0))

    def __len__(self):
        return len(self.send)

    def __str__(self):
        return 'MessageResponse: %s' % self.server_packet_id

class MessageCheck(MessageBase):

    def __init__(self, check: dict):
        self.id = int(check.get('id', 0))
        self.server_id = int(check.get('id', 0))
        self.status = int(check.get('status', ERROR))
        self.modified = check.get('modified')

    def is_delivered(self) -> bool:
        return self.status == DELIVERED

class MessageCheckResponse:
    def __init__(self, server_response: dict):
        self.raw_response = server_response
        self.check = [MessageCheck(check) for check in server_response.get('check', [])]

class UserInfo:
    def __init__(self, server_response):
        info = server_response.get('info', {})
        self.id = int(info.get('id'))
        self.tariff_id = info.get('tariff_id')
        self.email = info.get('email')
        self.phone = info.get('phone')
        self.name = info.get('name')
        self.balance = info.get('balance')
        self.date = info.get('date')
        self.senders = info.get('senders')
        self.default_sender = info.get('default_sender')
        self.any_sender = info.get('any_sender')

    def __str__(self):
        return 'UserInfo: %s' % self.name

    def __repr__(self):
        return str(self)

class PingResponseBase:
    ERROR = -2
    OUT_OF_SERVICE = -1
    ACCEPTED = 0
    SENT_TO_OPERATOR = 1
    PHONE_NUMBER_SERVED = 2
    STATUS_VERBOSE = (
        (ERROR, 'Запрос не принят, неправильный номер, ID не найден'),
        (OUT_OF_SERVICE, 'Номер не обслуживается'),
        (ACCEPTED, 'Запрос принят'),
        (SENT_TO_OPERATOR, 'Запрос передан оператору'),
        (PHONE_NUMBER_SERVED, 'Номер обслуживается'),
    )

    def __init__(self, server_response: dict):
        self.cost = float(server_response.get('cost', 0))
        self.balance = float(server_response.get('balance', 0))
        msg_obj = server_response.get('send', [{}])[-1]
        self.server_id = int(msg_obj.get('server_id', 0))
        self.phone = int(msg_obj.get('phone', 0))
        self.price = float(msg_obj.get('price', 0))
        self.status = int(msg_obj.get('status', 0))

    def is_served(self) -> bool:
        return self.status == self.PHONE_NUMBER_SERVED

    def is_ot_of_service(self) -> bool:
        return self.status == self.OUT_OF_SERVICE

    def get_verbose_status(self) -> str:
        status_dict = {code: description for code, description in self.STATUS_VERBOSE}
        return status_dict.get(self.status, '')

class HlrResponse(PingResponseBase):
    pass

class PingResponse(PingResponseBase):
    pass

class CheckResponse(PingResponseBase):
    def __init__(self, server_response: dict):
        super(CheckResponse, self).__init__(server_response)
        check = server_response.get('check', [{}])[-1]
        self.server_id = int(check.get('server_id', 0))
        phone = check.get('phone', 0)
        self.phone = int(phone) if phone != '' else None
        self.price = float(check.get('price', 0))
        self.status = int(check.get('status', 0))


import os
import sys
import smtplib
import datetime
import requests
import getopt
from time import time,ctime,sleep

G_senderEmail = ""
G_senderPassword = ""
G_receierList = ""
G_districtCode = ""
G_pincode = 0
G_age="999"

# https://myaccount.google.com/lesssecureapps to allow less secure apps to send email
def sendEmail(res):
    subject = 'Vaccine Slots Available'
    body = "Following vaccines centers are found \n\n" + res
    content = """\
From: %s
To: %s
Subject: %s
%s
""" % (G_senderEmail, ", ".join(G_receierList), subject, body)
    print (content)

    try:
        server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
        server.ehlo()
        server.login(G_senderEmail, G_senderPassword)
        server.sendmail(G_senderEmail, G_receierList, content)
        server.close()
        print ('Email sent to: ' + ", ".join(G_receierList))
    except Exception as exc:
        print ('Exception sending email. Exception')
        print (exc)

def availableCenters(result):
    retVal = []
    centers = result['centers']
    for center in centers:
        if G_pincode == 0 or center['pincode'] == int(G_pincode):
            sessions = center['sessions']
            for session in sessions:
                if session['available_capacity'] > 0 and session['min_age_limit'] <= int(G_age):
                   res = { 'name':center['name'], 'block_name':center['block_name'],'age_limit':session['min_age_limit'], 'vaccine_type':session['vaccine'] , 'date':session['date'],'available_capacity':session['available_capacity'] }
                   retVal.append(res)
    return retVal

def checkCenters(endPoint):
    print(ctime(time()))
    print (endPoint)
    userAgent = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
    response = requests.get(endPoint, headers = userAgent)

    if response.status_code == 200:
        result = response.json()
        jsonResult = availableCenters(result)
        if len(jsonResult) > 0:
            print ("Vaccine Slots Available")
            print('\007')
            resultStr = ""
            for center in jsonResult:
                resultStr = resultStr + center['name'] + "\n"
                resultStr = resultStr + "block:"+center['block_name'] + "\n"
                resultStr = resultStr + "vaccine count:"+str(center['available_capacity']) + "\n"
                resultStr = resultStr + "vaccine type:"+ center['vaccine_type'] + "\n"
                resultStr = resultStr + "Date:"+ center['date'] + "\n"
                resultStr = resultStr + "age_limit:"+str(center['age_limit'])+"\n"
                resultStr = resultStr + "-----------------------------------------------------\n"
            sendEmail(resultStr)
            return True
        else:
            print ("Vaccine slot not available on: " + _date +"\n")
            return False
    else:
        print("Could not check for vaccine availability at the moment. HTTP Code: " + str(response.status_code))
        return False

def parseArgs(argv):
    global G_senderEmail, G_senderPassword,  G_receierList, G_districtCode, G_pincode, G_age

    usageString = 'Usage: cowinNotifier.py [-e senderEmail][-p senderPwd][-r commaSeparatedReceiversEmail][-d districtCode][-c pinCode][-a age] \ne,p,r,d are necessary arguments'
    try:
      opts, args = getopt.getopt(argv,"he:p:r:d:c:a:",["help","senderEmail=","senderPwd=","receiverList=","districtCode=","pinCode=","age="])

    except getopt.GetoptError:
        print (usageString)
        sys.exit(1)

    for opt, arg in opts:
        if opt == '-h':
            print (usageString)
            sys.exit(2)
        elif opt in ("-e", "--senderEmail"):
            G_senderEmail = arg
        elif opt in ("-p", "--senderPwd"):
            G_senderPassword = arg
        elif opt in ("-r", "--receiverList"):
            G_receierList = arg
        elif opt in ("-d", "--districtCode"):
            G_districtCode = arg
        elif opt in ("-c", "--pinCode"):
            G_pincode = arg
        elif opt in ("-a", "--age"):
            G_age = arg
        else:
            print("Parameter " + opt + " is not supported")
            print(usageString)

    if (G_senderEmail == '' or G_senderPassword == '' or G_receierList == '' or G_districtCode == ''):
        print(usageString)
        sys.exit(3)
    G_receierList = G_receierList.split(',')
    G_receierList = [s.strip() for s in G_receierList]

    return 0

if __name__ == '__main__':
    parseArgs(sys.argv[1:])

    while True:
        periodicity = 5 # check every 5 minutes
        for x in range(7):
            _date = datetime.date.today() + datetime.timedelta(days=x)
            _date = str(_date.strftime("%d/%m/%Y")).replace("/","-")
            endPoint = "https://cdn-api.co-vin.in/api/v2/appointment/sessions/public/calendarByDistrict?district_id=" + G_districtCode + "&date="+ _date
            if checkCenters(endPoint) == True:
                periodicity = 1 #once vaccine is found, send status every 1 min.
                break
        print ("Sleeping for " + str(periodicity) + " mins")
        sleep(periodicity * 60)
        print ("Wake Up")


from library.extension import db
from library.library_ma import ProductSchema
from library.model import Products,Categories
from flask import request, jsonify
from sqlalchemy.sql import func
import json

product_schema = ProductSchema()
products_schema = ProductSchema(many=True)

def add_product_service():
    data = request.json
    if (data and ('ProductName' in data) and ('SupplierID' in data)
             and ('CategoryID' in data) and ('Unit' in data)
             and ('Price' in data)):
        ProductName = data['ProductName']
        SupplierID = data['SupplierID']
        CategoryID = data['CategoryID']
        Unit = data['Unit']
        Price = data['Price']

        try:
            new_product = Products(ProductName, SupplierID, CategoryID, Unit,Price)
            db.session.add(new_product)
            db.session.commit()
            return jsonify({"message": "Add success!"}), 200
        except IndentationError:
            db.session.rollback()
            return jsonify({"message": "Can not add product!"}), 400
    else:
        return jsonify({"message": "Request error"}), 400

def get_product_by_id_service(id):
    product = Products.query.get(id)
    if product:
        return product_schema.jsonify(product)
    else:
        return jsonify({"message": "Not found product"}), 404

def get_all_products_service():
    products = Products.query.all()
    if products:
        return products_schema.jsonify(products)
    else:
        return jsonify({"message": "Not found products!"}), 404

#update product name
def update_product_by_id_service(id):
    product = Products.query.get(id)
    data = request.json
    if product:
        if data and "ProductName" in data:
            try:
                product.ProductName = data["ProductName"]
                db.session.commit()
                return "Product Updated"
            except IndentationError:
                db.session.rollback()
                return jsonify({"message": "Can not update product!"}), 400
    else:
        return "Not found product"

def delete_product_by_id_service(id):
    product = Products.query.get(id)
    if product:
        try:
            db.session.delete(product)
            db.session.commit()
            return "Product Deleted"
        except IndentationError:
            db.session.rollback()
            return jsonify({"message": "Can not delete product!"}), 400
    else:
        return "Not found product"

def get_product_by_category_service(category):
    products = Products.query.join(Categories).filter(
        func.lower(Categories.CategoryName) == category.lower()).all()
    if products:
        return products_schema.jsonify(products)
    else:
        return jsonify({"message": f"Not found products by {category}"}), 404


import os

from django.contrib.auth.models import AbstractUser
from django.db import models

from reserve.models import ReserveRecord

# Create your models here.

class User(AbstractUser):
    GENDER_CHOICES = (
        (-1, '未知'),
        (1, '男'),
        (2, '女'),
    )
    name = models.CharField(max_length=50, verbose_name='姓名', default='', blank=True)
    gender = models.IntegerField(verbose_name='性别', choices=GENDER_CHOICES, default=-1)
    tel = models.CharField(max_length=60, verbose_name='电话号码', default='', blank=True)
    id_card_no = models.CharField(max_length=80, verbose_name='身份证号', default='', blank=True)

    avatar = models.ImageField(verbose_name='头像', upload_to='avatars/', default='default/default.png',
                               height_field='avatar_height', width_field='avatar_width')

    avatar_height = models.PositiveIntegerField(null=True, blank=True, editable=False, default="120",
                                                verbose_name='头像高度')
    avatar_width = models.PositiveIntegerField(null=True, blank=True, editable=False, default="120",
                                               verbose_name='头像宽度')

    def __str__(self):
        return '<User {}#>{}{}'.format(self.id, self.username, ' | {}'.format(self.name) if self.name else '')

    @classmethod
    def normalize_email(cls, email):
        """
        Normalize the email address by lowercasing the domain part of it.
        """
        email = email or ''
        try:
            email_name, domain_part = email.strip().rsplit('@', 1)
        except ValueError:
            pass
        else:
            email = email_name + '@' + domain_part.lower()
        return email

    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)
        if self.avatar:
            if not os.path.isfile(self.avatar.path):
                return
            from PIL import Image
            try:
                with Image.open(self.avatar.path) as im:
                    x, y = im.size
                    im.thumbnail((360, 360 * y / x), resample=Image.LANCZOS)
                    im.save(self.avatar.path)
            except OSError:
                pass

    def get_role(self):
        if self.is_superuser:
            return '超级管理员'
        elif self.is_staff:
            return '管理员'
        else:
            return '未知身份'

    class Meta:
        verbose_name = '用户'
        verbose_name_plural = verbose_name

class UserInfo(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE, verbose_name='用户')

    # 身份
    reserver = models.BooleanField(verbose_name='预约人', default=False)
    reservee = models.BooleanField(verbose_name='被预约人', default=False)

    # reserver
    auth_code = models.CharField(verbose_name='预约人编码', null=True, blank=True, max_length=80)
    auth_name = models.CharField(verbose_name='预约人姓名', null=True, blank=True, max_length=80)

    # reservee
    can_reserved = models.BooleanField(verbose_name='可被预约', default=False)
    RESERVEE_TYPE_CHOICES = (
        (1, '学导'),
        (2, '老师'),
    )
    reservee_type = models.IntegerField(verbose_name='被预约人种类', choices=RESERVEE_TYPE_CHOICES, default=1)
    GENDER_CHOICES = (
        (0, '不限'),
        (1, '仅限男性'),
        (2, '仅限女性'),
    )
    can_reserve_gender = models.IntegerField(verbose_name='可接受预约人性别', help_text='对于未定义性别的用户只有不限性别选项可被预约',
                                             choices=GENDER_CHOICES, default=0)
    short_intro = models.TextField(verbose_name='简介（显示于头像下方）', default='', blank=True)
    intro = models.TextField(verbose_name='介绍', default='', blank=True)
    default_address = models.CharField(verbose_name='默认地点', default='', blank=True, max_length=160)
    can_reserve_type = models.ManyToManyField('reserve.ReserveType', blank=True, verbose_name='可被预约种类')

    @property
    def reservee_rank_average(self):
        # TODO 缓存得分
        all = ReserveRecord.objects.filter(status=200, reserver_mark__isnull=False, main_time__reservee__userinfo=self)
        if not all.exists():
            return -2  # 暂无评分数据
        # elif len(all) < 10:
        #     return -1  # 评分数据不足

        return all.aggregate(models.Avg('reserver_mark')).get('reserver_mark__avg')

    @property
    def reservee_rank_average_msg(self):
        return {
            -2: '暂无评分数据',
            -1: '评分数据不足',
            0: '☆☆☆☆☆',
            1: '★☆☆☆☆',
            2: '★★☆☆☆',
            3: '★★★☆☆',
            4: '★★★★☆',
            5: '★★★★★',
        }.get(round(self.reservee_rank_average))

    # archive（存档数据，不显示）
    zhuanye = models.CharField(verbose_name='专业', max_length=80, default='', blank=True)
    banji = models.CharField(verbose_name='班级', max_length=80, default='', blank=True)

    class Meta:
        verbose_name = '用户额外信息'
        verbose_name_plural = verbose_name


# Import packages
import pandas as pd
import numpy as np
import netCDF4
import h5netcdf
import xarray as xr
from os.path import join
import joblib
from glob import glob
import datetime as dt
import sys
import pyresample
import itertools
from pathlib import Path

#Filters
from scipy.ndimage import uniform_filter, maximum_filter, gaussian_filter

#Custom Packages
sys.path.append('/home/samuel.varga/python_packages/WoF_post') #WoF post package
sys.path.append('/home/samuel.varga/python_packages/wofs_ml_severe/')
sys.path.append('/home/samuel.varga/python_packages/MontePython/')
sys.path.append('/home/samuel.varga/projects/deep_learning/')

from wofs.post.utils import (
    save_dataset,
    load_multiple_nc_files,
)
from data_utils.dl_2to6_data_pipeline import get_files, load_dataset
from collections import ChainMap

#Custom imports for target loading
from wofs_ml_severe.data_pipeline.storm_report_loader import StormReportLoader
from data_utils.MRMSutils import MeshGrabber
from wofs.plotting.util import decompose_file_path

class PatchExtractor:
    """
    Turns a forecast into a 2D predictor field, then returns multiple patches.
    Based heavily on the GridPointftracter class.
    --------------------------------------------------------------------------
    The PE does the following:
    1) Replaces NaNs with field average and Upscales the Field
    2) Calculates the time composite of every ensemble member
    3) Calculates pointwise ens. statistics to produce predictors
    4) Creates multiple binary target fields from LSR/MRMS data
    5) Returns n patches of predictors and targets from the field
    --------------------------------------------------------------------------

    Init Params:
    patch_shape: Diameter of patches given as a tuple of (Y,X)

    Class Methods:
    """

    def __init__(self, ncfile, ll_grid, env_vars, strm_vars, n_patches, patch_shape,
                forecast_window='2to6',
                target_sizes=[0.5, 2, 4],
                upscale_size=3, #Coarsens grid by 3x
                grid_spacing=3, #WoFS Grid Spacing (km)
                random_state=42,
                reports_path='/work/mflora/LSRS/StormEvents_2017-2022.csv',
                report_type='NOAA',
                verbose=False
                ):

        #Set Object parameters
        self._upscale_size=3
        self._inset = 2 #Number of points to count as "boundary"
        self._TARGET_SIZES=np.array(np.array(target_sizes)*2, dtype='int') #Convert from radius to diameter
        self._n_ens = 18 #Number of ens. members
        self._env_vars=env_vars
        self._strm_vars=strm_vars
        self._ncfile=ncfile
        self._DX = grid_spacing * self._upscale_size #Delta X of target grid
        self._reports_path = reports_path
        self._report_type = report_type
        self._forecast_window = forecast_window
        self._deltat = 5 #Time step of forecast in minutes
        self._original_grid = ll_grid
        self._target_grid = (self._original_grid[0][::self._upscale_size, ::self._upscale_size],
                             self._original_grid[1][::self._upscale_size, ::self._upscale_size])
        self._n_patches = n_patches #Number of patches to remove
        self._patch_shape = patch_shape #Patch shape in number of grid points (y, x)
        self.verbose=verbose #If true, print extra debug information

        if np.max(np.absolute(ll_grid[0]))>90:
            raise ValueError('Latitude values for ll_grid exceed 90 and are likely longitude values.')
        np.random.seed(random_state)

    def make_dataset(self, X_env, X_strm, predict=False):
        '''Creates the dataset and extracts patches'''

        #Apply a Mean filter to environmental fields and coarsen them
        X_env_upscaled = {v : self.upscaler(X_env[v], func=uniform_filter,
                        upscale_size=self._upscale_size) for v in self._env_vars}

        #Apply a maximum filter to intrastorm fields and coarsen them
        X_strm_upscaled = {v : self.upscaler(X_strm[v], func=maximum_filter,
                            upscale_size=self._upscale_size) for v in self._strm_vars}

        #Select subset of points based on patches
        X_strm_upscaled, X_env_upscaled=self.extract_patches([X_strm_upscaled, X_env_upscaled]) #{} of (n_patches, n_time, n_ens, y, x)

        #Time Composite
        X_env_time_comp=self.calc_time_composite(X_env_upscaled, func=np.nanmean, name='time_avg', keys=self._env_vars, ax=1) #{} of (n_patches, n_ens, y, x)
        X_strm_time_comp=self.calc_time_composite(X_strm_upscaled, func=np.nanmax, name='time_max', keys=self._strm_vars, ax=1)

        #Calc Ens. Stats
        X_env_stats=self.calc_ensemble_stats(X_env_time_comp, environ=True, ax=1) #{} of (n_patches, y, x)
        X_strm_stats=self.calc_ensemble_stats(X_strm_time_comp, environ=False, ax=1)

        #Add MRMS Comp DZ at t0
        dz = MeshGrabber(self._ncfile, self._upscale_size, np.zeros(np.shape(self._original_grid[0]))).load_comp_dz()
        upscaled_dz = {v : self.upscaler(dz, func=maximum_filter,
                            upscale_size=self._upscale_size, is_2d=True) for v in ['MRMS_DZ']}
        upscaled_dz=self.subset_patches([upscaled_dz], is_2D=True)[0]

        #Combine predictor dictionaries
        X_all={**X_strm_stats, **X_env_stats, **upscaled_dz}

        if predict:
            #If targets are not available (e.g. operational use)
            data = X_all
        else:
            #If targets are available (e.g. creating training data)
            y = self.get_targets(timescale=self._forecast_window)
            data = {**X_all, **y}

        #Combine predictors and targets into xarray dataset
        data = {v : (['patch_no','NY','NX'], data[v]) for v in data.keys()}
        ds = xr.Dataset(data)

        #Create Metadata of similar shape to data
        comps=decompose_file_path(self._ncfile)
        metadata={'run_date':[comps['VALID_DATE'] for i in range(self._n_patches)]}
        metadata['init_time']=[comps['INIT_TIME'] for i in range(self._n_patches)]
        metadata['patch_no']=[i+1 for i in range(self._n_patches)]
        metadata['NY']=(['patch','NY_ind'],[np.argwhere(self._patch_grid==i+1)[:,0] for i in range(self._n_patches)])
        metadata['NX']=(['patch','NX_ind'], [np.argwhere(self._patch_grid==i+1)[:,1] for i in range(self._n_patches)])
        metadata_ds = xr.Dataset(metadata)

        return ds, metadata_ds

###########################
## Preprocessing Methods ##
###########################

    def upscaler(self, X, func, upscale_size, is_2d=False):
        '''Applies a spatial filter per ensemble member and timestep and then
        subsamples the grid to reduce the number of grid points'''
        if is_2d:
            new_X = np.zeros((self._target_grid[0].shape[0], self._target_grid[1].shape[1]))
            fill_value = np.nanmean(X)
            X_ = np.nan_to_num(X, nan=fill_value)
            new_X=self.resampler(func(X_, upscale_size))

        else:
            new_X = np.zeros((X.shape[0], X.shape[1],
                          self._target_grid[0].shape[0], self._target_grid[1].shape[1]))
            fill_value = np.nanmean(X)

            for t,n in itertools.product(range(new_X.shape[0]), range(self._n_ens)):
                X_ = np.nan_to_num(X[t, n, :, :], nan=fill_value)
                new_X[t, n, :, :] = self.resampler(func(X_, upscale_size)) #(Time, ens_member, lat, lon)

        return new_X

    def resampler(self, variable):
        '''
        Resamples the original grid to the target grid using a nearest neighborhood approach

        '''

        #Create a pyresample object holding the original grid
        orig_def = pyresample.geometry.SwathDefinition(lons=self._original_grid[1], lats=self._original_grid[0])

        #Create another pyresample object for the target grid
        targ_def = pyresample.geometry.SwathDefinition(lons=self._target_grid[1], lats=self._target_grid[0])

        variable_nearest = pyresample.kd_tree.resample_nearest(orig_def, variable, targ_def, radius_of_influence=50000, fill_value=None)

        return variable_nearest

    def neighborhooder(self, field, func, size, is_2d=False, fill_method=0):
        '''Apply Neighborhood function. NaN Values are replaced with
        either a 0, or a generic, full-domain average based on fill_method'''

        new_field=field.copy()
        fill_value = np.nanmean(field) if str(fill_method)!='0' else 0
        if is_2d:
            for n in range(self._n_ens):
                field_ = np.nan_to_num(field[:,:], nan=fill_value)
                new_field[:,:] = func(field_, size)
        else:
            for n in range(self._n_ens): #Every Ens Member
                field_ = np.nan_to_num(field[n,:,:], nan=fill_value)
                new_field[n,:,:] = func(field_, size)
        return new_field

    def calc_time_composite(self, X, func, name, keys, ax=0):
        '''Compute the time-composite using the given function'''
        X_time_comp = {f'{v}__{name}' : func(X[v], axis=ax) for v in keys}
        return X_time_comp

    def calc_ensemble_stats(self, X, environ=True, ax=0): #Size 1
        '''Compute the ensemble statistics at each grid point

            Params:
            --------
            X: dic of variables with shape {variable : [n_ens, lat, lon]}
            environ: flag used to determine which statistics to compute. True if environment, otherwise false.
        '''
        keys = X.keys()
        X_final = []

        if environ:
            #Compute Statistics of Env. Variables
            X_ens_mean = {f'{v}__ens_mean' : np.nanmean(X[v], axis=ax) for v in keys}
            X_ens_std = {f'{v}__ens_std' : np.nanstd(X[v], axis=ax, ddof=1) for v in keys}
            X_ens_stats = {**X_ens_mean, **X_ens_std}
        else:
            #Compute Statistics of Storm Variables
            X_ens_mean = {f'{v}__ens_mean' : np.nanmean(X[v], axis=ax) for v in keys}
            #2nd Highest Member
            X_ens_16th = {f'{v}__ens_90th': np.nanpercentile(X[v], 16/18*100, axis=ax, method='higher') for v in keys}
            #2nd Lowest Member
            X_ens_2nd = {f'{v}__ens_10th': np.nanpercentile(X[v], 2/18*100, axis=ax, method='lower') for v in keys}
            #Interquartile range
            X_iqr = {f'{v}__ens_IQR': np.nanpercentile(X[v], 75, axis=ax, method='higher')-np.nanpercentile(X[v], 25, axis=ax, method='lower') for v in keys}
            X_ens_stats={**X_ens_mean, **X_ens_16th, **X_ens_2nd, **X_iqr}
        X_final.append(X_ens_stats)
        X_final = dict(ChainMap(*X_final))

        return X_final

###################
## Patch Methods ##
###################

    def check_patches(self, border_buffer=1):
            '''Checks to ensure that the forecast area can produce the requested number of patches'''

            patch_area = np.prod(self._patch_shape)
            total_patch_area=patch_area*self._n_patches
            forecast_area = np.prod(np.shape(self._target_grid[0]))
            self._max_patches = int(np.floor(forecast_area / patch_area))

            borderless_domain = (len(self._target_grid[0])-2*border_buffer, len(self._target_grid[0])-2*border_buffer)
            borderless_domain_area = np.prod(borderless_domain)

            if total_patch_area <= forecast_area:
                print('\nRequested patches are compatible with domain\n')
            else:
                print(f'\nRequested patches ({self._n_patches},{self._patch_shape[0]},{self._patch_shape[1]}) are incompatible with domain ({len(self._target_grid[0]),len(self._target_grid[1])})\n')

            if self.verbose:
                print('Verbose Readout')
                print('------------')
                print(f'Patch (Shape, Area): {self._patch_shape, (patch_area, total_patch_area)}')
                print(f'Forecast (Shape, Area): {np.shape(self._target_grid[0]), forecast_area}')
                print(f'Number of Unused Gridpoints: {forecast_area-total_patch_area}')
                print(f'Percent of Domain Used (%): {100*total_patch_area/forecast_area}')

                print(f'\nMaximum Number of patches (Area): {np.floor(forecast_area / patch_area)}')
                print(f'Maximum Number of patches (Grid): {np.floor(len(self._target_grid[0])/self._patch_shape[0])**2}')
                print(f'Number of Unused Gridpoints if Max Patches: {int(forecast_area-(np.floor(forecast_area/patch_area)*patch_area))}')

                print(f'\nBoundaryless Domain (N={border_buffer}) (Shape, Area): {borderless_domain, borderless_domain_area}')
                print(f'Max Patches in Boundaryless Domain: {np.floor(borderless_domain_area / patch_area)}')
                print(f'No. of Unused Gridpoints if Max Patches in Boundaryless Domain: {int(borderless_domain_area - (np.floor(borderless_domain_area / patch_area)*self._n_patches))}')
                print('------------')

    def draw_patches(self):
        '''Randomly Selects Patches'''
        patch_inds=[]

        #Number of possible patches, starting from top-left corner
        patch_no=np.product(((len(self._target_grid[0])-2*self._inset)//self._patch_shape[0],
                             (len(self._target_grid[1])-2*self._inset)//self._patch_shape[1]))

        #Choose n_patches from the possible patches
        patches = np.random.choice(patch_no, self._n_patches, replace=False)

        #Create 2D of possible patches
        patch_grid=np.arange(0,patch_no).reshape((len(self._target_grid[0])-2*self._inset)//self._patch_shape[0],
                                                 (len(self._target_grid[1])-2*self._inset)//self._patch_shape[1])

        #Find top-left (Y,X) of each patch
        for patch in patches:
            patch_inds.append(np.argwhere(patch_grid==patch).flatten()) #(Y, X)

        self._patch_centers = patch_inds

        if self.verbose:
            print(patches)
            print(patch_grid)

        return None

    def get_patch_centers(self):
        '''Converts patch coordinates to center, rather than top-left'''
        patch_centers=[]

        #Convert patch_center from top-left bound to center;
        #Doesn't react well to patches with odd dimensions
        for patch_center in self._patch_centers:
            patch_center = (((patch_center[0]*self._patch_shape[0])+self._patch_shape[0]//2)+self._inset,
                            ((patch_center[1]*self._patch_shape[1])+self._patch_shape[1]//2)+self._inset)
            patch_centers.append(patch_center)

        self._patch_centers=patch_centers

        if self.verbose:
            print(patch_centers)
        return None

    def patch_to_grid(self):
        '''Creates 2D grid of same shape as domain. Grid points not belonging to a patch are
        assigned 0, while grid points belonging to a patch are assigned their patch number.
        '''
        #Grid of zeroes of same shape as domain
        patch_grid=np.zeros_like(self._target_grid[0], dtype=int)

        #Assign patch no. to grid points inside patch
        for i, patch_cent in enumerate(self._patch_centers):
            patch_grid[patch_cent[0]-(self._patch_shape[0]//2):patch_cent[0]+(self._patch_shape[0]//2),
                  patch_cent[1]-(self._patch_shape[1]//2):patch_cent[1]+(self._patch_shape[1]//2)]=i+1

        if self.verbose:
            print([patch_cent for patch_cent in self._patch_centers])
            print(patch_grid)

        self._patch_grid =  patch_grid

        return None

    def stack_patches(self, var, is_2D=False):
        '''Finds and Stacks the Patches for a given field'''
        temp = []
        for i in range(1, self._n_patches+1):
            if is_2D:
                temp.append(np.reshape(var[self._patch_grid==i], (self._patch_shape[0], self._patch_shape[1])))
            else:
                temp.append(np.reshape(var[:,:,self._patch_grid==i], (np.shape(var)[0], np.shape(var)[1], self._patch_shape[0], self._patch_shape[1])))
        return np.stack(temp)

    def subset_patches(self, data, is_2D=False):
        '''Subsets the data to only include patches, rather than full fields '''
        out=[]
        for var_dic in data:
            out_dic={v : self.stack_patches(var_dic[v], is_2D=is_2D) for v in var_dic.keys()} #Reshapes to be (patches, time, ens, lat, lon)
            out.append(out_dic)
        return out

    def extract_patches(self, data=None, gen_patches=True):
        '''Selects and extracts Patches - calls other patch methods'''

        if gen_patches:
            #Check Compatibility of Requested Patches with Domain
            self.check_patches()

            #Select Patch Locations
            self.draw_patches()

            #Get Domain Locations of Patches
            self.get_patch_centers()

            #Map Patches onto Domain
            self.patch_to_grid()

        #Compress X based on patch_grid
        data_out = self.subset_patches(data)

        return data_out

####################
## Target Methods ##
####################

    def get_targets(self, timescale):
        '''Convert Storm reports to original grid, then upscale. Same Method used in GridPointExtracter'''
        comps=decompose_file_path(self._ncfile)

        #Compute the forecast start time
        start_time=(pd.to_datetime(comps['VALID_DATE']+comps['INIT_TIME'])+dt.timedelta(minutes=int(comps['TIME_INDEX'])*self._deltat)).strftime('%Y%m%d%H%M')

        forecast_length=180 if timescale=='0to3' else 240

        #(Down)load storm reports
        report = StormReportLoader(
                reports_path = '/work/mflora/LSRS/StormEvents_2017-2022.csv',
                report_type='NOAA',
                initial_time=start_time,
                forecast_length=forecast_length,
                err_window=15,
            )

        ds = xr.load_dataset(self._ncfile, decode_times=False)

        #Cast reports to original grid. Max filter of upscale size is applied during this step
        report_ds = report.to_grid(dataset=ds, size=self._upscale_size)

        keys = list(report_ds.data_vars)

        #Upscale to target Grid
        y = {v : report_ds[v].values[::self._upscale_size, ::self._upscale_size]
             for v in keys}

        ##Add MESH and Upscale
        mesh = MeshGrabber(self._ncfile, self._upscale_size, np.zeros(np.shape(self._original_grid[0])))()
        y.update({'MESH_severe': mesh[::self._upscale_size, ::self._upscale_size]}); keys.append('MESH_severe')

        #Force Binary Values
        for v in y.keys():
            y[v][y[v]>0] = 1

        y_final = []

        #Coarsen across target sizes
        for size in self._TARGET_SIZES:
            y_nghbrd = {f'{v}__{self._DX*size/2:.0f}km' : self.neighborhooder(y[v],
                                                                      func=maximum_filter,
                                                                     size=size, is_2d=True) for v in keys}
            y_final.append(y_nghbrd)

        y_final = dict(ChainMap(*y_final))

        #Use Hail/Wind/Tornado to create Any-Severe
        for severe, size in itertools.product([False,True],self._TARGET_SIZES):
            y_final.update(self.get_any_severe(y_final, severe, size))

        #Subset patches
        #(Subset patches returns a list, hence the 0 index)
        y_final=self.subset_patches([y_final], is_2D=True)[0]

        return y_final

    def get_any_severe(self, y_dics, sig, size):
        '''Converts all hazards into Any-Severe Grid'''
        y_out = np.stack([y_dics[f'{h}_{"sig_" if sig else ""}severe__{self._DX*size/2:.0f}km'] for h in ['wind','hail','tornado']]) #Need to extend to mesh
        y_out = np.any(y_out, axis=0).astype(int)

        return {f'any_{"sig_" if sig else ""}severe__{self._DX*size/2:.0f}km': y_out}


from django.contrib import admin

from .models import TemperatureReading

class TemperatureReadingAdmin(admin.ModelAdmin):
    fieldsets = [
        (None, {'fields': [
            'id', 'value', 'value_datetime',
            'mean', 'median', 'mode', 'std_deviation',
            'variance', 'percentile25', 'percentile50', 'percentile75'
            ]}),
    ]
    list_display = ('value_datetime', 'value')
    list_filter = ['value_datetime']
    search_fields = ['value']

admin.site.register(TemperatureReading, TemperatureReadingAdmin)


import requests
from ping.helper.apiHelper import check_errors, json_deserialize

def get(headers, base_url, payment_link_id):
    # Does a GET request to /api/v1/payment_links/{payment_link_id}/invoice.

    # Retrieves a specific Receipt.
    # Args (provided by the tenant):
    #    payment_link_id (String, required): The ID of the of the Payment Link from which you want a receipt.
    # Returns:
    #    Response: A json object with the response value as well as other
    #    useful information such as status codes, headers and potential errors.

    # Prepare and execute response
    _path = f'/api/v1/payment_links/{payment_link_id}/invoice'
    _url = base_url + _path
    response = requests.get(_url, headers=headers)

    # deserialize and check errors
    decoded = json_deserialize(response.text)
    _result = check_errors(response, decoded)
    return _result


import time

import board
import digitalio
import usb.util

# see circuit diagram in README.md

class Ft232hConnector:
    """
    Ft232hConnector class for interfacing with the opto-coupler signals and controlling frame processing.

    This class provides the necessary functionality to control the digital signals from opto-couplers,
    manage frame processing, and handle the End Of Film (EoF) signal.

    Args:
        opto_coupler_signal_subject (Subject): A subject that emits signals triggered by opto-coupler OK1.
    """

    # 15-m-Cassette about 3.600 frames (±50 frames due to exposure and cut tolerance at start and end)
    # 30-m-Cassette about 7.200 frames (±50 frames due to exposure and cut tolerance at start and end)
    MAXCOUNT = 1500  # 7250  # emergency break if EoF (End of Film) is not recognized by opto-coupler OK2

    def __init__(self, opto_coupler_signal_subject) -> None:
        """
        Initialize the Ft232hConnector instance with the provided subjects and set up necessary components.

        Args:
            opto_coupler_signal_subject (Subject): A subject that emits signals triggered by opto-coupler OK1.
        """

        self.__optoCouplerSignalSubject = opto_coupler_signal_subject
        self.__count = 0

        # Find the USB device with specified Vendor and Product IDs
        self.__dev = usb.core.find(idVendor=0x0403, idProduct=0x6014)
        print(self.__dev)

        # Set up the LED to indicate frame processing
        self.__led = digitalio.DigitalInOut(board.C1)
        self.__led.direction = digitalio.Direction.OUTPUT
        self.__led.value = False

        # Set up opto-coupler OK1 to trigger frame processing
        self.__optoCouplerOK1 = digitalio.DigitalInOut(board.C2)
        self.__optoCouplerOK1.direction = digitalio.Direction.INPUT

        # Set up opto-coupler OK2 to trigger End Of Film (EoF)
        self.__eof = digitalio.DigitalInOut(board.C3)
        self.__eof.direction = digitalio.Direction.OUTPUT
        # set initial eof value
        self.__eof.value = False

        # switch to INPUT mode
        self.__eof = digitalio.DigitalInOut(board.C3)
        self.__eof.direction = digitalio.Direction.INPUT

    def signal_input(self) -> None:
        """
        Process the input signals and trigger frame processing when opto-coupler OK1 is triggered.
        """
        while not self.__eof.value and self.__count < self.MAXCOUNT:
            if True or self.__optoCouplerOK1.value:
                self.__count += 1
                # turn on led to show processing of frame has started
                self.__led.value = True
                # Emit the frame count through the opto_coupler_signal_subject
                self.__optoCouplerSignalSubject.on_next(self.__count)
                #
                # Wait for self.__optoCouplerOK1 (OK1) to change to false
                # Latency of OK1 is about one millisecond
                #
                while self.__optoCouplerOK1.value:
                    time.sleep(0.0005)

                # turn off led to show processing of frame has been delegated to another thread or has been finished
                self.__led.value = False

        # Signal the completion of frame processing and EoF detection
        self.__optoCouplerSignalSubject.on_completed()


#!/usr/bin/env python
# coding=utf-8

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from future import standard_library
standard_library.install_aliases()

import socket
import dns.exception
import dns.resolver
import logging
import os
import requests
import sys
import time

from tld import get_tld

# Enable verified HTTPS requests on older Pythons
# http://urllib3.readthedocs.org/en/latest/security.html
if sys.version_info[0] == 2:
    requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()

logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.INFO)

ACME_DNS_PREFIX = '_acme-challenge'

try:
    DNSPOD_AUTH_PARAMS = {
        'login_token' : os.environ['DNSPOD_LOGIN_TOKEN'],
    }
except KeyError:
    logger.error(" + Unable to locate DNSPod credentials in environment!")
    sys.exit(1)

try:
    dns_servers = os.environ['DNSPOD_DNS_SERVERS']
    dns_servers = map(socket.gethostbyname, dns_servers.split())
except KeyError:
    dns_servers = False

def _has_dns_propagated(name, token):
    txt_records = []
    try:
        if dns_servers:
            custom_resolver = dns.resolver.Resolver()
            custom_resolver.nameservers = dns_servers
            dns_response = custom_resolver.query(name, 'TXT')
        else:
            dns_response = dns.resolver.query(name, 'TXT')
        for rdata in dns_response:
            for txt_record in rdata.strings:
                txt_records.append(txt_record)
    except dns.exception.DNSException as error:
        logger.exception("check TXT record failed")
        return False

    for txt_record in txt_records:
        if txt_record == token:
            return True

    return False

def _execute_dnspod_action(action, domain, params):
    res = get_tld('http://' + domain, as_object=True)
    sub_domain, tld = res.subdomain, res.tld
    if sub_domain:
        sub_domain = "%s.%s" % (ACME_DNS_PREFIX, sub_domain)
    else:
        sub_domain = ACME_DNS_PREFIX

    url = "https://dnsapi.cn/%s" % action
    payload = {
        'domain' : tld,
        'sub_domain' : sub_domain,
        'format' : 'json',
    }
    payload.update(params)
    payload.update(DNSPOD_AUTH_PARAMS)
    r = requests.post(url, data=payload)
    r.raise_for_status()
    return r

# https://www.dnspod.cn/docs/records.html#record-list
def _get_txt_record_id(domain, token):
    action = 'Record.List'
    payload = {
        'keyword' : token,
    }
    r = _execute_dnspod_action(action, domain, payload)
    try:
        for rec in r.json()['records']:
            if rec['type'] == 'TXT' and rec['value'] == token:
                return rec['id']
    except (IndexError, KeyError):
        logger.info(" + Unable to locate record named {0}".format(domain))
        return

# https://www.dnspod.cn/docs/records.html#record-create
def create_txt_record(args):
    domain, token = args[0], args[2]

    action = 'Record.Create'
    payload = {
        'record_type': 'TXT',
        'record_line': '默认',
        'value' :  token,
    }
    r = _execute_dnspod_action(action, domain, payload)

    record_id = r.json()['record']['id']
    logger.debug("+ TXT record created, ID: {0}".format(record_id))

    # give it 10 seconds to settle down and avoid nxdomain caching
    logger.info(" + Settling down for 10s...")
    time.sleep(10)

    name = "{0}.{1}".format(ACME_DNS_PREFIX, domain)
    while(_has_dns_propagated(name, token) == False):
        logger.info(" + DNS not propagated, waiting 30s...")
        time.sleep(30)

# https://www.dnspod.cn/docs/records.html#record-remove
def delete_txt_record(args):
    domain, token = args[0], args[2]
    if not domain:
        logger.info(" + http_request() error in letsencrypt.sh?")
        return

    record_id = _get_txt_record_id(domain, token)

    name = "{0}.{1}".format(ACME_DNS_PREFIX, domain)
    logger.debug(" + Deleting TXT record name: {0}".format(name))
    action = "Record.Remove"
    payload = {
        'record_id' : record_id
    }
    _execute_dnspod_action(action, domain, payload)

def deploy_cert(args):
    domain, privkey_pem, cert_pem, fullchain_pem, chain_pem, timestamp = args
    logger.info(' + ssl_certificate: {0}'.format(fullchain_pem))
    logger.info(' + ssl_certificate_key: {0}'.format(privkey_pem))
    return

def unchanged_cert(args):
    return

def main(argv):
    ops = {
        'deploy_challenge': create_txt_record,
        'clean_challenge' : delete_txt_record,
        'deploy_cert'     : deploy_cert,
        'unchanged_cert'  : unchanged_cert,
        'exit_hook'       : unchanged_cert,
    }
    logger.info(" + DNSPod hook executing: {0}".format(argv[0]))
    ops[argv[0]](argv[1:])

if __name__ == '__main__':
    main(sys.argv[1:])


#!/usr/bin/env python

from flask import Flask, render_template, url_for, request
import csv

class CodiceArticolo:
    def __init__(self):
        self.database=[]
        with open ("product-template.csv","r") as x:
            y = csv.reader(x, delimiter=',', quotechar='\"')
            for i in y:
                self.database.append("%s %s" % (i[1],i[7]))

    def cerca(self, codice):
        temp=[]
        for j in self.database:
            if codice in j.lower():
                temp.append(j.strip())

        return temp

app = Flask(__name__)

@app.route("/")
def home():

   return render_template("index.html")

@app.route("/search", methods = ['GET'])
def ricerca_json():

    codice = request.args.get("codice", None)
    c = CodiceArticolo()

    temp = c.cerca(codice)  #variabile di appoggio

    return temp  #return a json

@app.route("/ricerca", methods = ['GET'])
def ricerca():

    codice = request.args.get("codice", None)
    c = CodiceArticolo()

    temp = c.cerca(codice)  #variabile di appoggio

    return render_template("ricerca.html", lista_database = temp, codice_trovato = codice)


import torch.nn as nn
import torch.nn.functional as F
from base import BaseModel
import torchvision

class MnistModel(BaseModel):
    def __init__(self, num_classes=10):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50)
        self.fc2 = nn.Linear(50, num_classes)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

class ResNet18(BaseModel):
    def __init__(self,num_classes=10,pretrained=True):
        super().__init__()
        self.resnet = torchvision.models.resnet18(pretrained=pretrained)
        #for param in self.resnet.parameters():
        #    param.requires_grad=False
        self.resnet.fc = nn.Linear(self.resnet.fc.in_features, num_classes)

    def forward(self, x):
        return self.resnet(x)


import os
import yaml
from voidpp_tools.mocks.file_system import mockfs
from configpp.evolution import Evolution, Revision

from .utils import FileSystem

def test_init():
    template_file_path = Revision.ORIGINAL_TEMPLATE_FILE_PATH
    with open(template_file_path) as f:
        template_file_content = f.read()

    fs_data = {}

    fs = FileSystem(fs_data)
    fs.set_data(Revision.ORIGINAL_TEMPLATE_FILE_PATH, template_file_content, True)

    with fs.mock():
        ev = Evolution()

        assert ev.init('evolution')
        assert 'evolution' in fs_data
        assert 'evolution.yaml' in fs_data
        assert 'versions' in fs_data['evolution']
        config = yaml.load(fs_data['evolution.yaml'], Loader = yaml.FullLoader)
        assert 'revision_template_file' in config
        assert config['revision_template_file']
        assert fs.get_data(config['revision_template_file'])

@mockfs({'evolution': {'versions':{}}, 'evolution.yaml': 'script_location: evolution\nrevision_template_file: some\nconfigpp_urls: {}'})
def test_load():

    ev = Evolution()
    assert ev.load()

@mockfs({'evolution': {'versions':{}}, 'evolution.yaml': 'script_location: evolution42\nrevision_template_file: some\nconfigpp_urls: {}'})
def test_load_nonexists_folder():

    ev = Evolution()
    assert ev.load() is False


# -*- coding: utf-8 -*-

import unittest

from fluent import event, sender
from tests import mockserver

class TestException(BaseException): pass

class TestEvent(unittest.TestCase):
    def setUp(self):
        self._server = mockserver.MockRecvServer('localhost')
        sender.setup('app', port=self._server.port)

    def tearDown(self):
        from fluent.sender import _set_global_sender
        sender.close()
        _set_global_sender(None)

    def test_logging(self):
        # XXX: This tests succeeds even if the fluentd connection failed
        # send event with tag app.follow
        event.Event('follow', {
            'from': 'userA',
            'to': 'userB'
        })

    def test_logging_with_timestamp(self):
        # XXX: This tests succeeds even if the fluentd connection failed

        # send event with tag app.follow, with timestamp
        event.Event('follow', {
            'from': 'userA',
            'to': 'userB'
        }, time=int(0))

    def test_no_last_error_on_successful_event(self):
        global_sender = sender.get_global_sender()
        event.Event('unfollow', {
            'from': 'userC',
            'to': 'userD'
        })

        self.assertEqual(global_sender.last_error, None)
        sender.close()

    @unittest.skip("This test failed with 'TypeError: catching classes that do not "
                   "inherit from BaseException is not allowed' so skipped")
    def test_connect_exception_during_event_send(self, mock_socket):
        # Make the socket.socket().connect() call raise a custom exception
        mock_connect = mock_socket.socket.return_value.connect
        EXCEPTION_MSG = "a event send socket connect() exception"
        mock_connect.side_effect = TestException(EXCEPTION_MSG)

        # Force the socket to reconnect while trying to emit the event
        global_sender = sender.get_global_sender()
        global_sender._close()

        event.Event('unfollow', {
            'from': 'userE',
            'to': 'userF'
        })

        ex = global_sender.last_error
        self.assertEqual(ex.args, EXCEPTION_MSG)
        global_sender.clear_last_error()


from __future__ import absolute_import, division, print_function
from six.moves import range
#!/usr/bin/env python
# -*- mode: python; coding: utf-8; indent-tabs-mode: nil; python-indent: 2 -*-
#
# recompute_mosaicity.py
#
#  Copyright (C) 2017 Lawrence Berkeley National Laboratory (LBNL)
#
#  Author: Aaron Brewster
#
#  This code is distributed under the X license, a copy of which is
#  included in the root directory of this package.
#
# LIBTBX_SET_DISPATCHER_NAME cctbx.xfel.recompute_mosaicity
#
from dials.algorithms.indexing.nave_parameters import NaveParameters
from dials.array_family import flex
from dials.util import show_mail_on_error
import libtbx.load_env
from libtbx.phil import parse

help_message = '''
Recompute mosaic parameters for a set of experiments and apply outlier rejection.

Example:

  %s refined.expt refined.refl
''' % libtbx.env.dispatcher_name

# Create the phil parameters
phil_scope = parse("""
plot_changes = False
  .type = bool
  .help = If True, plot the change in the mosaic parameters
output {
  experiments = refined.expt
    .type = str
    .help = Name of output  experiments file
  reflections = refined.refl
    .type = str
    .help = Name of output reflections file
}
""")

from dials.command_line.stills_process import phil_scope as dsp_phil

class Script(object):
  ''' Class to parse the command line options. '''

  def __init__(self):
    ''' Set the expected options. '''
    from dials.util.options import ArgumentParser
    import libtbx.load_env

    # Create the option parser
    usage = "usage: %s refined.expt refined.refl" % libtbx.env.dispatcher_name
    self.parser = ArgumentParser(
      usage=usage,
      sort_options=True,
      phil=phil_scope,
      read_experiments=True,
      read_reflections=True,
      check_format=False,
      epilog=help_message)

  def run(self):
    ''' Parse the options. '''
    from dials.util.options import flatten_experiments, flatten_reflections
    # Parse the command line arguments
    params, options = self.parser.parse_args(show_diff_phil=True)
    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)
    assert len(reflections) == 1
    reflections = reflections[0]

    nvparams = dsp_phil.extract()

    domain_size = flex.double()
    mosaic_angle = flex.double()
    filtered_reflections = flex.reflection_table()

    for i in range(len(experiments)):
      refls = reflections.select(reflections['id'] == i)
      tmp = refls['id']
      refls['id'] = flex.int(len(refls), 0)
      try:
        nv = NaveParameters(params = nvparams, experiments=experiments[i:i+1], reflections=refls, refinery=None, graph_verbose=False)
        crystal_model_nv = nv()[0]
      except Exception as e:
        print("Error recomputing mosaicity for experiment %d: %s"%(i, str(e)))
        continue
      domain_size.append(experiments[i].crystal.get_domain_size_ang() - crystal_model_nv.get_domain_size_ang())
      mosaic_angle.append(experiments[i].crystal.get_half_mosaicity_deg() - crystal_model_nv.get_half_mosaicity_deg())
      experiments[i].crystal = crystal_model_nv

      refls['id'] = tmp
      refls = refls.select(nv.nv_acceptance_flags)
      filtered_reflections.extend(refls)

    print("Saving new experiments as %s"%params.output.experiments)
    experiments.as_file(params.output.experiments)

    print("Removed %d out of %d reflections as outliers"%(len(reflections) - len(filtered_reflections), len(reflections)))
    print("Saving filtered reflections as %s"%params.output.experiments)
    filtered_reflections.as_pickle(params.output.reflections)

    if params.plot_changes:
      from matplotlib import pyplot as plt
      domain_size = domain_size.select((domain_size >= -10) & (domain_size <= 10))
      mosaic_angle = mosaic_angle.select((mosaic_angle >= -0.1) & (mosaic_angle <= 0.1))

      for d in [domain_size, mosaic_angle]:
        f = plt.figure()
        plt.hist(d, bins=30)
      plt.show()

if __name__ == '__main__':
  with show_mail_on_error():
    script = Script()
    script.run()

{'ArgumentParser': 'dials.util.options.ArgumentParser', 'libtbx.load_env': 'libtbx.load_env', 'flatten_experiments': 'dials.util.options.flatten_experiments', 'flatten_reflections': 'dials.util.options.flatten_reflections', 'plt': 'matplotlib.pyplot'}


import torch
import torch.nn as nn
import numpy as np
from torch.autograd import Variable
from State1_DataProcessing.HIPM import *
from State1_DataProcessing.UVPM import *
from State3_Prediction.IRM import *
from State3_Prediction.PS import *
from State2_FetureExtraction.FExS import *
from State2_FetureExtraction.FEnS import *
from Training.LF import *

class BuildModule(nn.Module):
	def __init__(self, module_code, config, data_config, device):
		super(BuildModule, self).__init__()

		self.module_code = module_code
		self.numFactor = config["numFactor"]
		self.numItem = data_config["numItem"]
		self.numUser = data_config["numUser"]
		self.numCate = data_config["numCate"]
		self.numK = config["numK"]
		self.input_length = config["input_length"]
		self.lr = config["learning_rate"]
		if self.module_code["IRM"] == "IRM2":
			self.lr = 0.1 * self.lr
			print(self.module_code["IRM"])

		self.trainBatchSize = config["batchSize"]
		self.maxEpoch = config["maxEpochnum"]
		self.userEmbedding = Variable(torch.randn((self.numUser, self.numFactor), dtype=torch.float32).cuda(),
									  requires_grad=True)
		self.itemEmbedding = Variable(torch.randn((self.numItem, self.numFactor), dtype=torch.float32).cuda(),
									  requires_grad=True)
		# self.userEmbedding = nn.Embedding(self.numUser, self.numFactor)
		# self.itemEmbedding = nn.Embedding(self.numItem, self.numFactor)

		module_config = {
			"module_code":module_code,
			"numItem":self.numItem,
			"numUser":self.numUser,
			"numCate":self.numCate,
			"userEmbedding":self.userEmbedding,
			"itemEmbedding":self.itemEmbedding
		}

		self.HIPM = eval(self.module_code["HIPM"])(config, module_config, device)
		self.UVPM = eval(self.module_code["UVPM"])(config, module_config, device)
		self.IRM = eval(self.module_code["IRM"])(config, module_config, device)
		self.FExS1 = eval(self.module_code["1stFExS"])(config, module_config, device, "1st")
		self.FEnS1 = eval(self.module_code["1stFEnS"])(config, module_config, device, "1st")
		self.FExS2 = eval(self.module_code["2ndFExS"])(config, module_config, device, "2nd")
		self.FEnS2 = eval(self.module_code["2ndFEnS"])(config, module_config, device, "2nd")
		self.PS = eval(self.module_code["PS"])(config, module_config, device)
		self.LF1 = eval(self.module_code["1stLF"])(config, module_config, device)
		self.LF2 = eval(self.module_code["2ndLF"])(config, module_config, device)

	def get_parameters(self):
		HIPM_para = self.HIPM.get_parameters()
		UVPM_para = self.UVPM.get_parameters()
		IRM_para = self.IRM.get_parameters()
		FExS1_para = self.FExS1.get_parameters()
		FEnS1_para = self.FEnS1.get_parameters()
		FExS2_para = self.FExS2.get_parameters()
		FEnS2_para = self.FEnS2.get_parameters()
		PS_para = self.PS.get_parameters()
		LF1_para = self.LF1.get_parameters()
		LF2_para = self.LF2.get_parameters()
		para = HIPM_para + UVPM_para + IRM_para + FExS1_para + FEnS1_para + FExS2_para + FEnS2_para + PS_para + LF1_para + LF2_para
		para.append({'params': self.userEmbedding})
		#para.append({'params': self.itemEmbedding, 'lr': self.lr})
		para.append({'params': self.itemEmbedding})
		return para

	def forward(self, feed_dict, mode):
		#the new user--item matrix accrodding to the history information
		user_em = self.UVPM.uvpm(feed_dict)
		"(batchsize, numFactor)"

		# target_itm, item_em = self.IPM.ipm(feed_dict)
		# "(batchsize)  (batchsize, input_length, numFactor)"

		history_item_em = self.HIPM.hipm(feed_dict, user_em)
		"(batchsize, input_length, numFactor)"

		#the new user embedding with the extracted item information in
		extracted_feature1 = self.FExS1.fexs(feed_dict, history_item_em, user_em)
		"(batchsize, numFactor)"

		enhanced_feature1 = self.FEnS1.fens(extracted_feature1)
		"(batchsize, numFactor)"

		extracted_feature2 = self.FExS2.fexs(feed_dict, history_item_em, user_em)
		"(batchsize, numFactor)"

		enhanced_feature2 = self.FEnS2.fens(extracted_feature2)
		"(batchsize, numFactor)"

		extracted_feature = (extracted_feature1 + extracted_feature2)/2
		enhanced_feature = (enhanced_feature1 + enhanced_feature2)/2

		item_batch, item_em = self.IRM.irm(feed_dict)
		"(batchsize, 2)   (batchsize, 2, numFactor)"

		#predict with the original user embedding, the new user embedding and the whole item embedding
		self.prediction, extra_info = \
			self.PS.ps(feed_dict, user_em, history_item_em, item_batch, item_em, extracted_feature, enhanced_feature, mode)
		"(batchsize, target_length+neg_length)"

		# LOSS = Loss(self, item_batch, self.prediction)
		# loss = eval("LOSS." + self.module_code["LOSS"] + "()")
		# return loss
		if mode == "test" or mode == "val_test":
			loss = None
		else:
			loss1 = self.LF1.lf(user_em, item_batch, item_em, self.prediction, extra_info)
			loss2 = self.LF2.lf(user_em, item_batch, item_em, self.prediction, extra_info)
			loss = (loss1 + loss2)/2
		return loss


# -*- coding: utf-8 -*-
"""
Spyder Editor

Author: Ruby Li

This script includes how to raw peptide data cleaning and flanking sequences
generation

"""
import pandas as pd
import numpy as np
import xml.etree.cElementTree as et
import requests
import os

# Select peptides whose length is between 8-11
data = data.loc[(data.peptide.str.len() >= 8) & (data.peptide.str.len() <= 11)]

# Obtain data from Uniprot
FOLDER = 'uniprots\\'

# Obtain xml files from Uniprot
def get_files(resource):
    bol = False
    # Uniprot webservice
    sequence_file = requests.get('http://www.uniprot.org/uniprot/' + resource)

    # If response is empty
    if len(sequence_file.text) == 0:
        print('not available in .xml or does not exist')

    # http not 200
    elif sequence_file.status_code != 200:
        print('http error ' + str(sequence_file.status_code))

    # If response is html, then it's invalid
    else:
        html = False
        for line in sequence_file.iter_lines():
            if '<!DOCTYPE html' in line.decode():
                print('not available in .' + output_format + ' or does not exist')
                html = True

        with open(FOLDER + resource, "wb") as file_name:
            [file_name.write(line + '\n'.encode()) for line in sequence_file.iter_lines()]
            bol = True
#     return bol

# get one protein sequence
def get_protein_seq(file_name):
    tree = et.ElementTree(file = FOLDER + file_name)
    root = tree.getroot()
    for child in root:
        for grchild in child:
            if grchild.tag == '{http://uniprot.org/uniprot}sequence':
                seq = grchild.text.strip()
                return seq

# get all protein sequences
def get_all_protseq(df):
    seq_list = []
    for i in range(df.shape[0]):
        resource = df['protein id'].iloc[i] + '.xml'
#         bol = get_files(resource)
#         print(resource)
        if(os.path.exists('uniprots\\'+resource)):
            seq = get_protein_seq(resource)
        else:
            seq = ''
        seq_list.append(seq)
#         if(i%100 == 0):
#             print('%s sequences have been extracted' %(i))
    seq_df = pd.DataFrame(seq_list)
    new_df = pd.concat([df['protein id'], seq_df, df['sequence'], df[['flanking aa\n']]], axis = 1)
    new_df.columns = ['protein id','sequence', 'peptide', 'flanking aa']
    return new_df

# Compute flanking sequence
def compute_flanking_sequence(df):
    flanking_sequence_list = []
    for i in range(df.shape[0]):
#         print(i)
        s = df['sequence'].iloc[i]
        pp = df['peptide'].iloc[i]
        f = df['flanking aa'].iloc[i].replace('\n','')
        fs = ''
        for j in range(len(s)):
#             print(j)
            if (j+5+len(pp) == len(s)-1):
                fs = ''
                flanking_sequence_list.append(fs)
                break
            if(s[j+4] == f[0]) and (s[j+5:j+5+len(pp)] == pp):
                fs+=s[j:j+5]
                fs+=s[j+5+len(pp):j+10+len(pp)]
                flanking_sequence_list.append(fs)
#                 print(fs)
                break
    df['Flanking Sequence'] = pd.DataFrame(flanking_sequence_list)
    return df

Data Processing/peptides and flanking sequences.py


import matplotlib.pyplot as plt
import torch
import numpy as np
import cv2
import random
from scipy import ndimage

class Rescale(object):
    """Rescale the image in a sample to a given size.

    Args:
        output_size (tuple or int): Desired output size. If tuple, output is
            matched to output_size. If int, smaller of image edges is matched
            to output_size keeping aspect ratio the same.
    """

    def __init__(self, output_size):
        assert isinstance(output_size, (int, tuple)), 'output size should be int or tuple'
        self.output_size = output_size

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        h, w = image.shape[:2]
        if isinstance(self.output_size, int):
            if h > w:
                new_h, new_w = self.output_size * h / w, self.output_size
            else:
                new_h, new_w = self.output_size, self.output_size * w / h
        else:
            new_h, new_w = self.output_size

        new_h, new_w = int(new_h), int(new_w)

        image = cv2.resize(image, (new_h, new_w))
        mask = cv2.resize(mask, (new_h, new_w))

        return {'image': image, 'mask': mask}

class CenterCrop(object):
    """Crop the image in a sample.

    Args:
        output_size int: Desired output size. square crop.
    """

    def __init__(self, output_size):
        assert isinstance(output_size, (int, tuple)), 'output size should be int or tuple'
        if isinstance(output_size, int):
            self.output_size = (output_size, output_size)
        else:
            assert len(output_size) == 2
            self.output_size = output_size

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        width, height = image.shape[-1], image.shape[-2]

        # process crop width and height for max available dimension
        crop_width = self.output_size[0] if self.output_size[0] < image.shape[-1] else image.shape[-1]
        crop_height = self.output_size[1] if self.output_size[1] < image.shape[-2] else image.shape[-2]

        mid_x, mid_y = int(width / 2), int(height / 2)
        cw2, ch2 = int(crop_width / 2), int(crop_height / 2)

        image = image[mid_y - ch2:mid_y + ch2, mid_x - cw2:mid_x + cw2]
        mask = mask[mid_y - ch2:mid_y + ch2, mid_x - cw2:mid_x + cw2]

        return {'image': image, 'mask': mask}

class NegativeImage(object):
    """Returns the negative of the image"""

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        new_image = np.max(image) - image

        # fig, axs = plt.subplots(1, 2, figsize=(18, 4))
        # axs[0].imshow(image)
        # axs[0].set_title('original')
        # axs[1].imshow(new_image)
        # axs[1].set_title('negative')
        # plt.show()

        return {'image': new_image, 'mask': mask}

class ClaheImage(object):
    """Clahe on image.

    Args: clip limit, tileGridSize
    """

    def __init__(self, clipLimit=4., tileGridSize=(8, 8)):
        self.clipLimit = clipLimit
        self.tileGridSize = tileGridSize

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        clahe = cv2.createCLAHE(clipLimit=self.clipLimit, tileGridSize=self.tileGridSize)
        new_image = clahe.apply(image)

        # fig, axs = plt.subplots(1, 2, figsize=(18, 4))
        # fig.suptitle('original')
        # axs[0].imshow(image)
        # axs[1].hist(image.ravel(), np.max(image),[0,np.max(image)])
        # plt.show()
        #
        # fig, axs = plt.subplots(1, 2, figsize=(18, 4))
        # fig.suptitle('After Clahe')
        # axs[0].imshow(new_image)
        # axs[1].hist(new_image.ravel(),np.max(new_image),[0,np.max(new_image)])
        # plt.show()

        return {'image': new_image, 'mask': mask}

class RescalePixels(object):
    """Rescale the image's pixels value to a given range.

    Args:
        new_range = (new_min, new_max) (tuple): Desired pixels range.
    """

    def __init__(self, new_range):
        assert isinstance(new_range, tuple), 'range should be tuple'
        self.new_min = new_range[0]
        self.new_max = new_range[1]

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        old_min = np.min(image)
        old_max = np.max(image)

        new_image = (((image - old_min) * (self.new_max - self.new_min)) / (old_max - old_min)) + self.new_min
        new_image = np.max(new_image) - new_image

        # fig, axs = plt.subplots(1, 2, figsize=(18, 4))
        # axs[0].imshow(image)
        # axs[0].set_title('original')
        # axs[1].imshow(new_image)
        # axs[1].set_title('after scalse to [0 1]')
        # plt.show()

        return {'image': new_image, 'mask': mask}

class ExpendDim(object):
    """expend dim from [batch, *image_size] to [1, batch, *image_size]"""

    def __call__(self, sample):
        return {'image': np.expand_dims(sample['image'], axis=0),
                'mask': np.expand_dims(sample['mask'], axis=0)}

class RandomRotate(object):
    """Rotate the image and the mask by a random angle within a given range.

    Args:
        angle_range (tuple): the possible range of the angle.
    """

    def __init__(self, angle_range):
        assert isinstance(angle_range, (tuple)), 'range should be tuple'
        self.min_angle = angle_range[0]
        self.max_angle = angle_range[1]

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        angle = random.uniform(self.min_angle, self.max_angle)

        image = ndimage.rotate(image, angle, reshape=False)
        mask = ndimage.rotate(mask, angle, reshape=False)

        return {'image': image, 'mask': mask}

class RandomFlip(object):
    """Flip the image and the mask with probability 0.5."""

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        if random.uniform(0, 1) >= 0.5:
            image = np.fliplr(image)
            mask = np.fliplr(mask)

        return {'image': image, 'mask': mask}

class RandomCrop(object):
    """Randomly crop a portion of the image and mask, when the relationship between width and height is preserved."""

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        # random scale
        scale = random.uniform(0.5, 1)
        h = int(image.shape[0] * scale)
        w = int(image.shape[1] * scale)

        # random top-left point in image
        left = int(random.uniform(0, image.shape[0] - h -1))
        top = int(random.uniform(0, image.shape[1] - w - 1))

        # plt.figure()
        # plt.imshow(image)
        # plt.scatter(left, top, c='red')
        # plt.scatter(left+w, top+h, c='red')
        # plt.show()

        image = image[top:top+h, left:left+w]
        mask = mask[top:top+h, left:left+w]

        # plt.figure()
        # plt.imshow(image)
        # plt.show()
        return {'image': image, 'mask': mask}

class RandomBrightness(object):
    """ Random adjustment of the brightness  of the input image.

    Args:
        brightness_range (tuple): the possible range of the brightness adjustment.
    """

    def __init__(self, brightness_range):
        assert isinstance(brightness_range, (tuple)), 'range should be tuple'
        self.brightness_range = brightness_range

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        brightness = np.random.uniform(self.brightness_range[0], self.brightness_range[1])
        image = cv2.convertScaleAbs(image, alpha=brightness, beta=0)
        # image = np.clip(image * brightness, 0, 255)

        # plt.figure()
        # plt.imshow(image)
        # plt.title(f'after brightness {brightness}')
        # plt.show()

        return {'image': image, 'mask': mask}

class RandomContrast(object):
    """ Random adjustment of the contrast of the input image.

    Args:
        contrast_range (tuple): the possible range of the contrast adjustment.
    """

    def __init__(self, contrast_range):
        assert isinstance(contrast_range, (tuple)), 'range should be tuple'
        self.contrast_range = contrast_range

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        # mean = np.mean(image)
        # image = np.clip((image - mean) * np.random.uniform(self.contrast_range[0], self.contrast_range[1]) + mean, 0, 255)

        contrast = np.random.uniform(self.contrast_range[0], self.contrast_range[1])
        image = cv2.convertScaleAbs(image, alpha=contrast, beta=0)
        # image = np.clip((image - 511.5) * contrast + 511.5, 0, 1023)

        # plt.figure()
        # plt.imshow(image)
        # plt.title(f'after contrast {contrast}')
        # plt.show()

        return {'image': image, 'mask': mask}

class LinearStretch(object):
    """ Stretch the image pixles range to [0,255] - 8bit."""

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        # plt.figure()
        # plt.imshow(image)
        # plt.title('original')
        # plt.show()

        old_min = np.min(image)
        old_max = np.max(image)
        new_min = 0
        new_max = 255
        image = ((image - old_min) / (old_max - old_min)) * (new_max - new_min) + new_min

        # plt.figure()
        # plt.imshow(image)
        # plt.title('after stretch')
        # plt.show()

        return {'image': image, 'mask': mask}

class GaussNoise(object):
    """Add Gaussian noise to image.

    Args:
        mean (float):
        std (float):"
    """

    def __init__(self, mean, std):
        self.mean = mean
        self.std = std

    def __call__(self, sample):
        image, mask = sample['image'], sample['mask']

        image = image + np.random.normal(self.mean, self.std, size=image.shape)

        # plt.figure()
        # plt.imshow(image)
        # plt.title('after gaussian noise')
        # plt.show()

        return {'image': image, 'mask': mask}

class ToTensor(object):
    """Convert ndarrays in sample to Tensors."""

    def __call__(self, sample):
        return {'image': torch.from_numpy(sample['image'].astype(float)).float(),
                'mask': torch.from_numpy(sample['mask'].astype(float)).float()}