theta = float(input())
input()
alphabet = input().split()
input()
temp = input()
align = []
while temp!="*":
    align += [temp]
    temp = input()
# print(theta)
# print(alphabet)
# print(align)
n = len(align)
m = len(align[0])
matchOrInsert = []
for i in range(m):
    sum = 0
    for j in range(n):
        if align[j][i]== "-":
            sum+=1
    if sum/n <= theta:
        matchOrInsert += ["MD"]
    else:
        matchOrInsert += ["Ins"]
# print(matchOrInsert)
MIndex = 0
iIndex = 0
matrix = [["-" for i in range(m)] for j in range(n)]
# print(matrix)
for i in range(m):
    if matchOrInsert[i] == "MD":
        MIndex += 1
        iIndex += 1
        for j in range(n):
            if align[j][i] == "-":
                matrix[j][i] = "D"+str(MIndex)
            else:
                matrix[j][i] = "M"+str(MIndex)
    else:
        for j in range(n):
            if align[j][i] != "-":
                matrix[j][i] = "I"+str(iIndex)
# print(matrix)
InsCount = 0
for i in range(m):
    if matchOrInsert[i] == "Ins":
        InsCount +=1
probability = {"S":{"I0":0,"M1":0,"D1":0}}
for i in range(m-InsCount+1):
    if i == m-InsCount:
        probability["I" + str(i)] = {"I" + str(i): 0,"E":0}
    else:
        probability["I"+str(i)] = {"I"+str(i):0,"M"+str(i+1):0,"D"+str(i+1):0}
    if i < m-InsCount:
        if i==m-InsCount-1:
            probability["M" + str(i + 1)] = {"I" + str(i + 1): 0,"E":0}
            probability["D" + str(i + 1)] = {"I" + str(i + 1): 0,"E":0}
        else:
            probability["M"+str(i+1)] = {"I"+str(i+1):0,"M"+str(i+2):0,"D"+str(i+2):0}
            probability["D"+str(i+1)] = {"I"+str(i+1):0,"M"+str(i+2):0,"D"+str(i+2):0}

# print(probability)
for i in range(m):
    for j in range(n):
        if i==0 and matrix[j][i]!="-":
            probability["S"][matrix[j][i]]+=1
        elif i == 0 and matrix[j][i]=="-":
            probability["S"][matrix[j][i+1]]+=1
        elif i>0:
            if matrix[j][i]!="-":
                if matrix[j][i-1] != "-":
                    probability[matrix[j][i-1]][matrix[j][i]] +=1
                elif matrix[j][i-1] == "-"and matrix[j][i-2]!="-":
                    probability[matrix[j][i-2]][matrix[j][i]] +=1
                elif matrix[j][i-1] == "-" and matrix[j][i-2]=="-" and matrix[j][i-3]!="-":
                    probability[matrix[j][i-3]][matrix[j][i]] +=1
                elif matrix[j][i-1] == "-" and matrix[j][i-2]=="-"and matrix[j][i-3]=="-":
                    probability[matrix[j][i-4]][matrix[j][i]] +=1
for i in range(n):
    if matrix[i][m-1]!="-":
        probability[matrix[i][m-1]]["E"]+=1
    else:
        if matrix[i][m-2] != "-":
            probability[matrix[i][m-2]]["E"] +=1
        elif matrix[i][m-2] == "-"and matrix[i][m-3]!="-":
            probability[matrix[i][m-3]]["E"] +=1
        elif matrix[i][m-2] == "-" and matrix[i][m-3]=="-" and matrix[i][m-4]!="-":
            probability[matrix[i][m-4]]["E"] +=1
        elif matrix[i][m-2] == "-" and matrix[i][m-3]=="-"and matrix[i][m-4]=="-":
            probability[matrix[i][m-5]]["E"] +=1

# print(probability)
result = [[0 for i in range(3*(m-InsCount)+4)]for j in range(3*(m-InsCount)+4)]
result[0][0] = ""
result[0][1] = "S"
result[0][2] = "I0"
count = 0
for i in range(3,3*(m-InsCount)+3):
    # print(i)
    if i%3==0:
        count +=1
        result[0][i] = "M" + str(count)
        result[0][i+1] = "D"+str(count)
        result[0][i+2] = "I" + str(count)
result[0][3*(m-InsCount)+3] = "E"
result[1][0] = "S"
result[2][0] = "I0"
count = 0
for i in range(3,3*(m-InsCount)+3):
    # print(i)
    if i%3==0:
        count +=1
        result[i][0] = "M" + str(count)
        result[i+1][0] = "D"+str(count)
        result[i+2][0] = "I" + str(count)
result[3*(m-InsCount)+3][0] = "E"
# print(result)
sumNum = {}
for i in probability:
    sum = 0
    for j in probability[i]:
        sum += probability[i][j]
    sumNum[i] = sum
# print(sumNum)
for i in probability:
    for j in probability[i]:
        if sumNum[i]!=0 and probability[i][j]!=0:

            # print(probability[i][j])
            result[result[0].index(i)][result[0].index(j)] = round(probability[i][j]/sumNum[i],3)
for i in range(len(result)):
    print(*result[i] , sep=" ")
print("--------")
result2 = [[0 for j in range(len(alphabet)+1)]for i in range(3*(m-InsCount)+4) ]
result2[0][0] = ""
result2[1][0] = "S"
result2[2][0] = "I0"
count = 0
for i in range(1,len(alphabet)+1):
    result2[0][i] = alphabet[i-1]
for i in range(3,3*(m-InsCount)+3):

    # print(i)
    if i%3==0:
        count +=1
        result2[i][0] = "M" + str(count)
        result2[i+1][0] = "D"+str(count)
        result2[i+2][0] = "I" + str(count)
result2[3*(m-InsCount)+3][0] = "E"
alphCount = {}
count = 0
for i in range(2):
    for j in range(len(alphabet)):
        if i ==0:
            if "S" in alphCount.keys():
                alphCount["S"].update({alphabet[j]:0})
            else:
                alphCount["S"] = {alphabet[j]:0}
        elif i ==1:
            if "I0" in alphCount.keys():
                alphCount["I0"].update({alphabet[j]:0})
            else:
                alphCount["I0"]={alphabet[j]:0}
for i in range((m-InsCount)):
    count+=1
    for j in range(len(alphabet)):

        if "M"+str(count) in alphCount.keys():
            alphCount["M"+str(count)].update({alphabet[j]:0})
        else:
            alphCount["M"+str(count)] = {alphabet[j]:0}
        if "D"+str(count) in alphCount.keys():
            alphCount["D"+str(count)].update({alphabet[j]:0})
        else:
            alphCount["D"+str(count)] = {alphabet[j]:0}
        if "I"+str(count) in alphCount.keys():
            alphCount["I"+str(count)].update({alphabet[j]:0})
        else:
            alphCount["I"+str(count)] = {alphabet[j]:0}

for j in range(len(alphabet)):
    if "E" in alphCount.keys():
        alphCount["E"].update({alphabet[j]:0})
    else:
        alphCount["E"] = {alphabet[j]:0}
for i in range(m):
    for j in range(n):
        if align[j][i]!="-":
            alphCount[matrix[j][i]][align[j][i]] += 1
# print(alphCount)
countalph = {}
for i in alphCount:
    sum = 0
    for j in alphCount[i]:
        sum += alphCount[i][j]
    countalph[i] = sum
# print(countalph)
count = 0
# print(result2[15][0])
for i in (alphCount):
    count+=1
    # print(count)
    for j in (alphCount[i]):
        # print(j)
        if alphCount[i][j]!=0 and countalph[i]!=0:
            result2[count][result2[0].index(j)] = round(alphCount[i][j]/countalph[i],3)
for i in range(len(result2)):
    print(*result2[i] , sep=" ")

Different Bioinformatics Exercises/Profile HMM Problem.py


# Copyright (c) OpenMMLab. All rights reserved.
from mmengine.dataset.sampler import DefaultSampler

from mmagic.datasets.imagenet_dataset import ImageNet
from mmagic.datasets.transforms.aug_shape import Flip, Resize
from mmagic.datasets.transforms.crop import (CenterCropLongEdge,
                                             RandomCropLongEdge)
from mmagic.datasets.transforms.formatting import PackInputs
from mmagic.datasets.transforms.loading import LoadImageFromFile

# dataset settings
dataset_type = ImageNet

# different from mmcls, we adopt the setting used in BigGAN.
# We use `RandomCropLongEdge` in training and `CenterCropLongEdge` in testing.
train_pipeline = [
    dict(type=LoadImageFromFile, key='gt'),
    dict(type=RandomCropLongEdge, keys='gt'),
    dict(type=Resize, scale=(64, 64), keys='gt', backend='pillow'),
    dict(type=Flip, keys='gt', flip_ratio=0.5, direction='horizontal'),
    dict(type=PackInputs)
]

test_pipeline = [
    dict(type=LoadImageFromFile, key='gt'),
    dict(type=CenterCropLongEdge, keys='gt'),
    dict(type=Resize, scale=(64, 64), keys='gt', backend='pillow'),
    dict(type=PackInputs)
]

train_dataloader = dict(
    batch_size=None,
    num_workers=5,
    dataset=dict(
        type=dataset_type,
        data_root='./data/imagenet/',
        ann_file='meta/train.txt',
        data_prefix='train',
        pipeline=train_pipeline),
    sampler=dict(type=DefaultSampler, shuffle=True),
    persistent_workers=True)

val_dataloader = dict(
    batch_size=64,
    num_workers=5,
    dataset=dict(
        type=dataset_type,
        data_root='./data/imagenet/',
        ann_file='meta/train.txt',
        data_prefix='train',
        pipeline=test_pipeline),
    sampler=dict(type=DefaultSampler, shuffle=False),
    persistent_workers=True)

test_dataloader = val_dataloader


import graphene
from datetime import date
from core.models import Company, Version
from helpers.tst_helper import GraphQLTestCase
from helpers.functions import to_dict
from helpers.graphene import filter_fields_args

class HelperFunctionsTest(GraphQLTestCase):

    @classmethod
    def setUpTestData(cls):
        company = Company.objects.create(shortname="X", name="X-Company")
        Version.objects.create(company=company, shortname="V1", reporting_date=date(2010, 12, 31))
        super().setUpTestData()

    def test_toDict_with_existing_instance(self):
        print(""" - to_dict() works as expected with existing model instance """)
        version = Version.objects.all()[0]
        expected = {
            "pk": version.id,
            "id": version.id,
            "shortname": "V1",
            "reporting_date": version.reporting_date,
            "company": version.company,
            "compare_version": None,
            "matching_version": None,
            "copy_version": None,
            "description": "",
            "archived": False,
            "locked": False,
            "created_at": version.created_at,
            "updated_at": version.updated_at
        }
        self.assertEqual(to_dict(version), expected)

class HelperGrapheneTest(GraphQLTestCase):

    @classmethod
    def setUpTestData(cls):
        super().setUpTestData()

    def test_filterFieldsArgs(self):
        print(""" - filter_fields_args() works as expected """)
        filter_fields = {
            "company": {
                "field_type": graphene.String(),
                "filter": ["exact"]
            },
            "reporting_date__year": {
                "field_type": graphene.Int(),
                "filter": ["gt", "gte"]
            },
            "shortname": {
                "field_type": graphene.String(),
                "filter": ["icontains"]
            }
        }
        expected = {
            "company__exact": graphene.String(),
            "shortname__icontains": graphene.String(),
            "reporting_date__year__gt": graphene.Int(),
            "reporting_date__year__gte": graphene.Int()
        }
        self.assertEqual(filter_fields_args(filter_fields), expected)


#!/usr/local/bin/python
# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
# ---------------------------------------- SYSTEM DATABASE PARAMETERS -------------------------------------------------
# The name of the host where MongoDB is running.
mongodb_host = '127.0.0.1'
# The port where MongoDB is listening to.
mongodb_port = 27017

travel_requests_simulator_timeout = 100

travel_requests_simulator_max_operation_timeout = 600
travel_requests_simulator_min_number_of_documents = 10

travel_requests_simulator_max_number_of_documents = 100

travel_requests_simulator_datetime_distribution_weights = [
    1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1
]

maximum_bus_capacity = 100

average_waiting_time_threshold = 0
individual_waiting_time_threshold = 0
minimum_number_of_passengers_in_timetable = 10

look_ahead_timetables_generator_timeout = 100

look_ahead_timetables_generator_max_operation_timeout = 600

look_ahead_timetables_updater_timeout = 100

look_ahead_timetables_updater_max_operation_timeout = 600

testing_bus_stop_names = [

]

testing_bus_line_id = 1

now = datetime.now()
today = datetime(now.year, now.month, now.day, 0, 0, 0, 00000)
tomorrow = today + timedelta(days=1)

testing_travel_requests_min_departure_datetime = today
testing_travel_requests_max_departure_datetime = tomorrow

testing_timetables_starting_datetime = today
testing_timetables_ending_datetime = tomorrow


# -*- coding: utf-8 -*-
from abc import ABCMeta
from abc import abstractmethod
from datetime import datetime
from logging import getLogger
from typing import Dict
from typing import List
from typing import Optional

from moneybot.market import Order
from moneybot.market.history import MarketHistory
from moneybot.market.state import MarketState
from moneybot.trade import AbstractTrade

logger = getLogger(__name__)

class MarketAdapter(metaclass=ABCMeta):

    @classmethod
    @abstractmethod
    def reify_trades(
        cls,
        trades: List[AbstractTrade],
        market_state: MarketState,
    ) -> List[Order]:
        raise NotImplementedError

    def __init__(
        self,
        fiat: str,
        history: MarketHistory,
        initial_balances: Dict[str, float],
    ) -> None:
        self._fiat = fiat
        self._market_history = history
        self._market_state = MarketState(
            None,
            initial_balances,
            None,
            self.fiat,
        )

    @property
    def fiat(self):
        return self._fiat

    @property
    def market_history(self) -> MarketHistory:
        return self._market_history

    @property
    def market_state(self) -> MarketState:
        return self._market_state

    def update_market_state(self, time: datetime):
        # Get the latest chart data from the market
        charts = self.market_history.latest(time)
        balances = self.get_balances()
        self._market_state = MarketState(charts, balances, time, self.fiat)

    @abstractmethod
    def get_balances(self) -> Dict[str, float]:
        raise NotImplementedError

    @abstractmethod
    def execute_order(self, order: Order, attempts: int = 8) -> Optional[int]:
        """Execute an order, returning an order identifier.
        """
        raise NotImplementedError


import os
import re
import cv2
from torch import nn
from torch.nn import init
# from random import shuffle
import time
import numpy as np
from skimage import feature
from skimage.morphology import remove_small_holes, remove_small_objects

# ---------- make dir ----------
def make_dir(path):
    if not os.path.exists(path):
        os.makedirs(path)

# ---------- config ----------
def conf2dict(key_values):
    kv_dict = {}
    for kv in key_values:
        key = kv[0]
        value = get_value(kv[1])
        kv_dict[key] = value
    return kv_dict

def get_value(key_value):
    value = key_value
    float_p = re.compile(r"^[-+]?\d+\.\d+")
    float_ep = re.compile(r"\d+e[-+]?\d+")
    tuple_p = re.compile(r"\(\d+, *\d+\)")
    if value.isdigit():
        value = int(value)
    elif value.lower() == 'true':
        value = True
    elif value.lower() == 'false':
        value = False
    elif value.lower() == 'none':
        value = None
    elif float_p.match(value):
        value = float_p.match(value).group()
        value = float(value)
    elif float_ep.match(value):
        value = float_ep.match(value).group()
        value = float(value)
    elif tuple_p.findall(value):
        vlist = []
        for v in tuple_p.findall(value):
            value_tu = tuple(eval(v))
            value_tu = (int(value_tu[0]), int(value_tu[1]))
            vlist.append(value_tu)
        value = vlist
    return value

def print_config(config, save_path):
    mode = config.get('base_options', 'mode')

    sections = config.sections()
    for sec in sections:
        if (sec not in mode) & ((sec == 'train') | (sec == 'val') | (sec == 'test')):
            continue
        print('----------- ' + sec + ' ----------')
        for items in config.items(sec):
            print(items[0] + ' = ' + str(items[1]))
        print()
    config.write(open(save_path + 'config.conf', 'w'))

# ---------- name list ----------
def default_split(data_set):
    if data_set == 'ARAI':
        train_num = 143
    elif data_set == 'STARE':
        train_num = 5 * 2
    elif data_set == 'DRIVE':
        train_num = 20
    elif data_set == 'CHASE_DB1':
        train_num = 7 * 2
    else:
        train_num = 5 * 3
    return train_num

def get_name_list(data_set, img_dir=None, name_list=None, train_num=0, shuf=False):
    if name_list is None:
        name_list = []
        for _, _, f_n in os.walk(img_dir):
            name_list.extend(f_n)
        if data_set == 'CHASE_DB1':
            f_n = [k for k in name_list if '.jpg' in k]
            name_list = f_n
        # name_list.sort()
    val_num = len(name_list) - train_num
    # if shuf:
    #     shuffle(name_list)
    if val_num < train_num:
        split_num = val_num
        small_list = 'val_list'
    else:
        split_num = train_num
        small_list = 'train_list'

    val_list, train_list = split_nameList(name_list=name_list, split_num=split_num,
                                          data_set=data_set, small_list=small_list)
    if len(train_list) > 0:
        if type(train_list[0]) != list:
            train_list = [train_list]
    if len(val_list) > 0:
        if type(val_list[0]) != list:
            val_list = [val_list]

    return val_list, train_list

def split_nameList(name_list, split_num=0, data_set=None, small_list='val_list', shuf=False):
    if data_set == 'HRF':
        val_list = []
        train_list = []
        split_num = int(split_num / 3)
        healthy = [j for j in name_list if '_h' in j]
        glaucomstous = [j for j in name_list if '_g' in j]
        diabetic = [j for j in name_list if '_dr' in j]

        h_valList, h_trainList = split_list(healthy, split_num, small_list=small_list)
        g_valList, g_trainList = split_list(glaucomstous, split_num, small_list=small_list)
        dr_valList, dr_trainList = split_list(diabetic, split_num, small_list=small_list)

        if split_num == 0:
            if small_list == 'val_list':
                train_l = h_trainList + g_trainList + dr_trainList
                train_list.append(train_l)
            else:
                val_l = h_valList + g_valList + dr_valList
                val_list.append(val_l)
        else:
            for i in range(len(h_valList)):
                val_l = h_valList[i] + g_valList[i] + dr_valList[i]
                train_l = h_trainList[i] + g_trainList[i] + dr_trainList[i]
                val_list.append(val_l)
                train_list.append(train_l)

    elif data_set == 'STARE':
        # split_num = int(split_num / 2)
        h = ['im0077.ppm', 'im0081.ppm', 'im0082.ppm', 'im0162.ppm', 'im0163.ppm',
             'im0235.ppm', 'im0236.ppm', 'im0239.ppm', 'im0240.ppm', 'im0255.ppm']
        p = ['im0001.ppm', 'im0002.ppm', 'im0003.ppm', 'im0004.ppm', 'im0005.ppm',
             'im0044.ppm', 'im0139.ppm', 'im0291.ppm', 'im0319.ppm', 'im0324.ppm']
        # if shuf:
        #     shuffle(h)
        #     shuffle(p)
        total_names = [h, p]
        pic_names = [total_names[i][j] for j in range(len(h))
                     for i in range(len(total_names))]
        val_list, train_list = split_list(pic_names, split_num, small_list=small_list)
    else:

        val_list, train_list = split_list(name_list, split_num, small_list=small_list)

    return val_list, train_list

def split_list(name_list, split_num, small_list='val_list'):
    val_list = []
    train_list = []
    n = len(name_list)
    nameList_array = np.array(range(n))
    if small_list == 'val_list':
        print(n - split_num, 'for train, ', split_num, 'for validation')
        if split_num == 0:
            train_list.extend(name_list)
        else:
            for i in range(int(n / split_num) - 1, -1, -1):
                temp = nameList_array[i * split_num:(i + 1) * split_num]
                val_l = [name_list[j] for j in temp]
                train_l = [name_list[j] for j in range(n) if j not in temp]
                val_list.append(val_l)
                train_list.append(train_l)

    else:
        print(split_num, 'for train, ', n - split_num, 'for validation')
        if split_num == 0:
            val_list.extend(name_list)
        else:
            for i in range(int(n / split_num)):
                temp = nameList_array[i * split_num:(i + 1) * split_num]
                train_l = [name_list[j] for j in temp]
                val_l = [name_list[j] for j in range(n) if j not in temp]
                val_list.append(val_l)
                train_list.append(train_l)

    return val_list, train_list

# ---------- image information ----------
def get_imgInfo(mode, data_set, data_path):
    img_path = None
    gt_path = None
    mask_path = None
    img_num = 0
    img_size = (0, 0, 0)
    if data_set == 'DRIVE':
        if mode == 'train':
            img_path = 'DRIVE/training/images/'
            gt_path = 'DRIVE/training/1st_manual/'
            mask_path = 'DRIVE/training/mask/'
            img_num = 20
            img_size = (584, 565)
        elif mode == 'test':
            img_path = 'DRIVE/test/images/'
            gt_path = 'DRIVE/test/1st_manual/'
            mask_path = 'DRIVE/test/mask/'
            img_size = (584, 565)
            img_num = 20
    elif data_set == 'ARIA':
        img_path = 'ARIA/ARIA_all/'
        gt_path = 'ARIA/ARIA_all_GT/'
        img_num = 143
        img_size = (576, 768)
    elif data_set == 'CHASE_DB1':
        img_path = 'CHASE_DB1/'
        gt_path = img_path
        img_num = 28
        img_size = (960, 999)
    elif data_set == 'HRF':
        img_path = 'HRF/all/images/'
        gt_path = 'HRF/all/manual1/'
        mask_path = 'HRF/all/mask/'
        img_num = 45
        img_size = (2336, 3504)
    elif data_set == 'STARE':
        img_path = 'STARE/stare-images/'
        gt_path = 'STARE/labels-ah/'
        img_num = 20
        img_size = (605, 700)
    img_path = data_path + img_path
    if gt_path:
        gt_path = data_path + gt_path
    if mask_path:
        mask_path = data_path + mask_path
    return img_path, gt_path, mask_path, img_num, img_size

def get_name(dataSet, name, mode='train'):
    dSet = dataSet
    imgName = None
    gtName = None
    maskName = None

    if dSet == 'DRIVE':
        if mode == "test":
            imgNum = name.replace('_test.tif', '')
        else:
            imgNum = name.replace('_training.tif', '')
        imgName = imgNum
        gtName = imgNum + "_manual1.gif"
        maskName = name.replace('.tif', '_mask.gif')

    if dSet == "ARIA":
        imgNum = name[-6:-4]
        imgNum = imgNum.replace('_', '')
        gtName = name.replace('.tif', '')
        gtName = gtName + '_BDP.tif'
        imgName = imgNum

    if dSet == 'CHASE_DB1':
        imgName = name.replace('.jpg', '')
        gtName = imgName + '_1stHO.png'

    if dSet == 'HRF':
        imgName = name.replace('.jpg', '')
        imgName = imgName.replace('.JPG', '')
        gtName = imgName + '.tif'
        maskName = imgName + '_mask.tif'

    if dSet == 'STARE':
        imgName = filter(str.isdigit, name)
        gtName = name.replace('.ppm', '')
        gtName = gtName + '.ah.ppm'

    return gtName, maskName, imgName

def get_gt(gt):
    if gt.max() > 1:
        gt = gt / gt.max()
    gt_min = (gt == gt.min()) * 1
    gt_max = (gt == gt.max()) * 1
    if gt_min.sum() > gt_max.sum():
        return gt_max
    else:
        return gt_min

# get mask
def get_mask(name, img, dataSet):
    th = None
    fovTH = None
    before_FOV = None
    if dataSet == 'CHASE_DB1':
        fovTH = 300  # chase_DB1:300, stare: 430
        th = 10  # chase_db1: 10, stare_50
    elif dataSet == 'STARE':
        fovTH = 430  # chase_DB1:300, stare: 430
        th = 50  # chase_db1: 10, stare_50

    feature_sig = 3
    if name == 'im0004.ppm':
        fovTH = 1153
        feature_sig = 3

    img_lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)
    L = img_lab[:, :, 0]
    before = 0
    while True:
        inFOV = (L > th) * 1
        after = inFOV.sum()
        aaa = after - before
        if name == 'im0004.ppm':
            if aaa == 147:
                break
        if aaa <= fovTH:
            break
        before = after
        before_FOV = inFOV
        th = th - 1
    R = img[:, :, 0]
    edges = feature.canny(R, sigma=feature_sig)
    bbb = before_FOV.copy()
    bbb = (bbb == 1)
    bbb = (edges | bbb) * 1
    bbb = remove_small_holes(bbb, min_size=6000, connectivity=1)
    bbb = remove_small_objects(bbb, min_size=50)
    # bbb = bwareaopen(bbb, 1000)
    mask = bbb * 1
    return mask

# ---------- tiles ----------
def get_tile_size(length, ratio, num_down, pyr_level):
    max_down = max(num_down, pyr_level)
    tile_len = length * ratio
    tile_len = int(np.ceil(tile_len / (2 ** max_down)) * (2 ** max_down))
    return tile_len

def test_tiles(img, tile_size=(128, 128), stride=(64, 64), l_max=3):
    start = time.time()
    if not stride:
        stride = (int(tile_size[0] / 2), int(tile_size[1] / 2))

    dims = img.ndim
    length = img.shape[0]
    width = img.shape[1]
    tile_len = tile_size[0]
    tile_wid = tile_size[1]
    x_stride = stride[0]
    y_stride = stride[1]

    x_tNum = int(np.ceil((length - tile_len) * 1.0 / x_stride))
    y_tNum = int(np.ceil((width - tile_wid) * 1.0 / y_stride))
    tile_num = (x_tNum + 1) * (y_tNum + 1)

    new_len = x_stride * x_tNum + tile_len
    new_wid = y_stride * y_tNum + tile_wid

    pad_l = new_len - length + tile_len * (2 ** (l_max - 1) - 1)
    pl2 = pad_l / 2
    pad_w = new_wid - width + tile_wid * (2 ** (l_max - 1) - 1)
    pw2 = pad_w / 2

    x_padL = int(pl2)
    x_padR = int(pad_l - x_padL)
    y_padT = int(pw2)
    y_padB = int(pad_w - y_padT)

    print('pre:', time.time() - start)

    start = time.time()
    if dims == 3:
        channels = img.shape[2]
        img_padded = np.lib.pad(img, ((x_padL, x_padR), (y_padT, y_padB), (0, 0)), 'symmetric')
        tiles = np.zeros((tile_num * l_max, tile_len, tile_wid, channels))
    else:
        img_padded = np.lib.pad(img, ((x_padL, x_padR), (y_padT, y_padB)), 'symmetric')
        tiles = np.zeros((tile_num * l_max, tile_len, tile_wid))
    print('pad:', time.time() - start)

    for level in range(l_max):
        start = time.time()
        tiles[level::l_max] = make_test_tiles(img_padded, x_tNum, y_tNum, tile_len,
                                              tile_wid, level, l_max, x_stride, y_stride)
        print('make tile:', time.time() - start)

    return tiles, x_tNum + 1, y_tNum + 1
    # return tiles

def tile_combine(tiles, img_size, tile_size=(128, 128), x_stride=64, y_stride=64):
    t_len = tile_size[0]
    t_wid = tile_size[1]

    length = img_size[0]
    width = img_size[1]

    x_tNum = int(np.ceil((length - t_len) * 1.0 / x_stride))
    y_tNum = int(np.ceil((width - t_wid) * 1.0 / y_stride))

    new_len = x_stride * x_tNum + t_len
    new_wid = y_stride * y_tNum + t_wid

    pad_l = new_len - length
    pl2 = int(pad_l / 2)
    pad_w = new_wid - width
    pw2 = int(pad_w / 2)

    pred_padded = np.zeros((new_len, new_wid))
    count_padded = np.zeros((new_len, new_wid))
    for i in range(x_tNum + 1):
        for j in range(y_tNum + 1):
            pred_padded[i * x_stride:i * x_stride + t_len, j * y_stride:j * y_stride + t_wid] += tiles[
                i * (y_tNum + 1) + j]
            count_padded[i * x_stride:i * x_stride + t_len, j * y_stride:j * y_stride + t_wid] += 1

    pred_cols = pred_padded[pl2:new_len - pad_l + pl2]
    count_cols = count_padded[pl2:new_len - pad_l + pl2]
    pad_index = range(pl2)
    pred_cols[list(reversed(pad_index))] += pred_padded[pad_index]
    count_cols[list(reversed(pad_index))] += count_padded[pad_index]

    pad_index1 = range(length - pad_l + pl2, length)
    pad_index2 = range(new_len - pad_l + pl2, new_len)
    pred_cols[pad_index1] += pred_padded[list(reversed(pad_index2))]
    count_cols[pad_index1] += count_padded[list(reversed(pad_index2))]

    pred = pred_cols[:, pw2:new_wid - pad_w + pw2]
    count = count_cols[:, pw2:new_wid - pad_w + pw2]
    pad_index = range(pw2)
    pred[:, list(reversed(pad_index))] += pred_cols[:, pad_index]
    count[:, list(reversed(pad_index))] += count_cols[:, pad_index]

    pad_index1 = range(width - pad_w + pw2, width)
    pad_index2 = range(new_wid - pad_w + pw2, new_wid)
    pred[:, pad_index1] += pred_cols[:, list(reversed(pad_index2))]
    count[:, pad_index1] += count_cols[:, list(reversed(pad_index2))]

    pred = pred / count

    return pred

def make_test_tiles(img_padded, x_tNum, y_tNum, tile_len, tile_wid, level, l_max,
                    x_stride, y_stride):
    dims = img_padded.ndim
    x_s = int(x_stride / (2 ** level))
    y_s = int(y_stride / (2 ** level))

    x_pad = tile_len * (2 ** (l_max - 1) - 1) / 2
    y_pad = tile_wid * (2 ** (l_max - 1) - 1) / 2

    x_start = int(x_pad - tile_len * (2 ** level - 1) / 2)
    y_start = int(y_pad - tile_wid * (2 ** level - 1) / 2)

    if dims == 3:
        channels = img_padded.shape[2]
        new_img = img_padded[x_start::2 ** level][:, y_start::2 ** level, :]
        tiles = np.zeros(((x_tNum + 1) * (y_tNum + 1), tile_len, tile_wid, channels))
        for i in range(x_tNum + 1):
            for j in range(y_tNum + 1):
                tiles[i * (y_tNum + 1) + j] = new_img[i * x_s:i * x_s + tile_len, j * y_s:j * y_s + tile_wid, :]
    else:
        new_img = img_padded[x_start::2 ** level][:, y_start::2 ** level]
        tiles = np.zeros(((x_tNum + 1) * (y_tNum + 1), tile_len, tile_wid))
        for i in range(x_tNum + 1):
            for j in range(y_tNum + 1):
                tiles[i * (y_tNum + 1) + j] = new_img[i * x_s:i * x_s + tile_len, j * y_s:j * y_s + tile_wid]

    return tiles

# initilize weight
def init_weight(net):
    if isinstance(net, nn.Conv2d):
        init.xavier_normal(net.weight)
        init.constant(net.bias, 0)
    elif isinstance(net, nn.ConvTranspose2d):
        init.xavier_normal(net.weight)
        init.constant(net.bias, 0)
    else:
        for m in net.modules():
            if isinstance(m, nn.Conv2d):
                init.xavier_normal(m.weight)
                init.constant(m.bias, 0)
            elif isinstance(m, nn.ConvTranspose2d):
                init.xavier_normal(m.weight)
                init.constant(m.bias, 0)

def adjust_learning_rate(optimizer, decay_rate=0.1):
    for param_group in optimizer.param_groups:
        param_group['lr'] = param_group['lr'] * decay_rate


from sqlalchemy import *
from migrate import *

from migrate.changeset import schema
pre_meta = MetaData()
post_meta = MetaData()
bets = Table('bets', post_meta,
    Column('id', Integer, primary_key=True, nullable=False),
    Column('match_id', Integer),
    Column('bat', String(length=1)),
)

fixtures = Table('fixtures', post_meta,
    Column('id', Integer, primary_key=True, nullable=False),
    Column('hometeam', String(length=64)),
    Column('awayteam', String(length=64)),
    Column('date', DateTime),
)

games = Table('games', post_meta,
    Column('id', Integer, primary_key=True, nullable=False),
    Column('startdate', DateTime),
    Column('enddate', DateTime),
)

def upgrade(migrate_engine):
    # Upgrade operations go here. Don't create your own engine; bind
    # migrate_engine to your metadata
    pre_meta.bind = migrate_engine
    post_meta.bind = migrate_engine
    post_meta.tables['bets'].create()
    post_meta.tables['fixtures'].create()
    post_meta.tables['games'].create()

def downgrade(migrate_engine):
    # Operations to reverse the above upgrade go here.
    pre_meta.bind = migrate_engine
    post_meta.bind = migrate_engine
    post_meta.tables['bets'].drop()
    post_meta.tables['fixtures'].drop()
    post_meta.tables['games'].drop()


from interference.clusters.processor import Processor
import numpy as np
from typing import Any, Dict, Sequence, Tuple
from scipy.spatial.distance import mahalanobis

class ClusterNode:

    def __init__(self, id, embedding: np.ndarray, initial_std: float, dimensions: int) -> None:

        self.id = id
        self.dimensions = dimensions

        self.cov_matrix = np.eye(dimensions)
        self.std = initial_std
        self.mean = embedding
        self.instances = [embedding]

        i_observation = embedding.reshape((dimensions, 1))

        self.observations = i_observation

    def add_embedding(self, embedding) -> None:

        self.instances.append(embedding)

        self.observations = np.hstack([self.observations, embedding.reshape((self.dimensions, 1))])

        self.cov_matrix = np.cov(self.observations)

        self.mean = np.mean(self.instances, axis=0)

        std_vector = np.std(self.instances, axis=0)

        self.std = np.linalg.norm(std_vector)

class CovarianceCluster(Processor):

    def __init__(self, dimensions: int, initial_std: float = 0.01) -> None:

        self.initial_std = initial_std
        self.tag_to_cluster: Dict[str, int] = {}
        self.id = 0
        self.clusters: Dict[int, ClusterNode] = {}
        self.dimensions = dimensions

    def add_to_cluster(self, tag: str, embedding: np.ndarray) -> None:

        id = -1

        if len(self.clusters) == 0:

            id = self._create_node(embedding)

        else:

            distance, node = self.brute_search(embedding)

            if distance < node.std:

                node.add_embedding(embedding)
                id = node.id

            else:

                id = self._create_node(embedding)

        self.tag_to_cluster[tag] = id

    def remove_from_cluster(self, tag: str) -> None:

        pass

    def stat_distance(self, embedding: np.ndarray, node: ClusterNode) -> float:

        return mahalanobis(embedding, node.mean, node.cov_matrix)

    def brute_search(self, embedding: np.ndarray) -> Tuple[float, ClusterNode]:

        nodes = list(self.clusters.values())

        curr_node = nodes[0]
        distance = self.stat_distance(embedding, nodes[0])

        for node in nodes[1:]:

            c_distance = self.stat_distance(embedding, node)

            if c_distance < distance:

                distance = c_distance
                curr_node = node

        return (distance, curr_node)

    def _create_node(self, embedding: np.ndarray) -> int:

        id = self.id
        self.id += 1

        new_node = ClusterNode(id, embedding, self.initial_std, self.dimensions)

        self.clusters[id] = new_node

        return id

    def process(self, tag: str, embedding: np.ndarray) -> None:

        self.add_to_cluster(tag, embedding)

    def update(self, tag: str, embedding: np.ndarray) -> None:

        self.remove(tag)

        self.process(tag, embedding)

    def remove(self, tag: str) -> None:

        self.remove_from_cluster(tag)

    def get_cluster_by_tag(self, tag: str) -> int:

        return self.tag_to_cluster[tag]

    def get_tags_in_cluster(self, cluster_id: int) -> Sequence[str]:

        return [tag for tag, id in self.tag_to_cluster.items() if id ==
                cluster_id]

    def get_cluster_ids(self) -> Sequence[int]:

        return [
            id for id
            in self.clusters.keys()
        ]

    def predict(self, embedding: np.ndarray) -> int:

        return self.brute_search(embedding)[1].id

    def describe(self) -> Dict[str, Any]:

        return {
            "name": "Covariance Cluster",
            "parameters": {
                "initial_std": self.initial_std
            }
        }

    def safe_file_name(self) -> str:

        return f"CovCluster = initial_std={self.initial_std}"


import os
import tempfile
import boto3
from .localfile import deletefile
import controllers.configs as cfg

class S3EventsImages:
    def __init__(self):
        self.bucket = cfg.BUCKET
        self.client = boto3.client('s3')

    def download(self, event_id, image_id):
        try:
            fileobj = '%s/%s/%s.jpg' % (cfg.AWS_IMAGE_FOLDER_PREFIX, event_id, image_id)
            _, tmpfolder = os.path.split(tempfile.mkdtemp())
            tmpfolder = cfg.IMAGE_FILE_MOUNTPOINT + tmpfolder
            os.mkdir(tmpfolder)
            tmpfile = os.path.join(tmpfolder, event_id+"."+image_id)
            with open(tmpfile, 'wb') as f:
                self.client.download_fileobj(self.bucket, fileobj, f)
        except Exception as ex:
            deletefile(tmpfile)
            raise
        return tmpfile

    def delete(self, event_id, image_id):
        try:
            fileobj = '%s/%s/%s.jpg' % (cfg.AWS_IMAGE_FOLDER_PREFIX, event_id, image_id)
            if not self.__find(event_id, image_id):
                raise
            self.client.delete_object(Bucket=self.bucket, Key=fileobj)
        except Exception as ex:
            raise

    def upload(self, imagefile, event_id, image_id):
        try:
            fileobj = '%s/%s/%s.jpg' % (cfg.AWS_IMAGE_FOLDER_PREFIX, event_id, image_id)
            with open(imagefile, 'rb') as f:
                self.client.upload_fileobj(f, self.bucket, fileobj)
        except Exception as ex:
            raise

    def __find(self, event_id, image_id):
        try:
            fileobj = '%s/%s/%s.jpg' % (cfg.AWS_IMAGE_FOLDER_PREFIX, event_id, image_id)
            get_folder_objects = self.client.list_objects_v2(
                Bucket=self.bucket,
                Delimiter='',
                EncodingType='url',
                MaxKeys=1000,
                Prefix=fileobj,
                FetchOwner=False,
                StartAfter=''
            )
            if not get_folder_objects.get('Contents'):
                return False
        except Exception as ex:
            raise
        return True


import glob
import time
import sys
from dataclasses import dataclass

import torch
import torchvision
import numpy as np
import torch.utils.data
from torch import optim
from torch.nn import functional as F
from torch.utils.tensorboard import SummaryWriter

from src.preprocess.image_loader import ImgDataset, ImageTransform
from src.vae.vae_linear import VAE

"""
To do : データのpathがおかしい -> configの作業dirが間違っていた
"""

@dataclass
class Config:
    lr: float = 1e-4
    beta1: float = 0.9
    beta2: float = 0.9
    input_dim: int = 16384
    num_epoch: int = 1000
    num_stopping: int = 50
    batch_size: int = 64
    z_dim: int = 32
    save_path: str = '../../model/vae.pt'
    log_path: str = '../../log/vae_linear/vae_lr_8e3_nonorma'

def loss_function(recon_x, x, mu, logvar, config):
    bce = F.binary_cross_entropy_with_logits(recon_x, x.view(-1, config.input_dim), reduction='sum')
    kld = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
    return bce + kld

def train(train_dataloader, eval_dataloader, model, config):

    # check GPU
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print("Use device：", device)

    model = model.to(device)
    optimizer = optim.Adam(model.parameters(), lr=config.lr, betas=[config.beta1, config.beta2])

    writer = SummaryWriter(log_dir=config.log_path)

    models = []
    eval_loss = []
    generated_images = []
    for epoch in range(config.num_epoch):
        t_epoch_start = time.time()

        print('-------------')
        print('Epoch {}/{}'.format(epoch, config.num_epoch))
        print('-------------')

        # ---------------------------------------------------------------------------------

        model.train()
        train_epoch_loss = 0
        n_t = 0
        for images in train_dataloader:

            images = images.to(device)

            # For liner vae
            images = images.view(-1, config.input_dim)

            pred, mu, logvar = model(images)

            loss = loss_function(pred, images, mu, logvar, config)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            train_epoch_loss += loss.item()
            n_t += 1

        print('Train epoch loss:{:.4f}'.format(train_epoch_loss / train_dataloader.batch_size))

        # -------------------------------------------------------------------------------

        model.eval()
        eval_epoch_loss = 0
        n_e = 0
        for images in eval_dataloader:

            images = images.to(device)

            # For liner vae
            images = images.view(-1, config.input_dim)

            pred, mu, logvar = model(images)

            loss = loss_function(pred, images, mu, logvar, config)

            eval_epoch_loss += loss.item()
            n_e += 1

        models.append(model)

        # Early stopping -------------------------------------------------------

        eval_loss.append(eval_epoch_loss/n_e)

        # To tensor board
        writer.add_scalar('Train/loss', train_epoch_loss / n_t, epoch)
        writer.add_scalar('Eval/loss', eval_epoch_loss / n_e, epoch)

        if epoch >= config.num_stopping:
            if epoch == config.num_stopping:
                low_loss = np.min(eval_loss)
                low_index = np.argmin(eval_loss)
                if low_index == 0:
                    print('-------------------------------------------------------------------------------------------')
                    print("Early stopping")
                    print('Best Iteration:{}'.format(low_index+1))
                    print('Best evaluation loss:{}'.format(low_loss))
                    break

            elif epoch == low_index + config.num_stopping:
                low_loss_new = np.min(eval_loss[low_index:])
                low_index_new = np.argmin(eval_loss[low_index:])+low_index

                if low_loss <= low_loss_new:
                    print('-------------------------------------------------------------------------------------------')
                    print("Early stopping")
                    print('Best Iteration:{}'.format(low_index + 1))
                    print('Best evaluation loss:{}'.format(low_loss))
                    break
                else:
                    low_loss = low_loss_new
                    low_index = low_index_new
        else:
            pass

        t_epoch_finish = time.time()
        print('Eval_Epoch_Loss:{:.4f}'.format(eval_epoch_loss / n_e))
        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))

        # check generated image ---------------------------------------------------
        if epoch % 20 == 0:
            pred = pred.reshape(images.shape[0], 1, 128, 128)
            images = images.reshape(images.shape[0], 1, 128, 128)

            _generated = torchvision.utils.make_grid(pred[:10], nrow=5)
            _true = torchvision.utils.make_grid(images[:10], nrow=5)
            writer.add_image('Eval/generated', _generated, epoch)
            writer.add_image('Eval/true', _true, epoch)

    return models[low_index + 1], generated_images

def process(train_dir_path, eval_dir_path, config):

    # params of normalization
    _mean = 0
    _std = 1

    # read file path
    train_path_list = glob.glob(train_dir_path)
    eval_path_list = glob.glob(eval_dir_path)

    if train_path_list == [] or eval_path_list == []:
        print('FileNotFoundError: No such file or directory: ', file=sys.stderr)
        sys.exit(1)

    # mk dataloader
    train_dataset = ImgDataset(file_list=train_path_list, transform=ImageTransform(_mean, _std))
    train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True)

    eval_dataset = ImgDataset(file_list=eval_path_list, transform=ImageTransform(_mean, _std))
    eval_dataloader = torch.utils.data.DataLoader(eval_dataset, batch_size=config.batch_size, shuffle=True)

    vae = VAE(config.input_dim, config.z_dim)

    # train model
    model, generated = train(train_dataloader, eval_dataloader, vae, config)

    # save model
    torch.save(model.state_dict(), config.save_path)

    return generated

if __name__ == '__main__':
    t_dir_path = rf'..\..\figure\spectrogram_png\train\*.png'
    e_dir_path = rf'..\..\figure\spectrogram_png\test\*.png'
    process(t_dir_path, e_dir_path, Config())


import numpy as np
import torch
from tqdm import tqdm
from cheml.datasets import load_qm7
from sklearn import linear_model, model_selection
from scipy.spatial.distance import pdist
from scatwave.scattering import SolidHarmonicScattering
from scatwave.utils import compute_integrals

def evaluate_bilinear_regression(scat_0, scat_1, scat_2, target):
    x = np.concatenate([scat_0, scat_1, scat_2], axis=1)
    x = (x - x.mean()) / x.std()
    y_factor = (target.max() - target.min())
    y = (target - target.min()) / y_factor
    x_train, x_test, y_train, y_test = model_selection.train_test_split(x, y)

    x_train = torch.autograd.Variable(torch.from_numpy(x_train), requires_grad=False)
    x_test = torch.autograd.Variable(torch.from_numpy(x_test), requires_grad=False)
    y_train = torch.autograd.Variable(torch.from_numpy(y_train), requires_grad=False)
    y_test = torch.autograd.Variable(torch.from_numpy(y_test), requires_grad=False)

    n_x = x_train.size(1)
    bilinear = torch.nn.Bilinear(n_x, n_x, 1, bias=False)
    linear = torch.nn.Linear(n_x, 1)
    loss_fn = torch.nn.MSELoss(size_average=True)
    params = list(bilinear.parameters()) + list(linear.parameters())

    learning_rate = 1e-3
    optimizer = torch.optim.Adam(params, lr=learning_rate)

    for t in range(20000):
        y_train_pred = bilinear(x_train, x_train) + linear(x_train)
        loss = loss_fn(y_train_pred, y_train)
        if (t % 500) == 0:
            print(t, np.sqrt(loss.data[0])*y_factor)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    print('train RMSE : ', np.sqrt(loss.data[0])*y_factor)

    y_test_pred = bilinear(x_test, x_test) + linear(x_test)
    loss = loss_fn(y_test_pred, y_test)
    print('test RMSE: ', np.sqrt(loss.data[0])*y_factor)

def evaluate_linear_regression(scat_0, scat_1, scat_2, target):
    x_1 = np.concatenate([scat_0, scat_1], axis=1)
    x_1_2 = np.concatenate([x_1, scat_2], axis=1)
    lin_regressor = linear_model.LinearRegression()
    scat_1_prediction = model_selection.cross_val_predict(lin_regressor, x_1, target)
    scat_1_MAE = np.mean(np.abs(scat_1_prediction - target))
    scat_1_RMSE = np.sqrt(np.mean((scat_1_prediction - target)**2))
    print('''scattering order 1, linear regression,
          MAE: {}, RMSE: {} (kcal.mol-1)'''.format(scat_1_MAE, scat_1_RMSE))

    scat_1_2_prediction = model_selection.cross_val_predict(lin_regressor, x_1_2, target)
    scat_1_2_MAE = np.mean(np.abs(scat_1_2_prediction - target))
    scat_1_2_RMSE = np.sqrt(np.mean((scat_1_2_prediction - target)**2))
    print('''scattering order 1 and 2, linear regression,
          MAE: {}, RMSE: {} (kcal.mol-1)'''.format(scat_1_2_MAE, scat_1_2_RMSE))

def get_valence(charges):
    return (
        charges * (charges <= 2) +
        (charges - 2) * np.logical_and(charges > 2, charges <= 10) +
        (charges - 10) * np.logical_and(charges > 8, charges <= 18))

def renormalize(positions, charges, sigma, overlapping_precision=1e-1):
    # x_min, x_max = positions[:,:,0].min(), positions[:,:,0].max()
    # y_min, y_max = positions[:,:,1].min(), positions[:,:,1].max()
    # z_min, z_max = positions[:,:,2].min(), positions[:,:,2].max()

    # positions[:,:,0] -= 0.5*(x_max + x_min)
    # positions[:,:,1] -= 0.5*(y_max + y_min)
    # positions[:,:,2] -= 0.5*(z_max + z_min)

    min_dist = np.inf
    for i in range(positions.shape[0]):
        n_atoms = np.sum(charges[i] != 0)
        pos = positions[i, :n_atoms, :]
        min_dist = min(min_dist, pdist(pos).min())

    delta = sigma * np.sqrt(-8 * np.log(overlapping_precision))

    return positions * delta / min_dist

def get_qm7_positions_energies_and_charges(M, N, O, J, L, sigma):
    qm7 = load_qm7(align=True)
    positions = qm7.R
    charges = qm7.Z
    energies = qm7.T.transpose()
    valence_charges = get_valence(charges)

    positions = renormalize(positions, charges, sigma)

    return torch.from_numpy(positions), torch.from_numpy(energies), torch.from_numpy(charges), torch.from_numpy(valence_charges)

def generate_weighted_sum_of_gaussians(grid, positions, weights, sigma, cuda=False):
    _, M, N, O = grid.size()
    if cuda:
        signals = torch.cuda.FloatTensor(positions.size(0), M, N, O).fill_(0)
    else:
        signals = torch.FloatTensor(positions.size(0), M, N, O).fill_(0)

    for i_signal in range(positions.size(0)):
        n_points = positions[i_signal].size(0)
        for i_point in range(n_points):
            if weights[i_signal, i_point] == 0:
                break
            weight = weights[i_signal, i_point]
            center = positions[i_signal, i_point]
            signals[i_signal] += weight * torch.exp(
                -0.5 * ((grid[0]-center[0])**2 + (grid[1]-center[1])**2 + (grid[2]-center[2])**2) / sigma**2)
    return signals / ((2 * np.pi)**1.5 * sigma**3)

def main():
    """Trains a simple linear regression model with solid harmonic
    scattering coefficients on the atomisation energies of the QM7
    database.

    Achieves a MAE of ... kcal.mol-1
    """
    cuda = torch.cuda.is_available()
    batch_size = 32
    M, N, O = 192, 128, 96
    grid = torch.from_numpy(
            np.fft.ifftshift(np.mgrid[-M//2:-M//2+M, -N//2:-N//2+N, -O//2:-O//2+O].astype('float32'), axes=(1,2,3)))
    if cuda:
        grid = grid.cuda()
    sigma = 2.
    J, L = 2, 3
    integral_powers = [0.5, 1., 2., 3.]
    args = {'integral_powers': integral_powers}
    pos, energies, full_charges, valence_charges = get_qm7_positions_energies_and_charges(
            M, N, O, J, L, sigma)
    n_molecules = pos.size(0)
    n_batches = np.ceil(n_molecules / batch_size).astype(int)
    n_batches = 2

    scat = SolidHarmonicScattering(M=M, N=N, O=O, J=J, L=L, sigma_0=sigma)

    scat_0, scat_1, scat_2 = [], [], []
    print('Computing solid harmonic scattering coefficients of molecules \
            of QM7 database on {}'.format('GPU' if cuda else 'CPU'))
    print('L: {}, J: {}, integral powers: {}'.format(L, J, integral_powers))
    for i in tqdm(range(n_batches)):
        start, end = i*batch_size, min((i+1)*batch_size, n_molecules)
        pos_batch = pos[start:end]

        full_batch = full_charges[start:end]
        full_density_batch = generate_weighted_sum_of_gaussians(
                grid, pos_batch, full_batch, sigma, cuda=cuda)

        full_scat_0 = compute_integrals(full_density_batch, integral_powers)
        full_scat_1, full_scat_2 = scat(full_density_batch, True, 'integral', args)

        val_batch = valence_charges[start:end]
        val_density_batch = generate_weighted_sum_of_gaussians(
                grid, pos_batch, val_batch, sigma, cuda=cuda)
        val_scat_0 = compute_integrals(val_density_batch, integral_powers)
        val_scat_1, val_scat_2 = scat(val_density_batch, True, 'integral', args)

        core_density_batch = full_density_batch - val_density_batch
        core_scat_0 = compute_integrals(core_density_batch, integral_powers)
        core_scat_1, core_scat_2 = scat(core_density_batch, True, 'integral', args)

        scat_0.append(torch.stack([full_scat_0, val_scat_0, core_scat_0], dim=-1))
        scat_1.append(torch.stack([full_scat_1, val_scat_1, core_scat_1], dim=-1))
        scat_2.append(torch.stack([full_scat_2, val_scat_2, core_scat_2], dim=-1))

    n = batch_size * n_batches

    scat_0 = torch.cat(scat_0, dim=0)
    scat_1 = torch.cat(scat_1, dim=0)
    scat_2 = torch.cat(scat_2, dim=0)

    np_scat_0 = scat_0.numpy().reshape((n, -1))
    np_scat_1 = scat_1.numpy().reshape((n, -1))
    np_scat_2 = scat_2.numpy().reshape((n, -1))

    print('order 1 : {} coef, order 2 : {} coefs'.format(np_scat_1.shape[1], np_scat_2.shape[1]))
    target = energies.numpy()[:n]
    evaluate_linear_regression(np_scat_0, np_scat_1, np_scat_2, target)
    evaluate_bilinear_regression(np_scat_0, np_scat_1, np_scat_2, target)

    basename = 'qm7_L_{}_J_{}_sigma_{}_MNO_{}_powers_{}.npy'.format(
            L, J, sigma, (M, N, O), integral_powers)
    np.save('scat_0_'+basename, np_scat_0)
    np.save('scat_1_'+basename, np_scat_1)
    np.save('scat_2_'+basename, np_scat_2)

if __name__ == '__main__':
    main()


import json
import codecs
import sqlite3
from os.path import isfile
from os import remove
from address import Address

def main():
    save_json()
    save_sqlite()

def save_json():
    a = Address()
    with codecs.open('data/address.json', 'w', 'utf-8') as f:
        # save address datas in json format,
        # set ensure_ascii to False to write utf-8 character correctly
        f.write(json.dumps(a.datas, ensure_ascii=False, indent=4))

def save_sqlite():
    a = Address()

    filename = 'data/address.db'
    # remove exist database file
    if isfile(filename):
        remove(filename)

    conn = sqlite3.connect(filename)
    cursor = conn.cursor()
    cursor.execute('CREATE TABLE address (_id INTEGER PRIMARY KEY AUTOINCREMENT, city TEXT, cityarea TEXT, address TEXT)')
    counter = 0
    for city in a.datas:
        for cityarea in a.datas[city]:
            for address in a.datas[city][cityarea]:
                row = (counter, city, cityarea, address)
                cursor.execute('INSERT INTO address VALUES (?,?,?,?)', row)
                counter += 1
    conn.commit()
    conn.close()

if __name__ == '__main__':
    main()


from allennlp.predictors.predictor import Predictor
from allennlp.common.util import JsonDict, sanitize
from allennlp.data import Instance

@Predictor.register('conll_03_predictor')
class CoNLL03Predictor(Predictor):
    def predict_instance(self, instance: Instance) -> JsonDict:
        outputs = self._model.forward_on_instance(instance)
        label_vocab = self._model.vocab.get_index_to_token_vocabulary('labels')

        outputs['tokens'] = [str(token) for token in instance.fields['tokens'].tokens]
        outputs['predicted'] = [label_vocab[l] for l in outputs['logits'].argmax(1)]
        outputs['labels'] = instance.fields['label'].labels

        return sanitize(outputs)


#!/usr/bin/env python
# encoding: utf-8
"""
统计一个数字在排序数组中出现的次数。
"""

class Solution:
    def GetNumberOfK(self, data, k):
        """
        :param data: 有序的数组
        :param k: 数字k
        :return:
        """
        if not data:
            return 0
        index = self.binary_search(data, k)
        if index == -1:
            return 0
        # 向左搜寻
        left = index
        while left >= 0 and data[left] == k:
            left -= 1
        # 向右搜寻
        right = index
        while right < len(data) and data[right] == k:
            right += 1
        return (right - 1) - (left + 1) + 1

    @staticmethod
    def binary_search(data, k):
        start = 0
        end = len(data) - 1
        while start <= end:
            mid = (start + end) // 2
            if data[mid] == k:
                return mid
            elif data[mid] < k:
                start = mid + 1
            else:
                end = mid - 1
        return -1

剑指offer/37数字在排序数组中出现的次数.py

37数字在排序数组中出现的次数.py


import requests
from bs4 import BeautifulSoup
from urllib.request import urlretrieve

def run():
    for i in range (1,6):
        response = requests.get('https://xkcd.com/{}/'.format(i), verify = True)
        soup = BeautifulSoup(response.content,'html.parser')
        image_container = soup.find(id='comic')

        image_url = image_container.find('img')['src']
        image_name = image_url.split('/')[-1]
        print('Descargando la imagen {}'.format(image_name))
        urlretrieve('https:{}'.format(image_url),image_name)

if __name__ == '__main__':
    run()


# -*- coding: utf-8 -*-

import codecs
import json
import os
import sys
import time
import urllib2
from urllib import urlencode

from config import read_config
from downloader import Downloader
from memoize import Memoize
from messages import MessageWriter
from reporter import Reporter
import vk_auth

def _api(method, params, token):
    params.append(("access_token", token))
    url = "https://api.vk.com/method/%s?%s" % (method, urlencode(params))
    payload = urllib2.urlopen(url).read()
    payload_json = json.loads(payload)
    if not "response" in payload_json:
        sys.exit("Request failed:\nURL     = %s\nPAYLOAD = %s" % (url, payload))
    return payload_json["response"]

config = read_config()
reporter = Reporter.std_reporter()
downloader = Downloader(reporter, directory=config["export"]["output_directory"])

# auth to get token

try:
    reporter.progress("Authenticating as %s" % config["auth"]["username"], pad=True)
    token, user_id = vk_auth.auth(config["auth"]["username"], config["auth"]["password"], config["app"]["id"], 'messages')
    reporter.line("OK")
except RuntimeError:
    reporter.line("FAILED")
    sys.exit("Cannot authenticate, please check your credentials in .auth.ini")

# get some information about chat

selector = "chat_id" if config["export"]["is_group_chat"] else "uid"
messages = _api("messages.getHistory", [(selector, config["export"]["chat_id"])], token)

# prepare output

if config["export"]["output_directory"] is not None:
    if not os.path.exists(config["export"]["output_directory"]):
        os.makedirs(config["export"]["output_directory"])
output_filename = 'vk_exported_dialogue_%s%s.txt' % ('ui' if not config["export"]["is_group_chat"] else 'c', config["export"]["chat_id"])
output_path = Downloader.resolve_path(config["export"]["output_directory"], output_filename)
out = codecs.open(output_path, "w+", "utf-8")

def resolve_uid_details(uid):
    return _api("users.get", [("user_ids", uid)], token)[0]

resolve_uid_details = Memoize(resolve_uid_details)

message_writer = MessageWriter(out, downloader, lambda uid: resolve_uid_details(uid), save_photos=config["export"]["save_photos"])

mess = 0
max_part = 200  # Due to vk.api

cnt = messages[0]
reporter.line("Message count: %s" % cnt)

request_num = 0

while mess != cnt:
    # Try to retrieve info anyway

    if request_num % 5 == 0:
        time.sleep(1)
    request_num += 1

    while True:
        try:
            message_part = _api(
                "messages.getHistory",
                [(selector, config["export"]["chat_id"]), ("offset", mess), ("count", max_part), ("rev", 1)],
                token
            )
        except Exception as e:
            reporter.error_line('Got error %s, continue...' % e)
            continue
        break

    try:
        for i in range(1, 201):
            message_writer.write(message_part[i]["uid"], message_part[i])
    except IndexError:
        break

    result = mess + max_part
    if result > cnt:
        result = (mess - cnt) + mess
    mess = result
    reporter.line("Exported %s messages of %s" % (mess, cnt))

out.close()
reporter.line('Export done!')


#!/usr/bin/env  python3
# -*- coding: UTF-8 -*-
'''

插入排序
初始时⼿手⾥里里(有序区)只有⼀一张牌
每次(从⽆无序区)摸⼀一张牌，插⼊入到⼿手⾥里里已 有牌的正确位置
'''

def insert_sort(li):
    for i in range(1, len(li)): # 表示摸到的牌的下标
        tmp = li[i]
        j = i - 1 # j指的是手里的牌的下表
        while j >= 0 and tmp < li[j]:
            li[j + 1] = li[j]
            j = j - 1
        li[j + 1] = tmp
        print(li)

li = [3,2,4,6,8,1,5,9,7]

print(li)
insert_sort(li)


import os
import unittest

from yodeploy.test_integration.helpers import (
    build_sample, clear, deploy_sample)

class TestBasicAppDeploy(unittest.TestCase):

    def setUp(self):
        clear('basic-app')
        build_sample('basic-app')
        deploy_sample('basic-app')
        self.deployed_app = os.path.join(
            os.path.dirname(__file__),
            'filesys', 'deployed', 'basic-app', 'live')

    def test_is_deployable(self):
        deployed_file = os.path.join(self.deployed_app, 'hello-world.txt')
        self.assertTrue(os.path.exists(deployed_file))

class TestBasicConfiguredApp(unittest.TestCase):

    def setUp(self):
        build_sample('configs')
        build_sample('basic-configed')
        deploy_sample('basic-configed')
        self.deployed_app = os.path.join(
            os.path.dirname(__file__),
            'filesys', 'deployed', 'basic-configed', 'live')

    def test_is_deployable(self):
        deployed_config = os.path.join(self.deployed_app, 'configuration.json')
        self.assertTrue(os.path.exists(deployed_config))


from SearchEngine.spiders import models
from nltk.corpus import wordnet
import csv
import pandas as pd
import search_pb2

def clearCsvFile():
    f = open('/Users/sajjadaboutalebi/Documents/SearchEngine/Server/result.csv', "w+")
    f.close()
    with open('/Users/sajjadaboutalebi/Documents/SearchEngine/Server/result.csv', 'a') as result:
        fieldnames = ['link', 'score']
        writer = csv.DictWriter(result, fieldnames=fieldnames)
        writer.writeheader()

def sortCSVFile():
    df = pd.read_csv('result.csv')
    df = df.sort_values(by=('score'), ascending=False)
    df.to_csv('result.csv', index=False)

def writeResult(link, score):
    with open('/Users/sajjadaboutalebi/Documents/SearchEngine/Server/result.csv', 'a') as result:
        fieldnames = ['link', 'score']
        writer = csv.DictWriter(result, fieldnames=fieldnames)
        #writer.writeheader()
        writer.writerow({'link': link, 'score': score})
        sortCSVFile()

def doSearch(query):
    clearCsvFile()
    splitQ = query.split(':')
    shouldBe = splitQ[0].split(',')
    notBe = splitQ[1].split(',')
    mayBe = splitQ[2].split(',')
    for i in models.TopWords.select():
        l1 = []
        for j in eval(i.topWords):
            for k in shouldBe:
                wordFromList1 = wordnet.synsets(j[0])
                wordFromList2 = wordnet.synsets(k)
                if wordFromList1 and wordFromList2:
                    s = wordFromList1[0].wup_similarity(wordFromList2[0])
                    if s != None and s > 0.1:
                        l1.append(s)
        print(query)
        try:
            writeResult(i.link, str(max(l1)))
        except:
            pass

def RetrieveMessagesValuesFromFile():
    searchResponse = search_pb2.SearchResponse()

    with open('/Users/sajjadaboutalebi/Documents/SearchEngine/Server/result.csv', 'r') as result:
        reader = csv.DictReader(result)
        for row in reader:
            item = searchResponse.bars.add()
            item.link = row['link']
            item.score = row['score']
            #print(item.link)
    return searchResponse.SerializeToString()

#doSearch('love')
#a = RetrieveMessagesValuesFromFile()
#print(a)
#s = search_pb2.SearchResponse()
#s.ParseFromString(a)
#print(len(s.bars))


from selenium import webdriver
import os
import time
'''
используем   - это headless webkit браузер для селениум
update - отложено. надо разобраться как заходить на мобильную версию
'''

class Driver():
	def __init__(self):
		options = webdriver.ChromeOptions()
		options.add_argument('--headless')
		self.browser = webdriver.Chrome()#chrome_options=options)

		self.number = None

	def navigate(self, url):
		self.browser.set_window_size(450, 300)
		self.browser.get(url)
		ActionChains(browser).key_down(Keys.LEFT_CONTROL).key_down(Keys.LEFT_SHIFT).send_keys('i').perform()
		self.browser.get(url)
		self.browser.implicitly_wait(5)
		self.browser.find_element_by_class_name('BPWk2').click()
		self.number = self.browser.find_element_by_class_name('_3Ryy-').text
		return self.number

def main():
	pass

if __name__ == '__main__':
	main()


import os
import cv2
import numpy as np

from PIL import Image, ImageEnhance

def read_img(img_path):
    """
    读取图像,将RGB转为3通道的灰度图
    :param img_path:
    :return:
    """
    img = cv2.imread(img_path)
    gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    gray_3channel_img = cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB)
    return gray_3channel_img

def enhance_brightness(img, brightness):
    """
    亮度增强
    :param img:
    :param brightness:
    :return:
    """
    img = Image.fromarray(img)
    enh_bri = ImageEnhance.Brightness(img)
    image_brightened = enh_bri.enhance(brightness)
    return np.array(image_brightened)

def enhance_color(img, color):
    """
    色度增强
    :param img:
    :param color:
    :return:
    """
    img = Image.fromarray(img)
    enh_col = ImageEnhance.Color(img)
    image_colored = enh_col.enhance(color)
    return np.array(image_colored)

def enhance_contrast(img, contrast):
    """
    对比度增强
    :param img:
    :param contrast:
    :return:
    """
    img = Image.fromarray(img)
    enh_con = ImageEnhance.Contrast(img)
    image_contrasted = enh_con.enhance(contrast)
    return np.array(image_contrasted)

def enhance_sharpness(img, sharpness):
    """
    锐度增强
    :param img:
    :param sharpness:
    :return:
    """
    img = Image.fromarray(img)
    enh_sha = ImageEnhance.Sharpness(img)
    image_sharped = enh_sha.enhance(sharpness)
    return np.array(image_sharped)

if __name__ == "__main__":

    #img = Image.open('0.jpg')
    img = read_img('/home/tony/Pictures/ocr/rgb_original/test2.jpg')

    img_enhanced = enhance_contrast(img, 5)
    print(img_enhanced)
    num = np.array(img_enhanced)
    print(num)
    cv2.imshow('dwad',num)
    cv2.waitKey()
    # for i in range(8):
    #     img = Image.open(str(i)+'.jpg')
    #     img_enhanced = enhance_contrast(img, 5)
    #
    #     img_enhanced.save('/home/tony/Pictures/ocr/gray_contrast_original/'+str(i)+'_contrast.jpg')


# -*- coding: utf8 -*-
import json

from flask import send_from_directory, render_template, request, \
    make_response

from .helpers import add_datasets_to_model, \
    get_all_dataset_for_model, get_dataset, get_model_info, \
    get_models_pretty_list, get_rates, update_model

from .env import app, base, config

from .authorization import check_auth, authenticate, requires_auth, \
    hash_auth, is_admin

import pandas as pd
import numpy as np

from pandas import get_dummies

from wrapper.kerasclassifier import KerasClassifier
from etl.etl import ETL
from sklearn.model_selection import train_test_split

def add_default_values(elements, req):
    for element in elements:
        value = req.args.get(element['id'])
        if value:
            element['default'] = value

    return elements

@app.route('/files/<path:path>')
def send_file(path):
    return send_from_directory('files', path)

@app.route('/')
def main():
    return render_template(
        'menu.html',
        elements=[
            {
                'title': 'Просмотр моделей',
                'href': '/models',
            },
            {
                'title': 'Модель -> выборка',
                'href': '/chose_dataset'
            },
            {
                'title': 'Rates',
                'href': '/rates',
            },
            {
                'title': 'Моделирование',
                'href': '/modeling',
            },
            {
                'title': 'Авторизироваться как администратор',
                'href': '/authorize',
            },
        ],
    )

@app.route('/models')
def show_models():
    admin = is_admin(request)
    if request.args.get("update"):
        if admin:
            update_model(base, request.args.get("model_id"), request.args.get("model_name"), request.args.get("description"), request.args.get("model_type"));
        else:
            return authorize_as_admin()

    options = get_models_pretty_list(base)
    print(options)

    elements = [
        {
            'title': 'Модели',
            'id': 'models',
            'type': 'choice',
            'options': options,
            'default': options[0]['id'],
        },
    ]
    return_url = "/"

    if request.args.get('result'):
        model_info, datasets = get_model_info(base, request.args.get('models'))

        return render_template(
            "models.html",
            elements=add_default_values(elements, request),
            model_desc=model_info,
            datasets=datasets,
            return_url=return_url,
            is_admin=admin,
            model_id=request.args.get('models')
        )
    else:
        return render_template(
            "input.html",
            elements=elements,
            return_url=return_url,
        )

@app.route('/dataset')
def show_dataset():
    return_url = "/models?models=%s&result=1" % request.args.get('models')

    head, dataset = get_dataset(base, request.args.get('id'))

    return render_template(
        "dataset.html",
        table=dataset,
        name=request.args.get('name'),
        head=head,
        return_url=return_url,
    )

@app.route('/chose_dataset')
def show_chose_dataset():
    return_url = "/"

    options = get_models_pretty_list(base)

    elements = [
        {
            'title': 'Модели',
            'id': 'models',
            'type': 'choice',
            'options': options,
            'default': options[0]['id'],
        },
    ]

    admin = is_admin(request)

    if request.args.get('result') == '1':
        model_id = request.args.get('models')
        datasets, checked_datasets = get_all_dataset_for_model(base, model_id)

        return render_template(
            "checkbox_list.html",
            elements=add_default_values(elements, request),
            return_url=return_url,
            cb_list=datasets,
            datasets_ids=','.join(checked_datasets),
            result=2,
            is_admin=admin,
        )
    elif request.args.get('result') == '2':
        model_id = request.args.get('models')
        datasets_ids = request.args.get('datasets_ids').split(',')
        if is_admin == 1:
            add_datasets_to_model(base, model_id, datasets_ids)

        datasets, checked_datasets = get_all_dataset_for_model(
            base,
            request.args.get('models'),
        )

        return render_template(
            "checkbox_list.html",
            return_url=return_url,
            elements=add_default_values(elements, request),
            cb_list=datasets,
            datasets_ids=','.join(checked_datasets),
            result=2,
            is_admin=admin,
        )

    else:
        return render_template(
            "input.html",
            elements=elements,
            result=1,
            return_url=return_url,
        )

@app.route('/rates')
def rates():
    return_url = "/"
    category_id = request.args.get('node')
    rate_id = request.args.get('rate')
    cats, rate, tabs = get_rates(base, category_id, rate_id)

    if 'node' in request.args:
        return render_template(
            "tree.tmpl",
            tabs=tabs,
            cur_tab=rate,
            nodes=cats,
            category=category_id,
            cur_rate=rate_id,
            return_url=return_url,
        )
    else:
        return render_template(
            "tree.tmpl",
            tabs=tabs,
            cur_tab=rate,
            nodes=cats,
            return_url=return_url,
        )

@app.route('/authorize')
def authorize_as_admin():
    elements = [
        {
            'title': 'Логин',
            'id': 'login',
            'type': 'input',
        }, {
            'title': 'Пароль',
            'id': 'password',
            'type': 'input',
        }
    ]
    return_url = "/"

    if request.args.get('result'):
        auth = request.args.get("password") # не передавать прям вот в таком виде
        username = request.args.get("login")
        if check_auth(username, auth, True):
            resp = make_response(main())
            resp.set_cookie('auth', hash_auth(auth))
            resp.set_cookie('login', username)
            return resp
        else:
            return render_template(
                "login.html",
                elements=elements,
                return_url=return_url,
                error=1
            )
    else:
        return render_template(
            "login.html",
            elements=elements,
            return_url=return_url,
        )

def server_main():
    app.jinja_env.tests['equalto'] = lambda value, other: value == other
    app.run(
        host=config['server']['host'],
        port=int(config['server']['port']),
        use_reloader=False,
        debug=True,
        threaded=False,
    )

@app.route('/modeling')
def modeling():

    options = [
        {
            'title': 'iris',
            'id': 1,
        }
    ]
    print(options)

    elements = [
        {
            'title': 'Network layers number',
            'id': 'layers_n',
            'type': '',
            'default': 1,
        },
        {
            'title': 'Neuron number',
            'id': 'nn',
            'type': '',
            'default': 10,
        },

        {
            'title': 'Activation functions list',
            'id': 'func',
            'type': '',
            'default': 'sigmoid',
        },
        {
            'title': 'Metrics',
            'id': 'metrics',
            'type': '',
            'default': 'accuracy',
        },
        {
            'title': 'Loss',
            'id': 'loss',
            'type': '',
            'default': 'categorical_crossentropy',
        },
        {
            'title': 'Epoch number',
            'id': 'ep',
            'type': '',
            'default': '100',
        },
        {
            'title': 'Datasets',
            'id': 'dataset',
            'type': '',
            'options': options,
            'default': options[0]['id'],
        },
    ]
    return_url = "/"

    if request.args.get('result'):

        dataset_to_comps = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'] # more tables???
        model_info, datasets = get_model_info(base, request.args.get('models'))

        neurons = request.args.get('nn').split(',')
        input_dim = [len(dataset_to_comps)] +  [0] * (len(neurons) - 1)
        activation = request.args.get('func').split(',')

        etl = ETL(manager=base)
        load_data_instr = {"category_name": 'Iris Fisher'}
        path = 'local_files/iris.csv'
        etl.load_supervised_data(path=path, ctg_name=load_data_instr["category_name"])

#        x1 = base.get_raw_data(RateName=dataset_to_comps[0])
#        x1 = pd.DataFrame(x1[2].float_value)
#        x2 = base.get_raw_data(RateName=dataset_to_comps[1])
#        x2 = pd.DataFrame(x2[2].float_value)
#        x3 = base.get_raw_data(RateName=dataset_to_comps[2])
#        x3 = pd.DataFrame(x3[2].float_value)
#        x4 = base.get_raw_data(RateName=dataset_to_comps[3])
#        x4 = pd.DataFrame(x4[2].float_value)

        X = pd.read_csv(path)
        y = X['species']
        X = X.drop('species', axis=1)

        X = X.as_matrix()
        train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.7, random_state=42)
        train_y_ohe = np.array(get_dummies(train_y), dtype=np.float64)
        test_y_ohe = np.array(get_dummies(test_y), dtype=np.float64)

#        build_args = {
#            'build_args': [
#                {'neurons': neurons[i], 'input_dim': input_dim[i], 'activation': activation[i], 'init': 'normal'} for i in range(len(neurons))
##                {'neurons' : 16, 'input_dim' : 4, 'init' : 'normal', 'activation' : 'relu'},
##                {'neurons' : 3, 'input_dim' : 0, 'init' : 'normal', 'activation' : 'sigmoid'}
#                ],
#            'compile_args': {
#                    'loss': request.args.get('loss'),
#                    'optimizer': 'adam',
#                    'metrics': request.args.get('metrics')
#            }
#        }
#        compile_args = {
#                    'loss': request.args.get('loss'),
#                    'optimizer': 'adam',
#                    'metrics': request.args.get('metrics')
#            }
#        fit_args = {'nb_epoch': request.args.get('ep'), 'batch_size': 1, 'verbose': 0}
#        evaluate_args = {'verbose': 0}
#        predict_args = {}

        build_args = {
            'build_args': [
                {'neurons' : 16, 'input_dim' : 4, 'init' : 'normal', 'activation' : 'relu'},
                {'neurons' : 3, 'input_dim' : 0, 'init' : 'normal', 'activation' : 'sigmoid'}
                ],
            'compile_args': {'loss': 'categorical_crossentropy', 'optimizer': 'adam', 'metrics': 'accuracy'}
        }
        compile_args = {'loss': 'categorical_crossentropy', 'optimizer': 'adam', 'metrics': 'accuracy'}
        fit_args = {'epochs': 100, 'batch_size': 1, 'verbose': 1}
        evaluate_args = {'verbose': 0}
        predict_args = {}

        print(build_args)

        m = KerasClassifier(name='iris', args=build_args)
        history = m.fit(train_X, train_y_ohe, fit_args=fit_args)
        loss, accuracy = m.evaluate(test_X, test_y_ohe, evaluate_args)
        prediction = m.predict(train_X)

        loss_data = history.history['loss'][1:]

        return render_template(
            "modeling.html",
            elements=elements,
            return_url=return_url,
            loss=request.args.get('loss'),
            loss_data=list(zip(list(range(len(loss_data) - 1)), loss_data))
        )
    else:

        return render_template(
            "input.html",
            elements=elements,
            return_url=return_url,
        )

if __name__ == "__main__":
    server_main()


# collect-links-cursor.py
# MCW - 2/11/2021 / mperry - 3/10/2021
import sys
import json
import tweepy

# use coronavirus as default search term unless one provided
search_term = "coronavirus"
if len(sys.argv) > 1:
     search_term = str(sys.argv[1])

# number of links to collect
MAX_COUNT = 1200
count = 0

# load Twitter API keys from a file so they're not hard-coded
with open("secrets.json", "r") as secretsFile:
    secrets = json.load(secretsFile)
# print(secrets['consumer_key'])
# print(secrets['consumer_secret'])

# setup Twitter API with OAuth2 procedure
consumer_key = secrets['consumer_key']
consumer_secret = secrets['consumer_secret']
auth = tweepy.AppAuthHandler(consumer_key, consumer_secret)
api = tweepy.API(auth)

try:
  for page in tweepy.Cursor(api.search, q=search_term, tweet_mode='extended', lang='en').pages():
    for tweet in page:
         for link in tweet.entities["urls"]:
             print("%s" % link['expanded_url'])
             count = count + 1
    if count > MAX_COUNT:
         break
except tweepy.TweepError as e:
  print ("Tweepy Error: %s" % str(e))


#!/usr/bin/env python
import nltk
import re
import string
import json

class FileReader:
    @staticmethod
    def read_file(name):
        """ func to read from file """
        with open(name) as fl:
            return fl.readlines()

class JsonWriter:
    @staticmethod
    def write_json(name, jsn):
        with open(name, 'w') as file:                   # serialization
            json.dump(jsn, file)

class RemoverUnwantedWords:
    @staticmethod
    def remove_words(expr, line):
        """ func for cleaning by regex """
        return re.sub(expr, '', line)

class CheckerForMeaning:
    @staticmethod
    def find_trash(clean_line, inf_dict):
        """ find contextless word """
        tokens = clean_line.split(' ')

        clean_tokens = []
        for i in range(len(tokens)):                    # removing punctuation and " "
            if len(tokens[i]) < 2:
                continue
            while True:
                if tokens[i][-1] in string.punctuation:
                    tokens[i] = tokens[i][:-1]
                else:
                    clean_tokens.append(tokens[i])
                    break

        clean_tokens = set(clean_tokens)
        for i in clean_tokens:                          # check for meaning
            if nltk.wsd.lesk(tokens, i) is None:
                inf_dict['orphan_tokens'].append(i)
        return inf_dict

class UrlFinder:
    @staticmethod
    def find_urls(clean_line, raw_line, inf_dict):
        """ find URL """
        urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', raw_line)
        funk_for_clean = RemoverUnwantedWords.remove_words
        clean_line = funk_for_clean('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
                                    clean_line)

        if len(urls):
            inf_dict["metadata"].extend(urls)
        return clean_line, inf_dict

class TweetLineHandler:
    """ class for performance extraction useful inf from tweet"""

    def __init__(self, line):
        self.line = line

    def explore(self):
        inf_dict = {'body': '', "metadata": [], 'body_tags': [], 'orphan_tokens': []}

        dog_index = self.line.rfind('@') + 1
        inf_dict["metadata"].append(self.line[dog_index:-3])                        # find [@...]

        line = self.line[:dog_index - 4]
        inf_dict['body'] = line
        clean_line = RemoverUnwantedWords.remove_words(r'[\$]\w+', line)            # remove $-words

        inf_dict['body_tags'] = re.findall(r'[\#\@](\w+)', clean_line)
        clean_line = RemoverUnwantedWords.remove_words(r'[\#\@](\w+)', clean_line)  # clean line

        clean_line, inf_dict = UrlFinder.find_urls(clean_line, self.line, inf_dict)

        inf_dict = CheckerForMeaning.find_trash(clean_line, inf_dict)

        return inf_dict

if __name__ == "__main__":

    data = FileReader.read_file('input.txt')  # reading from file

    result = {'records': []}

    for i in data:                            # tweet handling
        hndlr = TweetLineHandler(i)
        result['records'].append(hndlr.explore)

    JsonWriter.write_json('output.json', result)


 # Number of Epochs for the experiment
epochs = 5

from model_32m import c3d_model
from gen_frame_list import gen_frame_list

from keras.backend.tensorflow_backend import set_session
from keras import backend as K
from keras.optimizers import SGD,Adam,Adagrad
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.utils import np_utils
from keras.layers import Dense,Flatten, Conv3D, MaxPool3D,Input,BatchNormalization , Dropout
from keras.optimizers import Adadelta
from keras.optimizers import SGD,adam, Adagrad
from keras.losses import categorical_crossentropy
from keras.models import Model
from keras.models import load_model
from keras.models import save_model
from keras.utils import multi_gpu_model
from keras.models import model_from_json

import tensorboard as tensorboard
import random
import matplotlib
from sklearn.model_selection import train_test_split
import numpy as np
import argparse
import cv2
import os
from math import floor
import matplotlib.pyplot as plt
import tensorflow as tf
#from tensorflow.keras.models import save_model

matplotlib.use('AGG')
#print("Entering with batch_size=32,img_size=150,120 having 4 biased classes+changed model")
#config = tf.ConfigProto(device_count={'GPU':0, 'CPU':4})

#----------------------- construct the argument parse and parse the arguments -----------------------
ap = argparse.ArgumentParser()
ap.add_argument("-c", "--checkpoints", required=False,
	help="path to output checkpoint directory")
ap.add_argument("-m", "--model", type=str,
	help="path to *specific* model checkpoint to load")
ap.add_argument("-s", "--start-epoch", type=int, default=0,
	help="epoch to restart training at")
args = vars(ap.parse_args())

#-------------------------------------- Set_clip_size and set_batch_size-------------------------------------
batch_size = 16
no_frame = 16

#------------------------------- Set_path_trainTest_direstory_Images -----------------------------

train_dir  = "/home/jupyter/Drive_2/Event_Detection/ucf_frames/train/"
test_dir=  "/home/jupyter/Drive_2/Event_Detection/ucf_frames/test/"

class_list_train,train_frame_list = gen_frame_list(train_dir,True)
print ('[INFO] >>> Training Class List: ', class_list_train)
print ('[INFO] >>> Training Frames List: ', class_list_train)
class_list_test,test_frame_list = gen_frame_list(test_dir,True)
print ('[INFO] >>> Testing Class List: ', class_list_test)
print ('[INFO] >>> Testing Frames List: ', class_list_test)

#-------------------------- Generate DATA --------------------------------------------------
def generate_data(files_list, categories, batch_size):
    print ("[INFO] >>> Generating Data:")
    """"Replaces Keras' native ImageDataGenerator."""
    if len(files_list) != 0:
        print("[INFO] >>> Total Files: ", len(files_list))
        cpe = 0
        while True:
            if cpe == floor(len(files_list)/ (batch_size * no_frame)):
                cpe = 0
#             for cpe in range(floor(len(files_list)/ (batch_size * no_frame))):
            x_train = []
            y_train = []
#             print('Cycle No: ', cpe)
            c_start  = batch_size * cpe
            c_end    = (c_start + batch_size)
#             print("C_Start:",c_start, " c_end: ", c_end)
            for b in range(c_start, c_end):
#                 print('  Frame Set: ',b)
                start = b *  no_frame
                end   = start + (no_frame)
                stack_of_16=[]
                for i in range(start,end):
#                     print('    Frame Index: ',i)

                    try:
                        image = cv2.imread(files_list[i])
                        image = cv2.resize(image,(100, 100))
                    except Exception as e:
                        print('[BROKEN IMAGE]: ',str(e))
                        print('Image File: ', files_list[i])
                        continue

                    image = image / 255.
                    stack_of_16.append(image)
#                   print("Path : ", files_list[i])
#                 print("Class: ", files_list[start].split("/")[4])
#                 print("Cat Index: ",categories.index(files_list[start].split("/")[4]))
                y_train.append(categories.index(files_list[start].split("/")[7]))
              #  print("y_train",y_train)
                x_train.append(np.array(stack_of_16))
            cpe += 1
#                 print("y_train",np_utils.to_categorical(y_train,2))

#            print("x_train",np.array(x_train).shape)
#            print("y_train",np.array(y_train).shape)
#            print("Total Frames:_x_train ", len(x_train))
            yield(np.array(x_train).transpose(0,1,2,3,4), np_utils.to_categorical(y_train, 2))

#----------------------------- Deploying Model Training on Multiple n-GPUs -----------------------------
print ('\n\n\n\n*******************************************************************************************')
print ('ALERT: Deploying Model Training on Multiple GPUs ...')
print("Train Frame List Shape: ", np.array(train_frame_list).shape)
print("Train Frame List Member Shape: ", np.array(train_frame_list[0]).shape)
print("Class List Train Shape: ", np.array(class_list_train).shape)
print("Test Frame List Shape: ", np.array(test_frame_list).shape)
print("Test Frame List Member Shape: ", np.array(test_frame_list[0]).shape)
print("Class List Test Shape: ", np.array(class_list_test).shape)
#print(len(test_frame_list)//(no_frame * batch_size))

# ------------- Implementing Checkpoint Restoration -------------
# if there is no specific model checkpoint supplied, then initialize the network and compile the model
if args["model"] is None:
    print("[INFO] compiling model...")
    model = c3d_model()

# otherwise, we're using a checkpoint model
else:
    # load the checkpoint from disk
    print("[INFO] loading {}...".format(args["model"]))
    model = load_model(args["model"])

#model = model_from_json(open('/home/student/usama_lahore/Ramna/Results/6-epochs-29%acc-overfit/ucf_crime.json', 'r').read())

#model=c3d_model()
#model.load_weights('/home/student/usama_lahore/Ramna/Results/6-epochs-29%acc-overfit/ucf_crime_weights_file.h5')

# Replicates `model` on n GPUs:
#model_multi = multi_gpu_model(model_1, gpus=2)

model.summary()

#original optimizer code
#le=0.001
#opt = SGD(lr=le, momentum=0.009, decay=le)
##model_1.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
#model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

#using the same optimizer as waqas sultani
#opt=Adagrad(lr=0.01, epsilon=1e-08)

#current optimizer
opt=Adam(learning_rate=0.00001, beta_1=0.9, beta_2=0.999, amsgrad=False)

model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
#train_frame_list1,class_list_train1,test_frame_list1,class_list_test1= train_test_split(generate_data(train_frame_list,class_list_train,batch_size),test_size=0.33, random_state=42)
print ('[INFO] >> Model Compiled Successfully !\n\n')

#print('********************** Model Weights (Format 1)**********************')
#model_weights = model.get_weights()
#print(model_weights)

print('********************** Model Configuration **********************')
total_layers = 0
for layer in model.layers:
    layer_weights = layer.get_weights()
    total_layers += 1
    print('Layer Name:',layer.name)
    print('Layer Configuration:',layer.get_config())
    print('Layer Input Shape:', layer.input_shape)
    print('Layer Output Shape:', layer.output_shape)
    #print('Layer Weights',layer_weights)
print('Total Layers in Network: ', total_layers)

#----------------------------- Training the Model -----------------------------

# construct the set of callbacks
#callbacks = [
#    EpochCheckpoint(args["checkpoints"], every=1,
#        startAt=args["start_epoch"])]

#filepath = '/home/student/usama_lahore/Ramna/trained_images/train/ucf_crime_weights_file.h5'
#checkpoint = ModelCheckpoint(filepath, monitor='val_accuraccy', verbose=1, save_best_only=True, mode = 'max')
#callbacks_list = [checkpoint]

# simple early stopping
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1)
#filepath = '/home/jupyter/Drive_2/Event_Detection/Saved_Model/weights_ucf_complete_16mp.h5'
#mc = ModelCheckpoint(filepath, monitor='val_accuracy', mode='max', verbose=1, save_best_only=True)
trainingResults = open('training_results.txt', 'w')
print ('\n\n\n\n*******************************************************************************************')
print ("ALERT >> Model Training Started ....")
trainingHistory = model.fit_generator(generate_data(train_frame_list,class_list_train,batch_size),
                    steps_per_epoch=floor(len(train_frame_list)/(no_frame * batch_size)),
                    epochs=epochs,
                    validation_data=generate_data(test_frame_list,class_list_test,batch_size),
                    validation_steps=floor(len(test_frame_list)/ (no_frame * batch_size )),
                    verbose=1,callbacks=[es])

iteration = 1
for epoch in trainingHistory.history['val_loss']:
    trainingResults.write("Validation Loss for Epoch #" + str(iteration) + " : " + str(epoch) + "\n")
    iteration += 1

iteration = 1
for epoch in trainingHistory.history['val_accuracy']:
    trainingResults.write("Validation Accuracy for Epoch #" + str(iteration) + " : " + str(epoch) + "\n")
    iteration += 1

iteration = 1
for epoch in trainingHistory.history['loss']:
    trainingResults.write("Training Loss for Epoch #" + str(iteration) + " : " + str(epoch) + "\n")
    iteration += 1

iteration = 1
for epoch in trainingHistory.history['accuracy']:
    trainingResults.write("Training Accuracy for Epoch #" + str(iteration) + " : " + str(epoch) + "\n")
    iteration += 1

trainingResults.close()

# plot training history
plt.plot(trainingHistory.history['loss'], label='train_loss')
plt.plot(trainingHistory.history['val_loss'], label='validation_loss')
plt.plot(trainingHistory.history['accuracy'], label='train_acc')
plt.plot(trainingHistory.history['val_accuracy'], label='validation_acc')
plt.legend()
plt.savefig('/home/jupyter/Drive_2/Event_Detection/plot.svg')
plt.show()

#----------------------------- Save Weights and Serialized Model to JSON -----------------------------

print ('\n\n\n\n*******************************************************************************************')
print ("ALERT >> Serializing Model to Disk ....")
model_json = model.to_json()
with open('/home/jupyter/Drive_2/Event_Detection/Saved_Model/arch_ucf_complete_16mp.json',"w") as json_file:
    json_file.write(model_json)
    json_file.close()
model.save_weights("/home/jupyter/Drive_2/Event_Detection/Saved_Model/weights_ucf_complete_16mp.h5") #Onlly Weights

# Saving/loading whole models (architecture + weights + optimizer state):
# model_1.save('/home/jupyter/Drive_2/Event_Detection/Saved_Model/ucf_hd5_arch_weights_optimizer_exp_1.h5')
model.save('/home/jupyter/Drive_2/Event_Detection/Saved_Model_1/weights_state_ucf_complete_16mp.h5', overwrite=True, include_optimizer=True)
#model.save_model
print ("ALERT >> Serialization Successful. Done ....")

#----------------------------- Model Evaluation -----------------------------

#loss,acc = model.evaluate_generator(
 #   generate_data(train_frame_list,class_list_train,batch_size),steps=floor(len(train_frame_list)/ (batch_size * no_frame)))
#print("loss_train_data",loss)
#print("accuracy_training",acc)

evaluationResults = open('evaluation_results.txt', 'w')
print ("\n\n ------ Model Evaluation Metrics ------ ", file=evaluationResults)
print (" ------ Model Evaluation Metrics ------ ", file=evaluationResults)

# Training Loss Metrics:
loss,acc= model.evaluate_generator(generate_data(train_frame_list,class_list_train,batch_size),steps=floor(len(train_frame_list)/ (batch_size * no_frame)))
print ("\n------ Training Evaluation Metrics ------", file=evaluationResults)
print("Model Training Loss: ",loss, file=evaluationResults)
print("Model Training Accuracy: ",acc, file=evaluationResults)

# Testing Loss Metrics:
loss, acc= model.evaluate_generator(generate_data(test_frame_list,class_list_test,batch_size),steps=floor(len(test_frame_list)/(batch_size * no_frame)))
print ("------ Testing Evaluation Metrics ------\n", file=evaluationResults)
print("Model Testing Loss: ",loss, file=evaluationResults)
print("Model Testing Accuracy: ",acc, file=evaluationResults)
evaluationResults.close()


# (C) Copyright 1996-2016 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http=//www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation nor
# does it submit to any jurisdiction.

#importing Magics module
from Magics.macro import *

import numpy

ref = 'geos-140'
#Setting of the output file name

png = output(output_formats = ['png'],
    output_name_first_page_number = "off",
    output_name = "%s"%(ref,))

#Setting the coordinates of the geographical area
projection = mmap(subpage_map_projection = 'geos',
        subpage_map_vertical_longitude = -140.,
        page_frame="off",
        subpage_frame="off",
	  )

#Import the z500 data
data =  mgrib(grib_input_file_name  = "2m_temperature.grib",)

#Define the simple contouring for z500
contour = mcont(
         legend                         = "on",
    contour                        = "off",
    contour_shade                  = "on",
    contour_shade_technique        = "cell_shading",
    contour_shade_max_level_colour = "red",
    contour_shade_min_level_colour = "blue",
    contour_shade_colour_direction = "clockwise"
    )

#Coastlines setting
coast = mcoast( map_grid =  "on",
			map_grid_colour  =  "tan",
			map_coastline  =  'on',
			map_coastline_colour =  "tan",
			map_coastline_style = "solid")

#Title settings
title = mtext(
      text_lines = [" Shading on geos"],
      text_justification = "left",
      text_font_size = 0.8,
      text_colour = "charcoal")

legend = mlegend(
        legend_display_type = "continuous",
        legend_box_mode="positional",
        legend_text_font_size=0.4,
        legend_box_y_position=1,
        legend_box_y_length=1.5,
        legend_entry_border="off",
        legend_label_frequency=10,
    )

plot(png, projection, data, contour, legend, coast, title)


import socket
from socketserver import ThreadingTCPServer, StreamRequestHandler

PORT = 9000

clientList = [] # 클라이언트 connSock 를 저장할 리스트

def sendAll(msg, exceptSock):
    "exceptSock 을 제외하고, 연결된 모든 클라이언트에 데이터를 전송"
    for clientSock in clientList:
        if clientSock != exceptSock:
            clientSock.send(msg.encode())

class RequestHandler(StreamRequestHandler): #상속을 받는다
    # 클라이언트 접속시 스레드가 생성되고
    # 해당 스레드는 handle 메서드를 동작시킨다
    def handle(self):
        print("Connection From ", self.client_address)
        connSock = self.request     # 클라이언트와 연결된 소켓; 연결 socket
        clientList.append(connSock) # 클라이언트 리스트에 소켓 저장
        while True:
            try:
                msg = connSock.recv(1024).decode("utf-8") # 데이터 수신
                if msg == "":       # 클라이언트가 close()로 접속 종료
                    connSock.close()
                    print("{} 정상 접속 종료".format(self.client_address))
                    clientList.remove(connSock)
                    break

                print("{}으로부터 수신 : {}".format(self.client_address, msg))
                sendAll(msg, connSock)

            except Exception as e:
                # 비정상적인 접속 종료(주로 클라이언트가 프로그램 바로 종료)
                connSock.close()
                print("{} 비정상 접속 종료 = {}".format(
                    self.client_address, e))
                clientList.remove(connSock)
                break

if __name__ == '__main__':
    # ('', PORT) : 현재 실행 프로세스가 동작하는 Host ip를 자동 할당해라
    #             9000포트 설정
    # 새로운 클라이언트가 접속할 때마다 스레드를 1개씩 할당하고
    # 해당 스레드가 RequestHandler 객체의 handle 메서드를 처리하도록 하라
    server = ThreadingTCPServer(('', PORT), RequestHandler) # thread 객체
    print('Server Start ', PORT)
    server.serve_forever()     # 서버가동 시작