import numpy as np
import cv2 as cv
import keras
import glob

def main():
    path = '/home/pi/.virtualenvs/tower/lib/python3.5/site-packages/tower/trainingData/'
    droneLetterNN = Sequential()
    droneLetterNN.add(Dense(100, input_shape(self.DRONE_IMAGE_Y, self.DRONE_IMAGE_X)))
    droneLetterNN.add(Activation('relu'))
    droneLetterNN.add(Dense(2))             #[0] - 'B', [1] - 'E'
    droneLetterNN.add(Activation('softmax'))
    droneLetterNN.compile(loss='categorical_cross_entropy', optimizer='sgd')

    X = Y = []

    LowerBlockFace = np.array([0, 0, 115])
    UpperBlockFace = np.array([180, 76, 255])

    Bpath = path + 'droneB/'
    Epath = path + 'droneE/'

    for filename in glob.glob(os.path.join(Bpath, '*.png')):
        image = cv.imread(filename)
        hsvImage = cv.cvtColor(image, cv.COLOR_BGR2HSV)

        if not((image is None) or (image.size == 0)):
            mask = cv.inRange(image, LowerBlockFace, UpperBlockFace)
            X.append(mask)
            Y.append([1, 0])
    #END OF for
    for filename in glob.glob(os.path.join(Epath, '*.png')):
        image = cv.imread(filename)
        hsvImage = cv.cvtColor(image, cv.COLOR_BGR2HSV)

        if not((image is None) or (image.size == 0)):
            mask = cv.inRange(image, LowerBlockFace, UpperBlockFace)
            X.append(mask)
            Y.append([0, 1])
    #END OF for

    droneLetterNN.fit(X, Y, epochs=150, batch_size=10, verbose=0)

    model_json = droneLetterNN.to_json()
    with open(path + "drone.json", "w") as json_file:
        json_file.write(model_json)
    # serialize weights to HDF5
    droneLetterNN.save_weights(path + "drone.h5")
#END OF main()

if __name__ == '__main__':
    main()


import mxnet as mx
import os, time, shutil
import logging
from mxnet import gluon, image, init, nd
from mxnet import autograd as ag
from mxnet.gluon import nn
from mxnet.gluon.data.vision import transforms
from gluoncv.model_zoo import get_model
import numpy as np
import gluoncv
from gluoncv.utils import export_block,viz,makedirs
from tools.f1_metric import f1_score

model_name = 'xception'
root_dir = '/home/lin/sheldon/my_mxnet'
# weights_name = 'gluoncv-xception'
num_classes = 25
num_gpus = 2
batch_per_gpu = 8
train_epoch = 10

batch_size = batch_per_gpu * num_gpus
rec_path = os.path.join(root_dir, 'img_file')

predict_path = os.path.join(root_dir, 'predict_img')

prefix = os.path.join(root_dir, 'weights/xception')
prefix_epoch = 10

def get_iterators(batch_size = batch_size, data_shape=(3, 299, 299),shuffle=True,root_path = rec_path):
    train_data = mx.io.ImageRecordIter(
        path_imgrec=os.path.join(root_path, 'train.rec'),
        path_imgidx=os.path.join(root_path, 'train.idx'),
        data_shape=data_shape,
        batch_size=batch_size,
        resize = 299,
        saturation = 0.2,
        contrast = 0.2,
        shuffle=shuffle,
        rand_mirror = True,
        brightness = 0.2,
        data_name='data',
        label_name='softmax_label',
        rotate = 180
    )
    val_data = mx.io.ImageRecordIter(
        path_imgrec=os.path.join(root_path, 'val.rec'),
        path_imgidx=os.path.join(root_path, 'val.idx'),
        data_shape=data_shape,
        data_name='data',
        label_name='softmax_label',
        batch_size=batch_size,
        resize = 299,
    )
    return (train_data, val_data)

def get_eval_metrics(metrics_list):
    eval_metrics = mx.metric.CompositeEvalMetric()
    eval_metrics_1 = mx.metric.Accuracy()
    eval_metrics_2 = mx.metric.TopKAccuracy(5)
    metrics_list.append([eval_metrics_1,eval_metrics_2])
    for child_metric in metrics_list:
        eval_metrics.add(child_metric)
    # (train, val) = get_iterators(batch_size)
    # a = mod.score(eval_data = val,eval_metric = eval_metrics)
    # a = dict(a)
    return eval_metrics

def predict(data,prefix = prefix,prefix_epoch = prefix_epoch,num_batch = None):
    mod = get_model(prefix=prefix, prefix_epoch=prefix_epoch,batch_size = 4)
    pred_result = mod.predict(data, num_batch = num_batch).asnumpy()
    pred_result = np.squeeze(pred_result)
    result = np.argsort(pred_result, axis=1)[:, -1]
    return result
    # return pred_result

def val_model():
    eval_metrics = get_eval_metrics([f1_score()])
    mod = get_model()
    (train, val) = get_iterators(batch_size)
    eval_result = dict(mod.score(eval_data=val, eval_metric=eval_metrics,num_batch = 1))
    return eval_result

def get_model(prefix = prefix,prefix_epoch = prefix_epoch,batch_size=batch_size):
    sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, prefix_epoch)
    devs = [mx.gpu(i) for i in range(num_gpus)]
    mod = mx.mod.Module(symbol=sym, context=devs,
                        data_names=["data"], label_names=None)
    mod.bind(for_training=False, data_shapes=[("data", (batch_size, 3, 299, 299))],
             label_shapes=[("softmax_label", (batch_size,))])
    # 设定模型参数
    mod.set_params(arg_params, aux_params, allow_missing=True)
    return mod

if __name__ == '__main__':
    predict_data = mx.io.ImageRecordIter(
        path_imgrec=os.path.join(predict_path, 'predict.rec'),
        path_imgidx=os.path.join(predict_path, 'predict.idx'),
        data_shape=(3,299,299),
        data_name='data',
        label_name='softmax_label',
        batch_size=1,
        resize=299,
    )
    print (predict(data = predict_data))


import sys
from PIL import Image

filelist = ['20.png', '40.png', '60.png', '80.png', '100.png', '120.png']

images = [Image.open(x) for x in filelist]
widths, heights = zip(*(i.size for i in images))

total_width = int(sum(widths) / 2)
total_height = int(max(heights) * 2)

new_im = Image.new('RGB', (total_width, total_height))

x_offset = 0
y_offset = 0
for i in range(len(images)):
  im = images[i]
  new_im.paste(im, (x_offset, y_offset))
  x_offset += im.size[0]
  if i == len(images) // 2 - 1:
    y_offset += im.size[1]
    x_offset = 0

new_im.save('combined.jpg')


from flask import Flask, request, jsonify
import fetch

app = Flask(__name__)

@app.route('/')
def station_details():
    response = jsonify({
        'data': fetch.todays()
    })
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response

@app.route('/history')
def aggregated_past():
    response = jsonify({
        'data': fetch.agg()
    })
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=12434)


from discord.ext import commands
import discord
from typing import Tuple, Dict, List
import re

class CustomHelp(commands.Cog):
    __slots__ = 'bot',
    def __init__(self, bot):
        self.bot = bot
        self.bot.remove_command('help')

    def commands_dict(self) -> Dict[str, List[commands.Command]]:
        final = {}
        for cogname, cog in self.bot.cogs.items():
            if hasattr(cog, 'overwrite_name'):
                cogname = cog.overwrite_name
            for cmd in cog.get_commands():
                if not cmd.hidden:
                    if cogname not in final:
                        final[cogname] = []
                    final[cogname].append(cmd)
        final = dict(sorted(final.items(), key=lambda i: len(i[1]), reverse=True))
        return final

    @commands.command(name='help', hidden=True)
    async def _help(self, ctx, lookup=None, *subcommands):
        """
        Shows help for given command or
        lists all/category commands
        Examples::
        >
        Sends an embed listing all commands
        > money
        Sends help and info for command *money*
        """
        # Show all cogs and their commands WITHOUT short doc
        if lookup is None:
            cmds = self.commands_dict()

            # Sort by name
            for category in cmds:
                cmds[category].sort(key=lambda c: c.name)

            embed = discord.Embed(title='**Commands**',colour=0x3498db)
            for cog in cmds:
                formatted_cmds = ' '.join(map(lambda cmd: f'`{cmd.name}`', cmds[cog]))
                embed.add_field(name=cog, value=formatted_cmds, inline=False)
            await ctx.send(embed=embed)

        # Look up command
        else:
            cmd = self.bot.get_command(' '.join([lookup.lower(), *map(str.lower, subcommands)]))
            if cmd:
                cmd_help = cmd.help
                examples = None
                if cmd_help:
                    examples = ''
                    final_help = ''
                    in_example = False
                    for line in cmd_help.splitlines():
                        if not in_example:
                            if line == 'Examples::':
                                in_example = True
                                final_help = final_help[:-1]
                            else:
                                final_help += line + '\n'
                        else:
                            if line[0] == '>':
                                examples += f'`> {ctx.prefix}{cmd.qualified_name} {line[2:]}`\n'
                            else:
                                examples += line + '\n'
                    cmd_help = final_help
                    examples = examples[:-1]

                embed = discord.Embed(
                    title=cmd.qualified_name,
                    description=cmd_help,
                    colour=0x87ef73
                    )
                embed.add_field(name='Syntax', value=f'`{ctx.prefix}{cmd.qualified_name} {cmd.signature}`')
                if cmd.aliases:
                    embed.add_field(name='Aliases', value=' '.join(map(lambda x: f'`{x}`', cmd.aliases)))
                if isinstance(cmd, commands.Group):
                    embed.add_field(name='Subcommands', value=' '.join(f'`{i.name}`' for i in cmd.commands))
                if examples:
                    embed.add_field(name='Examples', value=examples, inline=False)
                await ctx.send(embed=embed)
            else:
                return await ctx.send('Command not found')

async def setup(bot):
    await bot.add_cog(CustomHelp(bot))


#!/usr/bin/env python3

#Cunren Liang, 22-MAR-2018

#this command can run multiple times for a stack
#example command
#../../code/s1_select_ion.py -dir . -sn 34/38.5 -nr 5

import os
import sys
import glob
import shutil
import zipfile
import argparse
import datetime
import numpy as np
import xml.etree.ElementTree as ET

class sentinelSLC(object):
    """
        A Class representing the SLCs
    """
    def __init__(self, safe_file, orbit_file=None):
        self.safe_file = safe_file
        self.orbit_file = orbit_file

    def get_datetime(self):
        datefmt = "%Y%m%dT%H%M%S"
        safe = os.path.basename(self.safe_file)
        fields = safe.split('_')
        self.platform = fields[0]
        self.start_date_time = datetime.datetime.strptime(fields[5], datefmt)
        self.stop_date_time = datetime.datetime.strptime(fields[6], datefmt)
        self.date = (datetime.datetime.date(self.start_date_time)).isoformat().replace('-','')

    def get_param(self):
        #speed of light (m/s)
        c = 299792458.0

        #1. processing software version
        zf = zipfile.ZipFile(self.safe_file, 'r')
        manifest = [item for item in zf.namelist() if '.SAFE/manifest.safe' in item][0]
        xmlstr = zf.read(manifest)
        root = ET.fromstring(xmlstr)
        elem = root.find('.//metadataObject[@ID="processing"]')

        #setup namespace
        nsp = "{http://www.esa.int/safe/sentinel-1.0}"
        rdict = elem.find('.//xmlData/' + nsp + 'processing/' + nsp + 'facility').attrib
        self.proc_site = rdict['site'] +', '+ rdict['country']

        rdict = elem.find('.//xmlData/' + nsp + 'processing/' + nsp + 'facility/' + nsp + 'software').attrib
        #ver = rdict['name'] + ' ' + rdict['version']
        self.proc_version = rdict['version']

        #2. start ranges
        anna = sorted([item for item in zf.namelist() if '.SAFE/annotation/s1' in item])
        #dual polarization. for the same swath, the slant ranges of two polarizations should be the same.
        if len(anna) == 6:
            anna = anna[1:6:2]

        startingRange = []
        for k in range(3):
            xmlstr = zf.read(anna[k])
            root = ET.fromstring(xmlstr)
            startingRange.append(
                float(root.find('imageAnnotation/imageInformation/slantRangeTime').text)*c/2.0
                )

        self.startingRanges = startingRange

        #3. snwe
        map_overlay = [item for item in zf.namelist() if '.SAFE/preview/map-overlay.kml' in item][0]
        xmlstr = zf.read(map_overlay)
        xmlstr = xmlstr.decode('utf-8')
        start = '<coordinates>'
        end = '</coordinates>'
        pnts = xmlstr[xmlstr.find(start)+len(start):xmlstr.find(end)].split()

        lats=[]
        lons=[]
        for pnt in pnts:
           lons.append(float(pnt.split(',')[0]))
           lats.append(float(pnt.split(',')[1]))
        self.snwe=[min(lats),max(lats),min(lons),max(lons)]

def get_safe_from_group(group):

    safes = []
    ngroup = len(group)
    for i in range(ngroup):
        ngroupi = len(group[i])
        for j in range(ngroupi):
            safes.append(group[i][j].safe_file)

    return safes

def print_group(group):
    '''print group parameters
    '''
    print()
    print('slice                                                                    no   ver         IW1 (m)           IW2 (m)           IW3 (m)')
    print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')

    for i in range(len(group)):
        for j in range(len(group[i])):
            print_stuff = '%s %3d  %s  '%(os.path.basename(group[i][j].safe_file), i+1, group[i][j].proc_version)
            print_stuff += "{} {} {}".format(group[i][j].startingRanges[0], group[i][j].startingRanges[1], group[i][j].startingRanges[2])
            print(print_stuff)
        print()

def get_group(dir0):
    '''
    this routine group the slices, each group is an acquisition
    the returned result is a list (acquistions sorted by starting time) containing a number of lists (slices sorted by starting time)
    '''
    #sort by starting time
    zips = sorted(glob.glob(os.path.join(dir0, 'S1*_IW_SLC_*.zip')), key=lambda x: x.split('_')[-5], reverse=False)
    nzips = len(zips)

    group = []
    for i in range(nzips):
        safeObj=sentinelSLC(zips[i])
        safeObj.get_datetime()
        safeObj.get_param()

        datefmt = "%Y%m%dT%H%M%S"
        fields = zips[i].split('_')
        tbef = datetime.datetime.strptime(fields[-5], datefmt)
        taft = datetime.datetime.strptime(fields[-4], datefmt)

        if i == 0:
            #create new group
            tbef0 = tbef
            group0 = []

        #S-1A is capable of operating up to 25 min per orbit [21]
        #Yague-Martinez et al., "Interferometric Processing of Sentinel-1 TOPS Data,"
        #S1A/B revisit time is 6 days, here we use 1 day to check if from the same orbit
        if np.absolute((tbef - tbef0).total_seconds()) < 24 * 3600:
            group0.append(safeObj)
        else:
            group.append(group0)
            #create new group
            tbef0 = tbef
            group0 = []
            group0.append(safeObj)

        if i == nzips - 1:
            group.append(group0)

    return group

def check_redundancy(group, threshold=1):
    '''
    threshold: time difference threshold in seconds between two slices in second.
    this routine checks, for a slice, if there are multiple ones.
    '''

    print('\nchecking different copies of same slice')

    safe_removed_indices = []
    ngroup = len(group)
    for i in range(ngroup):
        ngroupi = len(group[i])
        if ngroupi == 1:
            continue
        else:
            for j in range(ngroupi-1):
                for k in range(j+1, ngroupi):
                    if np.absolute((group[i][j].start_date_time - group[i][k].start_date_time).total_seconds()) < threshold and \
                       np.absolute((group[i][j].stop_date_time - group[i][k].stop_date_time).total_seconds()) < threshold:

                        #determine which one to be removed
                        j_version = False
                        k_version = False
                        for l in range(ngroupi):
                            if l != j and l != k:
                                if group[i][j].proc_version == group[i][l].proc_version:
                                    j_version = True
                                if group[i][k].proc_version == group[i][l].proc_version:
                                    k_version = True
                        if j_version == k_version:
                            safe_removed_index = [i, j]
                        else:
                            if j_version == False and k_version == True:
                                safe_removed_index = [i, j]
                            else:
                                safe_removed_index = [i, k]
                        safe_removed_indices.append(safe_removed_index)

                        print('in acquistion {}, the following two slices are the same:'.format(i+1))
                        if safe_removed_index == [i, j]:
                            print(os.path.basename(group[i][j].safe_file) + ' (not to be used)')
                            print(os.path.basename(group[i][k].safe_file))
                        else:
                            print(os.path.basename(group[i][j].safe_file))
                            print(os.path.basename(group[i][k].safe_file) + ' (not to be used)')

    #remove redundant slices
    if safe_removed_indices != []:
        group_new = []
        for i in range(ngroup):
            ngroupi = len(group[i])
            group_new.append([group[i][j] for j in range(ngroupi) if [i,j] not in safe_removed_indices])

        print('slices removed:')
        for index in safe_removed_indices:
            print('%s %3d'%(os.path.basename(group[index[0]][index[1]].safe_file), index[0]+1))

    else:
        group_new = group
        print('no slices removed')

    return group_new

def check_version(group):
    print('\nchecking different slice versions of an acquisition')

    acquistion_removed_indices = []
    ngroup = len(group)
    for i in range(ngroup):
        ngroupi = len(group[i])
        for j in range(ngroupi):
            if group[i][0].proc_version != group[i][j].proc_version:
                print('different slice versions found in acquistion {}'.format(i+1))
                acquistion_removed_indices.append(i)
                break

    #remove acquistions
    if acquistion_removed_indices != []:
        group_new = [group[i] for i in range(ngroup) if i not in acquistion_removed_indices]
        print('acquistions removed:')
        for i in acquistion_removed_indices:
            for j in range(len(group[i])):
                print('%s %3d'%(os.path.basename(group[i][j].safe_file), i+1))
    else:
        group_new = group
        print('no acquistions removed')

    return group_new

def check_gap(group):
    print('\nchecking gaps in an acquistion')

    acquistion_removed_indices = []
    ngroup = len(group)
    for i in range(ngroup):
        ngroupi = len(group[i])
        if ngroupi == 1:
            continue
        else:
            for j in range(0, ngroupi-1):
                if (group[i][j+1].start_date_time - group[i][j].stop_date_time).total_seconds() > 0:
                    acquistion_removed_indices.append(i)
                    break

    #remove acquistions
    if acquistion_removed_indices != []:
        group_new = [group[i] for i in range(ngroup) if i not in acquistion_removed_indices]
        print('acquistions removed:')
        for i in acquistion_removed_indices:
            for j in range(len(group[i])):
                print('%s %3d'%(os.path.basename(group[i][j].safe_file), i+1))
    else:
        group_new = group
        print('no acquistions removed')

    return group_new

def acquistion_snwe(groupi):
    '''return snwe of an acquisition consisting a number of slices'''
    s = min([x.snwe[0] for x in groupi])
    n = max([x.snwe[1] for x in groupi])
    w = min([x.snwe[2] for x in groupi])
    e = max([x.snwe[3] for x in groupi])

    return [s, n, w, e]

def overlap(group):
    '''return snwe of the overlap of all acquistions'''

    s = max([(acquistion_snwe(x))[0] for x in group])
    n = min([(acquistion_snwe(x))[1] for x in group])
    w = max([(acquistion_snwe(x))[2] for x in group])
    e = min([(acquistion_snwe(x))[3] for x in group])

    if s >= n or w >= e:
       #raise Exception('no overlap among the acquistions')
       print('WARNING: there is no overlap among the acquistions, snwe: {}'.format([s, n, w, e]))

    return [s, n, w, e]

def check_aoi(group, s, n):
    '''
    check each group to see if it fully covers [s, n], if not remove the acquistion
    s: south bound
    n: north bound
    '''

    print('\nchecking if each acquistion fully covers user specifed south/north bound [{}, {}]'.format(s, n))

    acquistion_removed_indices = []
    ngroup = len(group)
    for i in range(ngroup):
        snwe = acquistion_snwe(group[i])
        if not (snwe[0] <= s and snwe[1] >= n):
            acquistion_removed_indices.append(i)

    #remove acquistions
    if acquistion_removed_indices != []:
        group_new = [group[i] for i in range(ngroup) if i not in acquistion_removed_indices]
        print('acquistions removed:')
        for i in acquistion_removed_indices:
            for j in range(len(group[i])):
                print('%s %3d'%(os.path.basename(group[i][j].safe_file), i+1))
    else:
        group_new = group
        print('no acquistions removed')

    return group_new

def check_different_starting_ranges(group):
    '''
    checking if there are different starting ranges in each acquistion.
    '''

    print('\nchecking if there are different starting ranges in each acquistion')

    acquistion_removed_indices = []
    ngroup = len(group)
    for i in range(ngroup):
        ngroupi = len(group[i])
        for j in range(1, ngroupi):
            if group[i][0].startingRanges != group[i][j].startingRanges:
                acquistion_removed_indices.append(i)
                #print('++++++++++++++{} {}'.format(group[i][0].safe_file, group[i][j].safe_file))
                break

    #remove acquistions
    if acquistion_removed_indices != []:
        group_new = [group[i] for i in range(ngroup) if i not in acquistion_removed_indices]
        print('acquistions removed:')
        for i in acquistion_removed_indices:
            for j in range(len(group[i])):
                print('%s %3d'%(os.path.basename(group[i][j].safe_file), i+1))
    else:
        group_new = group
        print('no acquistions removed')

    return group_new

def check_small_number_of_acquisitions_with_same_starting_ranges(group, threshold=1):
    '''
    for the same subswath starting ranges,
    if the number of acquistions < threshold, remove these acquistions
    '''

    print('\nchecking small-number of acquistions with same starting ranges')

    ngroup = len(group)

    starting_ranges = [x[0].startingRanges for x in group]

    #get unique starting_ranges
    starting_ranges_unique = []
    for i in range(ngroup):
        if starting_ranges[i] not in starting_ranges_unique:
            starting_ranges_unique.append(starting_ranges[i])

    #get number of acquistions for each unique starting ranges
    ngroup_unique = len(starting_ranges_unique)
    starting_ranges_unique_number = [0 for i in range(ngroup_unique)]
    for k in range(ngroup_unique):
        for i in range(ngroup):
            if starting_ranges_unique[k] == starting_ranges[i]:
                starting_ranges_unique_number[k] += 1

    #get starting ranges to be removed (number of acquistions < threshold)
    starting_ranges_removed = []
    for k in range(ngroup_unique):
        if starting_ranges_unique_number[k] < threshold:
            starting_ranges_removed.append(starting_ranges_unique[k])

    #remove acquistions
    if starting_ranges_removed != []:
        group_new = [group[i] for i in range(ngroup) if group[i][0].startingRanges not in starting_ranges_removed]

        print('acquistions removed:')
        for i in range(ngroup):
            if group[i][0].startingRanges in starting_ranges_removed:
                for j in range(len(group[i])):
                    print('%s %3d'%(os.path.basename(group[i][j].safe_file), i+1))

    else:
        group_new = group
        print('no acquistions removed')

    return group_new

def cmdLineParse():
    '''
    Command line parser.
    '''

    parser = argparse.ArgumentParser( description='select Sentinel-1A/B acquistions good for ionosphere correction. not used slices are moved to folder: not_used')
    parser.add_argument('-dir', dest='dir', type=str, required=True,
            help = 'directory containing the "S1*_IW_SLC_*.zip" files')
    parser.add_argument('-sn', dest='sn', type=str, required=True,
            help='south/north bound of area of interest, format: south/north')
    parser.add_argument('-nr', dest='nr', type=int, default=10,
            help = 'minimum number of acquisitions for same starting ranges. default: 10')

    if len(sys.argv) <= 1:
        print('')
        parser.print_help()
        sys.exit(1)
    else:
        return parser.parse_args()

if __name__ == '__main__':

    inps = cmdLineParse()
    s,n=[float(x) for x in inps.sn.split('/')]

    #group the slices
    group = get_group(inps.dir)
    safes_all = get_safe_from_group(group)

    #print overlap of group
    #print('overlap among acquisitions: {}'.format(overlap(group)))

    #print group before removing slices/acquistions
    print_group(group)

    #do checks and remove the slices/acquisitions
    group = check_redundancy(group, threshold=1)
    group = check_version(group)
    group = check_gap(group)
    group = check_aoi(group, s, n)
    group = check_different_starting_ranges(group)
    group = check_small_number_of_acquisitions_with_same_starting_ranges(group, threshold=inps.nr)

    #print group after removing slices/acquistions
    print_group(group)

    #move slices that are not used to 'not_used'
    safes_used = get_safe_from_group(group)
    not_used_dir = os.path.join(inps.dir, 'not_used')
    os.makedirs(not_used_dir, exist_ok=True)
    for safe in safes_all:
        if safe not in safes_used:
            shutil.move(safe, not_used_dir)


# This module 'punctuation' pre-insitialized string which holds all special characters
import string

# I decided to use Counter as I've read it's much more efficient than doing counts with loops
from collections import Counter

f = open("Book.txt")

# read file as string
text = f.read()
# convert everything to lower_case
text = text.lower()
# get rid of all special characters except
for symb in string.punctuation:
    text = text.replace(symb,' ')

# split text into words and count each word occurance
word_count = Counter(text.split())
for word in word_count:
    if word_count[word] > 1: # This is what I understood from lesson, but if we want all words we could simply comment out this line
        print(f"{word} {str(word_count[word])} time(s)")
f.close()


from cctbx import crystal, miller, sgtbx
from cctbx.array_family import flex
import numpy as np
from itertools import izip

def get_val_at_hkl(hkl, val_map, sg=sgtbx.space_group(" P 4nw 2abw")):
    """
    given a miller index, find its value in the miller dictionary array (val_map)

    :param hkl:  tuple
    :param val_map:  miller data dictionary
    :param sg: space group
    :return: miller value
    """
    poss_equivs = [i.h() for i in miller.sym_equiv_indices(sg, hkl).indices()]

    in_map = False
    for hkl2 in poss_equivs:
        if hkl2 in val_map:  # fast lookup
            in_map = True
            break
    if in_map:
        return hkl2, val_map[hkl2]
    else:
        return (None, None, None), None

def generate_table(complex_data, indices, numpy_args=False, anom=True):
    """

    :param complex_data: structure factors
    :param indices: miller indices in a list, Nx3
    :param numpy_args: are the complex data and indices numpy type, if not
        assume flex
    :param anom: return a miller array with +H and -H ?
    :return: dictionary whose keys are miller index tuple and values are structure fact
    """
    sg = sgtbx.space_group(" P 4nw 2abw")
    Symm = crystal.symmetry(unit_cell=(79, 79, 38, 90, 90, 90), space_group=sg)
    if numpy_args:
        assert type(indices) == tuple
        assert (type(indices[0]) == tuple)
        indices = flex.miller_index(indices)

    mil_set = miller.set(crystal_symmetry=Symm, indices=indices, anomalous_flag=anom)
    if numpy_args:
        complex_data = flex.complex_double(np.ascontiguousarray(complex_data))
    mil_ar = miller.array(mil_set, data=complex_data)

    mil_dict = {h: val for h, val in izip(mil_ar.indices(), mil_ar.data())}
    return mil_dict


import requests
import bs4
import json
import re

import os
from dotenv import load_dotenv
from celery import Celery
from celery.schedules import crontab

from database.crud import policyCrud
from database import database
from services import korea, alarm

load_dotenv()

engine = database.engineconn()
session = engine.sessionmaker()

app = Celery()

@app.on_after_configure.connect()
def setup_daily_tasks(sender, **kwargs):
    # 매일 자정에 실행
    sender.add_periodic_task(
        crontab(hour=4, minute=54),
        getPolicyInfo(),
    )

# def getPolicy(id):
#     policyCrud.getPolicyByPolicyId(session, id)

async def getPolicyInfo():
    policyList = []

    # 청년정책 정보 api metadata
    URL = os.getenv('POLICY_URL')
    params = {
        'openApiVlak': os.getenv('POLICY_KEY'),
        'pageIndex': '1',
        'display': '1'
    }

    # DB에서 마지막 값 조회
    lastPolicy = policyCrud.getLastPolicyId(session)
    if lastPolicy is None:
        idx = 1
    else:
        idx = lastPolicy[0] + 1
    print(idx)

    # 청년정책 정보 수집 및 저장
    while True:
        policyInfo = {}

        # 청년정책 파라미터 변경 및 요청
        params['pageIndex'] = idx
        response = requests.get(URL, params=params)
        content = response.text

        # item tag 분리
        xml_obj = bs4.BeautifulSoup(content, 'lxml-xml')
        rows = xml_obj.findAll('youthPolicy')

        # 더 이상 정책이 없는 경우 종료
        # if idx == 20:
        if len(rows) == 0:
            break

        # 컬럼 값 조회
        columns = rows[0].find_all()
        metadata = {}
        for i in range(0, len(columns)):
            infoType = columns[i].name  # 정책 정보 유형
            val = columns[i].text  # 정책 데이터

            if infoType == 'rnum':  # row 번호
                policyInfo["id"] = val
                metadata['id'] = val
            elif infoType == 'polyBizSjnm':  # 정책명
                policyInfo['text'] = val
                metadata['name'] = val
            elif infoType == 'bizId':  # 정책 ID
                metadata['policyId'] = val
            elif infoType == 'polyItcnCn':  # 정책 소개
                metadata['policyDetail'] = val
            elif infoType == 'sporCn':  # 지원 내용
                metadata['supportContent'] = val
            elif infoType == 'sporScvl':  # 지원 규모
                metadata['supportScale'] = val
            elif infoType == 'bizPrdCn':  # 사업 운영 기간
                metadata['policyOperationPeriod'] = val
            elif infoType == 'rqutPrdCn':  # 사업 신청 기간
                info = classifyApplicationPeriod(val)
                metadata['applicationStart'] = info[0]
                metadata['applicationEnd'] = info[1]
            elif infoType == 'ageInfo':  # 연령 정보
                info = classifyAge(val)
                metadata['startAge'] = info[0]
                metadata['endAge'] = info[1]
            elif infoType == 'majrRqisCn':  # 전공 요건
                metadata['major'] = val
            elif infoType == 'empmSttsCn':  # 취업 상태
                metadata['employeedStatus'] = val
            elif infoType == 'splzRlmRqisCn':  # 특화 분야
                metadata['specialized'] = val
            elif infoType == 'accrRqisCn':  # 학력 요건
                metadata['academicAbility'] = val
            elif infoType == 'prcpCn':  # 거주지 및 소득 조건
                metadata['ResiIncomeCondition'] = val
            elif infoType == 'aditRscn':  # 추가 단서 사항
                metadata['additional'] = val
            elif infoType == 'prcpLmttTrgtCn':  # 참여 제한 대상
                metadata['restriction'] = val
            elif infoType == 'rqutProcCn':  # 신청 절차
                metadata['applicationProcedure'] = val
            elif infoType == 'pstnPaprCn':  # 제출 서류
                metadata['submissionDocuments'] = val
            elif infoType == 'jdgnPresCn':  # 심사 발표 내용
                metadata['judge'] = val
            elif infoType == 'rqutUrla':  # 신청 사이트 주소
                metadata['applicationURL'] = val
            elif infoType == 'rfcSiteUrla1':  # 참고 사이트 1
                metadata['referenceURL1'] = val
            elif infoType == 'rfcSiteUrla2':  # 참고 사이트 2
                metadata['referenceURL2'] = val
            elif infoType == 'mngtMson':  # 주관 부처명
                metadata['department'] = val
                metadata['location'] = korea.getResidence(list(val))

        policyInfo['metadata'] = metadata
        policyList.append(policyInfo)
        idx += 1

        # print("===================================")

        # metadata db에 저장
        policyCrud.createPolicy(session, metadata)
        # print(metadata)

        # 정책에 적합한 회원 탐색
        alarm.findMatchMember(metadata)

        # json 형태로 반환
        data = {
            "documents": policyList
        }
        policyJson = json.dumps(data, ensure_ascii=False).encode("utf-8")

    return policyJson.decode("utf-8")

# 신청 기간 구분
# yyyy-mm-dd 형태로 저장
# 상시 정책의 경우 0000-00-00
def classifyApplicationPeriod(periodInfo):
    periodInfo = re.sub(r'\([^)]*\)', '', periodInfo)  # 괄호 제거
    periodInfo = re.sub(r'\d{1,2}:\d{2}', '', periodInfo)  # HH:MM 형태 제거
    periodInfo = re.sub(r'[가-힣a-zA-Z]+', '', periodInfo)  # 한글, 영어 제거
    periodInfo = periodInfo.strip()  # 공백 제거

    startDate = endDate = "0000-00-00"

    dateRange = periodInfo.split('~')
    # print(dateRange)

    # 시작 일, 종료 일이 모두 있는 경우
    if len(dateRange) == 2:
        # 20230807~20231231 형태의 데이터
        match = re.search(r'\d{8}', dateRange[0])
        if match is not None:
            startDate = formatDate(dateRange[0][:4], dateRange[0][4:6], dateRange[0][6:])
            endDate = formatDate(dateRange[1][:4], dateRange[1][4:6], dateRange[1][6:])
            return [startDate, endDate]

        # 그 외의 다양한 데이터 형태
        year, month, day = findDateFormat(dateRange[0])
        if day == 0:
            day = 1
        startDate = formatDate(year, month, day)

        year, month, day = findDateFormat(dateRange[1])
        if day == 0:
            day = 31
        endDate = formatDate(year, month, day)

    # 시작일 또는 종료 일만 있는 경우
    elif len(dateRange) == 1:
        year, month, day = findDateFormat(dateRange[0])
        if day == 0:
            startDate = formatDate(year, month, 1)
            endDate = formatDate(year, month, 31)

    return [startDate, endDate]

# 날짜 형태 파악
def findDateFormat(date):
    # yyyy.mm.dd 형태
    match = re.search(r'(\d{4})[\s.~-](\d{1,2})[\s.~-](\d{1,2})[\s.~-]?', date)
    if match:
        return match.group(1), match.group(2), match.group(3)

    # yy.mm.dd 형태
    match = re.search(r'(\d{2})[\s.~-](\d{1,2})[\s.~-](\d{1,2})[\s.~-]?', date)
    if match:
        return "20" + match.group(1), match.group(2), match.group(3)

    # yyyy.mm 형태
    match = re.search(r'(\d{4})[\s.~-](\d{1,2})[\s.~-]?', date)
    if match:
        return match.group(1), match.group(2), 0

    # mm.dd 형태
    match = re.search(r'(\d{1,2})[\s.~-](\d{1,2})[\s.~-]?', date)
    if match:
        return 2023, match.group(1), match.group(2)

    # mm 형태
    match = re.search(r'(\d{1,2})[\s.~-]?', date)
    if match:
        return 2023, match.group(1), 0

    else:
        return 0, 0, 0

# 날짜 형태 지정
def formatDate(year, month, date):
    return f"{int(year):04d}-{int(month):02d}-{int(date):02d}"

# 연령 정보 구분
def classifyAge(ageInfo):
    # print(ageInfo)
    start = 0
    end = 99

    data = re.findall(r'\d+', ageInfo);

    # '만 19세 ~ 34세' 형태의 데이터
    if '~' in ageInfo:
        return data

    # '만 18세 이상' 형태의 데이터
    if '이상' in ageInfo:
        end = data[0]

    # '만 39세 미만' 형태의 데이터
    if '미만' in ageInfo:
        start = data[0]

    return [start, end]


#!/usr/bin/env python
# coding:utf-8
"""
Name : pipe.py
Author  : anne
Time    : 2019-08-25 10:41
Desc:
"""
from multiprocessing import Process, Pipe

def f(conn):
    conn.send([42, None, 'hello'])
    conn.send(['from child'])
    print(conn.recv())
    conn.close()

if __name__ == '__main__':
    parent_conn, child_conn = Pipe()
    p = Process(target=f, args=(child_conn,))
    p.start()
    print(parent_conn.recv())  # prints "[42, None, 'hello']"
    print(parent_conn.recv())
    parent_conn.send('from child')

    p.join()


"""
cry_authorize
~~~~~~~~~~~~~

This module provides endpoints for client authorization and token verification within the application. It defines two
main routes: one for generating JWT tokens for authorized clients and another for verifying the validity of a token."""

import datetime

from flask_restx import Namespace, Resource, fields
from http import HTTPStatus
import logging

from globals import LOG_LEVEL, bucket_cache, SECRET_KEY
import jwt

from modules.cry_auth import verify_token

ns = Namespace('authorize', description='Authorisation Route Namespace')

authorize_model = ns.model('Authorize', {
    'app_name': fields.String(required=True, description='Application Name'),
    'bucket_name': fields.String(required=True, description='Bucket Name'),
    'client_id': fields.String(required=True, description='Client ID')
})

token_verify_model = ns.model('TokenVerify', {
    'token': fields.String(required=True, description='Token to verify')
})

logging.basicConfig(level=LOG_LEVEL)

@ns.route('/')
class AuthoriseResource(Resource):
    """
    Resource for client authorization based on their client_id and bucket_name.
    """
    @ns.expect(authorize_model, validate=True)
    @ns.doc(
        responses={
            HTTPStatus.OK: 'Token generated successfully.',
            HTTPStatus.BAD_REQUEST: 'Invalid client_id or bucket_name.',
            HTTPStatus.UNAUTHORIZED: 'Unauthorized access.',
            HTTPStatus.INTERNAL_SERVER_ERROR: 'Internal server error.'
        })
    def post(self):
        """
        Authorize a client based on their client_id and bucket_name and return a JWT token.

        :return: A dictionary containing the JWT token or an error message.
        """
        try:
            data = ns.payload
            client_id = data.get('client_id')
            bucket_name = data.get('bucket_name')
            app_name = data.get('app_name')

            # Check if the client_id and app_name + bucket_name combination is valid
            credentials = bucket_cache.get((app_name, bucket_name), None)
            if not credentials or credentials['client_id'] != client_id:
                return {'message': 'Invalid client_id or bucket_name'}, HTTPStatus.BAD_REQUEST

            token = jwt.encode({
                'client_id': client_id,
                'bucket_name': bucket_name,
                'app_name': app_name,
                'exp': datetime.datetime.utcnow() + datetime.timedelta(days=1)  # Token expires after 1 day
            }, SECRET_KEY, algorithm='HS256')

            return {'token': token}, HTTPStatus.OK

        except Exception as e:
            logging.error(f"Error authorizing client: {str(e)}")
            return {'error': 'Internal server error'}, HTTPStatus.INTERNAL_SERVER_ERROR

@ns.route('/verify_token')
class TokenVerifyResource(Resource):
    """
     Resource for verifying the validity of a provided JWT token.
     """
    @ns.expect(token_verify_model, validate=True)
    @ns.doc(
        responses={
            HTTPStatus.OK: 'Token is valid.',
            HTTPStatus.BAD_REQUEST: 'Invalid token format.',
            HTTPStatus.UNAUTHORIZED: 'Token is invalid or expired.',
            HTTPStatus.INTERNAL_SERVER_ERROR: 'Internal server error.'
        })
    def post(self):
        """
        Verify if a provided JWT token is still valid.

        :return: A dictionary indicating the validity of the token or an error message.
        """
        token = None  # Initialize token variable
        try:
            data = ns.payload
            token = data.get('token')

            # Utilize the verify_token function from cry_auth module
            is_valid, message = verify_token(token)

            if is_valid:
                return {'message': message}, HTTPStatus.OK
            else:
                return {'message': message}, HTTPStatus.UNAUTHORIZED

        except jwt.ExpiredSignatureError:
            logging.warning(f"Token expired for token: {token}")
            return {'message': 'Token has expired.'}, HTTPStatus.UNAUTHORIZED

        except jwt.DecodeError:
            logging.warning(f"Invalid token: {token}")
            return {'message': 'Invalid token format.'}, HTTPStatus.BAD_REQUEST

        except jwt.InvalidTokenError:
            logging.warning(f"Invalid token: {token}")
            return {'message': 'Token is invalid.'}, HTTPStatus.UNAUTHORIZED

        except Exception as e:
            logging.error(f"Error verifying token: {str(e)}")
            return {'error': 'Internal server error'}, HTTPStatus.INTERNAL_SERVER_ERROR


# As usual, a bit of setup
from __future__ import print_function
import time, os, json
import numpy as np
import matplotlib.pyplot as plt

from lib.gradient_check import eval_numerical_gradient, eval_numerical_gradient_array
from lib.layers.rnn_layers import *
from lib.solvers.captioning_solver import CaptioningSolver
from lib.classifiers.rnn import CaptioningRNN
from lib.utils.coco_utils import load_coco_data, sample_coco_minibatch, decode_captions
from lib.utils.image_utils import image_from_url

plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

def rel_error(x, y):
    """ returns relative error """
    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))

#Two type of data to use
data = load_coco_data(pca_features=True)

med_data = load_coco_data(max_train=1000)

for k, v in med_data.items():
    if type(v) == np.ndarray:
        print(k, type(v), v.shape, v.dtype)
    else:
        print(k, type(v), len(v))

# U can use either lstm or vanilla rnn
med_lstm_model = CaptioningRNN(
          cell_type='lstm',
          word_to_idx=med_data['word_to_idx'],
          input_dim=med_data['train_features'].shape[1],
          hidden_dim=512,
          wordvec_dim=256,
          dtype=np.float32,
        )

med_lstm_solver = CaptioningSolver(med_lstm_model, med_data,
           update_rule='adam',
           num_epochs=50,
           batch_size=50,
           optim_config={
             'learning_rate': 5e-3,
           },
           lr_decay=0.995,
           verbose=True, print_every=10,
         )

#achieved (Epoch 22 / 50) train acc: 0.992441; val_acc: 0.226580
#Overfitting, try to feed more data and increase regularization to improve accuracy
med_lstm_solver.train()

# Plot the training losses
plt.plot(med_lstm_solver.loss_history)
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.title('Training loss history')
plt.show()

for split in ['train', 'val']:
    minibatch = sample_coco_minibatch(med_data, split=split, batch_size=2)
    gt_captions, features, urls = minibatch
    gt_captions = decode_captions(gt_captions, data['idx_to_word'])

    sample_captions = med_lstm_model.sample(features)
    sample_captions = decode_captions(sample_captions, data['idx_to_word'])

    for gt_caption, sample_caption, url in zip(gt_captions, sample_captions, urls):
        plt.imshow(image_from_url(url))
        plt.title('%s\n%s\nGT:%s' % (split, sample_caption, gt_caption))
        plt.axis('off')
        plt.show()


from django.urls import path, include
from .views import *

urlpatterns = [
    path("", MarketPage.as_view(), name="market"),
    path("search", search, name="search"),
    path("create", create_product_card, name="create-card"),
    path("moderation", products_on_moderation, name="moderation"),
    path(
        "moderation/remove/<int:pk>",
        RemoveProductOnModeration.as_view(),
        name="moderation-remove",
    ),
    path(
        "moderation/approve/<int:pk>",
        ApproveProductModeration.as_view(),
        name="moderation-approve",
    ),
    path("<slug:slug_category>", category, name="category-detail"),
    path(
        "<slug:slug_category>/<int:product_id>",
        ProductCard.as_view(),
        name="product-card",
    ),
    path(
        "<slug:slug_category>/<slug:slug_subcategory>", category, name="category-detail"
    ),
]


import numpy as np
from sklearn.datasets import fetch_openml
from sklearn.model_selection import GridSearchCV
from sklearn.neighbors import KNeighborsClassifier

if __name__ == '__main__':
    mnist = fetch_openml('mnist_784', as_frame=False)
    x, y = mnist.data, mnist.target

    knn_clf = KNeighborsClassifier()

    x_train, x_test, y_train, y_test = x[:60000], x[60000:], y[:60000], y[60000:]

    param_grid = {
        'n_neighbors': [3,4,5,6],
        'weights': ["uniform", "distance"],
    }

    grid_search = GridSearchCV(knn_clf, param_grid, cv=3, n_jobs=4)
    grid_search.fit(x_train, y_train)
    tuned_accuracy = grid_search.score(x_test, y_test)

    print(grid_search.best_params_)
    print('grid search score: ', tuned_accuracy)
    # we get the best params {'n_neighbors': 6, 'weights': 'distance'}

    best_knn_clf = KNeighborsClassifier(n_neighbors=6, weights='distance')
    best_knn_clf.fit(x_train, y_train)

    accuracy = best_knn_clf.score(x_test, y_test)
    print('score with best parameters: ', accuracy)

    print('error rate: ')
    error_rate_change = (1 - accuracy) / (1 - tuned_accuracy) - 1
    print(f"error_rate_change = {error_rate_change:.0%}")


import numpy as np
import scipy as sp
import scipy.sparse as sparse
import scipy.linalg as linalg
import statistics as st
import matplotlib.pyplot as plt
import pandas
import math

Vol=0.2
int_Rate=0.05
Expiration=1.0
Strike=100
N_S=20
N_t= 20
dS=(2*Strike)/N_S
dt=Expiration/N_t

S = [x*dS for x in range (N_S)]
print(S)

V=[]
for i in range (N_S):
    V.append(max(S[i]-Strike,0.0))
print(V)

#Crank Nicolson Boundary Conditions 1st and last row
def CrankNicolson(Vol,S,int_Rate):
    nu1=dt/(dS**2)
    nu2=dt/dS
    A=[]
    B=[]
    C=[]
    nA=[]
    nB=[]
    nC=[]
    for i in range (1,N_S-1):
        ak=0.5*(Vol**2)*(S[i]**2)
        bk=int_Rate*S[i]
        ck=-int_Rate

        Ak=0.5*nu1*ak - 0.25*nu2*bk
        Bk=-nu1*ak + 0.5*dt*ck
        Ck=0.5*nu1*ak + 0.25*nu2*bk

        if (i==1):
            A.append(-Ak)
            B.append(1 - Bk -2*Ak)
            C.append(-Ck + Ak)

            nA.append(Ak)
            nB.append(1 + Bk + 2*Ak)
            nC.append(Ck - Ak)
        elif(i==N_S-2):
            A.append(-Ak + Ck)
            B.append(1 - Bk - 2*Ck)
            C.append(-Ck)

            nA.append(Ak - Ck)
            nB.append(1 + Bk + 2*Ck)
            nC.append(Ck)
        else:
            A.append(-Ak)
            B.append(1 - Bk)
            C.append(-Ck)
            nA.append(Ak)
            nB.append(1 + Bk)
            nC.append(Ck)
    A.pop(0)
    bndC=C.pop()
    nA.pop(0)
    nC.pop()
    return A,B,C,nA,nB,nC

#Crank Nicolson Boundary Conditions last row
def CrankNicolson2(Vol,S,int_Rate):
    nu1=dt/(dS**2)
    nu2=dt/dS
    A=[]
    B=[]
    C=[]
    nA=[]
    nB=[]
    nC=[]
    for i in range (1,N_S-1):
        ak=0.5*(Vol**2)*(S[i]**2)
        bk=int_Rate*S[i]
        ck=-int_Rate

        Ak=0.5*nu1*ak - 0.25*nu2*bk
        Bk=-nu1*ak + 0.5*dt*ck
        Ck=0.5*nu1*ak + 0.25*nu2*bk

        if(i==N_S-2):
            A.append(-Ak + Ck)
            B.append(1 - Bk - 2*Ck)
            C.append(-Ck)

            nA.append(Ak - Ck)
            nB.append(1 + Bk + 2*Ck)
            nC.append(Ck)
        else:
            A.append(-Ak)
            B.append(1 - Bk)
            C.append(-Ck)
            nA.append(Ak)
            nB.append(1 + Bk)
            nC.append(Ck)
    A.pop(0)
    bndC=C.pop()
    nA.pop(0)
    nC.pop()
    return A,B,C,nA,nB,nC

#Implicit Boundary Conditions last row
def Implicit(Vol,S,int_Rate):
    nu1=dt/(dS**2)
    nu2=dt/dS
    A=[]
    B=[]
    C=[]

    for i in range (1,N_S-1):
        ak= 0.5*(Vol**2)*(S[i]**2)
        bk= int_Rate*S[i]
        ck= -int_Rate

        Ak= -nu1*ak + 0.5*nu2*bk
        Bk= 2*nu1*ak - dt*ck
        Ck= -nu1*ak - 0.5*nu2*bk

        if(i==N_S-2):
            A.append(Ak - Ck)
            B.append(1 + Bk + 2*Ck)
            C.append(Ck)

        else:
            A.append(Ak)
            B.append(1 + Bk)
            C.append(Ck)
    A.pop(0)
    bndC=C.pop()
    return A,B,C

A,B,C,nA,nB,nC=CrankNicolson(Vol,S,int_Rate)

#Build LHS and RHS Matrices
def Build2Mat(A,B,C,nA,nB,nC):
    diag_lhs = [A,B,C]
    diag_rhs = [nA,nB,nC]
    lhs=sparse.diags(diag_lhs, [-1,0,1]).toarray()
    rhs=sparse.diags(diag_rhs, [-1,0,1]).toarray()
    return lhs,rhs

#Build LHS Matrix
def Build1Mat(A,B,C):
    diag_lhs = [A,B,C]
    lhs=sparse.diags(diag_lhs, [-1,0,1]).toarray()
    return lhs

lhs,rhs=Build2Mat(A,B,C,nA,nB,nC)

Vtemp=V[1:N_S-1]
Vtemp=np.array(Vtemp)
print(Vtemp)

#Solve Iteratively Crank Nicolson
def SolveCN(lhs,rhs,Vtemp,N_S):
    for i in range (N_S):
        Vmul=rhs@Vtemp
        Vres=linalg.solve(lhs,Vmul)
        Vtemp=Vres
        print(Vres)

#Solve Iteratively Implicit
def SolveImp(lhs,Vtemp,N_S):
    for i in range (N_S):
        Vres=linalg.solve(lhs,Vtemp)
        Vtemp=Vres
        print(Vres)

SolveCN(lhs,rhs,Vtemp,N_S)


import falcon
import re
import requests
import json
import datetime
import os
import jinja2
import shutil
from configparser import ConfigParser
from time import sleep
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session
from sqlalchemy.orm import sessionmaker

from scrapy.utils.project import get_project_settings
from sparepart.models import ScrapingJob
from crawler_queue import CrawlerQueue

SCRAPY_SETTINGS = get_project_settings()

CRAWLER_DELAY = {
    'isuzu': os.environ.get('DELAY_ISUZU', 1),
    'suzuki': os.environ.get('DELAY_SUZUKI', 1),
    'daihatsu': os.environ.get('DELAY_DAIHATSU', 1),
    'megazip': os.environ.get('DELAY_MEGAZIP', 15),
    'parts.com': os.environ.get('DELAY_PARTS', 15),
}

scrapyd_config = ConfigParser()
scrapyd_config.read('scrapyd.conf')

SCRAPYD_HOST = 'http://{}:{}'.format('localhost',
                                     scrapyd_config['scrapyd']['http_port'])

# create database session
CONNECTION_STRING = os.environ.get('CONNECTION_STRING')
if CONNECTION_STRING:
    engine = create_engine(CONNECTION_STRING, echo=True)
    session_factory = sessionmaker(bind=engine)
    Session = scoped_session(session_factory)

import logging
logger = logging.getLogger(__name__)

class SQLAlchemySessionManager:
    """
    Create a scoped session for every request and close it when the request
    ends.
    """

    def __init__(self, Session):
        self.Session = Session

    def process_resource(self, req, resp, resource, params):
        resource.session = self.Session()

    def process_response(self, req, resp, resource, req_succeeded):
        if hasattr(resource, 'session'):
            if not req_succeeded:
                resource.session.rollback()
            Session.remove()

class AuthMiddleware(object):

    def process_request(self, req, resp):
        auth_header = req.get_header('Authorization')
        if auth_header:
            req.context['authenticated'] = auth_header.split(' ')[-1] == 'dXNlcm5hbWU6cGFzc3dvcmQ='

def validate_auth(req, resp, resource, params):
    description = 'Please provide valid authentication header.'
    challenges = ['Authorization: Basic ...']
    if 'authenticated' not in req.context or not req.context['authenticated']:
        raise falcon.HTTPUnauthorized('Authentication required',
                                      description,
                                      challenges,
                                      href='http://docs.example.com/auth')

def save_job(session, job):
    """
    save new job to database
    """
    try:
        session.add(job)
        session.commit()

    except Exception as e:
        logger.error("{} - {}".format(type(e), str(e)))
        session.rollback()
        raise

    finally:
        session.close()

def get_job(session, job_id):
    """
    get job from database
    """
    try:
        return session.query(ScrapingJob.jobdir).filter_by(id=job_id).first()[0]
    except Exception as e:
        logger.error("{} - {}".format(type(e), str(e)))
        raise

def is_job_slot_available(spider, job_list=None):
    """
    check if job slot is available, max 1 job for each spider
    """
    if not job_list:
        response = requests.get('{}/listjobs.json?project=default'.format(SCRAPYD_HOST))
        data = response.json()
        job_list = data['pending'] + data['running']

    for job in job_list:
        for k, v in job.items():
            if k == 'spider' and v == spider:
                break
        else:
            continue

        # job slot for spider not available
        # response message here
        print('job slot not available')
        return False

    else:  # job slot available
        print('job slot available')
        return True

def schedule_job(spider, params, session):
    """
    schedule crawling job
    """
    scheduler_url = '{}/schedule.json'.format(SCRAPYD_HOST)
    if isinstance(params, bytes):
        params = params.decode()

    if not isinstance(params, dict):
        params = json.loads(params)

    options = params.copy()
    options.update({'spider': spider, 'project': 'default'})
    response = requests.post(scheduler_url, options)
    data = response.json()
    if 'status' in data:
        status = data['status']
    if status == 'ok':
        success = True
        msg = 'Scheduled crawling for spider {}. JobID ({})'.format(spider, data['jobid'])
        index = None
        for k, v in enumerate(params['setting']):
            if 'JOBDIR' in v:
                index = k
                break
        jobdir = params['setting'][index].split('=')[-1]

        job = ScrapingJob(**{'id': data['jobid'], 'spider': spider, 'start': datetime.datetime.now(),
                             'status': 'started', 'input_param': json.dumps(params), 'jobdir': jobdir})
        if session:
            save_job(session, job)
    else:
        status = data['status']
        msg = data['message']

    return success, status, msg

def get_valid_filename(s):
    str_ = str(s).strip().replace(' ', '_')
    return re.sub(r'(?u)[^-\w.]', '', str_)

class Crawler(object):
    job_directory = 'data/crawljobs'
    date_format = '%Y_%m_%d_%H_%M_%S'
    max_jobdir_length = int(os.environ.get('MAX_JOBDIR_LENGTH', 10))

    def __init__(self):
        self.queue = CrawlerQueue()

    @falcon.before(validate_auth)
    def on_get(self, req, resp):

        params = req.params

        spider = params.get('spider', None)
        merk = params.get('merk', None)

        if not spider or not merk:
            resp.body = json.dumps({'error': 'parameter `spider` and `merk` must not be empty'}, indent=4)
            resp.status = falcon.HTTP_400
            return

        if merk.lower() == 'suzuki' and isinstance(params.get('model', None), list):
            params['model'] = " ".join([a for a in params.get('model', [])])

        delay = params.get('delay', None)
        model = params.get('model', None)
        if not spider:
            raise falcon.HTTPBadRequest('Missing Spider', 'Missing spider in query string parameters.')
        if not merk:
            raise falcon.HTTPBadRequest('Missing Merk', 'Missing merk in query string parameters.')
        if not model:
            raise falcon.HTTPBadRequest('Missing Model', 'Missing model in query string parameters.')

        # cleanse params
        param_list = ['spider', 'merk', 'model', 'tahun', 'year', 'mesin', 'engine', 'varian', 'type', 'tipe', 'delay']
        param_list += ['region']
        for k in list(params.keys()):
            if k not in param_list:
                del params[k]

        # add delay parameter
        if not delay:
            delay = CRAWLER_DELAY[spider]
        params['setting'] = ['DOWNLOAD_DELAY={}'.format(delay)]
        job_dir = '{}/{}_{}_{}_{}'.format(self.job_directory, spider, merk, get_valid_filename(model),
                                          datetime.datetime.now().strftime(self.date_format))

        params['jobdir'] = job_dir
        params['setting'] += ['JOBDIR={}'.format(job_dir)]

        def sort_by_datetime(datestring):
            splitup = datestring.split('_')[-6:]
            dttm = '_'.join(splitup)
            return datetime.datetime.strptime(dttm, self.date_format)

        # housekeeping check for CRAWLING JOBDIR
        spiderjoblist = []
        for itm in os.listdir(self.job_directory):
            if itm.split('_')[0] == spider:
                spiderjoblist.append(itm)
        if len(spiderjoblist) > self.max_jobdir_length:
            spiderjoblist.sort(key=sort_by_datetime, reverse=True)
            while len(spiderjoblist) > self.max_jobdir_length:  # max jobdir length as settings
                shutil.rmtree('{}/{}'.format(self.job_directory, spiderjoblist.pop()))

        # add job for this spider to queue db
        self.queue.push(spider, params)

        # check if there is available job slot for current spider
        response = requests.get('{}/listjobs.json?project=default'.format(SCRAPYD_HOST))
        data = response.json()
        pending_running_jobs = data['pending'] + data['running']
        if not is_job_slot_available(spider, pending_running_jobs):  # queue crawling job
            success = True
            status = 'ok'
            msg = 'Crawling job has been added to queue list'

        else:  # schedule crawling job

            try:
                params, date_added = self.queue.pop(spider)  # get next job params
                session = self.session if hasattr(self, 'session') else None
                success, status, msg = schedule_job(spider, params, session)

            except Exception as e:
                success = False
                status = 'not ok'
                msg = str('{} - {}'.format(type(e), str(e)))

        resp.status = falcon.HTTP_200
        resp.body = json.dumps({'success': success, 'status': status, 'message': msg}, indent=4)

class CrawlJob(object):

    def on_get(self, req, resp):

        # limit scheduled crawler process
        response = requests.get('{}/jobs'.format(SCRAPYD_HOST))
        resp.status = falcon.HTTP_200
        resp.content_type = 'text/html'
        text = response.text
        text = re.sub('<title>Scrapyd</title>', '<title>Crawling Jobs</title>', text)
        text = re.sub('
Jobs
', '
Crawling Jobs
', text)
        text = re.sub('
.*Go back.*
', '', text)
        text = text.replace("
Pending
", '')
        resp.body = text

class Log(object):

    def on_get(self, req, resp, spider, log_file):
        response = requests.get('{}/logs/default/{}/{}'.format(SCRAPYD_HOST, spider, log_file))
        resp.status = falcon.HTTP_200
        text = response.text
        resp.body = text

def load_template(name):
    path = os.path.join('templates', name)
    with open(os.path.abspath(path), 'r') as fp:
        return jinja2.Template(fp.read())

class QueueList(object):
    """Crawl Queue Resource"""
    def __init__(self):
        self.queue = CrawlerQueue()

    def on_get(self, req, resp):
        # render template
        template = load_template('crawler_queue.html')

        resp.status = falcon.HTTP_200
        resp.content_type = 'text/html'
        resp.body = template.render(spiders=self.queue.get_all_queue())

class CancelQueue(object):
    """Cancel Crawling Queue"""
    def __init__(self):
        self.queue = CrawlerQueue()

    def on_get(self, req, resp, spider=None, queue_id=None):
        success, status, msg = False, 'ok', '{} queue with id:{} '.format(spider, queue_id)
        try:
            # remove queue from db
            if self.queue.delete(spider, queue_id):
                success = True
                msg += 'deleted'
            else:
                msg += 'not found'

        except Exception as e:
            success = False
            status = 'exception occured.'
            msg = str('{} - {}'.format(type(e), str(e)))
            print('{} - {}'.format(type(e), str(e)))

        resp.status = falcon.HTTP_200
        resp.body = json.dumps({'success': success, 'status': status, 'message': msg}, indent=4)

class CancelJob(object):
    """Cancel Crawling Job"""
    def on_get(self, req, resp, job_id=None):
        success, status, msg = False, 'ok', 'job with id:{} '.format(job_id)
        try:
            # stop running crawling
            cancel_url = '{}/cancel.json'.format(SCRAPYD_HOST)
            options = {'project': 'default', 'job': job_id}
            response = requests.post(cancel_url, options)
            data = response.json()
            if data['status'] == 'ok' and data['prevstate']:
                success = True
                msg += ' is being canceled. please wait a moment.'
            else:
                msg = 'cancelling job failed.'

        except Exception as e:
            success = False
            status = 'exception occured.'
            msg = str('{} - {}'.format(type(e), str(e)))
            print('{} - {}'.format(type(e), str(e)))

        resp.status = falcon.HTTP_200
        resp.body = json.dumps({'success': success, 'status': status, 'message': msg}, indent=4)

class CrawlerQueueCheck(object):
    def __init__(self):
        self.queue = CrawlerQueue()

    def on_get(self, req, resp, spider=None, job_id=None):

        delay = 5
        sleep(delay)  # add delay
        print('checking queue for {}...'.format(spider))
        success, status, msg = False, 'ok', 'crawler queue checked'
        try:
            if is_job_slot_available(spider):  # schedule new crawling job
                params, date_added = self.queue.pop(spider)  # get next job params
                session = self.session if hasattr(self, 'session') else None
                success, status, msg = schedule_job(spider, params, session)

        except Exception as e:
            success = False
            status = 'exception occured.'
            msg = str('{} - {}'.format(type(e), str(e)))

        resp.status = falcon.HTTP_200
        resp.body = json.dumps({'success': success, 'status': status, 'message': msg}, indent=4)

class TestResource(object):
    def on_get(self, req, resp):
        """Handles GET requests"""
        resp.status = falcon.HTTP_200  # This is the default status
        resp.body = ('\nTwo things awe me most, the starry sky '
                     'above me and the moral law within me.\n'
                     '\n'
                     '    ~ Immanuel Kant\n\n')

# falcon.API instances are callable WSGI apps
middleware_to_use = [AuthMiddleware()]
if CONNECTION_STRING:
    middleware_to_use.append(SQLAlchemySessionManager(Session))

app = falcon.API(middleware=middleware_to_use)

# Resources are represented by long-lived class instances
crawler = Crawler()
crawljob = CrawlJob()
log = Log()
crawler_queue_check = CrawlerQueueCheck()
queue_list = QueueList()
cancel_queue = CancelQueue()
cancel_job = CancelJob()

# will handle all requests to the '/crawl' URL path
app.add_route('/crawl', crawler)

# will handle all requests to the '/jobs' URL path
app.add_route('/jobs', crawljob)

# will handle all requests to the '/log' URL path
app.add_route('/logs/default/{spider}/{log_file}', log)

# check and update crawling queue
app.add_route('/crawler_queue_check/{spider}/{job_id}', crawler_queue_check)
app.add_route('/queue_list', queue_list)

# cancel crawl job
app.add_route('/cancel_job/{job_id}', cancel_job)

# cancel queue
app.add_route('/cancel_queue/{spider}/{queue_id}', cancel_queue)

# Resources are represented by long-lived class instances
test = TestResource()

# things will handle all requests to the '/test' URL path
app.add_route('/test', test)


import json
import boto3
import time
import math

dynamodb = boto3.client('dynamodb')

def lambda_handler(event, context):
    if not 'id' in event:
        return {
            'statusCode':400,
            'body': 'Bad Request'
        }
    travelId = event['id']
    timestamp = math.trunc(time.time())
    response = dynamodb.update_item(
        TableName = 'travel',
        Key = {
            'travelId':{
                'S':travelId
            }
        },
        UpdateExpression = 'SET #S = :f, #E = :e',
        ExpressionAttributeNames = {
            "#S":'status',
            "#E":'endtimestamp'
        },
        ExpressionAttributeValues = {
            ':f':{
                'BOOL':False
            },
            ':e':{
                'S':str(timestamp)
            }
        })
    if(response['ResponseMetadata']['HTTPStatusCode'] == 200):
        return {
            'statusCode': 200,
            'body': 'Status of '+ travelId +' change to false'
        }
    else:
        return {
            'statusCode':response['ResponseMetadata']['HTTPStatusCode']
        }


"""
1/21/2019
"""

from flask import Flask, make_response
#from config import DEBUG

__author__ = 'cardinalion'

app = Flask(__name__)
app.config.from_object('config')
#print(isinstance(app.config,dict))

"""
#another url registration method
app.add_url_rule('/hello',view_func=hello)
#基于类的视图（即插视图）
"""

#'/hello/' redirects to '/hello', with status number 301,302, but doesn't have unique url
#click CLTL + B to go to the source code
#route function also uses app.add_url_rule('',)
@app.route('/hello')
def hello():
    #status code
    #content-type http headers
    #content-type = text/html    text/plain,
    #Response object
    headers = {
        'content-type' : 'application/json',
        'location' : 'http://www.bing.com'
    }
    #response= make_response('', 301)
    #response.headers = headers
    #return response
    return '',301,headers
    #return ''

#debug mode on，no need to restart server，detailed error info
#use Ethernet IPv4 address in ipconfig
if __name__ == '__main__':
    #production enviroment, nginx + uswgi
    app.run(host ='0.0.0.0',debug=app.config['DEBUG'],port=81)


from itertools import combinations

lotto = []
k = []

while True:
    tmp = [ x for x in map(int, input().split()) ]
    if tmp[0] == 0:
        break
    k.append(tmp[0])
    del tmp[0]

    lotto.append(tmp)

for k in range(0, len(k)):
    result = list(combinations(lotto[k], 6))
    for i in range(0, len(result)):
        for j in range(0, len(result[i])):
            print(result[i][j], end=" ")
        print(end="\n")
    print()


import json
import random

from stellar_data import STARMAP, get_star_name
from util import filter_by_distance

print("There are {} stars".format(len(STARMAP)))

CURRENT_YEAR = 130

for _ in range(170):
    print("IT IS ", CURRENT_YEAR + 1950)
    for star in STARMAP:
        for planet in star.planets:
            planet.grow(CURRENT_YEAR)
            # if planet.population > 0:
                # print(planet.name, "has a population of", "{:,}".format(planet.population), "and is growing by",
                #       str(round((planet.growth_rate(CURRENT_YEAR) - 1) * 100, 2)) + "%")
            if planet.population > 1e8:
                colonists = int(planet.population * 0.05)
                planet.population -= colonists

                colonisation_candidates = [s for s in filter_by_distance(20, planet.star.closest_stars(STARMAP)) if any(p.habitable for p in s.planets)]
                while colonists > 0:
                    destination_star = random.choice(colonisation_candidates)
                    destination_planet = random.choice([p for p in destination_star.planets if p.habitable])
                    if destination_planet.founding_year is None:
                        destination_planet.founding_year = CURRENT_YEAR
                        if str(destination_star.hip) in destination_star.name:
                            destination_star.name = get_star_name()
                            print("HIP", destination_star.hip, "was renamed", destination_star.name, "when it was colonised!")
                        print(destination_planet.name, "was colonised!")
                    destination_planet.population += min(1e6, colonists)
                    colonists -= min(1e6, colonists)

    CURRENT_YEAR += 1

output = [star.to_json() for star in STARMAP]
with open("colony_map2.json", "w") as out_file:
    json.dump(output, out_file)

print()


# -*- coding: utf-8 -*-
"""Optimizer

These functions are for optimizers.

"""

import torch.optim as optim
import torch.nn as nn

SUPPORTED_OPTIMIZER = {
    "adam": None,
}

def get_optimizer(cfg: object, network: object) -> object:
    """Get optimizer function

    This is function to get optimizer.

    Args:
        cfg: Config of optimizer.
        network: Network of model.

    Returns:
        Optimizer object.

    Raises:
        NotImplementedError: If the optimizer you want to use is not suppoeted.

    """

    optimizer_name = cfg.train.optimizer.name

    if not optimizer_name:
        return None

    if optimizer_name not in SUPPORTED_OPTIMIZER:
        raise NotImplementedError('The optimizer is not supported.')

    if optimizer_name == "adam":
        return optim.Adam(network.parameters(),
                          lr=cfg.train.optimizer.lr,
                          weight_decay=cfg.train.optimizer.decay)

    else:
        return SUPPORTED_OPTIMIZER[optimizer_name](cfg, network)


import torch
import torch.nn as nn
import numpy as np
from torch.autograd import Variable
import os
import json
from decoder.LM_model import LanguageModel
from decoder.LM_loader import MyLoader
from settings import REVIEWS_FOLDER, MSD_SPLIT_FOLDER, MSD_NPY_FOLDER, DECODER_FOLDER, ENCODER_FOLDER
from decoder.LM_settings import embedding_dim, hidden_dim, music_dim, epochs
from decoder.LM_testing import generate_sample
from encoder.model import local_model, global_model
import pickle

from time import time

class Trainer():
    """
    A simple training cradle
    """

    def __init__(self, vocab_size, model, optimizer, train_data):
        self.model = model
        self.optimizer = optimizer
        self.train_data = train_data
        self.vocab_size = vocab_size
        self.loss_function = nn.CrossEntropyLoss()

    def save_model(self, path):
        torch.save(self.model.state_dict(), path)

    def init_weight(self, m):
        if type(m) == nn.Embedding:
            m.weight.data.uniform_(-0.1, 0.1)
        if type(m) == nn.LSTM:
            m.weight_ih_l0.data.uniform_(-1/np.sqrt(hidden_dim),
                                         1/np.sqrt(hidden_dim))
            m.weight_hh_l0.data.uniform_(-1/np.sqrt(hidden_dim),
                                         1/np.sqrt(hidden_dim))
            m.bias_ih_l0.data.fill_(0)
            m.bias_hh_l0.data.fill_(0)
        if type(m) == nn.Linear:
            smoothing = 0.1
            unigram = np.zeros(self.vocab_size)
            for article in self.train_data:
                for word in article:
                    unigram[word] += 1
            unigram = np.log(unigram/self.vocab_size)
            unigram = unigram*(1-smoothing)+smoothing/self.vocab_size
            unigram = torch.from_numpy(unigram)
            m.bias.data.copy_(unigram)

    def run(self, epochs):
        print("Begin training...")
        train_loader = MyLoader(self.train_data)
        step = 10000
        # print(len(train_loader))

        for e in range(epochs):

            start_time = time()
            epoch_loss = 0
            batch_number = 0
            self.model.train()
            current_threshold = step

            for i, (in_data, out_data) in enumerate(train_loader):
                batch_number += 1
                self.optimizer.zero_grad()
                X = Variable(torch.from_numpy(in_data).long()).cuda()
                Y = Variable(torch.from_numpy(out_data).long()).cuda()
                out = self.model(X)
                out = out.view(out.shape[0]*out.shape[1], out.shape[2])
                Y = Y.view(-1)
                loss = self.loss_function(out, Y)
                loss.backward()
                self.optimizer.step()
                epoch_loss += loss.data[0]

                # print(time() - start_time)

                if i == current_threshold:
                    print("{0} examples treated in {1:.4f}s".format(i, time() - start_time))
                    current_threshold += step

            total_loss = epoch_loss/batch_number
            print("Epoch: {0}, train_loss: {1:.8f}, time: {2:.4f}".format(
                e+1, total_loss, time() - start_time))
            generate_sample(self.model, cuda=True)
            self.save_model(os.path.join(DECODER_FOLDER, 'LanguageModel_'+str(e)+'.pt'))

def train():

    print('Loading datasets')
    train_reviews = np.load(os.path.join(REVIEWS_FOLDER, 'train_reviews.npy'))
    test_reviews = np.load(os.path.join(REVIEWS_FOLDER, 'test_reviews.npy'))
    dev_reviews = np.load(os.path.join(REVIEWS_FOLDER, 'dev_reviews.npy'))
    indexer = json.load(open(os.path.join(REVIEWS_FOLDER, 'indexer.json')))
    reverse_indexer = json.load(open(os.path.join(REVIEWS_FOLDER, 'reverse_indexer.json')))
    reviews = np.concatenate((train_reviews, dev_reviews, test_reviews))
    del train_reviews, dev_reviews, test_reviews
    print('Datasets loaded')

    # TRAIN MODELS
    vocab_size = len(indexer)
    model = LanguageModel(vocab_size, embedding_dim, hidden_dim, music_dim).cuda()
    optimizer = torch.optim.Adam(model.parameters())
    trainer = Trainer(vocab_size, model, optimizer, reviews)
    model.apply(trainer.init_weight)
    trainer.run(epochs)
    trainer.save_model(os.path.join(DECODER_FOLDER, 'LanguageModel.pt'))

def train_with_audio():
    indexer = json.load(open(os.path.join(REVIEWS_FOLDER, 'indexer.json')))
    reverse_indexer = json.load(open(os.path.join(REVIEWS_FOLDER, 'reverse_indexer.json')))
    vocab_size = len(indexer)

    language_model = LanguageModel(vocab_size, embedding_dim, hidden_dim, music_dim).cuda()
    language_model.load_state_dict(torch.load(os.path.join(DECODER_FOLDER, 'LanguageModel_4.pt')))
    language_model.train()
    # filter(lambda p: p.requires_grad, language_model.parameters()))
    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, language_model.parameters()))
    loss_function = nn.CrossEntropyLoss()

    n_inputs = 512+512+768
    segment_size_list = [18, 27, 54]
    local_models = []
    for segment_size in segment_size_list:
        loc_model = local_model(segment_size).cuda()
        loc_model.load_state_dict(torch.load(os.path.join(ENCODER_FOLDER, 'local_model_'+str(segment_size)+'.pt')))
        loc_model.eval()
        local_models.append(loc_model)
    model = global_model(n_inputs, 512).cuda()
    model.load_state_dict(torch.load(os.path.join(ENCODER_FOLDER, 'global_model_18_27_54_9051_123.pt')))
    model.eval()

    id7d_to_path = pickle.load(open(os.path.join(MSD_SPLIT_FOLDER, '7D_id_to_path.pkl'), 'rb'))
    idmsd_to_id7d = pickle.load(open(os.path.join(MSD_SPLIT_FOLDER, 'MSD_id_to_7D_id.pkl'), 'rb'))

    pairs = json.load(open(os.path.join(MSD_SPLIT_FOLDER, 'pairs.json')))

    for e in range(epochs):

        start_time = time()
        epoch_loss = 0
        batch_number = 0
        song_number = 0

        for track_id, value in pairs.items():
            song_number += 1

            if song_number < 6:
                print("for testing")
                continue

            review = value['review']
            review = np.expand_dims(np.array(review), axis=0)

            try:
                npy_path = os.path.join(
                    MSD_NPY_FOLDER+'new_pitchfork', id7d_to_path[idmsd_to_id7d[track_id]][:-9]+'.npy')
                X = np.load(npy_path)
            except KeyError:
                print("No key?")
                continue
            except FileNotFoundError:
                print("No audio?")
                continue

            try:
                X = torch.from_numpy(X[:, :1255]).unsqueeze(0)
                X = Variable(X).cuda().float()
                X = torch.cat([loc_model(X)[1] for loc_model in local_models], dim=1)
                _, music = model(X)
            except:
                print("Weird song (too short?)")
                print(X.shape)
                continue

            train_loader = MyLoader(review)
            for in_data, out_data in train_loader:
                batch_number += 1
                optimizer.zero_grad()
                X = Variable(torch.from_numpy(in_data).long()).cuda()
                Y = Variable(torch.from_numpy(out_data).long()).cuda()
                expanded_music = music.unsqueeze(1).expand(X.shape[0], X.shape[1], -1).contiguous()
                out = language_model(X, expanded_music)
                out = out.view(out.shape[0]*out.shape[1], out.shape[2])
                Y = Y.view(-1)
                loss = loss_function(out, Y)
                loss.backward(retain_graph=True)
                optimizer.step()
                epoch_loss += loss.data.item()

        total_loss = epoch_loss/batch_number
        print("Epoch: {0}, train_loss: {1:.8f}, time: {2:.4f}".format(
            e+1, total_loss, time() - start_time))
        generate_sample(language_model, cuda=True)
        torch.save(language_model.state_dict(), os.path.join(DECODER_FOLDER, 'LanguageModel_audio_'+str(e)+'.pt'))

    conn_pf.close()

if __name__ == "__main__":
    # train()
    train_with_audio()


#https://www.youtube.com/watch?v=mqhxxeeTbu0&list=PLzMcBGfZo4-n4vJJybUVV3Un_NFS5EOgX
from flask import Flask, render_template, request, redirect, url_for
import os
import man_db
import MyStorage
from datetime import datetime

Man_db = man_db.Man_Db(path="..\\ressources\\data\\db.db")
storage = MyStorage.Storage()

p = str(os.getcwd())
p = p.replace("\src", "")
p = p + "\\ressources\\templates"
app = Flask(__name__, template_folder=str(p))

@app.route("/")
def index():
	return render_template("index.html")

@app.route("/history/", methods=["POST", "GET"])
def history():

	page = request.args.get("page")
	prevp = request.args.get("previouspage")
	nextp = request.args.get("nextpage")
	pp = storage.getter(item="item1")
	if pp == None:
		pp = 1
	if page == None:
		if prevp and nextp == None:
			page = 1
		elif prevp == "True":
			page = int(pp) - 1
		elif nextp == "True":
			page = int(pp) + 1
		else:
			page = 1

	storage.setter("item1", page)

	p1=1
	p2=2
	p3=3
	p4=4
	p5=5

	url_history = []
	date_history = []
	history = Man_db.read_db_history(item="Url, Date", Order="ORDER BY Date DESC")
	for x in range(0, len(history)):
		url, date = history[x]
		url_history.append(url)
		date_history.append(date)
	for x in range(0, len(date_history)-1):
		if date_history[x] != "":
			date_history[x] = datetime.strptime(date_history[x], "%Y-%m-%d %H:%M:%S.%f")
			date_history[x] = date_history[x].strftime("%m/%d/%Y, %H:%M:%S")

	dates = []
	urls = []
	page = int(page)
	max = page*10
	for x in range(int(max)-11, int(max)-1):
		try:
			urls.append(url_history[x])
			dates.append(date_history[x])
		except:
			urls.append("")
			dates.append("")

	if urls[9] == "" or dates[9] == "":
		p1 = ""
		p2 = ""
		p3 = ""
		p4 = ""
		p5 = ""

	return render_template("history.html", url1=urls[0], url2=urls[1], url3=urls[2], url4=urls[3], url5=urls[4], url6=urls[5], url7=urls[6], url8=urls[7], url9=urls[8], url10=urls[9],
						   date1=dates[0], date2=dates[1], date3=dates[2], date4=dates[3], date5=dates[4], date6=dates[5], date7=dates[6], date8=dates[7], date9=dates[8], date10=dates[9],
						   p1=p1, p2=p2, p3=p3, p4=p4, p5=p5)

@app.route("/newv")
def newv():
	return render_template("newv.html")

@app.route("/darkmode")
def darkmode():
	return render_template("darkmode.html")

@app.route("/whitemode")
def whitemode():
	return render_template("whitemode.html")

@app.route("/https_warning")
def https_warning():
	return render_template("https_warning.html")

@app.route("/error_404")
def error_404():
	return render_template("404.html")

@app.route("/source")
def source():
	return render_template("cpage.html")

if __name__ == "__main__":
	app.run(host="localhost", port=80)
	app.jinja_env.auto_reload = True
	app.config['TEMPLATES_AUTO_RELOAD'] = True


import numpy as np
import cv2
import sys
import pandas as pd
import pickle
import math as m
import tkinter
import csv, glob
import yaml
import os, sys
from tkinter.filedialog import askopenfilename

annotation_type = input("[NOTE.....] If you are trying to store positive examples, enter yes; else type no: ")
with open("config.yml", "r") as stream:
    config_data= yaml.safe_load(stream)
path = config_data["root_dir"]
grab_size = int(config_data["annotation_size"])
ix,iy = -1,-1
# mouse callback function
def draw_circle(event,x,y,flags,param):
    global ix,iy
    if event == cv2.EVENT_LBUTTONDBLCLK:
        if ix>0 and iy>0:
            cv2.circle(img, (ix, iy), round(grab_size/8), (0, 255, 0), -1)
        ix,iy = x,y

# Read the yml configuration file to iddentify the root directory

root = tkinter.Tk()
movieName =  askopenfilename(filetypes=[("Video files","*")])
cap = cv2.VideoCapture(movieName)
nframe =cap.get(cv2.CAP_PROP_FRAME_COUNT)
i=0
steps= nframe/20
if annotation_type == "yes":
    while(cap.isOpened() & (i<(nframe-steps))):
      i = i + steps
      print("[UPDATING.....]{}th/{} frame detected and stored".format(i, nframe))
      cap.set(cv2.CAP_PROP_POS_FRAMES,i)
      ret, img = cap.read()
      cv2.namedWindow('image')
      cv2.setMouseCallback('image',draw_circle)
      counter = 0
      while(1):
          counter+=1
          cv2.imshow('image',img)
          k = cv2.waitKey(20) & 0xFF
          if k == 27:
              break
          elif k == ord('a'):
              crop_img = img[iy-grab_size:iy+(grab_size),ix-grab_size:ix+(grab_size)]
              cv2.imwrite(path + "yes/yes{}.jpg".format(counter), crop_img)
      cv2.destroyAllWindows()
elif annotation_type == "no":
    while(cap.isOpened() & (i<(nframe-steps))):
      i = i + steps
      print("[UPDATING.....]{}th/{} frame detected and stored".format(i, nframe))
      cap.set(cv2.CAP_PROP_POS_FRAMES,i)
      ret, img = cap.read()
      cv2.namedWindow('image')
      cv2.setMouseCallback('image',draw_circle)
      counter = 0
      while(1):
          counter+=1
          cv2.imshow('image',img)
          k = cv2.waitKey(20) & 0xFF
          if k == 27:
              break
          elif k == ord('a'):
              crop_img = img[iy-grab_size:iy+(grab_size),ix-grab_size:ix+(grab_size)]
              cv2.imwrite(path + "no/no{}.jpg".format(counter), crop_img)
      cv2.destroyAllWindows()


from typing import Tuple

import numpy as np #type: ignore

import colors

# Tile graphics structured type compatible with Console.tiles_rgb
graphic_dt = np.dtype(
    [("ch",np.int32), # Unicode codepoint,
     ("fg", "3B"),    # 3 unsigned bytes, for RGB colors
     ("bg","3B"),
    ]
)

# Tile struct used for statically defined tile data.
# (First notable departure from tutorial: I still plan to draw unseen tiles,
#  albeit black on black, because I want to use the character data from the
#  console for some git commands. Maybe there's a nicer way to do this, but
#  I'm too lazy to think about it atm.)
tile_dt = np.dtype(
    [("walkable",np.bool),    # True if can be walked over
     ("transparent",np.bool), # True if can be seen through
     ("dark",graphic_dt),     # graphics for when not in FOV
     ("light",graphic_dt),    # graphics for when in FOV
     ("unseen",graphic_dt),   # graphics for when unexplored
     ("name_index",np.int32), # index into list of tile names
    ]
)

def new_tile(
        *, # Enforce use of keywords
        walkable: int, transparent: int,
        dark: Tuple[int,Tuple[int,int,int],Tuple[int,int,int]],
        light: Tuple[int,Tuple[int,int,int],Tuple[int,int,int]],
        name:str,
        ) -> np.ndarray:
    # Automatically generate the "unseen" graphic
    """Helper for defining individual tile types."""
    unseen = (dark[0],colors.default_bg,colors.default_bg)
    # Possibly useful when designing UI (makes map boundary visible):
    #unseen = (dark[0],colors.test_bg,colors.test_bg)
    return np.array((walkable,transparent,dark,light,unseen,name),dtype=tile_dt)

# Define tiles here =========================
# (I know this is annoying, but the reason for this list is so that the
#  name can be stored as an int in the tile array, for numpy reasons)
tile_names = [
    "floor",
    "wall",
    "stairs (down)",
    "stairs (up)",
    "water",
]
floor = new_tile(
    walkable=True, transparent=True,
    dark=(ord(" "),colors.default_dim_fg,colors.default_bg),
    light=(ord(" "),colors.default_fg,colors.default_FOV_bg),
    name=tile_names.index("floor"),
)
wall = new_tile(
    walkable=False, transparent=False,
    dark=(ord("#"),colors.default_dim_fg,colors.default_bg),
    light=(ord("#"),colors.default_fg,colors.default_FOV_bg),
    name=tile_names.index("wall"),
)
water = new_tile(
    walkable=False, transparent=True,
    dark=(ord("~"),colors.water_dim_fg,colors.default_bg),
    light=(ord("~"),colors.water_fg,colors.default_FOV_bg),
    name=tile_names.index("water"),
)
down_stairs = new_tile(
    walkable=True, transparent=True,
    dark=(ord(">"),colors.default_dim_fg,colors.default_bg),
    light=(ord(">"),colors.default_fg,colors.default_FOV_bg),
    name=tile_names.index("stairs (down)"),
)
up_stairs = new_tile(
    walkable=True, transparent=True,
    dark=(ord("<"),colors.default_dim_fg,colors.default_bg),
    light=(ord("<"),colors.default_fg,colors.default_FOV_bg),
    name=tile_names.index("stairs (up)"),
)
tile_names = np.array(tile_names) # To support fancy indexing


from sklearn.model_selection import train_test_split
from sklearn.utils import indexable

def split_train_test(X, y, test_size=0.2):
    """Split a dataset into train and test sets.

    Args:
        X (np.array or pd.DataFrame): Features.
        y (np.array or pd.DataFrame): Labels.
        test_size (float): Percentage in the test set.

    Returns:
        list: List with the dataset splitted (X_train, X_test, y_train, y_test).

    Examples:
        >>> X = np.random.randint(0,10, (100,5))
        >>> y = np.random.randint(0,1, 100)
        >>> X_train, X_test, y_train, y_test = split_train_test(X, y, test_size=0.2)
        >>> print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
        (80, 5) (20, 5) (80,) (20,)
    """
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=test_size, random_state=42, stratify=y
    )
    return X_train, X_test, y_train, y_test

def split_train_val_test(*arrays, val_size=0.2, test_size=0.2, random_seed=42):
    """Split a dataset into train, validation and test sets.

    Args:
        *arrays (list, np.array, pd.DataFrame): Dataset to split, it can be one or two objects with the same number of
                                                rows.
        val_size (float): Percentage in the validation set.
        test_size (float): Percentage in the test set.
        random_seed (float): Seed.

    Returns:
        list: List with the dataset splitted. If the input is one element `X`, the output will be
            `X_train, X_val, X_test`. If the input is two elements `X,y`, the output will be
            `X_train, X_val, X_test, y_train, y_val, y_test`.

    Examples:
        >>> X = np.random.randint(0,10, (100,5))
        >>> y = np.random.randint(0,1, 100)
        >>> X_train, X_val, X_test, y_train, y_val, y_test = split_train_val_test(X, y, val_size=0.2, test_size=0.2)
        >>> print(X_train.shape, X_val.shape, X_test.shape)
        (60, 5) (20, 5) (20, 5)
        >>> print(y_train.shape, y_val.shape, y_test.shape)
        (60,) (20,) (20,)
        >>> df = pd.DataFrame(np.random.randn(100, 5), columns=list('ABCDE'))
        >>> df_train, df_val, df_test = split_train_val_test(df, val_size=0.2, test_size=0.2)
        >>> print(df_train.shape, df_val.shape, df_test.shape)
        (60, 5) (20, 5) (20, 5)
    """
    n_arrays = len(arrays)
    if n_arrays == 1:
        X_train, X_test = train_test_split(
            *arrays, test_size=test_size, random_state=random_seed
        )
        X_train, X_val = train_test_split(
            X_train, test_size=val_size / (1 - test_size), random_state=random_seed
        )
        return X_train, X_val, X_test
    elif n_arrays == 2:
        X, y = indexable(*arrays)
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=test_size, random_state=random_seed, stratify=y
        )
        X_train, X_val, y_train, y_val = train_test_split(
            X_train,
            y_train,
            test_size=val_size / (1 - test_size),
            random_state=random_seed,
            stratify=y_train,
        )
        return X_train, X_val, X_test, y_train, y_val, y_test
    else:
        raise ValueError("Wrong number of inputs")


"""
j.h.abel 25 Jan 2017

Using the cwt code based on Tanya Leise's code for analyzing rhythms of wheel
running. Follows the ideas in
"""

#
#
# -*- coding: utf-8 -*-
#basic packages
from __future__ import division
import sys
import os

#3rd party packages
import numpy as np
from scipy import stats, signal
from scipy.interpolate import UnivariateSpline
import matplotlib.pyplot as plt
from matplotlib import gridspec

import PlotOptions as plo
import Bioluminescence as bl

import pywt

example_path1 = 'data/example2/Channel1.csv'
example_path2 = 'data/example2/Channel2.csv'
example_path3 = 'data/example2/Channel9.csv'

def load_invivo_data(path, binning=None):
    """
    loads the wheel running rhythms from a csv located at path
    if binning is int, bins into that many minute units
    """
    # load the data and assemble parts
    data = np.loadtxt(path, delimiter=',', dtype=str)
    counts = data[1:,3]
    recordings = [loc for loc in range(len(counts)) if counts[loc].isdigit()]
    days   = data[1:,0].astype(float)
    hours  = data[1:,1].astype(float)
    mins   = data[1:,2].astype(float)
    times  = np.asarray([days[i]*24+(hours[i]+mins[i]/60)
                    for i in range(len(days))])

    # get into format times, counts
    ret_times = times[recordings]
    ret_counts = counts[recordings].astype(float)

    # if binning
    if type(binning) is int:
        new_times = times[::binning]
        bins = np.hstack([new_times,times[-1]])
        digitized = numpy.digitize(ret_times, bins)
        new_counts = np.array([ret_counts[digitized == i].sum()
                        for i in range(1, len(bins))])
        # re-assign
        ret_counts = new_counts
        ret_times = new_times

    return ret_times, ret_counts

def find_cwt0_region(region, x, mhw):
    """ finds the zero-cross of the mexican hat wavelet, as suggested in
    Leise 2013. """
    specific_region = mhw[region[0]:region[1]]
    specific_x_locs = np.arange(len(specific_region))+region[0]
    specific_times = x[specific_x_locs]

    # find peaks
    zeros_loc = numpy.where(numpy.diff(numpy.sign(specific_region)))[0]
    # spline interpolate for simplicity
    spl = UnivariateSpline(specific_times, specific_region, k=3, s=0)

    onset_time =spl.roots()
    if len(onset_time)>0:
        return onset_time[0]

def cwt_onset(times, counts):
    """ identifies wheel running onset from times and counts"""
    x = times
    y = counts
    bin_param = 15
    widths = np.arange((60/bin_param),
                       10*(60/bin_param),
                       0.05*(60/bin_param))

    #pywt version
    cwtmatr, freqs = pywt.cwt(y, widths, 'mexh')
    periods = 1/freqs/(60/bin_param)

    inst_per_loc = np.argmax(np.abs(cwtmatr.T),1)
    inst_ampl = np.asarray([cwtmatr.T[idx, loc]
                            for idx,loc in enumerate(inst_per_loc)])

    # identify regions of increasing activity
    maxs = signal.argrelextrema(inst_ampl, np.greater, order=20)[0]
    mins = signal.argrelextrema(inst_ampl, np.less, order=20)[0]

    # find the 0-crosses here
    inc_regions = []
    for mini in mins:
        try:
            maxi = np.min(maxs[np.where(maxs>mini)])
            inc_regions.append([mini,maxi])
        except ValueError:
            # if there is no following max
            pass

    # get the onset times
    onsets = []
    for region in inc_regions[:-1]:
        onsets.append(find_cwt0_region(region,x,inst_ampl))

    return_data = {
        'x'         : x,
        'y'         : y,
        'onsets'    : np.array(onsets),
        'dwt'       : cwtmatr,
        'regions'   : np.array(inc_regions),
        'days'      : np.array(onsets)//24
    }
    return return_data

example_paths = [example_path1, example_path2, example_path3]

all_cwts = []
for path in example_paths:
    times, counts = load_invivo_data(path, binning = 15)
    all_cwts.append(cwt_onset(times, counts))

names = ['Channel 1', 'Channel 2', 'Channel 9']

plt.figure()
plo.PlotOptions(ticks='in')
ax = plt.subplot()

ax.plot([4,11], np.array([15,15]), color='f', lw=2, label='Lo')
ax.plot([15,28], np.array([10,10]), color='f', ls='--', lw=2, label='Hi/Lo')

ax.plot( all_cwts[0]['days'], all_cwts[0]['onsets']%24,
'fx', mew=1, label='Channel1 (VIPCre/+;Ai32/+)')
ax.plot( all_cwts[1]['days'], all_cwts[1]['onsets']%24,
'f+', mew=1, label='Channel2 (VIPCre/+;Ai32/+)')
ax.plot( all_cwts[2]['days'],
        np.unwrap(all_cwts[2]['onsets']%24 *2*np.pi/24)*24/(2*np.pi),
        'k.', mew=1, label='Channel9 (Control)')
ax.set_xlim([-1,28])

#plo.format_2pi_axis(ax, y=True, x=False)
ax.set_xlabel('Day')
ax.set_ylabel('Activity Onset (h)')
plt.legend()
plt.tight_layout()

plt.plot(pulse_timing2[:7]/24, pulse_2_phases)

for i,wav in enumerate(wavs):
    plt.plot(wav['x']/24., wav['period'], label = names[i])
plt.legend()
plt.ylim([22,25])

# actogram test
import timbre.readdata as rd

cols = {'day':'Day', 'hour':'Hour', 'min':'Min', 'val':'Cnt/min'}

def my_coltypes(line):
    # line = {'date': '2010-01-01', 'time': '10:00:00', 'val': '50'}
    day = line['day']
    hour = line['hour']
    min_ = line['min']
    # convert values from string to int
    year, mon, day = int(0), int(0), int(day)
    hour, min_, sec = int(hour), int(min_), int(0)

    result = {}
    result['time'] = datetime.datetime(2016, 12, day, hour, min_, 0)
    result['cnt'] = int(line['val'])
    result['cnt'] = int(result['cnt'])
    return result

dataset = rd.CSVReader(example_path1, columns=cols, coltypes=my_coltypes,
                       delimiter=',', skiplines=1)

actogram(dataset, 'cnt')


import cv2

video_stream = 0
threshold = 80
circle_radius = 325

def convert_frame_to_grayscale(frame):
    return cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

def calculate_variance_of_laplacian(grayscale_frame):
    return cv2.Laplacian(grayscale_frame, cv2.CV_64F).var()

def add_center_circle_to_window(grayscale_frame, original_frame, text_color):
    M = cv2.moments(grayscale_frame)
    cX = int(M["m10"] / M["m00"])
    cY = int(M["m01"] / M["m00"])
    cv2.circle(original_frame, (cX, cY), circle_radius, text_color, 4)

def add_text_to_window(original_frame, display_text, text_color):
    cv2.putText(original_frame, display_text, (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, text_color, 2)

def is_tire_in_focus():
    cap = cv2.VideoCapture(video_stream)
    while True:
        ret, original_frame = cap.read()

        if not ret:
            break

        grayscale_frame = convert_frame_to_grayscale(original_frame)
        variance = calculate_variance_of_laplacian(grayscale_frame)

        if variance > threshold:
            text_color = (0, 255, 0)
            display_text = 'Tire in focus'
        else:
            display_text = 'Tire is out of focus'
            text_color = (0, 0, 255)

        add_text_to_window(original_frame, display_text, text_color)
        add_center_circle_to_window(grayscale_frame, original_frame, text_color)
        cv2.imshow("LiveStream Video", original_frame)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    cap.release()
    cv2.destroyAllWindows()

is_tire_in_focus()