import shutil
import subprocess
import os, sys
from utils import Platforms, fsl_assert, Stages, ShaderTarget, StageFlags, ShaderBinary, fsl_platform_assert
import tempfile, struct

fsl_basepath = os.path.dirname(__file__)

_config = {
    Platforms.DIRECT3D11: ('FSL_COMPILER_FXC', 'fxc.exe'),
    Platforms.DIRECT3D12: ('FSL_COMPILER_DXC', 'dxc.exe'),
    Platforms.VULKAN:     ('VULKAN_SDK', 'Bin/glslangValidator.exe'),
    Platforms.ANDROID:    ('VULKAN_SDK', 'Bin/glslangValidator.exe'),
    Platforms.SWITCH:     ('VULKAN_SDK', 'Bin/glslangValidator.exe'),
    Platforms.QUEST:      ('VULKAN_SDK', 'Bin/glslangValidator.exe'),
    Platforms.MACOS:      ('FSL_COMPILER_MACOS', 'metal.exe'),
    Platforms.IOS:        ('FSL_COMPILER_IOS', 'metal.exe'),
    Platforms.ORBIS:      ('SCE_ORBIS_SDK_DIR', 'host_tools/bin/orbis-wave-psslc.exe'),
    Platforms.PROSPERO:   ('SCE_PROSPERO_SDK_DIR', 'host_tools/bin/prospero-wave-psslc.exe'),
    Platforms.XBOX:       ('GXDKLATEST', 'bin/XboxOne/dxc.exe'),
    Platforms.SCARLETT:   ('GXDKLATEST', 'bin/Scarlett/dxc.exe'),
    Platforms.GLES:       ('VULKAN_SDK', 'Bin/glslangValidator.exe'),
}

def get_available_compilers():
    available = []
    for lang, compiler_path in _config.items():
        if get_compiler_from_env(*compiler_path, _assert=False):
            available += [lang.name]
    return available

def get_status(bin, params):
    # For some reason calling subprocess.getstatusoutput on these platforms fails
    if sys.platform == "darwin" or sys.platform == "linux":
        result = subprocess.run([bin] + params, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        return result.returncode, result.stderr.decode() + result.stdout.decode()
    else:
        return subprocess.getstatusoutput([bin] + params)

def get_compiler_from_env(varname, subpath = None, _assert=True):

    if os.name == 'posix' and 'metal.exe' in subpath:
        return 'xcrun'

    if sys.platform == "linux" and 'VULKAN_SDK' in varname:
        return "glslangValidator"

    if not varname in os.environ:
        print('WARN: {} not in env vars'.format(varname))
        if _assert: assert False
        return None

    compiler = os.environ[varname]
    if subpath: compiler = os.path.join(compiler, subpath)
    if not os.path.exists(compiler):
        print('WARN: {} doesn\'t exist'.format(compiler))
        if _assert: assert False
        return None
    return os.path.abspath(compiler)

def util_shadertarget_dx(stage, shader_target):
    stage_dx = {
        Stages.VERT: 'vs',
        Stages.FRAG: 'ps',
        Stages.COMP: 'cs',
        Stages.GEOM: 'gs',
        Stages.TESC: 'hs',
        Stages.TESE: 'ds',
    }
    return stage_dx[stage] + shader_target.name[2:]

def util_shadertarget_metal(shader_target):
    return shader_target.name[4:].replace("_",".")

def compile_binary(platform: Platforms, debug: bool, binary: ShaderBinary, src, dst):

    # remove any existing binaries
    if os.path.exists(dst): os.remove(dst)

    if sys.platform == 'win32': # todo
        if src.startswith("//"):
            src = src.replace("//", "\\\\", 1)

        if dst.startswith("//"):
            dst = dst.replace("//", "\\\\", 1)

    bin = get_compiler_from_env(*_config[platform])

    class CompiledDerivative:
        def __init__(self):
            self.hash = 0
            self.code = b''

    compiled_derivatives = []
    for derivative_index, derivative in enumerate(binary.derivatives[platform]):

        compiled_filepath = os.path.join(tempfile.gettempdir(), next(tempfile._get_candidate_names()))
        params = []

        if platform in [Platforms.VULKAN, Platforms.QUEST, Platforms.SWITCH, Platforms.ANDROID]:
            if debug: params += ['-g']
            params += ['-V', src, '-o', compiled_filepath, '-I'+fsl_basepath]
            params += ['-S', binary.stage.name.lower()]
            params += ['--target-env', 'vulkan1.1']

        elif platform == Platforms.DIRECT3D11:
            if debug: params += ['/Zi']
            params += ['/T', util_shadertarget_dx(binary.stage, ShaderTarget.ST_5_0)] # d3d11 doesn't support other shader targets
            params += ['/I', fsl_basepath, '/Fo', compiled_filepath, src]

        elif platform == Platforms.DIRECT3D12:
            if debug: params += ['/Zi', '-Qembed_debug']
            params += ['/T', util_shadertarget_dx(binary.stage, binary.target)]
            params += ['/I', fsl_basepath, '/Fo', compiled_filepath, src]

        elif platform == Platforms.ORBIS:
            # todo: if debug: ...

            params = ['-DGNM', '-O4', '-cache', '-cachedir', os.path.dirname(dst)]
            shader_profile = {
                Stages.VERT: 'sce_vs_vs_orbis',
                Stages.FRAG: 'sce_ps_orbis',
                Stages.COMP: 'sce_cs_orbis',
                Stages.GEOM: 'sce_gs_orbis',
                Stages.TESC: 'sce_hs_off_chip_orbis',
                Stages.TESE: 'sce_ds_vs_off_chip_orbis',
            }

            profile = shader_profile[binary.stage]

            # Vertex shader is local shader in tessellation pipeline and domain shader is the vertex shader
            if binary.stage == Stages.VERT:
                if StageFlags.ORBIS_TESC in binary.flags:
                    profile = 'sce_vs_ls_orbis'
                elif StageFlags.ORBIS_GEOM in binary.flags:
                    profile = 'sce_vs_es_orbis'

            params += ['-profile', profile]
            params += ['-I'+fsl_basepath, '-o', compiled_filepath, src]

        elif platform == Platforms.PROSPERO:
            # todo: if debug: ...
            params = ['-DGNM', '-O4']
            shader_profile = {
                Stages.VERT: 'sce_vs_vs_prospero',
                Stages.FRAG: 'sce_ps_prospero',
                Stages.COMP: 'sce_cs_prospero',
                Stages.GEOM: 'sce_gs_prospero',
                Stages.TESC: 'sce_hs_prospero',
                Stages.TESE: 'sce_ds_vs_prospero',
            }
            params += ['-profile', shader_profile[binary.stage]]
            params += ['-I'+fsl_basepath, '-o', compiled_filepath, src]

        elif platform == Platforms.XBOX:
            if debug: params += ['/Zi', '-Qembed_debug']
            params += ['/T', util_shadertarget_dx(binary.stage, binary.target)]
            params += ['/I', fsl_basepath, '/D__XBOX_DISABLE_PRECOMPILE', '/Fo', compiled_filepath, src]

        elif platform == Platforms.SCARLETT:
            if debug: params += ['/Zi', '-Qembed_debug']
            params += ['/T', util_shadertarget_dx(binary.stage, binary.target)]
            params += ['/I', fsl_basepath, '/D__XBOX_DISABLE_PRECOMPILE', '/Fo', compiled_filepath, src]

        elif platform == Platforms.GLES:
            params = [src, '-I'+fsl_basepath]
            params += ['-S', binary.stage.name.lower()]
            with open(compiled_filepath, "wb") as dummy:
                dummy.write(b'NULL')

        elif platform == Platforms.MACOS:
            # todo: if debug: ...
            if os.name == 'nt':
                params = ['-I', fsl_basepath]
                params += ['-dD', '-o', compiled_filepath, src]
            else:
                params = '-sdk macosx metal '.split(' ')
                params += [f"-std=macos-metal{util_shadertarget_metal(binary.target)}"]
                params += ['-I', fsl_basepath]
                params += ['-dD', src, '-o', compiled_filepath]

        elif platform == Platforms.IOS:
            # todo: if debug: ...
            if os.name == 'nt':
                params = ['-I', fsl_basepath]
                params += ['-dD','-std=ios-metal2.2', '-mios-version-min=8.0', '-o', compiled_filepath, src]
            else:
                params = '-sdk iphoneos metal'.split(' ')
                params += [f"-std=ios-metal{util_shadertarget_metal(binary.target)}"]
                params += ['-mios-version-min=11.0']
                params += ['-I', fsl_basepath]
                params += ['-dD', src, '-o', compiled_filepath]

        params += ['-D' + define for define in derivative ]

        cp = subprocess.run([bin] + params, stdout=subprocess.PIPE)
        fsl_platform_assert(platform, cp.returncode == 0, binary.fsl_filepath, message=cp.stdout.decode())

        with open(compiled_filepath, 'rb') as compiled_binary:
            cd = CompiledDerivative()
            cd.hash = derivative_index # hash(''.join(derivative)) #TODO If we use a hash it needs to be the same as in C++
            cd.code = compiled_binary.read()
            compiled_derivatives += [ cd ]

    with open(dst, 'wb') as combined_binary:

        # Needs to match:
        #
        # struct FSLMetadata
        # {
        #     uint32_t mUseMultiView;
        # };
        #
        # struct FSLHeader
        # {
        #     char mMagic[4];
        #     uint32_t mDerivativeCount;
        #     FSLMetadata mMetadata;
        # };

        num_derivatives = len(compiled_derivatives)
        combined_binary.write(struct.pack('=4sI', b'@FSL', num_derivatives) )
        combined_binary.write(struct.pack('=I', StageFlags.VR_MULTIVIEW in binary.flags))
        data_start = combined_binary.tell() + 24 * num_derivatives
        for cd in compiled_derivatives:
            combined_binary.write(struct.pack('=QQQ', cd.hash, data_start, len(cd.code)))
            data_start += len(cd.code)
        for cd in compiled_derivatives:
            combined_binary.write(cd.code)
    return 0


import mtbot.protocol as p
from hashlib import sha1
from base64 import b64encode
from time import time

# Class for seqnums
# And Packetbuffer later

class Seqnum:
    """docstring for Seqnum.
    Managing mt seqnums"""

    def __init__(self):
        self.seqs = {}
        self.next = p.seqnum_initial

    def pop(self, seq):
        key = str(int.from_bytes(seq, "big") - p.seqnum_initial)
        if key in self.seqs:
            self.seqs.pop(key)

    def get(self):
        s = self.next
        self.next += 1
        if self.next > p.seqnum_max:
            self.next = p.seqnum_initial

        return s.to_bytes(2, byteorder="big")

    def buffer(self, seq, buf):
        self.seqs[str((int.from_bytes(seq, "big") - p.seqnum_initial))] = (buf, time())

    def toresend(self):
        out = []
        for i in self.seqs:
            buf, t = self.seqs[i]
            if t+2 < time():
                out.append(buf)
                self.seqs[i] = (buf, time())

        return out

def translate_password(name, password):
    if len(password) == 0:
        return ""

    sr = name + password
    sh = sha1(sr).digest()
    return b64encode(sh)

def std_string(sr):
    n = sr
    if isinstance(sr, str):
        n = sr.encode()
    return numbtobyte(len(n), 2) + n

def from_std_wstring(string):
    newstring = ""
    for i in range(len(string[2:])):
        s = string[i + 2]
        if s != 0:
            newstring += chr(s)

    return newstring

def to_std_wstring(string):
    newbyte = b""
    len = 0
    for i in string:
        len += 1
        newbyte += b"\x00" + i.encode()

    return numbtobyte(len, 2) + newbyte

def std_stringtobyte(bytes):
    return bytes[2:]

def bytetonumb(bytes):
    return int.from_bytes(bytes, "big")

def numbtobyte(numb, size=1):
    return numb.to_bytes(size, byteorder="big")

# access the bytes in the array using byt[4:5] instead of byt[4] to get binary data.

def makePacket(peer_id, channel, data):
    byt = p.protocol_id
    byt += peer_id
    byt += p.channel[channel]
    byt += data
    return byt

def makeDataControl(controltype, cdata=b""):
    byt = p.packagetype["control"]
    byt += controltype
    if cdata != -1:
        byt += cdata

    return byt

def makeDataOriginal(command, data=-1):
    byt = p.packagetype["original"]
    byt += command
    if data != -1:
        byt += data

    return byt

def makeDataReliable(seqnum, data):
    byt = p.packagetype["reliable"]
    byt += seqnum
    byt += data
    return byt

def readPacket(data):
    if len(data) >= 9:
        if data[:4] == p.protocol_id:
            peer_id = data[4:6]
            channel = data[6]
            type = data[7:8]
            reliable = False
            seqnum = b"\x00"
            if type == p.packagetype["control"]:
                type, data = "control", readControl(data[7:])
            elif type == p.packagetype["original"]:
                type, data = "original", readOriginal(data[7:])
            elif type == p.packagetype["split"]:
                type, data = "split", data[7:]
            elif type == p.packagetype["reliable"]:
                reliable = True
                seqnum, type, data = readReliable(data[7:])

            return peer_id, channel, type, reliable, seqnum, data

    return False

def readControl(data):
    controltype = data[1:2]
    if controltype == p.controltype["ack"]:
        seqnum = data[2:4]
        return "ack", seqnum
    elif controltype == p.controltype["set_peer_id"]:
        peer_id = data[2:4]
        return "set_peer_id", peer_id
    elif controltype == p.controltype["disco"]:
        return "disco", "a"

def readOriginal(data):
    command = data[1:3]
    data = data[3:]
    return command, data

def readReliable(data):
    seqnum = data[1:3]
    type = data[3:4]
    if type == p.packagetype["control"]:
        return seqnum, "control", readControl(data[3:])
    elif type == p.packagetype["original"]:
        return seqnum, "original", readOriginal(data[3:])
    elif type == p.packagetype["split"]:
        return seqnum, "split", data[3:]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul  2 18:44:36 2020

@author: czhang
"""

# this is to create B0 map from 3D phases of 3D fully sampled images.

## delete h5 variables
import h5py
import numpy as np
import torch
from fastMRI.data import transforms
from training_utils import helpers
from training_utils.linear_mapping import LeastSquares
from pdb import set_trace as bp
import time
from skimage.restoration import unwrap_phase
from matplotlib import pyplot as plt

fname = '/home/czhang/lood_storage/divh/BMEPH/Henk/STAIRS/other/Chaoping/raw_data/phases.h5'

with h5py.File(fname, 'r') as data:
    data.keys()
    phases = data['phases'][:]
    mask_brain = data['mask_brain'][:]
    phase_unwrapped = np.zeros(phases.shape)
    for i in range(phases.shape[0]):
        phase_unwrapped[i] = unwrap_phase(np.ma.array(phases[i], mask=np.zeros(phases[i].shape)))

    TEs = (3.0, 11.5, 20.0, 28.5)
    phase_diff_set = []
    TE_diff = []
    TEnotused = 3
    # obtain phase differences and TE differences
    testplot = []
    for i in range(0, phase_unwrapped.shape[0] - TEnotused):
        phase_diff_set.append((phase_unwrapped[i + 1] - phase_unwrapped[i]).flatten())
        phase_diff_set[i] = phase_diff_set[i] - np.round(np.sum(phase_diff_set[i] * mask_brain.flatten())/np.sum(mask_brain.flatten())/2/np.pi) *2*np.pi
        TE_diff.append(TEs[i + 1] - TEs[i])

    phase_diff_set = np.stack(phase_diff_set, 0)
    TE_diff = np.stack(TE_diff, 0)

    # least squares fitting to obtain phase map
    scaling = 1e-3
    ls = LeastSquares()
    B0_map_tmp = ls.lstq_pinv(torch.from_numpy(np.transpose(np.expand_dims(phase_diff_set, 2), (1, 0, 2))), torch.from_numpy(np.expand_dims(TE_diff, 1) * scaling))
    B0_map = B0_map_tmp.reshape(phase_unwrapped.shape[1:4])
    B0_map = B0_map.numpy()

    data = h5py.File(fname, 'r+')
    data.__delitem__('B0_map')
    data.create_dataset('B0_map', data=B0_map)


from cx_Freeze import setup, Executable

exe=Executable(
     script="client.py",
     )
includefiles=["config.txt"]
includes=[]
excludes=[]
packages=['requests']
setup(

     version = "1.1",
     description = "No Description",
     author = "Name",
     name = "App name",
     options = {'build_exe': {'excludes':excludes,'packages':packages,'include_files':includefiles}},
     executables = [exe]
     )

# from cx_Freeze import setup, Executable

# setup(name='client',
#       version='1.1',
#       options={'build.exe':{'packages': ['requests'],
#                             'include_files': ['config.txt'],}},
#       executables = [Executable('client.py')]

# )

# options = {
#     'build_exe': {
#         'include_msvcr': True,
#       #   'excludes': excludes,
#         'includes': includes,
#         'zip_include_packages': zip_include_packages,
#         'build_exe': 'build_windows',
#         'include_files': include_files,
#     }
# }


'''Build the vocabulary for the yelp dataset'''

import json
from collections import Counter

# stop words are words that occur very frequently,
# and that don't seem to carry information
# about the quality of the review.
# we decide to keep 'not', for example, as negation is an important info.
# I also keep ! which I think might be more frequent in negative reviews, and which is
# typically used to make a statement stronger (in good or in bad).
# the period, on the other hand, can probably be considered neutral
# this could have been done at a later stage as well,
# but we can do it here as this stage is fast
stopwords = set(['.','i','a','and','the','to', 'was', 'it', 'of', 'for', 'in', 'my',
                 'that', 'so', 'do', 'our', 'the', 'and', ',', 'my', 'in', 'we', 'you',
                 'are', 'is', 'be', 'me'])

def process_file(fname, options):
    '''process a review JSLON lines file and count the occurence
    of each words in all reviews.
    returns the counter, which will be used to find the most frequent words
    '''
    print(fname)
    with open(fname) as ifile:
        counter = Counter()
        for i,line in enumerate(ifile):
            if i==options.lines:
                break
            if i%10000==0:
                print(i)
            data = json.loads(line)
            # extract what we want
            words = data['text']
            for word in words:
                if word in stopwords:
                    continue
                counter[word]+=1
        return counter

def parse_args():
    from optparse import OptionParser
    from base import setopts
    usage = "usage: %prog [options] <file_pattern>"
    parser = OptionParser(usage=usage)
    setopts(parser)
    parser.add_option("-n", "--nwords",
                      dest="nwords", default=20000, type=int,
                      help="max number of words in vocabulary, default 20000")
    (options, args) = parser.parse_args()
    if len(args)!=1:
        parser.print_usage()
        sys.exit(1)
    pattern = args[0]
    return options, pattern

if __name__ == '__main__':
    import os
    import glob
    import pprint
    from vocabulary import Vocabulary
    import parallelize

    options, pattern = parse_args()

    olddir = os.getcwd()
    os.chdir(options.datadir)

    fnames = glob.glob(pattern)

    nprocesses = len(fnames) if options.parallel else None
    results = parallelize.run(process_file, fnames, nprocesses, options)

    full_counter = Counter()
    for counter in results:
        full_counter.update(counter)

    vocabulary = Vocabulary(full_counter, n_most_common=options.nwords)
    vocabulary.save('index')

    pprint.pprint(full_counter.most_common(200))
    print(len(full_counter))
    print(vocabulary)
    os.chdir(olddir)


import os
import json

g = {
    "max-processes": 128,
    "output-size": 16,
    "compile-time": 5000,
    "compile-memory": 256,
    "mem-bonus": {
    },
    "time-bonus": {
    },
    "enabled": ['.c', '.cpp', '.py'],
}

def loadConfig():
    global g
    try:
        with open(os.path.dirname(__file__) + '/config.json') as f:
            fileconf = json.loads(f.read())
            for key in fileconf:
                g[key] = fileconf[key]
        print('Config loaded from config.json')
    except FileNotFoundError:
        print('Warning: config.json not found. Default config used.')

def generateConfig():
    with open(os.path.dirname(__file__) + '/Backend/config.sh', "w") as f:
        f.write('RAMDISKSIZE={}\n'.format(g['output-size'] + 8))


import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import rcParams
from matplotlib import style
import datetime
from os import path
style.use('ggplot')
rcParams.update({'font.size': 9})
fig, ax = plt.subplots(sharex=True, figsize=(8.5, 4.8))
fig_size = plt.rcParams["figure.figsize"]

df = pd.read_csv('U.S._Natural_Gas_Exports.csv')
df.dropna()
df = df.iloc[::-1]
print(df.columns)
mycolors = ['saddlebrown', '#3b5998', '#ffbf00']
a = df.plot(x = 'Month', y=['U.S. LNG Exports',\
                            'U.S. Natural Gas Pipeline Exports to Mexico',\
                            'U.S. Natural Gas Pipeline Exports to Canada',\
                             ], kind="area", color=mycolors, alpha=0.88, ax=ax)
for line in a.lines:
    line.set_linewidth(0)
a.xaxis.grid(False)
handles, labels = a.get_legend_handles_labels()
a.legend(reversed(handles), reversed(labels), bbox_to_anchor=(.85, -0.115), loc='best', ncol=1, fontsize='x-large')
plt.tick_params(axis='x', labelsize=8)
plt.xlabel("Date", color='black')
plt.ylabel("Billion Cubic Feet per Day (Bcf/d)",color='black')
plt.savefig(path.basename(__file__)+".png",bbox_inches='tight')
# plt.show()


# coding: utf-8

# In[89]:

import tarfile
import xml.etree.ElementTree as ET
import tqdm
import codecs

# In[87]:

members = []

tar = tarfile.open("unlabeled.tar.gz", "r:gz")
outfile = codecs.open("unlabeled.txt", 'w', 'utf-8')

for member in tar:
    f = tar.extractfile(member)

    if f is None:
        continue
    else:
        print(member.name)

    root = ET.fromstring(f.read())

    result = ''

    for para in root.iter('p'):
        result = ''

        if para.text is not None:
            result += para.text

        for child in para:
            if child is not None:
                if child.text is not None:
                    result += child.text
                if child.tail is not None:
                    result += child.tail

        outfile.write(result + "\n")

    outfile.flush()

outfile.close()
tar.close()


import math
import os
import random
import re
import warnings
from typing import Dict, List, Tuple, Union

import cv2
import numpy as np
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
import torch.optim as optim
from torch import nn
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader
from torch.utils.data.distributed import DistributedSampler
from tqdm.auto import tqdm

from .data import LDRDataset
from .models import EncoderDecoderModel, ImageSpaceLoss
from .tools import helper_functions

warnings.simplefilter("ignore")
TORCH_MAJOR = int(torch.__version__.split(".")[0])
TORCH_MINOR = int(torch.__version__.split(".")[1])
if TORCH_MAJOR == 1 and TORCH_MINOR < 8:
    from torch._six import container_abcs, int_classes, string_classes
else:
    import collections.abc as container_abcs

    int_classes = int
    string_classes = str

def setup(rank, world_size):
    os.environ["MASTER_ADDR"] = "localhost"
    os.environ["MASTER_PORT"] = "12355"

    # initialize the process group
    dist.init_process_group("gloo", rank=rank, world_size=world_size)

def cleanup():
    dist.destroy_process_group()

def seed_worker(worker_id):
    worker_seed = torch.initial_seed() % 2**32
    np.random.seed(worker_seed)
    random.seed(worker_seed)

class UnsupervisedHDRModel:
    def __init__(
        self,
        video_path: str,
        checkpoint_path: str = None,
        encoder: str = "SimpleEncoder",
        decoder: str = "SimpleDecoder",
        encoder_pretrained: bool = None,
        encoder_lr: float = 1e-4,
        decoder_lr: float = 1e-4,
        num_worker: int = 4,
        device_ids: Union[str, int, List[int]] = None,
        output_dir: str = "./",
        seed: int = 0,
    ) -> None:
        self.video_path = video_path
        self.encoder_lr = encoder_lr
        self.decoder_lr = decoder_lr
        self.num_worker = num_worker
        self.output_dir = output_dir
        self.seed = seed
        self.max_epoch = None
        self.iterator = 0
        self.epoch = 0

        seed = 0

        random.seed(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)
        torch.backends.cudnn.benchmark = False
        torch.backends.cudnn.deterministic = True
        g = torch.Generator()
        g.manual_seed(seed)

        if device_ids is None:
            self.device_ids = list(range(torch.cuda.device_count()))
            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        else:
            if device_ids == "cpu":
                self.device_ids = list()
                self.device = torch.device("cpu")
            else:
                self.device_ids = helper_functions.input2list(device_ids)
                self.device = torch.device(f"cuda:{self.device_ids[0]}")
        self.build_model(encoder, decoder, encoder_pretrained)
        self.configure_optimizers()
        self.train_dataloader = None
        self.predict_dataloader = None
        self.initialize_logger()
        if checkpoint_path:
            self.load_checkpoint(checkpoint_path)

    def build_model(
        self, encoder: str, decoder: str, encoder_pretrained: bool = False
    ) -> None:
        self.model = EncoderDecoderModel(encoder, decoder, encoder_pretrained)
        self.mse_loss = nn.MSELoss()
        self.image_space_loss = ImageSpaceLoss()

    def build_dataset(
        self,
        rank: int,
        world_size: int,
        train: bool,
        batch_size: int = 1,
        image_size: Tuple[int, int] = (512, 512),
    ) -> None:
        # TODO: dataloader
        dataset = LDRDataset(self.video_path, train=train, image_size=image_size)
        dataloader = DataLoader(
            dataset,
            batch_size=batch_size,
            shuffle=(rank == -1 and train),
            sampler=None
            if rank == -1
            else DistributedSampler(dataset, rank=rank, num_replicas=world_size),
            num_workers=self.num_worker,
            worker_init_fn=seed_worker,
            pin_memory=True,
            drop_last=train,
        )
        return dataloader

    def initialize_logger(self) -> None:
        self.train_results = {}
        self.lr = {}
        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)

        if not os.path.exists(os.path.join(self.output_dir, "checkpoints")):
            os.makedirs(os.path.join(self.output_dir, "checkpoints"))
        # TODO: save hyper parameters
        self.train_results["best_loss"] = float("inf")
        self.result_info = ""

    def forward(self, image: torch.Tensor) -> Dict[str, torch.Tensor]:
        """Forward pass. Returns logits."""
        outputs = {}
        outputs["pred_delta"] = self.model(image)
        outputs["pred_Ib"] = image * outputs["pred_delta"]
        return outputs

    def loss(
        self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
    ) -> Dict[str, torch.Tensor]:
        losses = {}
        losses["loss_delta"] = self.mse_loss(
            (batch["Ib"] + 1e-6) / (batch["Ih"] + 1e-6), outputs["pred_delta"]
        )
        losses["loss_image"] = self.image_space_loss(batch["Ib"], outputs["pred_Ib"])
        losses["loss"] = losses["loss_delta"] + losses["loss_image"]
        return losses

    def training_step(
        self, batch: Dict[str, torch.Tensor], batch_idx: int = None
    ) -> Dict[str, torch.Tensor]:
        step_output = {}
        outputs = self.forward(batch["Ih"])
        train_loss = self.loss(outputs, batch)
        step_output.update(train_loss)
        train_loss["loss"].backward()
        self.optimizer.step()
        return step_output

    def training_one_epoch(self, rank=-1, device=None) -> Dict[str, float]:
        self.model.train()
        losses = {}
        with tqdm(
            self.train_dataloader,
            position=0,
            leave=True,
            ascii=" ##",
            dynamic_ncols=True,
            disable=rank > 0,
        ) as t:
            for batch_idx, batch_data in enumerate(t):
                batch_data = self.cuda(batch_data, device=device if rank < 0 else rank)
                step_output = self.training_step(batch_data)
                t.set_description("Epoch %i Training" % self.epoch)
                print_losses = {}
                for key in step_output:
                    print_losses[key] = step_output[key].item()
                    if key in losses.keys():
                        losses[key].append(step_output[key].item())
                    else:
                        losses[key] = [step_output[key].item()]
                t.set_postfix(ordered_dict=dict(**print_losses))
                self.iterator += 1
        if rank <= 0:
            for key in losses:
                losses[key] = sum(losses[key]) / len(losses[key])
            if len(losses.values()) > 1 and not ("loss" in losses.keys()):
                losses["loss"] = sum(losses.values())
            for key in losses:
                self.train_results[key] = losses[key]
            if self.train_results["best_loss"] >= self.train_results["loss"]:
                self.train_results["best_loss"] = self.train_results["loss"]
                self.save_checkpoint(metric="loss")
            if self.epoch != 0:
                self.save_checkpoint()
            self.lr = {}
            self.lr["lr"] = [group["lr"] for group in self.optimizer.param_groups][0]
            self.result_info = ""
            for result_key, result_value in zip(
                self.train_results.keys(), self.train_results.values()
            ):
                self.result_info = (
                    self.result_info
                    + result_key
                    + ":"
                    + str(round(result_value, 4))
                    + " "
                )
            for lr_key, lr_value in zip(self.lr.keys(), self.lr.values()):
                self.result_info = (
                    self.result_info + lr_key + ":" + str(round(lr_value, 4)) + " "
                )
            print("Epoch %i" % self.epoch, self.result_info)
        self.epoch += 1
        return losses

    def fit_single(
        self, rank: int, max_epoch: int, world_size: int, batch_size: int = 1
    ) -> None:
        # TODO: early stopping
        self.train_dataloader = self.build_dataset(
            rank=rank,
            world_size=world_size,
            train=True,
            batch_size=batch_size,
        )
        use_ddp = world_size > 1
        if use_ddp:
            setup(rank, world_size)
            self.model = self.model.to(rank)
            self.model = DDP(self.model, device_ids=[rank])
        else:
            self.model = self.model.to(self.device)
        self.max_epoch = max_epoch
        for _ in range(max_epoch):
            self.training_one_epoch(rank=rank, device=self.device)
        if use_ddp:
            cleanup()

    def fit(self, max_epoch: int, batch_size: int = 1) -> None:
        if len(self.device_ids) > 1:
            mp.spawn(
                self.fit_single,
                args=(max_epoch, len(self.device_ids), batch_size),
                nprocs=len(self.device_ids),
                join=True,
            )
        else:
            self.fit_single(-1, max_epoch, len(self.device_ids), batch_size)

    def predict_step(
        self, batch: Dict[str, torch.Tensor], batch_idx: int = None
    ) -> Dict[str, np.ndarray]:
        with torch.no_grad():
            delta = self.model(batch["Ib"])  # (bs, 3, h, w)
            Il_2 = batch["Ib"] * delta
            delta_2l = self.model(Il_2)
            Il_4 = Il_2 * delta_2l
            Ih_2 = batch["Ib"] / delta
            delta_2h = self.model(Ih_2)
            Ih_4 = Ih_2 / delta_2h
        exposure_list = [
            Il_4,
            Il_2,
            batch["Ib"],
            Ih_2,
            Ih_4,
        ]
        exposure_list = np.stack(
            [
                (img.clone().detach().cpu().numpy().clip(0, 1) * 255).astype(np.uint8)
                for img in exposure_list
            ]
        ).transpose(
            1, 0, 3, 4, 2
        )  # (bs, 5, h, w, 3)

        merge_mertens = cv2.createMergeMertens()
        hdr_image = [merge_mertens.process(img_set) for img_set in exposure_list]
        output = {"exposure_list": exposure_list, "hdr_image": hdr_image}
        return output  # BGR

    def predict(
        self,
        frame_idx: Union[int, List[int]] = None,
        batch_size: int = 1,
        image_size=None,
    ) -> Dict[str, np.ndarray]:
        self.predict_dataloader = self.build_dataset(
            rank=-1,
            world_size=1,
            train=False,
            batch_size=batch_size,
            image_size=image_size,
        )
        best_checkpoint = os.path.join(self.output_dir, "checkpoints", "best_loss.pth")
        print(f"Start loading best checkpoint from {best_checkpoint}")
        self.load_checkpoint(best_checkpoint)
        self.cuda(self.model, device=self.device)
        self.model.eval()
        print("Finish loading!")
        output = {"exposure_list": [], "hdr_image": []}
        if frame_idx is None:
            for batch in tqdm(self.predict_dataloader):
                batch = self.cuda(batch, self.device)
                output_batch = self.predict_step(batch)
                output["exposure_list"].append(output_batch["exposure_list"])
                output["hdr_image"].append(output_batch["hdr_image"])
        else:
            frame_idx = helper_functions.input2list(frame_idx)
            for i in tqdm(
                range(math.ceil(len(frame_idx) / self.predict_dataloader.batch_size))
            ):
                batch = []
                for f in frame_idx[
                    i
                    * self.predict_dataloader.batch_size : (i + 1)
                    * self.predict_dataloader.batch_size
                ]:
                    batch.append(self.predict_dataloader.dataset[f])
                batch = self.cuda(batch, self.device)
                if len(batch) == 1:
                    batch = {k: b.unsqueeze(0) for k, b in batch[0].items()}
                else:
                    batch = helper_functions.concat_data(batch)
                output_batch = self.predict_step(batch)
                output["exposure_list"].append(output_batch["exposure_list"])
                output["hdr_image"].append(output_batch["hdr_image"])
        output["exposure_list"] = np.concatenate(output["exposure_list"])
        output["hdr_image"] = np.concatenate(output["hdr_image"])
        return output

    def load_checkpoint(self, checkpoint_path):
        self.model.load_state_dict(
            torch.load(
                checkpoint_path,
                map_location=torch.device("cpu"),
            )["model_state_dict"]
        )

    def save_checkpoint(self, metric=None):
        checkpoint = {}
        if metric is None:
            file_path = os.path.join(self.output_dir, "checkpoints", "last.pth")
        else:
            file_path = os.path.join(
                self.output_dir, "checkpoints", "best_" + metric + ".pth"
            )
            checkpoint = {
                "best_epoch": self.epoch,
                "best_" + metric: self.train_results["best_" + metric],
            }
        checkpoint["model_state_dict"] = (
            self.model.module.state_dict()
            if isinstance(self.model, DDP)
            else self.model.state_dict()
        )

        torch.save(checkpoint, file_path)

    def configure_optimizers(self) -> optim.Optimizer:

        self.optimizer = optim.Adam(
            [
                {"params": self.model.encoder.parameters(), "lr": self.encoder_lr},
                {"params": self.model.decoder.parameters(), "lr": self.decoder_lr},
            ],
            lr=self.decoder_lr,
            weight_decay=0.0001,
        )

    def cuda(self, x, device=None):
        np_str_obj_array_pattern = re.compile(r"[SaUO]")
        if torch.cuda.is_available():
            if isinstance(x, torch.Tensor):
                x = x.cuda(non_blocking=True, device=device)
                return x
            elif isinstance(x, nn.Module):
                x = x.cuda(device=device)
                return x
            elif isinstance(x, np.ndarray):
                if x.shape == ():
                    if np_str_obj_array_pattern.search(x.dtype.str) is not None:
                        return x
                    return self.cuda(torch.as_tensor(x), device=device)
                return self.cuda(torch.from_numpy(x), device=device)
            elif isinstance(x, float):
                return self.cuda(torch.tensor(x, dtype=torch.float64), device=device)
            elif isinstance(x, int_classes):
                return self.cuda(torch.tensor(x), device=device)
            elif isinstance(x, string_classes):
                return x
            elif isinstance(x, container_abcs.Mapping):
                return {key: self.cuda(x[key], device=device) for key in x}
            elif isinstance(x, container_abcs.Sequence):
                return [
                    self.cuda(np.array(xi), device=device)
                    if isinstance(xi, container_abcs.Sequence)
                    else self.cuda(xi, device=device)
                    for xi in x
                ]

    def to_cpu(self, x):
        if isinstance(x, torch.Tensor) and x.device != "cpu":
            return x.clone().detach().cpu()
        elif isinstance(x, np.ndarray):
            return x
        elif isinstance(x, container_abcs.Mapping):
            return {key: self.to_cpu(x[key]) for key in x}
        elif isinstance(x, container_abcs.Sequence):
            return [self.to_cpu(xi) for xi in x]
        else:
            return x


# http://localhost:3000/objects
import requests

url = "localhost:3000/objects"

payload = "{\n        \"id\": 5,\n        \"item\": \"The Fiancés\",\n        \"artist\": \"Pierre Auguste Renoir\",\n        \"collection\": \"Wallraf–Richartz Museum, Cologne, Germany\",\n        \"date\": \"1868\"\n    }"
headers = {
  'content-type': 'application/json'
}

response = requests.request("POST", url, headers=headers, data = payload)

print(response.text.encode('utf8'))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/11/30 14:47
# @Author  : qingping.niu
# @File    : StartTime.py
# @desc    :

import os,time,datetime
import uiautomator2 as u2

def getdevices():
    devices = []
    result = os.popen("adb devices").readlines()
    result.reverse()
    try:
        for line in result:
            li = line.strip()
            if not len(li) or "attached" in li or "?" in li or "offline" in li:
                continue
            else:
                devices.append(li.split()[0])
        return devices
    except Exception as e:
        print(e)

def start(devices,appName):
    d = u2.connect(devices)
    # d(scrollable=True).scroll.to(text=u"Notes")
    d(scrollable=True).fling.horiz.forward()
    d(text=u"Notes").click()
    begin = datetime.datetime.now()
    # end = None
    if d(text="Notes").exists:
        end = datetime.datetime.now()
        k = (end - begin)
        print(k.total_seconds())
    print("dddd")

if __name__=="__main__":
    devices = getdevices()
    print(devices[0])
    start(devices[0],appName="Notes")


import discord
import yaml

with open("data/users.yaml", "r") as ymlfile:
    users = yaml.load(ymlfile, Loader=yaml.BaseLoader)

def find(ctx, typ):
    for i in users[typ]:
        if str(i) == str(ctx.message.author):
            return True
    return False

async def check_admin(ctx):
    if find(ctx, "admin"):
        return True
    else:
        await ctx.send("You can't use **admin** commands.")
        return False

async def admin(ctx):
    return await check_admin(ctx)


"""Utility file to seed kindred database from Native-Land data"""

from sqlalchemy import func
from models import Tribe, Language, connect_to_db, db
from routes import app
import json

def json_reader(file_path):
	"""Opens & loads json files"""

	with open(file_path) as file:
		json_dict = json.load(file)

	return json_dict

def load_tribes():
	"""Load Tribe info from indigenousTerritories.json into
	database."""

	print('Tribes')

	Tribe.query.delete()

	tribe_file = "seed_data/indigenousTerritories.json"
	tribe_dict = json_reader(tribe_file)

	i = 0

	for key in tribe_dict['features']:
		name = tribe_dict['features'][i]['properties']['Name']
		region = None
		description = None
		i += 1

		tribe = Tribe(name=name,
					  region=region,
					  description=description
					  )

		db.session.add(tribe)

	db.session.commit()

def load_languages():
	"""Load language infor from indigenousLanguages.json into
	database."""

	print('Languages')

	Language.query.delete()

	lang_file = "seed_data/indigenousLanguages.json"
	lang_dict = json_reader(lang_file)

	i = 0

	for key in lang_dict['features']:
		language_name = lang_dict['features'][i]['properties']['Name']
		i += 1

		language = Language(language_name=language_name)

		db.session.add(language)

	db.session.commit()

def set_val_tribe_id():
	"""Set value for the next tribe_id after seeding database"""

	result = db.session.query(func.max(Tribe.tribe_id)).one()
	max_id = int(result[0])

	query = "SELECT setval('tribes_tribe_id_seq', :new_id)"
	db.session.execute(query, {'new_id': max_id + 1})
	db.session.commit()

if __name__ == "__main__":
    connect_to_db(app)

    # In case tables haven't been created, create them
    db.create_all()

    # Import different types of data
    load_tribes()
    load_languages()
    set_val_tribe_id()


from numpy import linspace
from xspec import *
from cstat_deviation import *
import matplotlib.pyplot as plt
import numpy as np

def compute_deviation(file_name):
    """
    This function computes the cstat deviation from an xcm file

    Args:
        file_name (.xcm): xcm file

    Returns:
        the cstat deviation for the given xcm file
    """
    Xset.restore(file_name)
    Fit.perform()
    xspec_cstat = Fit.statistic
    dof = Fit.dof

    # data values should be in counts and not counts/s
    t_e = AllData(1).exposure
    data_values = []
    for i in range (len(AllData(1).values)):
        # Alldata(1).values returns a tuple and I want a list
        data_values.append(AllData(1).values[i]*t_e)

    # model values should be in counts and limited to noticed bins
    Plot.device = '/null'
    Plot.xAxis = 'channel'
    Plot('counts')
    model_values = Plot.model()

    return db_compute_goodness_of_the_fit_from_cstat_v1(data_values,model_values,dof,xspec_cstat, verbose=False)

def select_models(file_names_list, deviation_min):
    """
    This function selects the models among a list that have a deviation > deviation_min

    Args:
        file_names_list (list): array of xcm file names to be loaded
        deviation_min (float): minimum deviation in unit of sigma

    Returns:
        An list of xcm file names where deviation > deviation_min
    """

    selected_models = []
    for file in file_names_list:
        cstat_dev = compute_deviation(file)

        if cstat_dev >= deviation_min:
            selected_models.append(file)

    return selected_models

def select_models_plot(file_names_list, deviation_min):
    """_summary_

    Args:
        file_names_list (_type_): _description_
        deviation_min (_type_): _description_
    """

    # list of deviation and file numbers
    cstat_deviations = []
    file_numbers = []
    for file in file_names_list:
        cstat_deviations.append(compute_deviation(file))
        file_numbers.append(int(file[1:4]))

    dev_min = [deviation_min]*len(file_names_list)

    plt.style.use(['science','no-latex'])

    plt.xlabel('model ID')
    plt.ylabel('deviation / sigma')

    plt.plot(file_numbers, cstat_deviations, 'ro')
    plt.plot(file_numbers, dev_min, '-b')
    plt.show()


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec

import torch
import torch.nn as nn
import torch.optim as optim

from pathlib import Path

from .model import EncoderModel, DecoderModel

class Modelrunner():

    def __init__(self, model_def, load_latest = False ):
        self.models_root = "models"
        self.model_def   = model_def

        last_session = self.getLastSession()
        if(load_latest):
            self.session = last_session
        else:
            self.session = last_session + 1

        self.encoders = {}
        self.metaencoder = self.setupEncoder()
        self.decoder = self.setupDecoder()

        self.loss_function = nn.SmoothL1Loss(reduction='sum')
        # self.fig, self.inputPlots, self.outputPlots = self.setup_plot()

    def save_all(self):
        print("Saving all models...")
        for encoder in self.encoders.values():
            encoder.save()

        self.metaencoder.save()
        self.decoder.save()
        print("All models saves")

    def run_step(self, sequences):
        # if( self.n_iter % 50 == 0):
        #     print("Epoc: {:06d}-{:02d} Loss: {}".format(self.n_iter, self.name, self.losses[-1]))
        #     self.writer.add_scalar(f"Loss/train_{self.name}", self.losses[-1], self.n_iter)
        #     self.plot_graph(batch, self.output)
        #     self.writer.add_figure(f"Samples/train_{self.name}", self.fig, self.n_iter)
        outputs = {}

        decoder = self.decoder
        metaencoder = self.metaencoder

        decoder.model.train()
        metaencoder.model.train()

        decoder.zero_grad()
        metaencoder.zero_grad()
        for device, sequence in sequences.items():

            batch_t = torch.from_numpy(sequence.astype(np.float32)).cuda()

            if(batch_t.ndim == 2):
                batch_t = batch_t.reshape(
                    (self.model_def.sequence_length, 1,  self.model_def.num_channels)
                )

            if( device != 0):
                if( device not in self.encoders):
                    self.encoders[device] = self.setupEncoder(f"encoder_{device:02d}")

                encoder = self.encoders[device]
                encoder.model.train()

                # Encoder
                encoder.zero_grad()
                loss, z, _ = self.run_model(encoder, decoder, batch_t)
                loss.backward()
                encoder.optimizer.step()

                # Metaencoder
                meta_loss, _, _ = self.run_model(metaencoder, decoder, batch_t)
                meta_loss.backward()

                z = z.squeeze().detach().cpu().numpy()
                loss = loss.item()
                outputs[device] = (loss, z)

            else:
                # Metaencoder
                meta_loss, meta_z, _ = self.run_model(metaencoder, decoder, batch_t)
                meta_loss.backward()

                loss = meta_loss.item()
                outputs[device] = (loss, None)

        metaencoder.optimizer.step()
        decoder.optimizer.step()

        return outputs

    def evaluate(self, sequences):
        outputs = {}
        meta_outputs = {}

        decoder = self.decoder
        metaencoder = self.metaencoder

        for device, sequence in sequences.items():

            batch_t = torch.from_numpy(sequence.astype(np.float32)).cuda()

            if(batch_t.ndim == 2):
                batch_t = batch_t.reshape(
                    (self.model_def.sequence_length, 1,  self.model_def.num_channels)
                )

            if( device not in self.encoders):
                self.encoders[device] = self.setupEncoder(f"encoder_{device:02d}")

            encoder = self.encoders[device]

            # Encoder
            loss, z, output = self.run_model(encoder, decoder, batch_t)

            z = z.squeeze().detach().cpu().numpy()
            output = output.squeeze().detach().cpu().numpy()
            loss = loss.item()

            outputs[device] = (loss, z, output)

            # Metaencoder
            meta_loss, meta_z, _ = self.run_model(metaencoder, decoder, batch_t)

            meta_z = meta_z.squeeze().detach().cpu().numpy()
            meta_loss = meta_loss.item()
            meta_outputs[device] = (loss, meta_loss, meta_z)

        return outputs, meta_outputs

    def run_model(self, encoder, decoder, batch_t):
        mu, logvar =  encoder.model(batch_t)
        z = self.reparameterize(mu, logvar)

        output = decoder.model(z)

        loss = self.loss_function(output, batch_t)

        return loss, z, output

    def reparameterize(self, mu, logvar):
        std = logvar.mul(0.5).exp_()
        # return torch.normal(mu, std)
        esp = torch.randn(*mu.size()).cuda()
        z = mu + std * esp
        return z

    def setupEncoder(self, name = None):

        if(name):
            path = Path(self.models_root, f"Session_{self.session:04d}", name)
        else:
            path = Path(self.models_root, "metaencoder")

        model = EncoderModel(path, self.model_def)
        model.load()

        return model

    def setupDecoder(self):
        path = Path(self.models_root, "decoder")
        model = DecoderModel(path, self.model_def)
        model.load()

        return model

    def getLastSession(self):
        path = Path(self.models_root)

        sessions = []
        for p in path.glob('Session_*'):
            if(p.is_dir()):
                try:
                    sessions.append(int(p.stem[-4:]))
                except:
                    pass
        if(len(sessions)):
            return max(sessions)
        else:
            return 0

    # def setup_plot(self):
    #     fig = plt.figure()

    #     gs = gridspec.GridSpec(1, 2, figure=fig)

    #     inputPlot  = gs[0].subgridspec(self.num_channels, 1)
    #     inputPlots = [0.5] * self.num_channels
    #     for ch in range(self.num_channels):
    #         inputPlots[ch] = fig.add_subplot(inputPlot[ch,:])
    #         inputPlots[ch].set_ylim(0.0, 1.0)
    #         inputPlots[ch].plot(range(self.seq_length ), np.zeros(self.seq_length ), 'b-')

    #     outputPlot  = gs[1].subgridspec(self.num_channels, 1)
    #     outputPlots = [0.5] * self.num_channels
    #     for ch in range(self.num_channels):
    #         outputPlots[ch] = fig.add_subplot(outputPlot[ch,:])
    #         outputPlots[ch].set_ylim(0.0, 1.0)
    #         outputPlots[ch].plot(range(self.seq_length ), np.zeros(self.seq_length ), 'r-')

    #     # zAx     = fig.add_subplot(gs[3])

    #     return fig, inputPlots, outputPlots

    # def plot_graph(self, batch, output ):

    #     for ch in range(self.num_channels):
    #         input_y = batch[:,0,ch]
    #         line = self.inputPlots[ch].get_lines()[0]
    #         line.set_ydata(input_y)

    #         output_y =  output[:,0,ch]
    #         line = self.outputPlots[ch].get_lines()[0]
    #         line.set_ydata(output_y)


from selenium.webdriver.firefox.options import Options as FirefoxOptions
import filesHandler as fh
import lpCrawler as lp

def scrape_charts(filepath):
    """ Scrape the data of all the charts of Polish Radio Program 3. """
    ff_options = FirefoxOptions()
    ff_options.add_argument('--headless')
    driver = lp.LPCrawler(options=ff_options)
    data = driver.get_charts()
    fh.write_json(filepath, data)
    driver.quit()


import logging

import pandas as pd
from zenml import step

class IngestData:
    """
    Ingesting data from the data_path provided.
    """
    def __init__(self, path: str):
        """
        Args:
            data_path: path to the data.
        """
        self.path = path

    def get_data(self):
        """
        Getting the data from the path provided.
        Returns:
            pd.DataFrame: The data in a pandas dataframe.
        """
        logging.info(f"Getting data from {self.path}")
        return pd.read_csv(self.path)

@step
def ingest_df(data_path: str) -> pd.DataFrame:
    """
    Ingesting the data from the data_path provided.

    Args:
        data_path: Path to the data.

    Return:
        pd.DataFrame: The data in a pandas dataframe.
    """
    try:
        data = IngestData(data_path)
        df = data.get_data()
        return df
    except Exception as e:
        logging.error(f"Error while ingesting data: {e}")
        raise e


import re
import sys
from deep_translator import GoogleTranslator

# Split the text into smaller groups up to 5000 characters without breaking lines so need to split on new lines
def split_text(text, max_len=4500):
    # Split the text into lines
    lines = text.split("\n")
    # List of chunks
    chunks = []
    # Chunk buffer
    chunk = ""

    # Loop through the lines
    for line in lines:
        # If the chunk is too long, add it to the list and reset the chunk
        if len(chunk + line) > max_len:
            chunks.append(chunk)
            chunk = ""
        # Add the line to the chunk
        chunk += line + "\n"

    # Add the last chunk to the list
    if chunk:
        chunks.append(chunk)

    # Return the list of chunks
    return chunks

def translate(text, source, target):
    # Split the text into smaller groups up to 5000 characters without breaking lines so need to split on new lines
    chunks = split_text(text)

    # Translate the text
    translated_text = ''
    for chunk in chunks:
        if source is None:
            source = 'auto'
        translated = GoogleTranslator(source=source, target=target).translate(text=chunk)
        translated_text += translated + "\n"

    # Remove extra new lines
    translated_text = re.sub(r'[\n]{3,}', '\n\n', translated_text.strip())
    translated_text = translated_text.strip()

    # Return the translated text
    return translated_text

if __name__ == "__main__":
    args = sys.argv

    # Handling the case when no file is passed
    if len(args) == 1:
        print("Please pass a file to translate")
        sys.exit(1)
    elif len(args) == 2:
        print("Please pass a target language")
        sys.exit(1)
    elif len(args) >= 3:
        text = str(open(args[1], encoding='utf8', newline='\n').read()).strip()

        # Handling `<filename> <target>`
        if len(args) == 3:
            translated = translate(text, None, args[2])
            print(translated)
        # Handling `<filename> <from> <target>`
        elif len(args) == 4:
            translated = translate(text, args[2], args[3])
            print(translated)


'''
爬虫主程序

从工信部获取并下载《免征车辆购置税的新能源汽车车型目录》，《新能源汽车推广应用推荐车型目录》，《道路机动车辆生产企业及产品》文件，文件储存至 './data/origin_data'

@Author: KivenChen
@Date: 2019-04-10
'''
import os
import asyncio
from . import output
from .spider import spider_1, spider_2

def main():
    base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    data_path = os.path.join(base_path, 'data')
    data = {}
    loop = asyncio.get_event_loop()
    data.update(spider_1(loop, pageNum=8))
    # 数据修正，网页上链接出错
    # 根据规律找到修正网址
    data[
        '新能源汽车推广应用推荐车型目录（2018年第1批）'] = 'http://123.127.164.29:18082/CVT/Jsp/zjgl/nerds/201801.html'
    data[
        '新能源汽车推广应用推荐车型目录（2018年第4批）'] = 'http://123.127.164.29:18082/CVT/Jsp/zjgl/nerds/201804.html'
    data[
        '新能源汽车推广应用推荐车型目录（2017年第1批）'] = 'http://123.127.164.29:18082/CVT/Jsp/zjgl/nerds/201701.htm'
    data[
        '新能源汽车推广应用推荐车型目录（2017年第2批）'] = 'http://123.127.164.29:18082/CVT/Jsp/zjgl/nerds/201702.htm'
    print('Spider 1 is done!')
    data.update(spider_2(loop, pageSize=1000))
    print('Spider 2 is done!')
    output.to_files(data, os.path.join(data_path, 'origin_data'))
    loop.close()
    print('=' * 30)
    print('链接爬取已完成!')
    print('=' * 30)

if __name__ == "__main__":
    main()


import pygame
import time
import random

pygame.init()

white = (255, 255, 255)
black = (0, 0, 0)
red = (255, 0, 0)
blue = (0, 0, 255)

dis_width = 800
dis_height = 600

dis = pygame.display.set_mode((dis_width, dis_height))
pygame.display.set_caption('collect the trash')

background = pygame.image.load("naturebackground.jpg")
nature = pygame.transform.rotozoom(background,0,1.75)

recycling = pygame.image.load("recycling.jpg")
soda = pygame.image.load("crushedcan.jpg")
banana = pygame.image.load("bananapeel.png")

##compost =
bag = pygame.image.load("trash.png")

compost = pygame.image.load("compost.png")
##candy =
##garbage =

Bin = pygame.image.load("trashBin.jpg")
clock = pygame.time.Clock()

bin_block = 25
bin_speed = 15

font_style = pygame.font.SysFont(None, 30)

enviro_facts = ["The average college student produces 640 pounds of solid waste each year :o",
"By 2050, the ocean will contain more plastic by weight than fish",
"In 2018, Americans disposed of 146.2 million tons of trash - 24% was food waste",
"The Great Pacific Garbage Patch contains almost 3.5 million tons of trash :'(",
"Roughly 80% of the items in landfill could be recycled :o",
"The average person generates over 4 pounds of trash every day :(",
"1/3 of all the food produced globally goes to waste :("]

def message(msg, color):
    mesg = font_style.render(msg, True, color)
    dis.blit(mesg, [dis_width/3, dis_height/3])

def dis_score(score):
    value = font_style.render("Score: " + str(score), True, red)
    dis.blit(value, [0,0])

def instruct():
    instructions = font_style.render("W = trash, E = compost, R = recycling", True, red)
    dis.blit(instructions, [0,20])

def recyclebin(x,y):
    binimage = pygame.transform.rotozoom(recycling,0,0.075)
    dis.blit(binimage,(x,y))

def sodacan(x,y):
    crushsoda = pygame.transform.rotozoom(soda,0,0.050)
    dis.blit(crushsoda,(x,y))

def bananapeel(x,y):
    bananaskin = pygame.transform.rotozoom(banana,0,0.037)
    dis.blit(bananaskin,(x,y))

def garbage_bag(x,y):
    trash_bag = pygame.transform.rotozoom(bag,0,0.1)
    dis.blit(trash_bag,(x,y))

def garbageBin(x,y):
    garbage = pygame.transform.rotozoom(Bin, 0, 0.1)
    dis.blit(garbage,(x,y))

def compostBin(x,y):
    compBin = pygame.transform.rotozoom(compost, 0, 0.25)
    dis.blit(compBin,(x,y))

def gameLoop():  # creating a function
    game_over = False
    game_close = False

    x1 = dis_width / 2
    y1 = dis_height - 100

    x1_change = 0
    trash_change = 0

    trash_x = round(random.randrange(0, dis_width - bin_block) / 10.0) * 10.0
    trashy = 0

    bintype = 0 ##0 for recycling, 1 for compost, 2 for garbage
    trashtype = 0

    score = 1

    while not game_over:

        if game_close == True:
            index = random.randint(0,6)

        while game_close == True:
            dis.fill(white)
            dis.blit(nature,(0,0))

            end = font_style.render("You Lost! Press Q-Quit or C-Play Again", True, red)
            pygame.draw.rect(dis,red,[0,0,800,80])
            fact = font_style.render(enviro_facts[index], True, white)
            end_score = font_style.render("Final score: " +str(score-1), True, red)
            dis.blit(end, [270,420])
            dis.blit(end_score, [270,450])
            dis.blit(fact, [0,30])

            pygame.display.update()

            for event in pygame.event.get():
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_q:
                        game_over = True
                        game_close = False
                    if event.key == pygame.K_c:
                        gameLoop()

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                game_over = True
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_r:
                    ##change bin
                    bintype = 0
                elif event.key == pygame.K_e:
                    bintype = 1
                elif event.key == pygame.K_w:
                    bintype = 2
                if event.key == pygame.K_LEFT:
                    x1_change = -bin_block
                    trash_change = 10*score**0.2

                elif event.key == pygame.K_RIGHT:
                    x1_change = bin_block
                    trash_change = 10*score**0.2

            if event.type == pygame.KEYUP:
                x1_change = 0
                trash_change = 10*score**0.2

        if x1 >= dis_width :
            x1_change = -bin_block

        if x1<0:
            x1_change = bin_block

        if trashy >= dis_height:
            game_close = True

        x1 += x1_change
        trashy += trash_change
        dis.fill(white)
        dis.blit(nature,(0,0))

        if bintype == 0:
            recyclebin(x1,y1)
        elif bintype == 1:
            compostBin(x1,y1)
        elif bintype == 2:
            garbageBin(x1, y1)
        if trashtype == 0:
            sodacan(trash_x,trashy)
        elif trashtype == 1:
            bananapeel(trash_x,trashy)
        elif trashtype == 2:
            garbage_bag(trash_x,trashy)

        dis_score(score-1)
        instruct()
        pygame.display.update()

        if abs(x1 - trash_x)<=25 and abs(y1-trashy)<= 15 and trashtype == bintype:
            trash_x = round(random.randrange(100, dis_width - bin_block - 100) / 10.0) * 10.0
            trashy = 0
            trashtype = random.randrange(0,3)
            score +=1
        clock.tick(bin_speed)

    pygame.quit()
    quit()

gameLoop()


from flask import Flask

from flask import request # `request` is a global object that can be used to access the query string of a url

# Create a new Flask instance (passing the current namespace to the Flask initializer)
app = Flask(__name__)

# Create a route
# Note this uses 'pie syntax' to implement a decorator
# A decorator is a way of wrapping a function (here `app.route('/')`) around another function
# (here `index()`) in order to modify the latter's behaviour.
# It replaces the latter function with the result of calling the former function and passing the latter function to it.
#
# We can have multiple decorators for a single function.
# Flask is designed to be able to handle multiple routes (using multiple decorators) with a single view
#
# In the second decorator we have created a capture group, called `name`.
# What this means is that Flask will take whatever the string after the `/` is and put it into the `name` variable.
@app.route('/')
@app.route('/<name>')
def index(name='Default Person'):

    # request is a global object and represents the query string (if any) included in the url
    # the components of the query string are in a dictionary-like structure
    # and we can access these using the .get(key, default=None) method
    #
    # Here, if there is a 'name' argument, we take that, otherwise we get the existing name variable
    # and put that into the name variable.
    #
    # So if we access the page 127.0.0.1:localhost without a query string, we'll get 'Hello Default Person'
    # If we access the page 127.0.0.1:localhost/?name=SpecialPerson, we'll get 'Hello SpecialPerson'
    name = request.args.get('name', name)

    pet = request.args.get('pet', None)

    first_line = "Hello {}.".format(name)
    second_line = None
    if pet == None:
        second_line = "I understand you don't have a pet at the moment, have you considered getting a labrador? They are the best."
    elif pet == 'labrador':
        second_line = "You have a labrador! What a great choice! They are the best, aren't they?"
    else:
        second_line = "Ah, so you have a {}.".format(pet)

    return first_line + "\n" + second_line

# run the app
# `debug=True`: automatically restart the server if there is an error
# `port=8000`: listen on port 8000 (a common port for web servers)
# `host='0.0.0.0': listen on all addresses that can reach this app
app.run(debug=True, port=8000, host='0.0.0.0')


# coding=utf-8
from openpyxl import Workbook

#在内存中创建一个workbook对象
wb = Workbook()

ws = wb.get_active_sheet()
print(ws.title)
ws.title = u'99乘法表'

#设置单元格的值
for row in range(1,10):
    for col in range(1,10):
        ws.cell(row = row, column = col).value = row * col

#最后保存
wb.save(filename='99乘法表.xls')


import requests
from bs4 import BeautifulSoup as bs
import arrow
import csv
import pandas as pd
from time import sleep

def forager():
    venue = "Forager"
    prices = "NaN"

    month = input("Please enter the number of the month in the form '1':  ")
    for l in range(1, 13):
        if month == str(l):
            month_name = arrow.get(month, "M").format("MMMMM")
            month_name = month_name.replace(month, "")
            break
        elif (l == 12) & (str(month) != l):
            print("Please enter a valid option")
            forager()

    year = input("Please enter a year in the form '18'")
    for yr in range(18, 100):
        if year == str(yr):
            year_name = arrow.get(year, 'YY').format('YYYY')
            break
        elif (yr == 99) & (str(yr) != year):
            print("Please enter a valid input")
            forager()

    site_url = "https://www.sjforager.com/new-events/?view=calendar&month=" + month_name + "-" + year_name
    site = requests.get(site_url)
    soup = bs(site.content, 'html.parser')
    events = [x.text for x in soup.find_all('h1')]
    links = soup.find_all('h1')

    base_link = site_url.replace("/new-events/?view=calendar&month=" + month_name + "-" + year_name, "")

    dates = []
    times = []
    for i in links:
        sleep(1)
        a = str(i.find('a'))
        stripped = a.replace('<a href="', '')
        stripped = stripped.split('"')
        stripped = stripped[0]

        full_link = base_link + stripped
        sleep(1)
        site = requests.get(full_link)
        item_info = bs(site.content, 'lxml')

        try:
            event_date = (item_info.find(class_="event-date")).text
            event_date = event_date.split(',')
            year = event_date[-1]
            mday = event_date[1]
            date = mday.lstrip() + year
            dates.append(arrow.get(date, "MMMM D YYYY").format("MM/DD/YY"))
        except AttributeError:
            dates.append('NaN')
        try:
            start_time = (item_info.find(class_="event-time-12hr-start")).text
            end_time = (item_info.find(class_="event-time-12hr-end")).text
            times.append(start_time + " - " + end_time)
        except AttributeError:
            times.append('NaN')

    # t = ""
    # # tries to read csv, if not creates or empty them headers are added
    # try:
    #   df = pd.read_csv('forager.csv')
    # except FileNotFoundError:
    #   t = "NaN"
    # except pd.errors.EmptyDataError:
    #   t = 'NaN'
    # if t == 'NaN':
    with open('forager.csv', 'w') as ttable:
      filewriter = csv.writer(ttable)
      filewriter.writerow(["Venue", "Event", "Date", "Time", "Price"])
    # Appends all of the elements in our lists to the csv
    with open('forager.csv', 'a') as ttable:
      filewriter = csv.writer(ttable)
      for i in range(0, len(dates)):
          filewriter.writerow([venue, events[i], dates[i], times[i], prices])

forager()


# coding=utf-8
import logging
import os
import uuid
import xlwt
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.db import IntegrityError
from django.db import transaction
from BanBanTong import constants
from BanBanTong.db import models
from BanBanTong.db import utils as db_utils
from BanBanTong.forms.lesson_teacher import LessonTeacherForm
from BanBanTong.forms.lesson_teacher import LessonTeacherUploadForm
from BanBanTong.forms.lesson_teacher import LessonTeacherUploadVerifyForm
from BanBanTong.utils import create_failure_dict
from BanBanTong.utils import create_success_dict
from BanBanTong.utils import decorator
from BanBanTong.utils import get_page_info
from BanBanTong.utils import model_list_to_dict
from BanBanTong.utils import model_to_dict
from BanBanTong.utils import simplecache

logger = logging.getLogger(__name__)

@decorator.authorized_user_with_redirect
@decorator.authorized_privilege('system_class_lesson')
@transaction.atomic
def add(request, *args, **kwargs):
    if request.method == 'POST':
        try:
            f = LessonTeacherForm(request.POST)
            if f.is_valid():
                lesson_teacher = f.save()
                simplecache.LessonTeacher.update(lesson_teacher.class_uuid.uuid)
                data = model_to_dict(lesson_teacher)
                school_year = lesson_teacher.class_uuid.grade.term.school_year
                data['class_uuid__grade__term__school_year'] = school_year
                term_type = lesson_teacher.class_uuid.grade.term.term_type
                data['class_uuid__grade__term__term_type'] = term_type
                grade_name = lesson_teacher.class_uuid.grade.name
                data['class_uuid__grade__name'] = grade_name
                data['class_uuid__name'] = lesson_teacher.class_uuid.name
                data['lesson_name__name'] = lesson_teacher.lesson_name.name
                data['teacher__name'] = lesson_teacher.teacher.name
                try:
                    stat_obj = models.Statistic.objects.get(key=lesson_teacher.class_uuid.pk)
                    stat_obj.cala('teacher_num', True)
                except Exception as e:
                    logger.exception(e)
                return create_success_dict(msg='添加班级课程授课教师记录成功！',
                                           data=data)

            return create_failure_dict(msg='添加班级课程授课教师记录失败！',
                                       errors=f.errors)
        except IntegrityError:
            msg = '已有相同的年级班级-课程-教师姓名，无法重复添加！'
            return create_failure_dict(msg=msg)
        except Exception as e:
            logger.exception(e)
            return create_failure_dict(msg=u'添加班级课程授课教师记录失败！', debug=str(e))

@decorator.authorized_user_with_redirect
@decorator.authorized_privilege('system_class_lesson')
def delete(request, *args, **kwargs):
    if request.method == 'POST':
        if not cache.get('sudo'):
            return create_failure_dict(msg=u'请输入正确的超级管理员admin密码！')
        uu = request.POST.get('uuid')
        if uu:
            try:
                obj = models.LessonTeacher.objects.get(uuid=uu)
                objs = obj.teacherloginlog_set.count()
                if objs > 0:
                    return create_failure_dict(msg=u'该教师已经产生登录授课记录,仅支持编辑.')
            except:
                return create_failure_dict(msg=u'错误的uuid！')
            try:
                if not obj.class_uuid.grade.term.allow_import_lesson():
                    return create_failure_dict(msg=u'该信息学年学期已过期,不能删除')
                obj.delete()
                simplecache.LessonTeacher.update(obj.class_uuid.uuid)
            except Exception as e:
                logger.exception('')
                return create_failure_dict(msg=u'内部错误', debug=str(e))
        return create_success_dict(msg=u'删除班级课程授课教师记录成功！')

@decorator.authorized_user_with_redirect
@decorator.authorized_privilege('system_class_lesson')
@transaction.atomic
def edit(request):
    if request.method == 'POST':
        uu = request.POST.get('uuid')
        try:
            l = models.LessonTeacher.objects.get(uuid=uu)
        except:
            return create_failure_dict(msg='错误的uuid！')
        if not l.class_uuid.grade.term.allow_import_lesson():
            return create_failure_dict(msg='该信息学年学期已过期,不能编辑')
        form = LessonTeacherForm(request.POST, instance=l)
        if form.is_valid():
            form.save()
            simplecache.LessonTeacher.update(l.class_uuid.uuid)
            return create_success_dict(msg='编辑班级课程授课教师记录成功！')

        return create_failure_dict(msg='编辑班级课程授课教师记录失败！',
                                   errors=form.errors)

@decorator.authorized_user_with_redirect
@decorator.authorized_privilege('system_class_lesson')
def export(request, *args, **kwargs):
    '''导出所有班级课程授课教师信息'''
    xls = xlwt.Workbook(encoding='utf8')
    title = u'学期班级课程授课老师信息'
    sheet = xls.add_sheet(title)
    header = [u'姓名', u'生日', u'授课年级', u'授课班级',
              u'授课课程', u'计划课时']
    for i in range(len(header)):
        sheet.write(0, i, header[i])
    row = 1
    try:
        t = models.Term.get_current_term_list()[0]
    except:
        return create_failure_dict(msg='当前时间不在任何学期内')
    q = models.LessonTeacher.objects.filter(class_uuid__grade__term=t)
    grade_name = request.REQUEST.get('grade_name', None)
    class_name = request.REQUEST.get('class_name', None)
    lesson_name = request.REQUEST.get('lesson_name', None)
    teacher_name = request.REQUEST.get('teacher_name', None)
    if grade_name:
        q = q.filter(class_uuid__grade__name=grade_name)
    if class_name:
        q = q.filter(class_uuid__name=class_name)
    if lesson_name:
        q = q.filter(lesson_name__name=lesson_name)
    if teacher_name:
        q = q.filter(teacher__name__contains=teacher_name)

    q = q.values('teacher__name', 'teacher__birthday',
                 'class_uuid__grade__name', 'class_uuid__name',
                 'lesson_name__name', 'schedule_time')
    for i in q:
        sheet.write(row, 0, i['teacher__name'])
        sheet.write(row, 1, str(i['teacher__birthday']).replace('-', ''))
        sheet.write(row, 2, i['class_uuid__grade__name'])
        sheet.write(row, 3, i['class_uuid__name'])
        sheet.write(row, 4, i['lesson_name__name'])
        sheet.write(row, 5, i['schedule_time'])
        row += 1
    cached_id = str(uuid.uuid1())
    tmp_file = os.path.join(constants.CACHE_TMP_ROOT, cached_id)
    xls.save(tmp_file)
    filename = u'%s.xls' % title
    return create_success_dict(url=reverse('base:xls_download',
                                           kwargs={'cached_id': cached_id,
                                                   'name': filename}))

@decorator.authorized_user_with_redirect
@decorator.authorized_privilege('system_class_lesson')
def import_from(request, *args, **kwargs):
    if request.method == 'POST':
        f = LessonTeacherUploadForm(request.POST, request.FILES)
        if f.is_valid():
            objs = f.save()
            simplecache.LessonTeacher.update()

            # 更新一下Statistic表中的登记教师总数
            klasses = map(lambda i: i.class_uuid.uuid, objs)
            klasses = models.Class.objects.filter(uuid__in=klasses)
            for cls in klasses:
                try:
                    stat_obj = models.Statistic.objects.get(key=cls.pk)
                    stat_obj.cala('teacher_num', True)
                except Exception as e:
                    logger.exception(e)

            return create_success_dict(data=model_list_to_dict(objs))
        return create_failure_dict(msg='导入班级课程授课老师失败！',
                                   errors=f.errors)

@decorator.authorized_user_with_redirect
@decorator.authorized_privilege('system_class_lesson')
def list_current(request, *args, **kwargs):
    page_info = get_page_info(request)
    grade_name = request.GET.get('grade_name')
    class_name = request.GET.get('class_name')
    lesson_name = request.GET.get('lesson_name')
    teacher_name = request.GET.get('teacher_name')

    terms = models.Term.objects.filter(deleted=False)
    q = models.LessonTeacher.objects.filter(class_uuid__grade__term__in=terms)
    if grade_name:
        q = q.filter(class_uuid__grade__name=grade_name)
    if class_name:
        q = q.filter(class_uuid__name=class_name)
    if lesson_name:
        q = q.filter(lesson_name__name=lesson_name)
    if teacher_name:
        q = q.filter(teacher__name__contains=teacher_name)
    order_lst = (
        'class_uuid__grade__number',
        'class_uuid__number',
        'lesson_name__name'
    )
    q = q.order_by(*order_lst)
    q = q.values('uuid', 'class_uuid__grade__term__school_year',
                 'class_uuid__grade__term__term_type',
                 'class_uuid__grade__name', 'class_uuid__name',
                 'lesson_name__name', 'teacher__name', 'teacher__uuid',
                 'teacher__birthday',
                 'schedule_time')
    grade_class_assigned_time_dict = {}
    q = list(q)
    for one in q:
        key = u'%s_%s' % (one['class_uuid__grade__name'], one['class_uuid__name'])
        if not grade_class_assigned_time_dict.has_key(key):
            uuid = one.get('uuid')
            one_class = models.LessonTeacher.objects.get(uuid=uuid).class_uuid
            one['remain_time'] = int(one_class.grade.term.schedule_time) - int(one_class.cala_assigned_time())
            grade_class_assigned_time_dict[key] = one['remain_time']
        else:
            one['remain_time'] = grade_class_assigned_time_dict[key]

    page_data = db_utils.pagination(q, **page_info)

    return create_success_dict(data={
        'records': model_list_to_dict(page_data['records']),
        'page': page_data['page_num'],
        'page_size': page_data['page_size'],
        'record_count': page_data['record_count'],
        'page_count': page_data['page_count'],
    })

@decorator.authorized_user_with_redirect
@decorator.authorized_privilege('system_class_lesson')
def verify(request, *args, **kwargs):
    if request.method == 'POST':
        f = LessonTeacherUploadVerifyForm(request.POST, request.FILES)
        if f.is_valid():
            objs = f.save()
            return create_success_dict(data={'records': model_list_to_dict(objs)})

        return create_failure_dict(msg='验证班级课程授课老师失败！',
                                   errors=f.errors)

@decorator.authorized_user_with_redirect
@decorator.authorized_privilege('system_class_lesson')
def get_remain_time(request):
    '''获取每个班级的剩余可分配课时'''
    if request.method == 'GET':
        data = request.GET
    else:
        data = request.POST
    grade_name = data.get('grade_name')
    class_name = data.get('class_name')

    try:
        term = models.Term.get_current_term_list()[0]
        class_obj = models.Class.objects.get(name=class_name, grade__name=grade_name, grade__term=term)
    except Exception as e:
        logger.exception(e)
        return create_failure_dict(msg='错误的班级名称')

    remain_time = class_obj.cala_remain_time()

    return create_success_dict(data={'grade_name': class_obj.grade.name, 'class_name': class_obj.name, 'remain_time': remain_time})


import numpy as np
import tensorflow as tf
import extra_keras_datasets.emnist as centr_emnist  # See https://github.com/machinecurve/extra_keras_datasets
import nest_asyncio

nest_asyncio.apply()

tf.compat.v1.enable_v2_behavior()
tf.compat.v1.graph_util.extract_sub_graph

# Final evaluation in test dataset
print("Evaluating the entire test dataset:")
# Load the model
final_model = tf.keras.models.load_model('./models/model_EMNIST_100_0.75.h5')
print(final_model.get_weights())
# Load the centralized version of the EMNIST dataset
(x_train, y_train), (x_test, y_test) = centr_emnist.load_data(type='digits')
print(f"Evaluating the model in {x_test.shape[0]} test samples")
# Evaluate the final model on the test dataset
x_train = x_train.reshape(-1, 784).astype('float32') / 255
x_test = x_test.reshape(-1, 784).astype('float32') / 255
final_eval = final_model.evaluate(x_test, y_test)
# print(final_eval)
# print('Test loss:', final_eval[0])
# print('Test accuracy:', final_eval[1])

# SAVE THE RESULTS
# # Final accuracy on the entire test dataset
# np.savetxt('final_test_loss_K' + str(m) + '_' + str(percentage) + '.txt', np.reshape(final_eval[0], (1,1)))
# np.savetxt('final_test_accuracy_K' + str(m) + '_' + str(percentage) + '.txt', np.reshape(final_eval[1], (1,1)))

Code & Results/TensorFlow code/evaluate_models_test_dataset.py


import os

from setuptools import setup, find_packages

exec(open('hops/version.py').read())

def read(fname):
    try:
        return open(os.path.join(os.path.dirname(__file__), fname), encoding='utf8').read() #python3
    except:
        return open(os.path.join(os.path.dirname(__file__), fname)).read() #python2

setup(
    name='hopsworks-cloud-sdk',
    version=__version__,
    install_requires=[
        'requests',
        'numpy',
        'pandas',
        'pyhopshive[thrift]',
        'boto3>=1.9.226',
        'SQLAlchemy',
        'PyMySQL',
        'pyopenssl',
        'idna'
    ],
    extras_require={
        'docs': [
            'sphinx',
            'sphinx-autobuild',
            'recommonmark',
            'sphinx_rtd_theme',
            'jupyter_sphinx_theme'
        ],
        'test': [
            'mock',
            'pytest',
        ],
        'plotting': ['matplotlib', 'seaborn']
    },
    author='Steffen Grohsschmiedt',
    author_email='steffen@logicalclocks.com',
    description='An SDK to integrate cloud solutions such as SageMaker and Databricks with Hopsworks.',
    license='Apache License 2.0',
    keywords='Hopsworks, SageMaker, Databricks',
    url='https://github.com/logicalclocks/hopsworks-cloud-sdk',
    download_url='http://snurran.sics.se/hops/hopsworks-cloud-sdk/hops-' + __version__ + '.tar.gz',
    packages=find_packages(exclude=['tests']),
    long_description=read('README.rst'),
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Topic :: Utilities',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python :: 3',
    ]
)


import websocket, json, pprint,talib, numpy

SOCKET = "wss://stream.binance.com:9443/ws/ethusdt@kline_1m"
avg = 0
open_total = []
closed_total = []
high_total = []
low_total = []
SSL_PERIOD =  14
STOCK = "ETHUSD"
PAY = 100.00
POSTION = False
TRADE_QUANTITY = 0
profit = 0

def on_open(webS):
    print(" opened connection")
def on_close(webS):
    print(" closed connection")
def on_message(webS,message):
    print("received connection")
    json_message = json.loads(message)
    # pprint.pprint(json_message)
    # global closed_total

    candle = json_message["k"]
    final_value_tick = candle["x"]
    open_price = candle["o"]# Open price
    closed_price = candle["c"]#Close price
    high_price = candle["h"]#High price
    low_price = candle["l"]#Low price
    base_vol = candle["v"]#Base_asset_volume
    num_trades = candle["n"]#Number_of_trades
    quote_vol = candle["q"]#Quote_asset_volume
    taker_base = candle["V"]#Taker buy base asset volume
    taker_quote = candle["Q"]#Taker_buy_quote_asset_volume

    if final_value_tick:
        open_total.append(float(open_price))
        high_total.append(float(high_price))
        low_total.append(float(low_price))
        print("closed at this price {}".format(closed_price))
        closed_total.append(float(closed_price))
        print("Open: {}".format(open_price))
        print('------')
        print("High: {}".format(high_price))
        print('------')
        print("Low: {}".format(low_price))
        print('------')

        if len(closed_total) > SSL_PERIOD:

            np_highs = numpy.array(high_total)
            np_lows = numpy.array(low_total)
            np_closes = numpy.array(closed_total)
            np_opens = numpy.array(open_total)
            rsi = talib.RSI(np_closes,SSL_PERIOD)
            atr = talib.ATR(np_highs,np_lows,np_closes,SSL_PERIOD)
            avg = talib.AVGPRICE(np_opens,np_highs,np_lows,np_closes)
            ema = talib.EMA(np_closes,SSL_PERIOD)
            dp = talib.PLUS_DM(np_highs,np_lows,SSL_PERIOD)
            dm = talib.MINUS_DM(np_highs,np_lows,SSL_PERIOD)
            adx = talib.ADX(np_highs,np_lows,np_closes,SSL_PERIOD)
            ema += atr
            print('---------')
            print("The last rsi is {}".format(rsi[-1]))
            print('----')
            print("The last atr is {}".format(atr[-1]))
            print('----')
            print("last Average price is {}".format(avg[-1]))
            print('----')
            print("last  true EMA  is {}".format(ema[-1]))
            print('----')
            print("last DMI+  is {}".format(dp[-1]))
            print('----')
            print("last DMI-  is {}".format(dm[-1]))
            print('----')
            print("last ADX-  is {}".format(adx[-1]))
            # print('--RSI--')
            # print(rsi)
            # print('--ATR--')
            # print(atr)
            # print('--AVG--')
            # print(avg)
            # print('----')
            # print('--EMA--')
            # print(ema)
            # print('----')
            # print('--DM(+)--')
            # print(dp)
            # print('----')
            # print('--DM(-)--')
            # print(dm)
            # print('--ADX--')
            # print(adx)
            print('----')
            if dp[-1] > dm[-1]:
                total = float(dp[-1] - dm[-1]) * 100
                print("Uptrending: % {:.3f}".format(total))
                if (90 > rsi[-1] > 60) and (adx[-1] >25):

                    print("now we invest.Calculate price to buy")
                    if((ema[-1]) < closed_total[-1]):
                        profit += ema[-1]
                        print("BUY!!!")
                        print("Bought: ${}".format(ema[-1]))
                        print("------")

                else:
                    print('Current RSI: {:.2}\nCurrent ADX: {:.2}\nCurrent true EMA: ${}'.format(rsi[-1],adx[-1],ema[-1]))
            # if dp[-1] < dm[-1]:
            #     print("downtrend")
            if dm[-1] > dp[-1]:
                print("downtrend")

    print("Current closed price: {}".format(closed_total[-1]))
    print("update")
    profit -= closed_total[-1]
    print("Profit: ${}".format(profit))

# def ema(webS):
#     day = 50
webS = websocket.WebSocketApp(SOCKET,on_open=on_open,on_close=on_close, on_message=on_message)

webS.run_forever()


#!/usr/bin/env python3
# publisher for a specific img folder

import rospy
from std_msgs.msg import String
from detection_only.msg import Bbox_6, Bbox6Array
from sensor_msgs.msg import Image
import json
import time
import cv2
from os import listdir
from os.path import isfile, join
from cv_bridge import CvBridge
import pdb

def img_list(img_folder):
    file_list = []
    img_name_list = []
    for image in listdir(img_folder):
        img_name = int(image.split('.')[0])
    #     while len(img_name) != 4:
    #         img_name = '0' + img_name
        img_name_list.append(img_name)
    # pdb.set_trace()
    img_name_list = sorted(img_name_list)
    for n in img_name_list:
        f = str(n) + '.png'
        if isfile(join(img_folder, f)):
            file_list.append(join(img_folder, f))
    # for f in listdir(img_folder):
    #     if isfile(join(img_folder, f)):
    #         file_list.append(join(img_folder, f))
    # pdb.set_trace()
    return file_list

if __name__ == '__main__':

    try:
        rospy.init_node('img_test_node', anonymous=False)

        freq = 30
        timer = rospy.timer.Rate(freq)
        # source_folder = "/home/ziyan/Yolov5_DeepSort_Pytorch_ros/Yolov5_DeepSort_Pytorch/test_imgs_less"
        source_folder = "/home/ziyan/Downloads/exp1"
        img_list = img_list(source_folder)
        pdb.set_trace()
        img_pub = rospy.Publisher('raw_image', Image, queue_size=1)
        br = CvBridge()
        n = len(img_list)
        # while not rospy.is_shutdown():
        for i in range(len(img_list)):
            img = cv2.imread(img_list[i])
            img = img[:,:,[2,1,0]]
            data = Image()
            data = br.cv2_to_imgmsg(img)
            img_pub.publish(data)
            print("image publish!")
            # if i > 85:
            #     pdb.set_trace()
            timer.sleep()
    except RuntimeError:
        rospy.logfatal("get runtime error")
    except rospy.ROSInterruptException:
        pass


import z3
import random

special_var_table = {}

_var_count = 0
def new_variable(var_type):
    global _var_count
    _var_count += 1
    if var_type == "Bool":
        return z3.Bool("Bool" + str(_var_count))
    elif var_type == "Int":
        return z3.Int("Int" + str(_var_count))
    else:
        assert False

def _recursively_build_only_one(variable_list, constraint_list):
    if len(variable_list) == 1:
        return variable_list[0]
    mid = len(variable_list) // 2
    left = _recursively_build_only_one(variable_list[:mid], constraint_list)
    right = _recursively_build_only_one(variable_list[mid:], constraint_list)
    constraint_list.append(z3.Not(z3.And(left, right)))
    return_value = new_variable("Bool")
    constraint_list.append(return_value == z3.Or(left, right))
    return return_value

def build_only_one(variable_list, constraint_list, one_value=True):
    assert len(variable_list) > 0
    result = _recursively_build_only_one(variable_list, constraint_list)
    constraint_list.append(result == one_value)

def get_random(variable_type):
    if variable_type == "Int":
        return random.randint(-10 ** 8, 10 ** 8)
    elif variable_type == "Bool":
        return random.randint(0, 1) == 0
    else:
        assert False

def _list_to_string(expr):
    if type(expr) == str:
        return expr
    if type(expr) == tuple:
        return str(expr[1])
    if type(expr) == list:
        string_expr = list(map(lambda x: _list_to_string(x), expr))
        return "(" + " ".join(string_expr) + ")"
    assert False

def print_function(function_result, function_info):
    arg_list = list(map(lambda x: [x.name, x.type], function_info.arg_list))
    result = ["define-fun", function_info.name, arg_list, function_info.return_type, function_result]
    print(_list_to_string(result))

def get_new_symbolic_input(input_list):
    inp = {}
    for var in input_list:
        variable_name = var.name
        variable_type = var.type
        inp[variable_name] = new_variable(variable_type)
    return inp

def parse_input_from_model(arg_list, symbolic_inp, model):
    inp = {}
    for var_info in arg_list:
        symbolic_var = symbolic_inp[var_info.name]
        if model[symbolic_var] is not None:
            if var_info.type == "Int":
                inp[var_info.name] = model[symbolic_var].as_long()
            elif var_info.type == "Bool":
                inp[var_info.name] = z3.is_true(model[symbolic_var])
            else:
                assert False
        else:
            inp[var_info.name] = get_random(var_info.type)
    return inp


# This demo shows how to plot the contour plot for given function, with resultant of the partial derivatives displayed
# as arrows

import numpy as np
import matplotlib.pyplot as plt
import math

def func(x, y):
    z = y - x - 2 * x ** 2 - 2 * x * y - y ** 2
    return z

x = np.linspace(-2, 0, 20)
y = np.linspace(1, 3, 20)
[X, Y] = np.meshgrid(x, y)
Z = func(X, Y)
[fx, fy] = np.gradient(Z, 0.25)
vs1 = plt.contour(x, y, Z)
vs2 = plt.quiver(x, y, -fx, -fy)
plt.title("Contour plot & partial derivatives")
plt.show()