'''
Created on 25 Jan 2013

@author: Dave Wilson
'''

import wx
import enhanced_status_bar

FRAME_STYLE = (wx.FRAME_FLOAT_ON_PARENT | wx.SYSTEM_MENU | wx.CAPTION |
                        wx.CLOSE_BOX | wx.RESIZE_BORDER | wx.MAXIMIZE_BOX |
                        wx.CLIP_CHILDREN | wx.FRAME_NO_TASKBAR)

FRAME_DIALOG_STYLE = (wx.FRAME_FLOAT_ON_PARENT | wx.CAPTION |
                      wx.FRAME_NO_TASKBAR)
FRAME_DIALOG_STYLE2 = (FRAME_DIALOG_STYLE | wx.SYSTEM_MENU | wx.CLOSE_BOX)

FRAME_DIALOG_STYLE3 = (FRAME_DIALOG_STYLE2 | wx.RESIZE_BORDER |
                       wx.MAXIMIZE_BOX | wx.RESIZE_BOX)

class GaugeStatusBar(enhanced_status_bar.EnhancedStatusBar):
    def __init__(self, parent):
        super(GaugeStatusBar, self).__init__(parent, style=wx.SB_FLAT)
        self.SetSize((-1, 23))
        self.SetFieldsCount(2)
        self.SetStatusWidths([-1, 100])

        self.progress_gauge = wx.Gauge(self, size=(100, 15))
        self.progress_gauge.Show(False)
        self.AddWidget(self.progress_gauge, pos=1)

        self.timer = wx.Timer(self)
        self.Bind(wx.EVT_TIMER, self._on_timer)

    def _on_timer(self, event):
        self.progress_gauge.Pulse()

    def pulse_gauge(self, pulse=True):
        isRunning = self.timer.IsRunning()
        if not isRunning and pulse:
            self.timer.Start(500)
        elif isRunning and not pulse:
            self.timer.Stop()
        self.progress_gauge.Show(pulse)

class GaugeFrame(wx.Frame):
    def __init__(self, *args, **kwargs):
        super(GaugeFrame, self).__init__(*args, **kwargs)
        self.sizer = wx.BoxSizer(wx.VERTICAL)
        self.SetSizer(self.sizer)
        self.panel = wx.Panel(self)
        self.sizer.Add(self.panel, 1, wx.EXPAND)
        self.statusBar = GaugeStatusBar(self)
        self.SetStatusBar(self.statusBar)

    @classmethod
    def modal(cls, *args, **kwargs):
        if not kwargs.get('style'):
            kwargs['style'] = FRAME_DIALOG_STYLE2
        gauge_frame = cls(*args, **kwargs)
        gauge_frame.CenterOnParent()
        gauge_frame.MakeModal()
        gauge_frame.Bind(wx.EVT_CLOSE, gauge_frame.close_modal)
        return gauge_frame

    @classmethod
    def modal_resize(cls, *args, **kwargs):
        kwargs['style'] = FRAME_DIALOG_STYLE3
        return cls.modal(*args, **kwargs)

    def close_modal(self, event):
        event.Skip()
        self.MakeModal(False)
        self.Destroy()

    def set_panel(self, panel):
        self.Freeze()
        self.panel.Destroy()
        self.panel = panel
        self.sizer.Add(self.panel, 1, wx.EXPAND)
        self.Layout()
        self.Thaw()

    def set_gauge_pulse(self, pulse=True):
        self.statusBar.pulse_gauge(pulse)

    def set_status_text(self, status_text=''):
        self.statusBar.SetStatusText(status_text)

    def set_gauge_and_status_text(self, pulse=True, status_text=''):
        self.set_gauge_pulse(pulse)
        self.set_status_text(status_text)

    def show_error_dialog(self, title_text, error_text, icon=None):
#         title_text, error_text = str(title_text), str(error_text)
        if not icon:
            icon = wx.ICON_ERROR
        elif icon == 'information':
            icon = wx.ICON_INFORMATION
        dialog = wx.MessageDialog(self, error_text, title_text,
                                  icon | wx.OK | wx.CENTER)
        dialog.ShowModal()
        dialog.Destroy()

    def set_panel_enable(self, enable=True):
        self.panel.Enable(enable)

if __name__ == '__main__':
    wx_app = wx.App(None)
    frame = GaugeFrame(None, title='Testing GaugeFrame')
    frame.Show()
    frame.set_gauge_and_status_text(True, 'Test status')
    wx_app.MainLoop()


import sys
from collections import deque

def bfs(i, j):
    di = [1, 0, 0, -1]
    dj = [0, 1, -1, 0]
    visited = [[-1] * M for _ in range(N)]
    queue = deque()
    queue.append([i, j])
    visited[i][j] = 0
    result = []
    while queue:
        x, y = queue.popleft()
        for d in range(4):
            ni = x + di[d]
            nj = y + dj[d]
            if 0 <= ni < N and 0 <= nj < M:
                if maze[ni][nj] == 1 and visited[ni][nj] == -1:
                    queue.append([ni, nj])
                    visited[ni][nj] = visited[x][y] + 1
                if ni == N - 1 and nj == M - 1:
                    result.append(visited[x][y] + 1)
    return min(result) + 1

N, M = map(int, sys.stdin.readline().split())

maze = [[int(i) for i in sys.stdin.readline().rstrip()] for _ in range(N)]

print(bfs(0, 0))


from torchvision import models
from torch import nn
from classBlock import ClassBlock
import torch

class resnet50(nn.Module):
    def __init__(self, class_num, pretrained=False, model_name='resnet50'):
        super(resnet50, self).__init__()
        self.model_name = model_name
        self.class_num = class_num
        self.pretrained = pretrained

        model_ft = getattr(models, self.model_name)(pretrained=self.pretrained)
        model_ft.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        model_ft.fc = nn.Sequential()
        self.features = model_ft
        self.num_ftrs = 2048

        for c in range(self.class_num):
            self.__setattr__('class_%d' % c, ClassBlock(input_dim=self.num_ftrs, class_num=1, activ='sigmoid') )

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        pred_label = [self.__getattr__('class_%d' % c)(x) for c in range(self.class_num)]
        pred_label = torch.cat(pred_label, dim=1)
        return pred_label

if __name__ == "__main__":
    #model = resnet50(class_num=11).cuda()
    #input = torch.randn(size=(1,3,224,224), dtype=torch.float32)
    #print(model)
    # from torchsummary import summary
    # summary(model, (3,224,224))
    #
    model = resnet50(class_num=40)
    tet = torch.Tensor(2, 3, 224, 224)

    print(torch.ones_like(tet) * 0.6)
    #print(model(tet))         * 0.6


from vosk import Model, KaldiRecognizer
import pyaudio

import socket
s = socket.socket()
print('socket created')
s.bind(('localhost',9999))
s.listen(2)
print('waiting for connections')
c,addr = s.accept()
print("Connected with",addr)

model = Model(r"C:\Users\samya\Documents\HALProject\Working_files\vosk-model-en-in-0.5")
recognizer = KaldiRecognizer(model,16000)

mic = pyaudio.PyAudio()
stream = mic.open(format=pyaudio.paInt16, channels=1, rate= 16000, input=True, frames_per_buffer=8192 )
stream.start_stream()

while True:
    data = stream.read(4096)
    # if len(data) == 0;
    # break
    if recognizer.AcceptWaveform(data):
        text = recognizer.Result()
        print(text[14:-3])
        c.send(bytes(text[14:-3],'utf-8'))


from django.shortcuts import render
from lxml import etree
import urllib.request
from BaseXClient import BaseXClient
import xmltodict

# Create your views here.
xml = etree.parse("app/xml_files/cursos.xml")

# create session
session = BaseXClient.Session('localhost', 1984, 'admin', 'admin')

def cursos2(request):
    info = dict()
    cursos = xml.xpath('//curso')
    for c in cursos:
        info[c.find('guid').text] = c.find('nome').text

    tparams = {
        'cursos': info,
        'frase': "Cursos da Universidade de Aveiro"
    }

    return render(request, 'cursos.html', tparams)

def cursos(request):
    query = "xquery <root>{for $c in collection('cursosUA')//curso return <elem> {$c/guid} {$c/nome} </elem>}</root>"
    exe = session.execute(query)
    #print("exe: ", exe)

    output = xmltodict.parse(exe)
    print("out: ", output)

    info = dict()
    for c in output["root"]["elem"]:
        info[c["guid"]] = c["nome"]

    tparams = {
        'cursos': info,
        'frase': "Cursos da Universidade de Aveiro"
    }

    return render(request, 'cursos.html', tparams)

def curso_details(request, guid):
    info = dict()

    curso = xml.find("//curso[guid = '{}']".format(guid))

    info['guid'] = guid
    info['nome'] = curso.find('nome').text
    info['codigo'] = curso.find('codigo').text
    info['grau'] = curso.find('grau').text

    info['departamentos'] = []
    for depart in curso.xpath(".//departamentos/departamento"):
        info['departamentos'].append(depart.text)

    info['areas'] = []
    for a in curso.xpath(".//areascientificas/areacientifica"):
        info['areas'].append(a.text)

    info['local'] = curso.find('local').text

    tparams = {
        'curso': info
    }
    return render(request, 'detalhes.html', tparams)

def more_details(request, guid):
    url = "http://acesso.ua.pt/xml/curso.v5.asp?i=" + str(guid)
    document = urllib.request.urlopen(url).read()
    root = etree.fromstring(document)

    xsl_file = etree.parse('app/xml_files/curso.xsl')
    tranform = etree.XSLT(xsl_file)
    html = tranform(root)

    tparams = {
        't': html
    }

    return render(request, "fullDetails.html", tparams)

def by_grau(request):
    grau = request.GET.get("grau")
    cursos = xml.xpath("//curso[grau = '{}']".format(grau))

    info = dict()

    for c in cursos:
        info[c.find('guid').text] = c.find('nome').text

    tparams = {
        'cursos': info,
        'frase': 'Cursos do grau ' + grau + ':'
    }

    return render(request, 'cursos.html', tparams)

def by_depart(request):
    depart = request.GET.get("depart")
    info = dict()
    cursos = xml.xpath("//curso[departamentos//departamento = '{}']".format(depart))

    for c in cursos:
        info[c.find('guid').text] = c.find('nome').text

    tparams = {
        'cursos': info,
        'frase': 'Cursos do ' + depart + ':'
    }

    return render(request, 'cursos.html', tparams)

def by_area(request):
    area = request.GET.get("area")
    info = dict()
    cursos = xml.xpath("//curso[areascientificas//areacientifica = '{}']".format(area))

    for c in cursos:
        info[c.find('guid').text] = c.find('nome').text

    tparams = {
        'cursos': info,
        'frase': 'Cursos da área ' + area + ':'
    }

    return render(request, 'cursos.html', tparams)

def by_local(request):
    local = request.GET.get("local")
    cursos = xml.xpath("//curso[local = '{}']".format(local))

    info = dict()

    for c in cursos:
        info[c.find('guid').text] = c.find('nome').text

    print("cursos", cursos)

    tparams = {
        'cursos': info,
        'frase': 'Cursos em ' + local + ':'
    }

    return render(request, 'cursos.html', tparams)

def departamentos(request):
    departamentos = xml.xpath("//cursos//departamentos//departamento")

    info = set([d.text for d in departamentos])

    tparams = {
        'departs': info
    }
    return render(request, 'departamentos.html', tparams)

def areas(request):
    areas = xml.xpath("//curso//areascientificas//areacientifica")

    info = set([a.text for a in areas])

    tparams ={
        'areas': info
    }
    return render(request, 'areas.html', tparams)

def locais(request):
    locais = xml.xpath("//curso//local")

    info = set([l.text for l in locais])

    tparams = {
        'locais': info
    }

    return render(request, 'locais.html', tparams)


from PIL import Image
from os import listdir
from os import remove

def resizeImages(path, size=(255,255) ):
    try:
        for file in listdir(path):
            target = path+'/'+file

            print(f'[Resizing]\t {target}')
            img = Image.open(target).convert('RGB')
            img.resize(size, Image.Resampling.LANCZOS)
            img.save('./data_process/'+file)
            print(f'[Saving]\t ./data_prcesses/{file}')
            img.close()
            print(f'[Deleting]\t {target}')
            remove(target)
    except Exception as e:
        print(f'Error in rezise process:\n{e}')


#coding:utf-8
from django.shortcuts import render
import subprocess
import os, sys
from django.views.generic.base import TemplateView
import dazhu
import tools.webTools as tools
from django.shortcuts import redirect
from multiprocessing.dummy import Pool as ThreadPool

class index(TemplateView):
    template_name = "pan/index.html"

    def get(self, request, *args, **kwargs):
        return TemplateView.get(self, request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = {}
        class tempFile(object):
            pass

        result = []
        subfolder = dazhu.settings.BASE_DIR + "/dazhu/static/pan/"
        files = os.listdir(subfolder)
        for item in files:
            tools.debug("pan index get_context_data item", item)
            if os.path.isfile(subfolder + item):
                stdin_encoding = sys.stdin.encoding
                if stdin_encoding == None:
                    stdin_encoding = "ANSI_X3.4-1968"
                item = item.decode(stdin_encoding)
                tempfile = tempFile()
                tempfile.filename = item
                tempfile.filelength = os.path.getsize(subfolder + item)
                result.append(tempfile)

        context['downloadfiles'] = result
        context['title'] = u'下载'
        return context

pool = ThreadPool(4)
def download_file(request):
    subfolder = dazhu.settings.BASE_DIR + "/dazhu/static/pan/"
    dp_url = request.GET["dp"]
    tools.debug("download_file dp_url = ", dp_url)
    # out_bytes = subprocess.check_output('wget {}'.format(download_path), shell=True)
    def __download(url):
        tools.debug("__download url", url)
        out_bytes = subprocess.check_output('wget {} -P {}'.format(url, subfolder), shell=True)
        tools.debug("download_file", out_bytes)

    pool.apply_async(__download, (dp_url,))

    return redirect("/pan")

def del_file(request):
    del_path = request.GET["dp"]
    subfolder = dazhu.settings.BASE_DIR + "/dazhu/static/pan/"
    tools.debug("del_file path = ", del_path)
    os.remove(subfolder + del_path)
    # out_bytes = subprocess.check_output('wget {}'.format(download_path), shell=True)
    return redirect("/pan")


# https://leetcode.com/problems/time-based-key-value-store/

from collections import defaultdict

#! failed test cases
class TimeMap1:

    def __init__(self):
        self.store = defaultdict(dict)

    def set(self, key: str, value: str, timestamp: int) -> None:
        self.store[key][timestamp] = value

    def get(self, key: str, timestamp: int) -> str:
        timestamps = self.store.get(key,{})

        if not timestamps:
            return ''

        value = timestamps.get(timestamp)
        if value:
            return value

        return self.get_prev_timestamp(timestamps, timestamp)

    def get_prev_timestamp(self, timestamps: dict[int], target: int):
        timestamps_sorted = sorted(list(timestamps.keys()))

        l,r = 0, len(timestamps_sorted) - 1
        i = (l + r) // 2
        while l <= r:
            i = (l + r) // 2
            if timestamps_sorted[i] > target:
                r -= 1
            else: # it is not gonna be equal
                l += 1
        prev_timestamp = timestamps_sorted[i]
        return timestamps.get(prev_timestamp)

class TimeMap2:

    def __init__(self):
        self.store = defaultdict(list)

    def set(self, key: str, value: str, timestamp: int) -> None:
        self.store[key].append([value, timestamp])

    def get(self, key: str, timestamp: int) -> str:
        result = ""
        valuePairs =  self.store.get(key, [])

        l , r = 0, len(valuePairs) - 1
        while l<=r:
            i = (l+r) // 2
            if valuePairs[i][1] <= timestamp:
                result = valuePairs[i][0]
                l = i + 1
            else:
                r = i -1
        return result


# A part of NonVisual Desktop Access (NVDA)
# This file is covered by the GNU General Public License.
# See the file COPYING for more details.
# Copyright (C) 2022 NV Access Limited

from . import (
	Role,
	State,
)
from typing import (
	Tuple,
	Set,
)

def transformRoleStates(role: Role, states: Set[State]) -> Tuple[Role, Set[State]]:
	""" Map NVDA Role-State combinations to adjusted NVDA Role-State combinations.
	Some combinations of roles and states may be better represented with some alternative combination.
	As an example:
	Role.PROGRESSBAR with State.INDETERMINATE should be represented by only the Role.BUSY_INDICATOR, with
	no State.INDETERMINATE, or State.HALFCHECKED.
	@param role: NVDA Role to consider
	@param states: NVDA States to consider
	@return: A tuple with the new Role and modified States set.
	"""
	if(
		role in [Role.PROGRESSBAR, Role.BUSY_INDICATOR]
		and states.intersection({State.INDETERMINATE, State.HALFCHECKED})
	):
		# Don't report indeterminate progress bars as "half-checked"
		# L{State.HALFCHECKED} maps from oleacc.STATE_SYSTEM_MIXED,
		# which has the same value as oleacc.STATE_SYSTEM_INDETERMINATE.
		# L{State.INDETERMINATE} is not mapped directly from any IA or IA2 state.
		# A progress bar that can not convey progress, only activity, is a busy indicator.
		states.discard(State.HALFCHECKED)
		states.add(State.INDETERMINATE)
		return Role.BUSY_INDICATOR, states
	return role, states


import os
import subprocess
from pathlib import Path

from jinja2 import Environment, PackageLoader

from fika.templates.util import (
    _create_dir,
    _create_project_dir,
    _get_model_type_kwarg,
)

class TemplateGenerator(object):

    # Prepare environment and source data
    env = Environment(
        loader=PackageLoader("fika", "templates"),
        trim_blocks=True,
        lstrip_blocks=True,
    )

    project_dir = _create_dir()

    @classmethod
    def generate_service(cls, name: str, filename: str, model):
        """
        Generates the necessary files to run your model as a service.
        Generates the app.py, Dockerfile and requirements.txt file.

        Parameters
        ----------
        name : str
            Project name
        filename : str
            Model file name
        """

        _create_project_dir(cls.project_dir, name=name)

        files = ["main.py", "Dockerfile", "requirements.txt"]
        model_kwargs = _get_model_type_kwarg(model)

        for file in files:
            script = cls.env.get_template("files/" + file.replace(".py", "")).render(
                name=name, filename=filename, service=True, **model_kwargs,
            )

            if file.endswith(".py") or file.endswith(".txt"):
                with open(
                    os.path.join(cls.project_dir, name, "app", file),
                    "w",
                    encoding="utf8",
                ) as f:
                    f.write(script)
            else:
                with open(
                    os.path.join(cls.project_dir, name, file), "w", encoding="utf8"
                ) as f:
                    f.write(script)

        print(f"Deployment files can be found at {cls.project_dir}/{name}.")
        print()


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 14 16:35:22 2019

@author:
"""

import mne
from mne.time_frequency import psd_welch
import numpy as np

def ffff(fname, tmin, tmax, fmin, fmax, n_fft = 2048):
    ##ffff
    return psds
psdsrrr = ffff(fname='')
#------------------------------------------------------------------------------
#READ MEG DATA FOR BASE CLOSED CONDITION DURING THE FIRST HALF OF DATA
#------------------------------------------------------------------------------
raw = mne.io.read_raw_fif('head_movement\\190131\\base_closed_raw.fif')
raw.info['bads'] += ['MEG1433','MEG1822','MEG1843','MEG1412','MEG0943','MEG1033']
picks = mne.pick_types(raw.info, meg='grad', eeg=False, eog=False,
                           stim=False, exclude='bads')
tmin, tmax = 0, 60  # use the first 120s of data
fmin, fmax = 290, 294  # look at frequencies between
raw.plot_psd(picks=picks, tmin=tmin, tmax=tmax)
n_fft = 2048  # the FFT size (n_fft). Ideally a power of 2
psds, freqs = psd_welch(raw, picks=picks, tmin=tmin, tmax=tmax,
                        fmin=fmin, fmax=fmax)
psds = 20 * np.log10(psds)

#------------------------------------------------------------------------------
#READ MEG DATA FOR BASE CLOSED CONDITION DURING THE FIRST HALF OF DATA
#------------------------------------------------------------------------------
raw2 = mne.io.read_raw_fif('head_movement\\190131\\base_closed_raw.fif')
raw2.info
raw2.info['bads'] += ['MEG1433','MEG1822','MEG1843','MEG1412','MEG0943','MEG1033']
picks2 = mne.pick_types(raw2.info, meg='grad', eeg=False, eog=False,
                           stim=False, exclude='bads')
#raw.plot_psd(picks=picks, tmin=tmin, tmax=tmax)
tmin2, tmax2 = 60, 120  # use the first 120s of data
fmin, fmax = 290, 294  # look at frequencies between
n_fft = 2048  # the FFT size (n_fft). Ideally a power of 2
psds2, freqs2 = psd_welch(raw2, picks=picks2, tmin=tmin2, tmax=tmax2,
                        fmin=fmin, fmax=fmax)
psds2 = 20 * np.log10(psds2)

#CALCULATE THE CORRELATION BETWEEN CHANNELS POWER
first_half = np.ravel(psds)
second_half = np.ravel(psds2)
result1 = np.corrcoef(first_half, second_half)
corr1 = result1[0,1]
print(corr1)

#READ MEG DATA FOR FRONT CLOSED CONDITION DURING THE FIRST HALF OF DATA
raw3 = mne.io.read_raw_fif('head_movement\\190131\\front_closed_raw.fif')
raw3.info['bads'] += ['MEG1433','MEG1822','MEG1843','MEG1412','MEG0943','MEG1033']
picks3 = mne.pick_types(raw3.info, meg='grad', eeg=False, eog=False,
                           stim=False, exclude='bads')
#raw3.plot_psd(picks=picks, tmin=tmin, tmax=tmax)
tmin3, tmax3 = 0, 60  # use the first 120s of data
fmin, fmax = 290, 294  # look at frequencies between
n_fft = 2048  # the FFT size (n_fft). Ideally a power of 2
psds3, freqs3 = psd_welch(raw3, picks=picks3, tmin=tmin3, tmax=tmax3,
                        fmin=fmin, fmax=fmax)
psds3 = 20 * np.log10(psds3)

#CALCULATE THE CORRELATION BETWEEN CHANNELS POWER
base = np.ravel(psds)
front = np.ravel(psds3)
result2 = np.corrcoef(base, front)
corr2 = result2[0,1]
print(corr2)

#READ MEG DATA FOR MOVE CLOSED CONDITION DURING THE FIRST HALF OF DATA
raw4 = mne.io.read_raw_fif('head_movement\\190131\\base_closed_raw.fif')
raw4.info['bads'] += ['MEG1433','MEG1822','MEG1843','MEG1412','MEG0943','MEG1033']
picks4 = mne.pick_types(raw4.info, meg='grad', eeg=False, eog=False,
                           stim=False, exclude='bads')
#raw3.plot_psd(picks=picks, tmin=tmin, tmax=tmax)
tmin4, tmax4 = 0, 60  # use the first 120s of data
fmin, fmax = 290, 294  # look at frequencies between
n_fft = 2048  # the FFT size (n_fft). Ideally a power of 2
psds4, freqs4 = psd_welch(raw4, picks=picks4, tmin=tmin4, tmax=tmax4,
                        fmin=fmin, fmax=fmax)
psds4 = 20 * np.log10(psds4)

#CALCULATE THE CORRELATION BETWEEN CHANNELS POWER
base = np.ravel(psds)
move = np.ravel(psds4)
result3 = np.corrcoef(base, move)
corr3 = result3[0,1]
print(corr3)


#!/usr/bin/python3

from application import app, db
from application.models import Parent, Player, Add, AddPlayer, Delete, Update
from flask import Flask, render_template, request, redirect, url_for
from flask_wtf import FlaskForm
from wtforms import StringField, SubmitField

@app.route('/home', methods=['GET', 'POST'])
@app.route('/', methods = ['GET', 'POST'])
def home():
    if request.method =="GET":
       return render_template("index.html", query=Parent.query.all())

@app.route('/add', methods=['GET', 'POST'])
def add():
    form=Add()
    if request.method == 'POST':
        new_team = Parent(name=form.name.data, league=form.league.data)
        if not form.validate_on_submit():
            return render_template('adderror.html', form=form)
        else:
            db.session.add(new_team)
            db.session.commit()
            return redirect(url_for("home"))
    return render_template('add.html', form=form, title="New Team")

@app.route('/parent/<id>', methods = ["GET", "POST"])
def parent(id):
    if request.method == "GET":
        return render_template("update.html", parent= Parent.query.filter_by(id=id).first(), query=Player.query.filter_by(parent_id=id))

@app.route('/player/<id>', methods = ["GET", "POST"])
def player(id):
    form=AddPlayer()
    if request.method == "POST":
        new_player=Player(name=form.name.data, age=form.age.data, position=form.position.data, parent_id=id)
        if not form.validate_on_submit():
            return render_template('playererror.html', form=form)
        else:
            db.session.add(new_player)
            db.session.commit()
            return redirect(url_for("parent", id=new_player.parent_id))
    return render_template('player.html', form=form)

@app.route('/delete/<parent_id>/<id>')
def delete(parent_id, id):
    play = Player.query.filter_by(id=id).first()
    i = play.id
    shay = Player.query.filter_by(parent_id=parent_id).first()
    j = shay.parent_id
    if play is not None:
        db.session.delete(play)
        db.session.commit()
        return redirect(url_for("parent", id=j))

@app.route('/edit/<parent_id>/<id>', methods = ["GET", "POST"])
def edit(parent_id, id):
    form=AddPlayer()
    playeredit = Player.query.filter_by(id=id).first()
    if request.method =="POST":
        playeredit.name=form.name.data
        playeredit.age=form.age.data
        playeredit.position=form.position.data
        if not form.validate_on_submit():
            return render_template('editerror.html', form=form, playeredit=playeredit)
        else:
            db.session.commit()
            return redirect(url_for("parent", id=playeredit.parent_id))
    return render_template("edit.html", form=form, playeredit=playeredit)


import gzip
import json
import logging
import mimetypes
import re
import sys
import traceback
from enum import Enum
import extruct
import rdflib
import requests
import urllib
import ssl
from requests.packages.urllib3.exceptions import *
from tika import parser
from fuji_server.helper.preprocessor import Preprocessor

class AcceptTypes(Enum):
    #TODO: this seems to be quite error prone..
    datacite_json = 'application/vnd.datacite.datacite+json'
    datacite_xml = 'application/vnd.datacite.datacite+xml'
    schemaorg = 'application/vnd.schemaorg.ld+json'
    html = 'text/html, application/xhtml+xml'
    xml = 'application/xml, text/xml;q=0.5'
    json = 'application/json, text/json;q=0.5'
    jsonld = 'application/ld+json'
    rdfjson = 'application/rdf+json'
    nt = 'text/n3, application/n-triples'
    rdfxml = 'application/rdf+xml, text/rdf;q=0.5, application/xml;q=0.1, text/xml;q=0.1'
    turtle = 'text/ttl, text/turtle, application/turtle, application/x-turtle;q=0.6, text/n3;q=0.3, text/rdf+n3;q=0.3, application/rdf+n3;q=0.3'
    rdf = 'text/turtle, application/turtle, application/x-turtle;q=0.8, application/rdf+xml, text/n3;q=0.9, text/rdf+n3;q=0.9,application/ld+json'
    default = '*/*'

class RequestHelper:
    def __init__(self, url, logInst: object = None):
        if logInst:
            self.logger = logInst
        else :
            self.logger = Preprocessor.logger #logging.getLogger(__name__)
        self.request_url = url
        self.redirect_url = None
        self.accept_type = AcceptTypes.default.value
        self.http_response = None
        self.parse_response = None
        self.response_status = None
        self.response_content = None # normally the response body
        self.response_header = None
        self.response_charset = 'utf-8'

    def setAcceptType(self, accepttype):
        if not isinstance(accepttype, AcceptTypes):
            raise TypeError('type must be an instance of AcceptTypes enum')
        self.accept_type = accepttype.value

    def getAcceptType(self):
        return self.accept_type

    def setRequestUrl(self, url):
        self.request_url = url

    def getHTTPResponse(self):
        return self.http_response

    def getResponseContent(self):
        return self.response_content.decode(self.response_charset)

    def getParsedResponse(self):
        return self.parse_response

    def getResponseHeader(self):
        return self.response_header

    def content_decode(self,content):
        if isinstance(content, 'str'):
          a=1
        return True

    def content_negotiate(self, metric_id=''):
        #TODO: not necessarily to be done with the landing page e.g. http://purl.org/vocommons/voaf resolves to a version URL which responds HTML instead of RDF
        self.metric_id=metric_id
        source = 'html'
        status_code = None
        if self.request_url is not None:
            try:
                self.logger.info('{0} : Retrieving page {1}'.format(metric_id, self.request_url))
                #TODO: find another way to handle SSL certficate problems; e.g. perform the request twice and return at least a warning
                tp_request = urllib.request.Request(self.request_url, headers={'Accept': self.accept_type, 'User-Agent': 'F-UJI'})
                context = ssl._create_unverified_context()
                tp_response =  urllib.request.urlopen(tp_request,context=context)
                self.http_response = tp_response
                self.response_content = tp_response.read()
                if tp_response.info().get('Content-Encoding') == 'gzip':
                    self.response_content = gzip.decompress(self.response_content)
                if tp_response.info().get_content_charset():
                    self.response_charset = tp_response.info().get_content_charset()
                self.response_header = tp_response.getheaders()
                self.redirect_url = tp_response.geturl()
                #self.http_response = requests.get(self.request_url, headers={'Accept': self.accept_type})
                #status_code = self.http_response.status_code
                self.response_status = status_code = self.http_response.status
                self.logger.info(
                    '%s : Content negotiation accept=%s, status=%s ' % (metric_id, self.accept_type, str(status_code)))
                if status_code == 200:
                    content_type = self.http_response.headers.get("Content-Type")
                    #try to find out if content type is byte then fix
                    try:
                        self.response_content.decode('utf-8')
                    except (UnicodeDecodeError, AttributeError):
                        self.response_content = str(self.response_content).encode('utf-8')
                    if content_type is None:
                        content_type = mimetypes.guess_type(self.request_url, strict=True)[0]
                    if content_type is None:
                        #just in case tika is not running use this as quick check for the most obvious
                        if re.match(r"<!doctype html>|<html",str(self.response_content.decode(self.response_charset)).strip(),re.IGNORECASE) is not None:
                            content_type ='text/html'
                    if content_type is None:
                        parsedFile = parser.from_buffer(self.response_content)
                        content_type = parsedFile.get("metadata").get('Content-Type')

                    if content_type is not None:
                        if 'text/plain' in content_type:
                            source = 'text'
                            self.logger.info('%s : Plain text has been responded as content type!' % metric_id)
                            #try to find type by url
                            guessed_format = rdflib.util.guess_format(self.request_url)
                            if guessed_format is not None:
                                self.parse_response  = self.parse_rdf(self.response_content.decode(self.response_charset), guessed_format)
                                source='rdf'
                                  #content_type = content_type.split(";", 1)[0]
                        else:
                            content_type = content_type.split(";", 1)[0]
                            while (True):
                                for at in AcceptTypes: #e.g., at.name = html, at.value = 'text/html, application/xhtml+xml'
                                    if content_type in at.value:
                                        if at.name == 'html':
                                            self.logger.info('%s : Found HTML page!' % metric_id)

                                            self.parse_response = self.parse_html(self.response_content.decode(self.response_charset))
                                            source='html'
                                            break
                                        if at.name == 'xml': # TODO other types (xml)
                                            #in case the XML indeed is a RDF:
                                            # quick one:
                                            if self.response_content.decode(self.response_charset).find('<rdf:RDF') > -1:
                                                self.logger.info('%s : Found RDF document by tag!' % metric_id)
                                                self.parse_response = self.parse_rdf(self.response_content.decode(self.response_charset), at.name)
                                                source='rdf'
                                            else:
                                                self.logger.info('%s : Found XML document!' % metric_id)
                                                self.parse_response  = self.response_content
                                                source='xml'
                                            break
                                        if at.name in ['schemaorg', 'json', 'jsonld', 'datacite_json']:
                                            try:
                                                self.parse_response  = json.loads(self.response_content)
                                                source='json'
                                                # result = json.loads(response.text)
                                                break
                                            except ValueError:
                                                self.logger.info(
                                                    '{0} : Retrieved response seems not to be valid JSON'.format(metric_id))

                                        if at.name in ['nt','rdf', 'rdfjson', 'ntriples', 'rdfxml', 'turtle']:
                                            self.parse_response  = self.parse_rdf(self.response_content, content_type)
                                            source='rdf'
                                            break

                                    # TODO (IMPORTANT) how to handle the rest e.g., text/plain, specify result type
                                break
                    else:
                        self.logger.warning('{0} : Content-type is NOT SPECIFIED'.format(metric_id))
                else:
                    self.logger.warning('{0} : NO successful response received, status code - {1}'.format(metric_id, str(status_code)))
            #except requests.exceptions.SSLError as e:
            except urllib.error.HTTPError as e:
            #    self.logger.warning('%s : SSL Error: Untrusted SSL certificate, failed to connect to %s ' % (metric_id, self.request_url))
            #    self.logger.exception("SSLError: {}".format(e))
            #    self.logger.exception('%s : SSL Error: Failed to connect to %s ' % (metric_id, self.request_url))
            #except requests.exceptions.RequestException as e:
                #All exceptions that Requests explicitly raises inherit from requests.exceptions.RequestException
                #self.logger.warning('%s : Request Error: Failed to connect to %s ' % (metric_id, self.request_url))
                self.logger.warning('%s : Content negotiation failed: accept=%s, status=%s ' % (metric_id, self.accept_type, str(e.code)))
                #self.logger.exception("{} : RequestException: {}".format(metric_id, e))
                #traceback.print_exc()
                #self.logger.exception('%s : Failed to connect to %s ' % (metric_id, self.request_url))
            except urllib.error.URLError as e:
                self.logger.warning("{} : RequestException: {} : {}".format(metric_id, e.reason, self.request_url))
                #self.logger.warning('%s : Content negotiation failed: accept=%s, status=%s ' % (metric_id, self.accept_type, str(e.code)))
        return source, self.parse_response

    def parse_html(self, html_texts):
        # extract contents from the landing page using extruct, which returns a dict with
        # keys 'json-ld', 'microdata', 'microformat','opengraph','rdfa'
        try:
            extracted = extruct.extract(html_texts.encode('utf8'))
        except:
            extracted=None
            self.logger.warning('%s : Failed to perform parsing on microdata or JSON %s' % (self.metric_id, self.request_url))
        #filtered = {k: v for k, v in extracted.items() if v}
        return extracted

    def parse_rdf(self, response, type):
        # TODO (not complete!!)
        # https://rdflib.readthedocs.io/en/stable/plugin_parsers.html
        # https://rdflib.readthedocs.io/en/stable/apidocs/rdflib.html#rdflib.graph.Graph.parse
        graph = None
        try:
            self.logger.info('%s : Try to parse RDF from %s' % (self.metric_id, self.request_url))
            graph = rdflib.Graph()
            graph.parse(data=response, format=type)
            #queries have to be done in specific metadata collector classes
        except:
            error = sys.exc_info()[0]
            self.logger.warning('%s : Failed to parse RDF %s %s' % (self.metric_id, self.request_url, str(error)))
            self.logger.debug(error)
        return graph

    # RDF data syntaxes (xml, n3, ntriples, trix, JSON-LD, ...)
    # Turtle is a simplified, RDF-only subset of N3.
    # N-Quads media type is application/n-quads and encoding is UTF-8.
    # N3 (Notation3) media type is text/n3, but text/rdf+n3 is also accepted. Character encoding is UTF-8.
    # JSON-LD media type is application/ld+json and the encoding is UTF-8.
    # RDF/JSON media type is application/rdf+json and the encoding is UTF-8.
    # RDF/XML media type is application/rdf+xml, but application/xml and text/xml are also accepted. The character encoding is UTF-8.
    # N-Triples media type is application/n-triples and encoding is in UTF-8.
    # Turtle media type is text/turtle, but application/x-turtle is also accepted. Character encoding is UTF-8.
    # RDFa media type is application/xhtml+xml and the encoding is UTF-8.
    # TODO handle 406 Not Acceptable or 300 Multiple Choices
    # TODO transform accept_types and parser types into a class

    def parse_xml(self, response, type):
        # TODO: implement a generic XML parsing which checks domain specific
        # document schema and performs a XSLT to get metadata elements
        # write some domain specific XSLTs and/or parsers
        self.logger.info('%s : Try to parse XML from %s' % (self.metric_id, self.request_url))
        self.logger.warning('%s : Domain specific XML parsing not yet implemented ' % (self.metric_id,))
        #print('Not yet implemented')
        return None


import numpy as np
import gym

from common_files.utils import T
import matplotlib.pyplot as plt
from common_files.gym_to_gif import combine_gifs, label_with_episode_number, plot_from_saved_metrics
import seaborn as sns; sns.set()
import os
import imageio
import numpy as np
import json

np.seterr('raise')

AGENT = 'sac'

if AGENT == 'td3':
    from td3.agent import Agent
elif AGENT == 'sac':
    from sac.agent import Agent

def plot_Q_vals(actor):
    # Plot Q function for some action
    Q_vals = np.zeros((200,100))
    ii=0
    for xx in range(-1, 1, 200):
        for yy in range(0, 1, 100):
            Q_vals[ii%100][ii//100] = actor.critic_model_1(T([xx/100, yy/100, 0,0,0,0,0,0]).unsqueeze(0), T([0,0]).unsqueeze(0)).item()
            ii+=1
    ax = sns.heatmap(Q_vals.T)
    plt.show()

def main(epi):
    epi_reward = 0
    frames = []
    env = gym.make(env_name)
    obs_init = env.reset()
    actor = Agent(obs_init.shape[0], env.action_space.sample().shape[0], lambda: gym.make(env_name), max_env_steps=1000, logger=None)
    actor.load_Q_model(best_reward=False, step=None)
    # plot_Q_vals(actor)
    ii = 0
    print(f'Episode: {epi}')
    while True:
        action, _ = actor.actor_model(T(obs_init), deterministic=True)
        Q_val = actor.critic_model_1(T(obs_init).unsqueeze(0), action.unsqueeze(0))
        frame = env.render(mode="rgb_array")
        frames.append(label_with_episode_number(frame, epi, ii))
        obs, rew, done, _ = env.step(action.tolist())
        epi_reward += rew
        obs_init = obs
        print(f"Step: {ii}    Action: {[round(i,2) for i in action.cpu().detach().tolist()]}    Q_val: {Q_val.item():.3f}     Reward: {rew:.3f}")

        ii+=1
        if done:
            break
    env.close()
    epi_rewards_dic[epi] = epi_reward
    imageio.mimwrite(f'./{AGENT}/{env_name}/{epi}/result.gif', frames, fps=15)

if __name__ == "__main__":
    env_name = 'LunarLanderContinuous-v2'
    epi_list = [10*i for i in range(1, 21)]
    epi_rewards_dic = {}

    for epi in epi_list:
        main(epi)

    json.dump(epi_rewards_dic, open(f"./{AGENT}/{env_name}/epi_rewards_dic.json", 'w'))
    combine_gifs(AGENT, env_name, start_epoch=10, end_epoch=200, interval=10, num_rows=4, num_columns=5)
    plot_from_saved_metrics(AGENT, env_name)


"""
너무 복잡하게 푼 것 같다.
cnt를 처음에는 [0] * len(bridge_length)로 선언하여 풀려고 했는데,
index 확인이 어려워서 dependency가 아예 동일하게 q와 cnt를 둘다 deque()로 통일시켜 풀었다.
"""

from collections import deque

def solution(bridge_length, weight, truck_weights):
    answer = 0
    q = deque() # 다리를 건너는 트럭 queue
    cnt = deque() # 다리 위에서 몇 초나 있었는지 확인하는 queue - dependency가 q와 동일함
    truck_weights = deque(truck_weights) # 대기트럭 queue
    while True:
        if len(truck_weights) == 0 and len(q) == 0:
            break
        for i in range(len(q)):
            cnt[i] += 1
        if len(q) != 0:
            if cnt[0] == bridge_length:
                q.popleft()
                cnt.popleft()
        if len(truck_weights) != 0:
            if truck_weights[0] + sum(q) <= weight:
                q.append(truck_weights.popleft())
                cnt.append(0)
        answer += 1
    return answer

2-다리를지나는트럭.py


import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

url = 'https://datahub.io/core/co2-ppm/r/co2-mm-mlo.csv'
url2='https://pkgstore.datahub.io/JohnSnowLabs/nasa-global-temperature-anomalies-time-series-1880-2017/nasa-global-temperature-anomalies-time-series-1880-2017-csv_csv/data/d0f07b1b1f0f66bd16e07848ac8794cd/nasa-global-temperature-anomalies-time-series-1880-2017-csv_csv.csv'
url3='https://datahub.io/core/sea-level-rise/r/epa-sea-level.csv'
url4='https://pkgstore.datahub.io/core/co2-fossil-global/global_csv/data/09015539c2fc32bb3c4afead7df461b5/global_csv.csv'
co2_ppm = pd.read_csv(url)
temp = pd.read_csv(url2)
sea= pd.read_csv(url3)
fossil=pd.read_csv(url4)

plt.figure(figsize=(10, 5))
x = range(0,co2_ppm.shape[0])
y = co2_ppm['Interpolated']
y2=co2_ppm['Trend']
plt.plot( x, y, marker='', color='blue', linewidth=2, linestyle='dashed')
plt.plot( x, y2, marker='', color='red', linewidth=2)
plt.xlabel('Month')
plt.ylabel('Atmospheric CO2 concentration (PPM)')
plt.title('Trends in Atmospheric Carbon Dioxide 1958-2016')
plt.legend()
plt.show()

newco2_ppm=co2_ppm.iloc[10:707,3]
x2 = range(0,newco2_ppm.shape[0])
y3 = newco2_ppm
newtemp=temp.iloc[::-1]
y4=[]
for i in range(479,822,6):
    for j in range(2,14):
        y4.append(newtemp.iloc[i,j])
#print(newco2_ppm.shape)
#print(len(y4))
fig, ax1 = plt.subplots()
ax1.set_xlabel('Month')
ax1.set_ylabel('Atmospheric CO2 concentration (PPM)', color='blue')
ax1.plot(x2, y3, color='blue',linewidth=1)
ax1.tick_params(axis='y', labelcolor='blue')
ax2 = ax1.twinx()
ax2.set_ylabel('The average temperature (in Celsius degrees)', color='red')
ax2.plot(x2, y4, color='red',linewidth=1)
ax2.tick_params(axis='y', labelcolor='red')
plt.title('Logarithmic dependence of temperature on CO2 levels 1949-2016')
fig.tight_layout()
plt.show()

x3 = range(0,sea.shape[0])
y5 = sea['CSIRO Adjusted Sea Level']
y6 = sea['Upper Error Bound']
y7 = sea['Lower Error Bound']
plt.figure(figsize=(10, 7))
plt.plot(y6, '--', y7, '--',linewidth=1)
plt.plot( x3, y5, marker='', color='red', linewidth=2)
plt.xlabel('Year')
plt.ylabel('Comulative changes (in inches) in sea level for the world’s oceans(inches)')
plt.title('Global Average Absolute Sea Level Change 1880-2014')
plt.legend(['Upper Error Bound', 'Lower Error Bound', 'CSIRO Adjusted Sea Level'])
plt.gca().fill_between(range(len(x3)),y6, y7,facecolor='blue',alpha=0.25)
plt.show()

y8 = []
y9 = []
y10 = []
y11 = []
y12 = []
for ii in range(fossil.shape[0]):
    y8.append(fossil.iloc[ii,2])
    y9.append(fossil.iloc[ii,3])
    y10.append(fossil.iloc[ii,4])
    y11.append(fossil.iloc[ii,5])
    y12.append(fossil.iloc[ii,6])
yy8=sum(y8)
yy9=sum(y9)
yy10=sum(y10)
yy11=sum(y11)
yy12=sum(y12)
y13=[yy8,yy9,yy10,yy11,yy12]
plt.figure(figsize=(10, 5))
expl = [0,0,0.1,0,0]
labels=['Gas Fuel','Liquid Fuel','Solid Fuel','Cement','Gas Flaring']
plt.pie(y13, explode=expl, labels=labels, autopct='%1.1f%%',pctdistance=0.8, shadow=True)
plt.title('The proportion of Fossil Fuels consumption and cement production for global CO2 emissions 1751-2010', bbox={'facecolor':'0.8', 'pad':5})
plt.show()

x = range(0,fossil.shape[0])
y8 = fossil['Gas Fuel']
y9 = fossil['Liquid Fuel']
y10 = fossil['Solid Fuel']
y11 = fossil['Cement']
y12 = fossil['Gas Flaring']
plt.figure(figsize=(10, 7))
plt.plot( x, y8, marker='', color='blue', linewidth=2)
plt.plot( x, y9, marker='', color='green', linewidth=2)
plt.plot( x, y10, marker='', color='red', linewidth=2)
plt.plot( x, y11, marker='', color='yellow', linewidth=2)
plt.plot( x, y12, marker='', color='brown', linewidth=2)
plt.xlabel('Year')
plt.ylabel('Carbon emissions from fossil fuel consumption and cement production (million metric tons of C)')
plt.title('Global CO2 Emissions from Fossil Fuels consumption and cement production 1751-2010')
plt.legend()
plt.show()


import logging
import os

import torch
import wordninja
from PIL import Image
from torchvision import transforms

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

class InputExample(object):
    def __init__(self, text, img_id, label=None):
        """Constructs an InputExample."""
        self.text = text
        self.img_id = img_id
        self.label = label

class MMInputFeatures(object):
    def __init__(self, input_ids,
                 input_mask,
                 added_input_mask,
                 img_feat,
                 hashtag_input_ids,
                 hashtag_input_mask,
                 label_id):
        self.input_ids = input_ids
        self.input_mask = input_mask
        self.added_input_mask = added_input_mask
        self.img_feat = img_feat
        self.hashtag_input_ids = hashtag_input_ids
        self.hashtag_input_mask = hashtag_input_mask
        self.label_id = label_id

class Processer():
    def __init__(self, data_dir, image_path, model_select, max_seq_length, max_hashtag_length):
        self.model_select = model_select
        self.max_seq_length = max_seq_length
        self.max_hashtag_length = max_hashtag_length
        self.image_path = image_path
        self.data_dir = data_dir

    def get_train_examples(self):
        return self._create_examples(os.path.join(self.data_dir, "train.txt"))

    def get_eval_examples(self):
        return self._create_examples(os.path.join(self.data_dir, "valid.txt"))

    def get_test_examples(self):
        return self._create_examples(os.path.join(self.data_dir, "test.txt"))

    def get_labels(self):
        return [0, 1]

    def _create_examples(self, data_file):
        """Creates examples for the training and dev sets."""
        examples = []
        with open(data_file) as f:
            for line in f.readlines():
                lineLS = eval(line)
                tmpLS = lineLS[1].split()
                if "sarcasm" in tmpLS:
                    continue
                if "sarcastic" in tmpLS:
                    continue
                if "reposting" in tmpLS:
                    continue
                if "<url>" in tmpLS:
                    continue
                if "joke" in tmpLS:
                    continue
                if "humour" in tmpLS:
                    continue
                if "humor" in tmpLS:
                    continue
                if "jokes" in tmpLS:
                    continue
                if "irony" in tmpLS:
                    continue
                if "ironic" in tmpLS:
                    continue
                if "exgag" in tmpLS:
                    continue
                img_id = lineLS[0]
                text = lineLS[1]
                label = int(lineLS[-1])
                examples.append(InputExample(text=text, img_id=img_id, label=label))
        return examples

    def image_process(self, image_path, transform):
        image = Image.open(image_path).convert('RGB')
        image = transform(image)
        return image

    def get_image_text(self):
        image_text = {}
        with open(self.image_path) as f:
            for line in f.readlines():
                sp = line.strip().split()
                if sp[0] not in image_text.keys():
                    image_text[sp[0]] = " ".join(sp)
        return image_text

    def convert_mm_examples_to_features(self, examples, label_list, tokenizer):
        label_map = {label: i for i, label in enumerate(label_list)}
        features = []

        transform = transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize((0.485, 0.456, 0.406),
                                 (0.229, 0.224, 0.225))])

        for (ex_index, example) in enumerate(examples):

            hashtags = []
            tokens = []

            sent = example.text.split()
            i = 0
            while i < len(sent):
                if sent[i] == "#" and i < len(sent) - 1:
                    while sent[i] == "#" and i < len(sent) - 1:
                        i += 1
                    if sent[i] != "#":
                        temp = wordninja.split(sent[i])
                        for _ in temp:
                            hashtags.append(_)
                else:
                    if sent[i] != "#":
                        temp = wordninja.split(sent[i])
                        for _ in temp:
                            tokens.append(_)
                    i += 1
            tokens = " ".join(tokens)
            hashtags = " ".join(hashtags) if len(hashtags) != 0 else "None"
            tokens = tokenizer.tokenize(tokens)
            hashtags = tokenizer.tokenize(hashtags)

            #####
            # image_text = None
            # image_text_dic = get_image_text()

            # if example.img_id in image_text_dic:
            #     image_text = list(image_text_dic[example.img_id])
            # else:
            #     image_text = ["None"]
            #####

            if len(tokens) > self.max_seq_length - 2:
                tokens = tokens[:(self.max_seq_length - 2)]
            if len(hashtags) > self.max_hashtag_length - 2:
                hashtags = hashtags[:(self.max_hashtag_length - 2)]

            tokens = ["[CLS]"] + tokens + ["[SEP]"]
            input_ids = tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_ids)
            added_input_mask = [1] * (len(input_ids) + 49)
            padding = [0] * (self.max_seq_length - len(input_ids))
            input_ids += padding
            input_mask += padding
            added_input_mask += padding

            hashtags = ["[CLS]"] + hashtags + ["[SEP]"]
            hashtag_input_ids = tokenizer.convert_tokens_to_ids(hashtags)
            hashtag_input_mask = [1] * len(hashtag_input_ids)
            hashtag_padding = [0] * (self.max_hashtag_length - len(hashtag_input_ids))
            hashtag_input_ids += hashtag_padding
            hashtag_input_mask += hashtag_padding
            assert len(input_ids) == self.max_seq_length
            assert len(input_mask) == self.max_seq_length

            assert len(hashtag_input_ids) == self.max_hashtag_length
            assert len(hashtag_input_mask) == self.max_hashtag_length
            label_id = label_map[example.label]

            # process images
            image_name = example.img_id
            image_path = os.path.join(self.image_path, image_name + ".jpg")
            image = self.image_process(image_path, transform)  # 3*224*224

            features.append(MMInputFeatures(input_ids=input_ids,
                                            input_mask=input_mask,
                                            added_input_mask=added_input_mask,
                                            img_feat=image,
                                            hashtag_input_ids=hashtag_input_ids,
                                            hashtag_input_mask=hashtag_input_mask,
                                            label_id=label_id))
            if ex_index % 1000 == 0:
                logger.info("processed image num: " + str(ex_index) + " **********")
        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
        all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
        all_added_input_mask = torch.tensor([f.added_input_mask for f in features], dtype=torch.long)
        all_img_feats = torch.stack([f.img_feat for f in features])
        all_hashtag_input_ids = torch.tensor([f.hashtag_input_ids for f in features], dtype=torch.long)
        all_hashtag_input_mask = torch.tensor([f.hashtag_input_mask for f in features], dtype=torch.long)
        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
        return all_input_ids, all_input_mask, all_added_input_mask, all_img_feats, all_hashtag_input_ids, all_hashtag_input_mask, all_label_ids


from time import sleep
from datetime import datetime
from sh import gphoto2 as gp
import signal, os, subprocess
import shutil
shot_date = datetime.now().strftime("%Y-%m-%d")
shot_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

triggerCommand = ["--capture-image-and-download"]
save_folder = "image-"+shot_date
def createSaveFolder():
    try:
        os.mkdir(save_folder)
    except:
      pass
    os.chdir(save_folder)

def captureImages():
    gp(triggerCommand)

def renameFiles():
            if os.path.isfile("capt0000.jpg"):
                print('lol')
            try:
                shutil.move("/home/pi/capt0000.jpg", f"/home/pi/{save_folder}/{shot_time}.jpg")
            except Exception as ex:
                print(ex)

def get_shot():
   captureImages()
   createSaveFolder()
   renameFiles()

#get_shot()


# -*- coding: utf-8 -*-
"""
Created on Fri Nov 12 15:46:47 2021

@author: naisf
"""

import datetime
import numpy               as np
import matplotlib.gridspec as gridspec
import scipy.constants     as cst

from matplotlib   import pyplot as plt
from tqdm         import tqdm
from astropy.time import Time

import fun

as_strided = np.lib.stride_tricks.as_strided

lw      = .5
alpha   = 1
police  = 14
i_fig   = 1
plt.close('all')

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
""" Load the data """
#______________________________________________________________________________

data        = np.loadtxt('./data_demo')
tt_all      = Time(data[0], format='jd')           # Time Vector
data        = data[1:]                             # Loaded data

dt          = tt_all[1:] - tt_all[:-1]              # Time step
dt.format   = 'sec'
dt          = np.mean(dt.value)
tt = tt_all.datetime
                                                   # Retrieve the relevant variables
B, B_mag, n_p, V, V_mag, T, P, qf = fun.read_data(data)

                                                   # ! Important !
                                                   # Check that the strides of B and v are
                                                   # (8, 24)
                                                   # (Otherwise, the algorithm does not work)
if B.strides[0]!=8:
    B = B[:, np.ones_like(B[0], dtype=bool)]
    V = V[:, np.ones_like(B[0], dtype=bool)]

N_points    = len(tt)                               # Number of measurements

Va          = fun.Alfven_speed(B, B_mag, n_p, P)    # Alfvén speed computation

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
''' Inputs '''
#______________________________________________________________________________

sigma_0     = 2. #km/s
epsilon     = 0.1 #km/s / points
prior       = 0.01
dV_min      =.3
current_lim = 0.04

print('__________________________________________________')
print('___________________ PARAMETERS ___________________')
print('sigma = ' + str(sigma_0))
print('eps = ' + str(epsilon))
print('prior = ' + str(prior))
print('dV = ' + str(dV_min))
print('current = ' + str(current_lim))

scale   = np.linspace(25,850,100)         # Scales (in seconds) to span
print(
      '\nInvestigating scales from '
    + str(scale[0])  + ' to '
    + str(scale[-1]) + ' seconds'
)

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
""" Test the Walen relation """
#______________________________________________________________________________

tt_jet      = []                          # Initialise the jet list

for j in tqdm(range(len(scale))):         # Sweep the different scales

    h = int(scale[j] / dt)                # Number of points in the window (should be ODD)
    if h % 2 == 0:
        h += 1

    sigma   = (
        sigma_0 +                  #Define the noise profile sigma
        epsilon * h * (1 - np.exp(-0.5 * (np.linspace(-h/2, h/2, h))**2 / h**2) )
    )

                                          # Reshape the vectors into (..., h, N-h+1) Arrays
                                          # to facilitate the analysis
    Va_reshaped = as_strided(Va, (3,h,N_points-h+1), (8, 24, 24))
    V_reshaped  = as_strided(V,  (3,h,N_points-h+1), (8, 24, 24))
    tt_reshaped = as_strided(tt, (h,N_points-h+1),   (8, 8))

                                          # Substract the central value for each window
    dV          = V_reshaped - V_reshaped[:, h//2, :][:, None]
    dVa         = Va_reshaped - Va_reshaped[:, h//2, :][:, None]

                                          # Compute the posterior ratio
    post_ratio  = fun.compare_posteriors(prior, dVa, dV, h, sigma)

                                          # Memorise the post ratio of extreme scales
                                          #(for plotting purposes)
    if j==0:
        post_ratio_min_scale  = fun.compare_posteriors(prior, dVa, dV, h, sigma)
    elif j==len(scale)-1:
        post_ratio_max_scale  = fun.compare_posteriors(prior, dVa, dV, h, sigma)

                                         # find potential jets
                                         # (post_ratio>1 over two conescutive points)
    i_jet_temp  = np.argwhere(post_ratio > 1)[:, 0]
    i_jet       = fun.consecutive(i_jet_temp, 2)

    n_jets      = len(i_jet)             # Number of potential jets flagged
    if n_jets != 0:
        if len(i_jet[0]) == 0:
            n_jets = 0

    for p in range(n_jets):              # Check the flagged intervals

        add_points  = h - len(i_jet[p])  # adjust to window length
        i_0         = i_jet[p][0] - add_points//2
        i_f         = np.min((i_jet[p][-1] + add_points//2 + 1, len(tt) - 1))

                                         # Rotate to the lmn frame
        lmn_matrix  = np.asarray(
            fun.mva(B[:, i_0: i_f].T, noprint=True).results['Matrice_Passage']
        )
        B_mva       = np.dot(lmn_matrix.T, B[:,i_0:i_f])
        V_mva       = np.dot(lmn_matrix.T, V[:,i_0:i_f])

        #If the interval was not detected at lower scales
        if fun.not_detected(tt_jet, tt, i_0, i_f):

            #If B_L reverses consistently over the interval
            if fun.Bl_reversal(B_mva[0]):
                if fun.stable_sign(B_mva[0], h):

                    #If a velocity jet is observed
                    if fun.velocity_jet(V_mva, dV_min, h,):

                        #If a current sheet is observed
                        if fun.current_sheet(B_mva[0], V_mva[2], scale[j], current_lim, h):

                            #Then add the interval to the detection list
                            tt_jet.append(
                                [[Time(tt[i_0]).jd],
                                 [Time(tt[i_f]).jd],
                                 [np.log(np.max(post_ratio[i_jet[p]]))]]
                                )

list.sort(tt_jet, key=lambda x: x[0])     # Sort the detected intervals
tt_jet = np.array(tt_jet)

if len(tt_jet)==0:                       # Case where no jet is detected
    N_JETS=0
    final_tt_jet = []

else:                                    # Else, concatenate the overlapping intervals
    tt_jet = tt_jet[:, :, 0]

    tt_jet_temp = Time(tt_jet[:, :2], format='jd').datetime
    proba_temp = tt_jet[:, 2]

    detection_vect = np.zeros(N_points)  # Creation of a detection vector, size (N,),
                                         # 1 if in a jet, 0 elsewhere
    proba = np.zeros(N_points)           # initialize the associated log-posterior probability vector

    for i in (range(len(tt_jet[:, 0]))):
        detection_vect[
              (tt >= tt_jet_temp[i, 0])
            & (tt <= tt_jet_temp[i, 1])
        ] = 1

        proba[
              (tt >= tt_jet_temp[i, 0])
            & (tt <= tt_jet_temp[i, 1])
        ] = max(
            proba_temp[i],
            max(
                proba[
                      (tt >= tt_jet_temp[i, 0])
                    & (tt <= tt_jet_temp[i, 1])
                ]
            )
        )

                                          # Number of jets
    i_detection = fun.consecutive(np.argwhere(detection_vect == 1)[:,0],1)
    N_JETS = len(i_detection)

    final_tt_jet = []                     # Final jet timetable
    final_proba  = []                     # Final associated log-posterior probability

    for p in range(N_JETS):               # Re-perform the most obvious checks after concatenation:

        i_0         = i_detection[p][0]   # Start and End of the interval
        i_f         = i_detection[p][-1]

                                          # Transform to the lmn frame
        lmn_matrix  = np.asarray(
            fun.mva(B[:, i_0: i_f].T, noprint=True).results['Matrice_Passage']
        )
        B_mva       = np.dot(lmn_matrix.T, B[:, i_0:i_f])
        V_mva       = np.dot(lmn_matrix.T, V[:, i_0:i_f])
        size = len(B_mva[0])

        # If B_L reverses consistently over the interval
        if fun.Bl_reversal(B_mva[0]):
            if fun.stable_sign(B_mva[0], size):

                # If the slopes of Vl are consistent with a velocity jet
                pente_1 = V_mva[0, size//2] - V_mva[0,  0]
                pente_2 = V_mva[0, size//2] - V_mva[0, -1]
                if np.sign(pente_1) * np.sign(pente_2) == 1:

                    # Then add the interval to the detection list
                    final_tt_jet.append(
                        [[Time(tt[i_detection[p][0]]).jd],
                         [Time(tt[i_detection[p][-1]]).jd]]
                    )
                    final_proba.append(
                        np.max(
                            (proba[i_detection[p][ 0]],
                             proba[i_detection[p][-1]])
                        )
                    )

    final_tt_jet = np.array(final_tt_jet)
    final_proba  = np.array(final_proba)

    #Convert jd to datetime
    final_tt_jet_time = Time(final_tt_jet[:, :, 0], format = 'jd')
    final_tt_jet = final_tt_jet_time.datetime
    N_JETS = len(final_tt_jet)

print('__________________________________________________\n')
print('########## ! ' +str(N_JETS) + ' jet(s) detected ! ##########')
print('__________________________________________________\n')

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
''' Visualize the results '''
#______________________________________________________________________________

fig = plt.figure(i_fig, figsize = (13,9))
i_fig +=1
gs = gridspec.GridSpec(7,1)

ax1 = fig.add_subplot(gs[0])
# ax1.set_title(str(N_JETS) + ' jet(s) detected !')
ax1.plot(tt, B[0],  lw=lw, c='b', alpha = alpha, label = r'$B_R$')
ax1.plot(tt, B_mag, lw=lw, c='k', alpha = alpha, label = r'$B_{mag}$')
plt.setp(ax1.get_xticklabels(), visible=False)

ax2 = fig.add_subplot(gs[2], sharex = ax1)
ax2.plot(tt, B[1],  lw=lw, c='g', alpha = alpha, label = r'$B_T$')
ax2.plot(tt, B_mag, lw=lw, alpha = alpha, c='k')
plt.setp(ax2.get_xticklabels(), visible=False)

ax3 = fig.add_subplot(gs[4], sharex = ax1)
ax3.plot(tt, B[2],  lw=lw, c='r', alpha = alpha, label = r'$B_N$')
ax3.plot(tt, B_mag, lw=lw, alpha = alpha, c='k')
plt.setp(ax3.get_xticklabels(), visible=False)

ax4 = fig.add_subplot(gs[1], sharex = ax1)
ax4.plot(tt, V[0], lw=lw, c='b', alpha = alpha, label = r'$V_R$')
plt.setp(ax4.get_xticklabels(), visible=False)

ax5 = fig.add_subplot(gs[3], sharex = ax1)
ax5.plot(tt, V[1], lw=lw, c='g', alpha = alpha, label = r'$V_T$')
plt.setp(ax5.get_xticklabels(), visible=False)

ax6 = fig.add_subplot(gs[5], sharex = ax1)
ax6.plot(tt, V[2],   lw=lw, c='r', alpha = alpha, label = r'$V_N$')
ax1.set_xlim(tt[0], tt[-1])
plt.setp(ax6.get_xticklabels(), visible=False)

ax7 = fig.add_subplot(gs[6], sharex = ax1)
ax7.plot(tt, n_p,   lw=.5, c='k', alpha = alpha, label = r'$n$')
ax7.set_ylabel(r'${cm^{-3}}$', fontsize=police)
ax77 = ax7.twinx()
ax77.plot(tt, T,   lw=lw, c='purple', alpha = alpha, label = r'$T$')
ax77.set_ylabel(r'${eV}$', fontsize=police, c='purple')

ax1.set_xlim(tt[0], tt[-1])

[ax.set_ylabel(r'${nT}$', fontsize=police)   for ax in [ax1, ax2, ax3]]
[ax.set_ylabel(r'${km/s}$', fontsize=police) for ax in [ax4, ax5, ax6]]
[ax.axhline(0,c='grey', lw=lw)          for ax in [ax1, ax2, ax3]]
[ax.legend(prop={'size': police},  loc='center left',
            bbox_to_anchor=(1,0.5)) for ax in [ax1, ax2, ax3, ax4, ax5, ax6]]

alpha=.2

if N_JETS>0:
    for i in range(final_tt_jet[:,0].size):
        duration = Time(final_tt_jet[i,1]) - Time(final_tt_jet[i,0])
        duration.format = 'sec'
        print(
            '#' + str(i) + ' - '
                + str(int(np.round(duration.value))) + 's - '
                + str(Time(final_tt_jet).iso[i, 0])  + ' - '
                + str(Time(final_tt_jet).iso[i, 1])  + ' - '
                + str(np.round(final_proba[i], 2))
        )
        [ax.axvspan(
            final_tt_jet[i,0],final_tt_jet[i,1], color = 'b', alpha=alpha
        ) for ax in [ax1, ax2, ax3, ax4, ax5, ax6, ax7]]

alpha = 1

[ax.tick_params(labelsize = police) for ax in [ax1, ax2, ax3, ax4, ax5, ax6, ax7]]
plt.tight_layout()
plt.subplots_adjust(hspace=0.3)
plt.show()


import os

import logging
import telegram
import openai
from flask import Flask, redirect, request, url_for
from dotenv import load_dotenv

load_dotenv()

# Initialize Flask app
app = Flask(__name__)

# Set environment variables
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
TELEGRAM_API_KEY = os.getenv("TELEGRAM_API_KEY")

# Set other constants
MODEL_NAME = "text-davinci-003"
TEMPERATURE = 0.6

# Set OpenAI API key
openai.api_key = OPENAI_API_KEY

try:
    # Initialize Telegram bot
    bot = telegram.Bot(token=TELEGRAM_API_KEY)
except Exception as e:
    # Raise error if initialization failed
    raise ValueError(f"Error initializing Telegram bot: {e}")

@app.route('/start', methods=['POST'])
def start():
    update = telegram.Update.de_json(request.get_json(force=True), bot)
    chat_id = update.message.chat_id
    username = update.message.from_user.username
    bot.send_message(chat_id=chat_id, text=f"Привет, {username}! Я бот. Твой chat_id: {chat_id}")
    return 'ok'

@app.route('/animal', methods=['POST'])
def index():
    # Get update from Telegram or form data
    update = telegram.Update.de_json(request.get_json(force=True), bot)
    animal = request.form.get('animal', update.message.text)

    try:
        # Call OpenAI API to generate response
        response = openai.Completion.create(
            model=MODEL_NAME,
            prompt=generate_prompt(animal),
            temperature=TEMPERATURE,
        )

        # Extract text from top suggestion
        result = response.choices[0].text
    except Exception as e:
        # Handle errors by returning empty result
        result = ''

    # Respond to user message in Telegram and redirect to index
    update.message.reply_text(result)
    return redirect(url_for('index', result=result))

def generate_prompt(animal):
    return f"{animal.capitalize()}"

app.logger.setLevel(logging.DEBUG)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 8000)))


# https://leetcode.com/problems/insertion-sort-list/
# tags: #linked_list, #sorting
#
# Solution: Sort by swapping nodes
# Using insertion sort we need to remove current from its original position and insert it at its correct position:
# * Update current.next to behind which is the position before which current needs to be inserted.
# * Update next pointer of previous behind node (behind_prev.next) because current is now inserted before behind.
# * Update next pointer of previous current node (curr_prev.next) because current was removed previously from here.
# Time complexity: O(n^2), Space complexity O(1)
import sys
from typing import Optional

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

    def __str__(self):
        return str(self.val)

class Solution:
    def insertionSortList(self, head: Optional[ListNode]) -> Optional[ListNode]:
        dummy = ListNode(-(sys.maxsize - 1), head)
        curr_prev, current = head, head.next

        while current:
            curr_next = current.next

            if current.val > curr_prev.val:
                curr_prev = current
            else:
                behind_prev, behind = dummy, dummy.next

                while behind.val < current.val:
                    behind_prev, behind = behind, behind.next

                current.next = behind
                behind_prev.next = current
                curr_prev.next = curr_next

            current = curr_next

        return dummy.next

if __name__ == "__main__":
    sol = Solution()
    h1 = ListNode(4)
    h1.next = ListNode(2)
    h1.next.next = ListNode(1)
    h1.next.next.next = ListNode(3)
    h1 = sol.insertionSortList(h1)

    c = h1
    while c:
        print(c, end=" -> ")
        c = c.next
    print("", end="\n")


import sys
import random
from random import shuffle
from tqdm import tqdm
import os.path
import urllib
import urllib.error
import urllib.request
import imghdr
from ssl import CertificateError
from socket import timeout as TimeoutError
from socket import error as SocketError
import time
import pickle
import hashlib
import itertools

random.seed(42)

min_size = 7000
# api_url = "http://www.image-net.org/api/text/imagenet.synset.geturls?wnid=%s"
api_url = "http://www.image-net.org/api/text/imagenet.synset.geturls.getmapping?wnid=%s"

def hash(content):
    m = hashlib.md5()
    m.update(content)
    hex_value = m.hexdigest()
    return hex_value

class DownloadError(Exception):
    """Base class for exceptions in this module."""
    def __init__(self, message=""):
        self.message = message

def download(url, timeout, retry, sleep=0.8):
    """Downloads a file at given URL."""
    count = 0
    while True:
        try:
            f = urllib.request.urlopen(url, timeout=timeout)
            if f is None:
                raise DownloadError('Cannot open URL' + url)
            content = f.read()
            f.close()
            break
        except (urllib.error.HTTPError, CertificateError) as e:
            if isinstance(e, urllib.error.HTTPError):
                if e.code >= 400: raise DownloadError("No Access")
            count += 1
            if count > retry:
                raise DownloadError("Retry limit")
        except (TimeoutError, SocketError, IOError) as e:
            count += 1
            if count > retry:
                raise DownloadError("Retry limit")
            time.sleep(sleep)
    return content

def get_urls(id_to_synset_dict, limit=-1):
    urls = []

    print("Downloading urls from %s" % api_url)

    progress_bar = tqdm(total=len(id_to_synset_dict))
    for synset_id, i in id_to_synset_dict.items():
        url = api_url % synset_id
        content = download(url, timeout=15, retry=5, sleep=0.1)
        content = content.decode('ISO-8859-1')
        content = content.split('\n')[:-1] # last line is blank

        if limit is not None:
            shuffle(content)
            content = content[:limit]

        content = [(i, *c.split()) for c in content]
        urls += content
        progress_bar.set_postfix({"num_urls": len(urls)})
        progress_bar.update(1)

    return urls

def find_extension(content):
    extension = imghdr.what('', content) #check if valid image
    if extension == "jpeg":
        extension = "jpg"
    if extension == None:
        raise DownloadError()
    return extension

def write_file(content, file_name):
    with open(file_name, 'wb') as f:
        f.write(content)

if __name__ == "__main__":
    if len(sys.argv) == 1:
        print("""Specify number of images:
        > python download_imagenet_images.py [num_imgs]""")
        exit()

    # Id to Class information
    with open('./synset_id_to_class', 'r') as f:
        id_to_class = {}
        for l in f:
            c, i = l.strip().split()
            id_to_class[i] = c

    n_imgs = int(sys.argv[1])
    if os.path.exists('valid_urls.pkl'):
        with open('valid_urls.pkl', 'rb') as f:
            urls = pickle.load(f)
    else:
        urls = get_urls(id_to_class, limit=None)

        print(f"Storing urls in 'valid_urls.pkl'")
        with open('valid_urls.pkl', 'wb') as f:
            pickle.dump(urls, f)

    shuffle(urls)
    print(urls[:10])

    with open('flickr_empty.png', 'rb') as f:
        empty = f.read()

    # Hash of empty image.
    empty_hex = hash(empty)

    os.makedirs('images', exist_ok=True)

    print("Starting image download")
    progress_bar = tqdm(total=n_imgs)
    i = j = 0
    good = [True] * len(urls)
    while i < n_imgs and j < len(urls):
        try:
            cl      = urls[j][0]
            name    = urls[j][1]
            url     = urls[j][2]

            response = download(url, 2, 2)

            if hash(response) == empty_hex:
                raise DownloadError()

            ext = find_extension(response)
            file_name = os.path.join("images", cl + "_" + name + "." + ext)
            write_file(response, file_name)
            progress_bar.update(1)
            j += 1
            i += 1

        except DownloadError as d:
            # print("Download error", d)
            good[j] = False
            j += 1

        except Exception as e:
            # print("Exception", e, e.__class__.__name__)
            good[j] = False
            j += 1

    progress_bar.close()
    print(f"i: {i}, j: {j}, skipped: {j - i}")

    len_before = len(urls)
    urls = list(itertools.compress(urls, good))
    len_after = len(urls)

    print("Updating valid urls to only contain urls, that actually gave a good result")
    print(f"Went from {len_before} to {len_after} elements")
    with open('valid_urls.pkl', 'wb') as f:
        pickle.dump(urls, f)


# -*- coding: utf-8 -*-

import os

from pkg_resources import parse_version
import pkginfo

def _egg_info(path_to_egg='../../src/'):
    path_to_egg = os.path.join(
        os.path.dirname(__file__), path_to_egg)
    egg_info = pkginfo.Develop(path_to_egg)
    release = egg_info.version
    version = '%s.%s' % tuple(map(int, parse_version(release)[0:2]))
    return egg_info.name, egg_info.author, version, release

project, author, version, release = _egg_info()
copyright = '2009, %s' % author

# Extension
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    ]
intersphinx_mapping = {'http://docs.python.org/': None}

# Source
master_doc = 'index'
templates_path = ['_templates']
source_suffix = '.rst'
exclude_trees = []
pygments_style = 'sphinx'

# html build settings
html_theme = 'default'
html_static_path = ['_static']

# htmlhelp settings
htmlhelp_basename = '%sdoc' % project

# latex build settings
latex_documents = [
  ('index', '%s.tex' % project, u'%s Documentation' % project,
   author, 'manual'),
]


import json
import logging

logging.basicConfig(filename='app.log', filemode='w', format='%(name)s - %(levelname)s - %(message)s')

def unpack_json(file):
    try:
        with open(file) as json_file:
            data = json.load(json_file)
            return data
    except FileNotFoundError:
        logging.error(f"File {file} doesn't exists")

def parse_user(output_file, *args):
    data = []
    for items in args:
        json = unpack_json(items)
        if json != None:
            for item in json:
                if "name" in item.keys():
                    repeated = 0
                    if len(data) > 0:
                        for items in data:
                            if item["name"] == items["name"]:
                                repeated += 1
                    if repeated == 0:
                        data.append(item)
    return create_json(output_file, data)

def create_json(file, data):
    with open(file, "w") as json_file:
        json.dump(data, json_file, indent=4)

    return file

print(parse_user('user3.json' , 'user1.json', 'user2.json'))

sprint 06 ( With (in, out). Serialization )/Task 2/Task 2.py


import cv2
import omni2

values = [25, 25, 0, 65, 65, 65, 0, 170]
thrower = 100
speed = 40

cv2.namedWindow("Controller")

while True:
    k = cv2.waitKey(1) & 0xFF

    if k == ord("w"):
        print("Forward")
        omni2.forward(values, speed)
    if k == ord("s"):
        print("Backward")
        omni2.forward(values, speed * -1)
    if k == ord("d"):
        print("Right")
        omni2.rotate(values, speed * -1)
    if k == ord("a"):
        print("Left")
        omni2.rotate(values, speed)
    if k == ord("t"):
        print("Throw")
        omni2.startThrow(values, thrower)
    if k == ord("r"):
        print("Stop throw")
        omni2.endThrow(values)
    if k == ord("e"):
        print("Stop")
        omni2.stopAll(values)
    if k == ord("q"):
        print("Break")
        omni2.stopAll(values)
        break

cv2.destroyAllWindows()


from bs4 import BeautifulSoup
import copy
import re

def getTabmeta(tabstr):#type是否引入加粗,等于1不加粗；way是否以#区分，1是不区分；method是百分号是否加前面的东西
    """
    把带表格的html字符串，从中提取内容和描述信息，还原为list。
    先以
或
字符串表行标识，拆分表格字符串，其中
部分为表头部分。
    以字符串为标识，逐行找寻列数据，以	为标识，拆分列信息。
    获取列的行合并，列合并属性。
    获取列的文字属性，判断是否存在空格或tab。
    完成表格元数据获取的第一步
    """
    try:
        soup_all = BeautifulSoup(tabstr, 'lxml')  # 首先是将具有表格描述的内容进行解析，tabstr是带有html标签的
        tabMeta = []

        # 获取表头或普通行的数据和描述信息
        def metas(rows, tabMeta):
            # 没有行标识的字符串不处理。
            t_list = [' ', '\t', '\u00A0', '\u0020', '\u3000', '\u0009','\n']  # 这些都代表着空格
            if len(rows) > 0:  # 看一下表格包含的行数
                for rowind, row in enumerate(rows):  # 先从每一行看,找到的是带有tr标签的内容
                    # 没有列标识的行不处理。
                    rowdata = []
                    cols = row.find_all('td')  # 找到有td标签的全部内容，也就是每一行有多少列
                    if len(cols) > 0:  # 每一行必须有内容
                        for colind, col in enumerate(cols):  # 再从每一列看，col是html标签内容
                            # 没有列标识的不处理
                            coldata = []
                            try:
                                rowspan = int(col['rowspan'])  # 直接获取td标签中的'rowspan'属性
                            except:
                                rowspan = 1
                            coldata.append(rowspan)  # 得到行合并数

                            try:
                                colspan = int(col['colspan'])
                            except:
                                colspan = 1
                            coldata.append(colspan)  # 得到列合并数

                            text = col.get_text(strip=False).rstrip()  # 只得到每一个单元格的文字部分

                            # 解决某些带有同期增长的表格中，行列标题没有说明时间，只有比去年同期增长但是并没有去年同期是啥
                            if text.find('%') != -1 :#or text.find('增') != -1
                                if colind > 0:
                                    text = cols[colind - 1].get_text(strip=False).rstrip() + '_' + text

                            result = [tmp in t_list for tmp in text]  # 判断所有的表格中是否有缩进的
                            t_count = 0
                            for b in result:
                                if b:
                                    t_count = t_count + 1
                                else:
                                    break

                            if t_count > 0:
                                coldata.append(True)
                            else:
                                coldata.append(False)
                            coldata.append(t_count)
                            coldata.append(text)
                            rowdata = rowdata + [coldata]
                    tabMeta = tabMeta + [rowdata]
            return tabMeta

        heads = soup_all.find_all('th')
        tabMeta = tabMeta + metas(heads, tabMeta)  # 表头
        rows = soup_all.find_all('tr')  # 表格
        tabMeta = tabMeta + metas(rows, tabMeta)

        # 解决某些表格中只有tr没有td的问题,每行的元素个数都不一致，所以需要找到列数最多的行然后将其余行补齐
        colnum = [sum(list(map(lambda x: x[1], row))) for row in tabMeta]
        median = max(colnum)

        for rowind, row in enumerate(tabMeta):
            col_num = sum(list(map(lambda x: x[1], row)))
            if row == []:
                tabMeta[rowind] = [[1, 1, False, 0, '']] * int(median)
            elif col_num<median:
                tabMeta[rowind] = tabMeta[rowind]+[[1, 1, False, 0, '']] * int(median-col_num)
            else:
                tabMeta[rowind] = tabMeta[rowind]
    except:
        tabMeta = {'900':'该页面中表格结构存在问题！'}
    return tabMeta

def getFulltab(tabmeta):
    """
    根据网页表格带描述信息的数据，还原整个表格全貌。
    主要处理三个事情：
    1、出现行合并的，拆分到行。
    2、出现列合并的，拆分到列。
    3、包含空格或TAB的，拼装上级信息。
    入参为第一步的识别结果。
    出参为拼接后的完整表格。
    """
    try:
        fullTab = copy.deepcopy(tabmeta)
        # 先处理列合并
        for rowind, row in enumerate(tabmeta):
            rowdata = []
            for col in row:
                colspan = col[1]  # 列合并数
                value = col[4]  # 对应的每个单元格中的名称
                for i in range(colspan):
                    rowdata.append(value)
            fullTab[rowind] = rowdata

        # 处理行合并
        for rowind, row in enumerate(tabmeta):
            for i in range(len(fullTab[rowind])):  # 每行的数量
                tmps = fullTab[rowind][i]  # 具体到每一个描述
                realrowin = -1
                for ss in row:
                    if rowind >= 1:
                        if ss[4] == tmps:  # 这里有问题#and fullTab[rowind-1][i]!=tmps
                            realrowin = row.index(ss)
                            break
                    if rowind < 1:
                        if ss[4] == tmps:
                            realrowin = row.index(ss)
                            break
                if realrowin != -1:
                    rowspan = row[realrowin][0]  # 行合并数
                    value = row[realrowin][4]  # 对应值
                    if rowspan > 1:
                        for j in range(rowspan - 1):
                            fullTab[rowind + j + 1].insert(i, value)
        # 最后再更新第一列
        # 首先看一下是不是行标题之前有空白列
        coldata_str = [list(map(lambda x: x[i], fullTab)) for i in range(len(fullTab[0]))]
        coldata = list(map(lambda x: ''.join(x), coldata_str))  # 合并
        for i, data in enumerate(coldata_str) :
            if len(''.join(data)) != 0 and data.count(' ')/len(data)<0.1:
                startcol = i
                break
        # 然后再补充
        for rowind, row in enumerate(tabmeta):
            # 从第二行开始
            if rowind > 0:
                for colind, col in enumerate(row):
                    if colind == startcol:  # 暂时先对第一列处理
                         # 如果第一个单元格非空才可以添加上去
                        if col[2]:  # 该条件表示如果存在空格
                            des_rowid = rowind - 1
                            while True:
                                if tabmeta[des_rowid][startcol][3] < col[3] - 1 or tabmeta[des_rowid][startcol][
                                    3] == col[3] - 1:  # 这里将 等于 改为 小于等于
                                    fullTab[rowind][colind] = fullTab[des_rowid][colind] + '_' + fullTab[rowind][
                                        colind].strip()
                                    break
                                else:
                                    des_rowid = des_rowid - 1
                                    if des_rowid < 1:  # 防止前面两行只是简单的缩进并没有字段名包含关系，寻找到第一行也没有符合条件的单元格就跳出便可以了
                                        break
                            if len(''.join(fullTab[0][0])) != 0:
                                fullTab[rowind][colind] = fullTab[0][0] + '_' + fullTab[rowind][colind]  # 将最开始的第一个格的内容利用上
                            else:
                                fullTab[rowind][colind] =  fullTab[rowind][colind]

    except:
        fullTab = {'901':'表格合并单元格的分列过程中出现问题！'}
    return fullTab

def getStructdata(tablist):
    """
    将转换后的报表列表，转换为结构化数据。
    格式为[行信息，列信息，数值]
    默认从第二行第二列开始。替换掉其中的英文逗号，如果能参与计算，即为数值，设定为需要获取的单元格。
    :param tablist:转换后的table列表
    :return: 大的list，每个子list是一个单元格的数据，及其描述信息。
    :rwoid:起始的行号，默认从第二行开始。程序因为是从0开始的，因此，使用中需要减去1.
    :colid:起始的列号，默认从第二列开始。程序因为是从0开始的，因此，使用中需要减去1.
    :type:是否判断计量单位，配合前面行列起始号使用。用于数值中带有计量单位的情况。
    默认为1，即不含计量单位信息，为2时不判断单元格信息是否为数值，从指定的行列直接拼结果。
    """
    # 将整行是空格的删除
    try:
        fullTab_clear = []
        for i in tablist:
            if len(''.join(i)) > 0:
                fullTab_clear.append(i)
        tablist = copy.deepcopy(fullTab_clear)
        danwei = [str(j)[str(j).find('单位：'):].strip() for x in tablist for j in x if
                  str(j).replace('\xa0', '').replace('\u3000', '').replace(' ', '').replace('\n', '').find('单位：') != -1]
        unit = '_' + danwei[0] if danwei != [] else ''
        #获取单位列表
        try:
            unit_list = list(set([re.findall('\d+[\(](.*?)[\)]',str(j))[0]  for  x in tablist for j in x  if re.findall('[\(](.*?)[\)]',str(j))!=[]]))
        except:
            unit_list=[]
            unit_list.append(unit.replace('_',''))

        strudata_result = []
        colStart = 2  # 默认开始的列
        rowStart = 2  # 默认开始的行

        def cantransdigit(strdata,x):
            """
            用于判断给定的字符串能不能转换为数值，需要去除里面的英文逗号，中文句号。
            空值或者‘-’转换为0处理。
            :param strdata:传入的字符串
            :return:能够转换为数字。
            """
            strdata = ''.join(strdata.split())  # 为了去除特殊符号\u3000,\xa0,\n
            tmp_s = strdata.replace(',', '')  # 数字中出现逗号分隔 1,000这种写法
            tmp_s = tmp_s.replace('(百分点)', '')  # 这个括号有什么用呢，
            tmp_s = tmp_s.replace('（百分点）', '')
            tmp_s = tmp_s.replace('倍', '')
            # tmp_s = tmp_s.replace(tmp_s[tmp_s.find('('):tmp_s.find(')') + 1], '')#这个括号有什么用呢，
            tmp_s = tmp_s.replace('。', '')  # 有的时候是小数点输入有误，输入成了句号
            tmp_s = tmp_s.replace('-', '0')  # 某些表格不存在内容以-代替，英文状态的-
            tmp_s = tmp_s.replace('—', '0')  # 某些表格不存在内容以-代替，中文状态的-
            tmp_s = tmp_s.replace('.', '0')  # 将小数点以0代替
            if len(tmp_s) == 0:  # 避免某个单元格就是一个没有内容的空值，这种被识别为不是数字，影响之后的操作
                result = True
            else:
                result = tmp_s.isdigit()
            if result == False:
                tmp_s = tmp_s.replace('(','')
                tmp_s = tmp_s.replace(')', '')
                tmp_s = tmp_s.replace('（','')
                tmp_s = tmp_s.replace('）', '')
                for i in x:
                    tmp_str = re.findall('\d+'+ i ,tmp_s)
                    if tmp_str !=[]:
                        result = True
                        break
                    else:
                        result == False

            return result  # 如果字符串只包含数字返回true
        # 从指定的列开始，取一列数据判断是否均能转换为数值。
        while True:
            colmeta = [tablist[rowind][colStart - 1] for rowind in range(rowStart - 1, len(tablist))]
            coldatabool = [cantransdigit(tablist[rowind][colStart - 1],unit_list) for rowind in
                           range(rowStart - 1, len(tablist))]  # 看一下每一行均从第二列开始是不是都是可以数字化的，遍历了所有的均不可以就从第三列开始依次类推
            if sum(coldatabool) / len(coldatabool) > 0.2 and len(''.join(colmeta)) != 0:
                # 因为有些表的表尾包含一些说明信息，所以只要包含数字，就暂定为这列不是列头。
                break
            else:
                colStart = colStart + 1
        # 从指定的行开始，取一行数据判断是否均能转换为数值。
        while True:
            endcol = min([len(row) for row in tablist])
            rowmeta = [tablist[rowStart - 1][colind] for colind in range(colStart - 1, endcol)]  # 遍历每一行
            rowdatabool = [cantransdigit(tablist[rowStart - 1][colind],unit_list) for colind in
                           range(colStart - 1, len(tablist[0]))]
            if False in rowdatabool or len(''.join(rowmeta)) == 0:
                rowStart = rowStart + 1
            else:
                break

        for rowind in range(rowStart - 1, len(tablist)):
            colcount = endcol  # 每一行有多少列
            for colind in range(colStart - 1, colcount):
                strudata = []

                # 需要算法推断表头所在的行列，以指定的起始行列开始，
                # 剩下对应行或者列数据出去空值和‘-’均能转换为数值型的，则认为不是表头。
                # 替换数据中的英文逗号或者中文句号（人为失误）
                value = tablist[rowind][colind].replace(',', '').replace('。', '').replace('_','')  # 去掉逗号和句号，#前两步处理好的结果,该步处理的只是数字部分
                col_tmp_info = [
                    tablist[i][colind].replace('\xa0', '').replace('\u3000', '').replace(' ', '').replace('\n', '')
                    for i in range(rowStart - 1)]  # 表头部分，同一列是否有重复的指标名称
                col_info = []
                for s in col_tmp_info:
                    s = ''.join(s.split())
                    s_words = s.split('_')
                    for s_word in s_words:
                        if s_word not in col_info:
                            col_info.append(s_word)
                # row_tmp_info = [tablist[rowind][i].replace('\xa0', '').replace('\u3000', '').replace(' ','').replace('\n','') for i in range(colStart - 1)]
                row_tmp_info = [
                    tablist[rowind][i].replace('\xa0', '').replace('\u3000', '').replace(' ', '').replace('\n', '')
                    for i in range(colStart - 1)]
                row_info = []
                for s in row_tmp_info:
                    s = ''.join(s.split())
                    s_words = s.split('_')
                    for s_word in s_words:
                        if s_word not in row_info:
                            row_info.append(s_word)
                strudata.append('_'.join(row_info) + unit)  # 行标题的连接,在这里加上的单位
                strudata.append('_'.join(col_info))  # 以—连接列标题
                try:
                    # 避免表尾的说明信息以及空值转换为无用的结构化数据。
                    if len(value) > 0:  # 这里如果值不能转化为数字就被忽略了
                        value_str = value.replace(value[value.find('('):value.find(')') + 1], '') if value.find(
                            '(') != -1 else value
                        strudata.append(str(float(value_str))+value[value.find('('):value.find(')') + 1])
                        strudata_result = strudata_result + [strudata]
                except:
                    pass

    except:
        strudata_result = {'902':'识别表头部分出现错误'}
    return strudata_result


import json

import pytest
from dbt.tests.util import run_dbt

ref_models__table_comment_sql = """
{{
  config(
    materialized = "table",
    persist_docs = {"relation": true, "columns": true},
  )
}}

select
    'foo' as first_name,
    'bar' as second_name

"""

ref_models__view_comment_sql = """
{{
  config(
    materialized = "view",
    persist_docs = {"relation": true, "columns": true},
  )
}}

select
    'foo' as first_name,
    'bar' as second_name

"""

ref_models__schema_yml = """
version: 2

models:
  - name: table_comment
    description: "YYY table"
    columns:
      - name: first_name
        description: "XXX first description"
      - name: second_name
        description: "XXX second description"
  - name: view_comment
    description: "YYY view"
    columns:
      - name: first_name
        description: "XXX first description"
      - name: second_name
        description: "XXX second description"
"""

class TestBaseComment:
    @pytest.fixture(scope="class")
    def models(self):
        return {
            "schema.yml": ref_models__schema_yml,
            "table_comment.sql": ref_models__table_comment_sql,
            "view_comment.sql": ref_models__view_comment_sql,
        }

    @pytest.mark.parametrize(
        'model_name',
        ['table_comment', 'view_comment'],
    )
    def test_comment(self, project, model_name):
        run_dbt(["run"])
        run_dbt(["docs", "generate"])
        with open("target/catalog.json") as fp:
            catalog_data = json.load(fp)

        assert "nodes" in catalog_data
        column_node = catalog_data["nodes"][f"model.test.{model_name}"]
        for column in column_node["columns"].keys():
            column_comment = column_node["columns"][column]["comment"]
            assert column_comment.startswith("XXX")

        assert column_node['metadata']['comment'].startswith("YYY")