# ClimbingAnalysis application
# Written by Andrew Watkins
# watkins(dot)andrewb(at)gmail(dot)com
#
# This dash application gives a basic exploration
# of USA Climbings initial results for regional qualification
# for Virtual Qualifiers in the bouldering season of 2020

import dash
import dash_core_components as dcc
import dash_html_components as html
import dash_table
import plotly.express as px
import pandas as pd

from dash.dependencies import Input, Output

fname ='assets/current_results.csv'
df = pd.read_csv(fname)

app = dash.Dash(__name__)
app.index_string = '''
<!DOCTYPE html>
<html>
    <head>
        {%metas%}
        <title>USAC Virutal QE Climbing Analysis</title>
        {%favicon%}
        {%css%}
    </head>
    <body>
        {%app_entry%}
        <footer>
            {%config%}
            {%scripts%}
            {%renderer%}
        </footer>
    </body>
</html>
'''
rlist = [ x for x in range(10,100) if x%10==1 or x%10==2]
catList=['FJR', 'MJR', 'FYA', 'MYA', 'FYB', 'MYB', 'FYC', 'MYC', 'FYD', 'MYD']
catColors ={'FJR':"#ffa822", 'MJR':"#b36b00",
            'FYA':"#134e6f", 'MYA':"#071e2c",
            'FYB':"#ff6150", 'MYB':"#ff1a00",
            'FYC':"#1ac0c6", 'MYC': "#128387",
            'FYD':"#b6bbc8", 'MYD': "#8b93a7"}
bgcolor = '#f2f2f2'

def genericScatter(df, compScore=0, compCat=None):
    if compCat != None:
        df = df[df['Category'].isin(compCat)]
    fig = px.scatter(df, x="Region", y="Score", color="Category",
                hover_data=['Score'],
                color_discrete_map=catColors,
                category_orders ={'Region':rlist, 'Category':catList},
                title="Qualifying Event Scores by Region and Category",
                )
    if compScore>0:
        fig.add_hline(y=compScore, annotation_text = 'Your Climber\'s Score',annotation_position='top left',
                    line_dash='dash')
    fig.update_xaxes(type='category')
    fig.update_layout(paper_bgcolor=bgcolor,
                        #plot_bgcolor='#eff8fb',
                        font_family="Futura")
    return fig

def avgScatter(df, compScore =0, compCat = None):
    gf = df.groupby(['Region', 'Category'], as_index=False)['Score'].mean()
    fig = px.scatter(gf if compCat == None else gf[gf['Category'].isin(compCat)], x="Region", y="Score", color="Category",
                     hover_data=['Score'],
                     color_discrete_map=catColors,
                    category_orders ={'Region':rlist, 'Category':catList},
                    labels=dict(Score='Average Score'),
                    title='Average QE Score by Region and Category')
    fig.update_xaxes(type='category')
    if compScore>0:
        fig.add_hline(compScore, annotation_text = 'Your Climber\'s Score', annotation_position='top left',
                        line_dash='dash')
    fig.update_layout(paper_bgcolor=bgcolor,
                    font_family="Futura")
    return fig

def catHistogram(df, compScore = 0, compCat = None):
    fig = px.histogram(df if compCat == None else df[df['Category'].isin(compCat)], x="Score", color='Category',
                        color_discrete_map=catColors,
                       category_orders ={'Region':rlist, 'Category':catList},
                       title="Histogram of Scores by Climbing Category (200 point bins)")
    if compScore>0:
        fig.add_vline(compScore, annotation_text = 'Your Climber\'s Score',
                        annotation_position="top left",
                        line_dash='dash')
    fig.update_layout(paper_bgcolor=bgcolor,
                    font_family="Futura")
    return fig

def allHistogram(df, compScore = 0, compCat = None):
    fig=px.histogram(df,x='Score', title='Histogram of QE Scores (200 point bins)')
    if compScore>0:
        fig.add_vline(compScore, annotation_text = 'Your Climber\'s Score',
                        annotation_position="top left",
                        line_dash = 'dash')
    fig.update_layout(paper_bgcolor=bgcolor)
    return fig

def findPlace(df, testScore=0, testRegions=None, testCats = None):
    retdf = pd.DataFrame(columns=['Region','Category', 'Place', 'Number of Climbers'])
    if(testScore<=0):
        return retdf
    if testRegions==None:
        testRegions = [ x for x in range(10,100) if x%10==1 or x%10==2]
    if testCats == None:
        testCats = ['FJR', 'MJR', 'FYA', 'MYA', 'FYB', 'MYB', 'FYC', 'MYC', 'FYD', 'MYD']
    for reg in testRegions:
        for cat in testCats:
            tf = df[df['Region']==reg]
            tf = tf[tf['Category']==cat]
            totalClimbers = len(tf)
            tf = tf[tf['Score']>testScore]
            place = len(tf)+1
            if totalClimbers>0:
                row = [reg,cat,place,totalClimbers]
                retdf = retdf.append(pd.Series(row,index=retdf.columns), ignore_index = True)
    return retdf

# Not really using the region_drop down any more
region_drop = html.Div([
    dcc.Markdown('''**Choose one or more regions** (or leave blank for all) '''),
    dcc.Dropdown(
        id='dropdown-reg',
        options=[{'label':i,'value':i} for i in rlist],
        multi=True,
        placeholder='Select one or more regions',
        ),])

cat_drop = html.Div([
    dcc.Markdown('''**Choose your climber's category**'''),
    dcc.Dropdown(
        id='dropdown-cat',
        options =[{'label':i,'value':i} for i in catList],
        multi=True,
        #value=None,
        placeholder='Select one or more categories',
    ),], className='offset-by-two columns three columns')
score_input = html.Div([
    dcc.Markdown('''**Enter your climber's score**'''),
    dcc.Input(id='inputScore', type='number', min=0, step=50,
    placeholder='2000', value=2000,
    autoFocus = True),
], className = 'two columns')

table_results = html.Div([
    html.Div(id='score-output-container'),
    dcc.Markdown('''your climber would have placed as follows in the selected categories '''),
    dash_table.DataTable(id='compTable',
        style_cell={'font-family': 'Futura'},
        style_data_conditional=[
        {
            'if': {'row_index': 'odd'},
            'backgroundColor': bgcolor
            }
            ]
        )
    ], className ='four columns')

visuals = html.Div([
    html.Div([
    dcc.Markdown('''Hover over the graphs to see more information about each data point
    '''),
    dcc.Graph(
        id='genScat',
    ),
    dcc.Graph(
        id='avgScat',
    ),
    dcc.Graph(
        id='catHist',
    ),
#    dcc.Graph(
#        id='allHist',
#        figure=allHFig
#    )
    ],className='eight columns',),
])

app.layout = html.Div(children=[
    html.Div([
    html.H2(children='Virtual Bouldering Regional Qualifier Events Across USAC Regions', style={"text-align": "center"}),
    dcc.Markdown(children='''Based off of file: <http://www.usaclimbing.org/Assets/Regional+Ranking-Preliminary-201220d.pdf> (as of 1/5/2021)''', style={"text-align": "center"}),
    html.P(style={"text-align": "center"},children='Use this tool to compare a climber\'s score to others or to see general trends among climbers at QEs across the country'),
    html.Br(),
    ],),

    html.Div([
    score_input, cat_drop,], className='offset-by-one columns row'),
    html.Hr(), html.Hr(),
    html.Div([table_results
    , visuals,
        ],className="row"),
   html.Div([
    dcc.Markdown(children='''Source code can be found on github: <https://github.com/ABWatkins/USACVirtualQEClimbingAnalysis>''', style={"text-align": "center"}),
    dcc.Markdown(children='''Developed by Andrew Watkins: watkins dot andrewb at gmail dot com''', style={"text-align": "center"})]),
])

@app.callback(
    Output('compTable', 'data'),
    Output('compTable', 'columns'),
   Output('score-output-container', 'children'),
    Input('dropdown-cat', 'value'),
    Input('inputScore', 'value')
)
def getDataTable(compCat = None, compScore = 0):
    if compCat == []:
        compCat = None
    if compScore == None:
        compScore = 0
    compDF=findPlace(df, testScore= compScore, testRegions=None, testCats=compCat)
    data=compDF.to_dict('records')
    columns=[{'id': c, 'name': c} for c in compDF.columns]
    return data,columns, 'With a score of {}'.format(compScore)

@app.callback(
    Output('genScat', 'figure'),
    Input('dropdown-cat', 'value'),
    Input('inputScore', 'value'))
def getGenScat(compCat = None, compScore = 0):
    if compCat == []:
        compCat = None
    if compScore == None:
        compScore = 0
    return genericScatter(df, compScore,compCat)

@app.callback(
    Output('avgScat', 'figure'),
    Input('dropdown-cat', 'value'),
    Input('inputScore', 'value'))
def getAvgScat(compCat = None, compScore = 0):
    if compCat == []:
        compCat = None
    if compScore == None:
        compScore = 0
    return avgScatter(df, compScore,compCat)

@app.callback(
    Output('catHist', 'figure'),
    Input('dropdown-cat', 'value'),
    Input('inputScore', 'value'))
def getCatHist(compCat = None, compScore = 0):
    if compCat == []:
        compCat = None
    if compScore == None:
        compScore = 0
    return catHistogram(df, compScore,compCat)

# @app.callback(
#     Output('allHist', 'figure'),
#     #Input('dropdown-reg', 'value'),
#     Input('dropdown-cat', 'value'),
#     Input('inputScore', 'value'))
# def getAllHist(compCat = None, compScore = 0):
#     if compCat == []:
#         compCat = None
#     if compScore == None:
#         compScore = 0
#     return allHistogram(df, compScore,compCat)

if __name__ == '__main__':
    app.run_server(debug=True,host='127.0.0.1')


from collections import defaultdict
import json
import numpy as np
try:
    from tqdm import tqdm
except ImportError:
    tqdm = lambda x, *args, **kwargs: x

from utils import check_dir_or_create, word_clean, get_clean_words

class ErrorModel:

    def __init__(self):
        self.stat = defaultdict(lambda: defaultdict(int))
        # {'orig' : {'fix' : P(orig|fix)}}
        self._stat_size = 0

    @staticmethod
    def make_bigrams(word):
        word = '^' + word + '_'
        bigrams = []
        for i in range(len(word) - 1):
            bigrams.append(word[i:i + 2])
        return bigrams

    @staticmethod
    def _levenshtein_matrix(a, b):
        n, m = len(a), len(b)
        need_transpose = False
        if n > m:
            a, b = b, a
            n, m = m, n
            need_transpose = True

        current = list(range(n + 1))
        previous = current
        lv_matrix = np.array([current])
        for i in range(1, m + 1):
            prev_previous, previous, current = previous, current, [i] + [0] * n
            for j in range(1, n + 1):
                a_ne_b = int(a[j - 1] != b[i - 1])
                add    = previous[j] + 1
                delete = current[j - 1] + 1
                change = previous[j - 1] + a_ne_b

                # transpose = change
                # if j > 1 and i > 1 and a[j - 2] == b[i - 1] and a[j - 1] == b[j - 2]:
                #     transpose = prev_previous[j - 2] + a_ne_b

                current[j] = min(add, delete, change)#, transpose)
            lv_matrix = np.vstack((lv_matrix, [current]))
        return lv_matrix.T if need_transpose else lv_matrix

    def _add_stats(self, orig, fix):
        self.stat[orig][fix] += 1
        self._stat_size += 1

    def _get_stats(self, orig, fix):
        if orig in self.stat and fit in self.stat[orig]:
            return self.stat[orig][fit]
        return 1.0 / self._stat_size

    def _fill_stats(self, a, b, lv_matrix):
        i, j = len(a), len(b)
        cur_distance = lv_matrix[len(b), len(a)]

        while cur_distance != 0:
            add = lv_matrix[j - 1, i] if j > 0 else np.inf
            delete = lv_matrix[j, i - 1] if i > 0 else np.inf
            change = lv_matrix[j - 1, i - 1] if j > 0 and i > 0 else np.inf

            operation = np.argmin([change, add, delete]).item()

            if operation == 0:
                i -= 1
                j -= 1
                if cur_distance != change:
                    cur_distance = change
                    self._add_stats(a[i], b[j])

            elif operation == 1:
                j -= 1
                if cur_distance != add:
                    cur_distance = add
                    self._add_stats(a[i - 1][1] + '~', b[j])
            else:
                i -= 1
                if cur_distance != delete:
                    cur_distance = delete
                    self._add_stats(a[i], b[j - 1][1] + '~')

    def normalize_stat(self):
        for orig in self.stat:
            for fix in self.stat[orig]:
                self.stat[orig][fix] /= self._stat_size

    def make_model(self, filename="queries_all.txt"):
        self.stat = defaultdict(lambda: defaultdict(int))
        self._stat_size = 0
        with open(filename, 'r') as f:
            if filename == 'queries_all.txt':
                tqdm_cur = lambda x: tqdm(x, total=2000000)
            else:
                tqdm_cur = tqdm
            for line in tqdm_cur(f):
                if '\t' not in line:
                    continue
                wrong, right = line.lower().split('\t')
                wrong = get_clean_words(wrong)
                right = get_clean_words(right)

                if len(wrong) != len(right):  # join or split
                    continue

                for wrong_word, right_word in zip(wrong, right):
                    wrong_bigrams = self.make_bigrams(wrong_word)
                    right_bigrams = self.make_bigrams(right_word)
                    lv_matrix = self._levenshtein_matrix(wrong_bigrams, right_bigrams)
                    self._fill_stats(wrong_bigrams, right_bigrams, lv_matrix)

        self.normalize_stat()

    def save_model(self, filename='stats.json', directory='prepared_data'):
        check_dir_or_create(directory)
        with open(directory + '/' + filename, 'w') as f:
            f.write(json.dumps((self._stat_size, self.stat)))

    def load_model(self, filename='stats.json', directory='prepared_data'):
        check_dir_or_create(directory)
        with open(directory + '/' + filename, 'r') as f:
            self._stat_size, self.stat = json.loads(f.read())


from ursina.prefabs.trail_renderer import TrailRenderer

from .class_.player import *

class ROOM(Entity):
    def __init__(self,ch=None,room=None,server = None):
        super().__init__(
            enabled=False
        )
        self.change = ch
        self.pool_obj = server
        self.room = room
        self.pool = []

    def Start(self) -> None:
        self.enabled = True

        # self.room = load_blender_scene('room1', load=False,reload=True,)
        self.room = Entity(model='cube',scale=(30,.1,30),color=color.gray)
        trail_renderer = TrailRenderer(parent=self.room)
        self.pool.append(self.room)
        # for e in self.room.children:
        #     e.collider = 'mesh'

        self.player = Player(scn=self, name="IAM",)
        self.pool.append(self.player)
        self.am_l = AmbientLight(color=color.rgba(100, 100, 100, 0.2))
        self.pool.append(self.am_l)

    def Stop(self):
        for i in self.pool:
            destroy(i,delay=0)
        self.enabled = False
        self.visible = False
        mouse.locked = False

    def input(self, key):
        if key == 'space':
            self.change.Change('MM')

if __name__ == '__main__':
    app = Room()
    app.run()


import cv2
import numpy as np

from utils import get_four_points

image = cv2.imread('data/images/book1.jpg')

# cv2.imshow('Original', image)

# image 좌표 4곳
# 시계방향으로 찍을 것
# 좌표만 알아내고 주석처리함
# 주석처리 안하고는 좌표찍으면 결과 이미지 나옴
point_src = get_four_points(image)
print(point_src)

# image의 4개의 점은 마우스 클릭으로 해결완료
# point_original = np.array([[318., 256.],[535., 374.],[318., 669.],[ 74., 475.]])
point_dst = np.array([[0, 0], [300, 0], [300, 400], [0, 400]])
# image_dst의 4개의 점은 우리가 구할것
# 행렬 구하기
matrix, status = cv2.findHomography(point_src, point_dst)
# 변환함수를 통해 이미지를 가져올 것
# 새로운 이미지의 사이즈는 x=400, y=300
result = cv2.warpPerspective(src = image, M = matrix, dsize = (300, 400))

cv2.imshow('result', result)

cv2.waitKey(0)
cv2.destroyAllWindows()


import math
import time
import sys
import mimetypes

import numpy as np
import cv2

class Converter:
    CHARACTERS = (" ", ".", "-", "+", "*", "&", "#", "@")
    MIN_ASPECT_CHARS = 50
    FONT = cv2.FONT_HERSHEY_PLAIN
    COLOR_RGB = (0, 255, 0)

    def __init__(self, filename) -> None:
        type = self.getMediaType(filename)
        if not type:
            print("Error: Not a valid filetype")
            return
        self.isVid = True if type == "video" else False

        if self.isVid:
            self.file = cv2.VideoCapture(filename)
            self.fps = self.file.get(cv2.CAP_PROP_FPS)
            self.ms_per_frame = 1 / self.fps
            frame = self.file.read()[1]
        else:
            self.file = cv2.imread(filename)
            frame = self.file

        self.og_h = frame.shape[0]
        self.og_w = frame.shape[1]
        self.aspect = self.MIN_ASPECT_CHARS / min(frame.shape[:2])
        self.h = round(frame.shape[0] * self.aspect)
        self.w = round(frame.shape[1] * self.aspect)
        self.ascii_frame = self.imgToAscii(frame)

        if self.isVid:
            self.ascii_frames = self.asciiVidGenerator()

    def getMediaType(self, filename):
        mimestart = mimetypes.guess_type(filename)[0]
        if mimestart != None:
            type = mimestart.split("/")[0]
            if type in ["video", "image"]:
                return type
        return False

    def getCharacter(self, pixel):
        intensity = pixel / 256
        return self.CHARACTERS[math.floor(intensity * len(self.CHARACTERS))]

    def imgToAscii(self, img):
        img = cv2.resize(img, (0, 0), fx=self.aspect, fy=self.aspect)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        cols = []
        for y in range(self.h):
            row = ""
            for x in range(self.w):
                row += self.getCharacter(img[y, x]) * 2
            cols.append(row)
        return cols

    def asciiVidGenerator(self):
        while self.file.isOpened():
            ret, frame = self.file.read()
            if ret == True:
                yield self.imgToAscii(frame)
            else:
                break

    def asciiToImg(self, ascii):
        img = np.zeros((self.n_height, self.n_width, 3), np.uint8)
        for i, row in enumerate(ascii):
            y = (i + 1) * self.y_increment
            for j, char in enumerate(row):
                x = j * self.x_increment
                cv2.putText(
                    img=img,
                    text=char,
                    org=(x, y),
                    fontFace=self.FONT,
                    fontScale=self.font_scale,
                    color=self.COLOR_RGB,
                    thickness=1,
                )
        return img

    def saveAscii(self):
        self.x_increment = round(self.og_w / len(self.ascii_frame[0]))
        self.y_increment = round(self.og_h / len(self.ascii_frame))
        self.n_width = self.x_increment * len(self.ascii_frame[0])
        self.n_height = self.y_increment * len(self.ascii_frame)

        for scale in range(0, 60, 1):
            char_width = cv2.getTextSize(
                text=self.CHARACTERS[-1],
                fontFace=self.FONT,
                fontScale=scale / 20,
                thickness=1,
            )[0][0]
            if char_width >= self.x_increment:
                self.font_scale = scale / 20
                break

        if self.isVid:
            out = cv2.VideoWriter(
                "ascii_vid.avi",
                cv2.VideoWriter_fourcc("M", "J", "P", "G"),
                self.fps,
                (self.n_width, self.n_height),
            )
            for frame in self.ascii_frames:
                img = self.asciiToImg(frame)
                out.write(img)
            out.release()
        else:
            img = self.asciiToImg(self.ascii_frame)
            cv2.imwrite("ascii_pic.png", img)

    def displayInTerminal(self):
        if self.isVid:
            gotime = time.time()
            for ascii_frame in self.ascii_frames:
                while True:
                    if time.time() >= gotime:
                        gotime += self.ms_per_frame
                        for row in ascii_frame:
                            print(row)
                        for _ in range(len(ascii_frame)):
                            sys.stdout.write("\x1b[1A")
                            sys.stdout.write("\x1b[2K")
                        break
        else:
            for row in self.ascii_frame:
                print(row)
            input()
            for _ in range(len(self.ascii_frame) + 1):
                sys.stdout.write("\x1b[1A")
                sys.stdout.write("\x1b[2K")

if __name__ == "__main__":
    # pic = Converter("pic.png")
    # pic.saveAscii()
    vid = Converter("vid.mp4")
    vid.displayInTerminal()
    # vid.saveAscii()


from flask import Blueprint, flash, g, redirect, render_template, request, session, url_for, current_app
from src.db import get_db

from src.auth import login_required

bp = Blueprint('page', __name__, url_prefix='/')

@bp.route('/movie', methods=('GET', 'POST'))
@login_required
def movie_handler():
	mid = request.args.get('id', type=int)
	db = get_db()
	uid = g.user['uid']
	if not mid:
		mid = 1
	else:
		mid += 1

	userlist = db.execute('SELECT mid FROM watchlist WHERE uid = ?', (uid,)).fetchall()
	while exists(userlist, mid):
		mid += 1

	movie = db.execute('SELECT * FROM movie WHERE mid = ?', (mid,)).fetchone()
	print(movie)

	return render_template('movie.html', mid=mid, movie=movie)

@bp.route('/add', methods=('GET', 'POST'))
@login_required
def add():
	mid = request.args.get('id', type=int)
	uid = session.get('user-id')
	db = get_db()
	db.execute('INSERT INTO watchlist (uid, mid) VALUES (?, ?)', (uid, mid))
	db.commit()

	return redirect(url_for('page.movie_handler', id=mid))

@bp.route('/watchlist', methods=('GET', 'POST'))
@login_required
def watchlist():
	user = g.user
	db = get_db()
	movies = db.execute(
            '''SELECT m.mid, m.title, m.picture, m.genre FROM
            movie m INNER JOIN watchlist w ON m.mid = w.mid
            INNER JOIN user u on w.uid = u.uid
            where u.uid = ?''',
            (str(user[0]),)
        ).fetchall()

	return render_template('watchlist.html', movies=movies)

@bp.route('/remove', methods=('GET', 'POST'))
@login_required
def wremover():
	user = g.user
	db = get_db()
	mid = request.args.get('id', type=int)
	uid = user['uid']
	print(uid, mid)

	db.execute(
		'''
		DELETE FROM watchlist
		WHERE uid = ? AND mid = ?

		''',
		(uid, mid)
	)
	db.commit()

	return redirect(url_for('page.watchlist'))

def exists(ulist, id):
	for row in ulist:
		if row['mid'] == id:
			return True

	return False


from bs4 import BeautifulSoup
import requests

class Community(object):
    def __init__(self, url):
        self.url = url

    @classmethod
    def get_soup(cls, url):
        r = requests.get(url)
        url_contents = r.text
        url_soup = BeautifulSoup(url_contents, 'html.parser')
        return url_soup

    @classmethod
    def find_keyword(cls, url, keyword):
        """Returns True if the keyword is found within the post.

            :param url: Url of the post to be iterated
            :type url: str
            :param keyword: Word to look for in the post
            :type keyword: str
            :return: True
            """
        title = Community.get_title(url)
        body = Community.get_body(url)
        for word_t in title.split():
            if word_t.lower() == keyword.lower():
                return True
        for word_b in body.split():
            if word_b.lower() == keyword.lower():
                return True

    @classmethod
    def find_keywords(cls, url, keywords):
        """Returns True if all keywords are found within the post.

        :param url: Url of the post to be iterated
        :type url: str
        :param keywords: Words to look for in the post
        :type keywords: str
        :return: True
        """
        if type(keywords) is str:
            multi_keywords = keywords.split()
        else:
            multi_keywords = keywords
        match = []
        if len(multi_keywords) == 0:
            if False in match:
                return False
            else:
                return True
        else:
            first_keyword = Community.find_keyword(url, multi_keywords[0])
            match.append(first_keyword)
            smaller_list = multi_keywords[1:]
            return Community.find_keywords(url, smaller_list)

    @classmethod
    def get_posts(cls, url):
        """Returns a list of url of all the posts in a single page in forum.

        :return: A list of url of a single post in the page
        :rtype: list
        """
        return []

    @classmethod
    def get_next_page(cls, current_url, delimiter):
        """Returns url of the next page in forum.

        :param current_url: Url of the current page
        :type current_url: str
        :param delimiter: Delimiter to split up the url of the current page
        :type delimiter: str
        :return: Url of the next page
        :rtype: str
        """
        current_url = current_url.split(delimiter)
        current_url[-1] = str(int(current_url[-1]) + 1)
        current_url = delimiter.join(current_url)
        return current_url

    @classmethod
    def reach_last_page(cls, url):
        """Returns True if the last page in forum is reached

        :param url: Url of the current page
        :type url: str
        :return: True
        """
        return False

    @classmethod
    def get_title(cls, url):
        """Returns title of a post

        :param url: Url of the post
        :type url: str
        :return: Title of the post
        :rtype: str
        """
        return ''

    @classmethod
    def get_body(cls, url):
        """Returns body content of a post

        :param url: Url of the post
        :type url: str
        :return: Body content of the post
        :rtype: str
        """
        return ''

class BRCANowForum(Community):
    def __init__(self, url):
        super().__init__(url)
        self.url = 'https://forum.breastcancernow.org/t5/forums/recentpostspage/post-type/thread/page/1'

    def get_posts(self, url):
        url_soup = Community.get_soup(url)
        posts = url_soup.find_all('', {'class', 'page-link lia-link-navigation lia-custom-event'})
        url_list = []
        for post in posts:
            post_url = 'http://forum.breastcancernow.org' + post['href']
            url_list.append(post_url)
        return url_list

    def reach_last_page(self, url):
        current = url[url.rfind('/') + 1:]
        if int(current) != 1:
            r = requests.get(url)
            url_contents = r.text
            url_soup = BeautifulSoup(url_contents, 'html.parser')
            next_page = url_soup.find('', {'class',
                                           'lia-link-navigation lia-js-data-pageNum-' + current + ' lia-link-disabled'})
            if next_page is None:
                return False
        else:
            return False

    def get_title(self, url):
        url_soup = Community.get_soup(url)
        return url_soup.title.string.replace('<title>', '').rsplit('-', 2)[0]

    def get_body(self, url):
        url_soup = Community.get_soup(url)
        return str(url_soup.find('', {'class', 'lia-message-body-content'}))

class BRCAOrg(Community):
    def __init__(self, url):
        super().__init__(url)
        self.url = 'https://community.breastcancer.org/'

    def find_keyword(self, url, keyword):
        title = Community.get_title(url)
        body = Community.get_body(url)
        for word_t in title.split():
            if word_t.lower() == keyword.lower():
                return True
        for content in body:
            for word_b in content.text.split():
                if word_b.lower() == keyword.lower():
                    return True

    @classmethod
    def get_forums(cls, url):
        """Returns a list of url of forums

        :param url: Url of the index page
        :type url: str
        :return: A list of url of forums in the same category
        """
        url_soup = Community.get_soup(url)
        forums = url_soup.find_all('h3')[3:57]
        link_list = []
        for forum in forums:
            link = forum.find('a').get('href')
            link = 'https://community.breastcancer.org' + link
            link_list.append(link)
        return link_list

    def get_posts(self, url):
        url_soup = Community.get_soup(url)
        posts = url_soup.find_all(title="View this Topic and its replies")
        url_list = []
        for post in posts:
            post_url = 'https://community.breastcancer.org' + post.get('href')
            url_list.append(post_url)
        return url_list

    def reach_last_page(self, url):
        url_soup = Community.get_soup(url)
        next_page = url_soup.find('', {'class', 'next_page disabled'})
        if next_page is None:
            return False

    def get_title(self, url):
        url_soup = Community.get_soup(url)
        return url_soup.h1.string[7:]

    def get_body(self, url):
        url_soup = Community.get_soup(url)
        return url_soup.find('', {'class', "user-post"}).find_all('p')[1:]

class Macmillan(Community):
    def __init__(self):
        super().__init__(url)
        self.url = 'https://community.macmillan.org.uk/cancer_types/breast-cancer/discussions?pi5752=1'

    def get_posts(self, url):
        url_soup = Community.get_soup(url)
        posts = url_soup.find_all('h4', {'class', 'post-name'})
        url_list = []
        for post in posts:
            if len(post.get('class')) != 1:
                posts.remove(post)
            else:
                post_url = 'https://community.macmillan.org.uk/' + post.find('a').get('href')
                url_list.append(post_url)
        return url_list

    def reach_last_page(self, url):
        url_soup = Community.get_soup(url)
        next_page = url_soup.find('a', {'class', 'next'})
        if next_page is not None:
            return False

    def get_title(self, url):
        url_soup = Community.get_soup(url)
        return url_soup.find('h4', {'class', 'post-name'}).string

    def get_body(self, url):
        url_soup = Community.get_soup(url)
        tags = url_soup.find(lambda tag: tag.get('class') == ['content'])
        body = ''
        for tag in tags:
            if tag.string is not None:
                body = body + tag.string
        return body

class AUSBRCAForum(Community):
    def __init__(self, url):
        super().__init__(url)
        self.url = 'https://www.bcaus.org.au/phpBB3'

    @classmethod
    def get_forums(cls, url):
        """Returns a list of url of forums

        :param url: Url of the index page
        :type url: str
        :return: A list of url of forums in the same category
        """
        url_soup = Community.get_soup(url)
        forums = url_soup.find_all('a', {'class', 'forumtitle'})[1:6]
        del forums[3]
        link_list = []
        for forum in forums:
            link = forum.get('href')
            link = 'https://www.bcaus.org.au/phpBB3' + link[1:]
            link_list.append(link)
        return link_list

    def get_posts(self, url):
        url_soup = Community.get_soup(url)
        posts = url_soup.find_all(class_='icon topic_read') + url_soup.find_all(class_='icon topic_read_hot')
        url_list = []
        for post in posts:
            if post.find(class_='pagination') is not None:
                print(post.find(class_='pagination').find_all('li')[-1])
                post_url = 'https://www.bcaus.org.au/phpBB3' + post.find(class_='pagination').find_all('li')[-1].find(
                    'a').get('href')[1:]
            else:
                post = post.find(class_='topictitle')
                post_url = 'https://www.bcaus.org.au/phpBB3' + post.get('href')[1:]
            url_list.append(post_url)
        return url_list

    def get_next_page(self, url, delimiter):
        delimiter = '='
        url = url.split(delimiter)
        if len(url) == 2:
            url = url[0] + '&start=50'
        else:
            url[-1] = str(int(url[-1]) + 50)
            url = delimiter.join(url)
        return url

    def reach_last_page(self, url):
        url_soup = Community.get_soup(url)
        next_page = url_soup.find(class_='next')
        if next_page is not None:
            return False

    def get_title(self, url):
        url_soup = Community.get_soup(url)
        return url_soup.find(class_='topic-title').string

    def get_body(self, url):
        url_soup = Community.get_soup(url)
        return str(url_soup.find_all(lambda tag: tag.get('class') == ['content'])[-1])

class KoreaVenus(Community):
    def __init__(self, url):
        super().__init__(url)
        self.url = 'http://www.koreavenus.com/db/index.html?table=noh_qna'

class CancerSuvivorsNetwork(Community):
    def __init__(self, url):
        super().__init__(url)
        self.url = 'https://csn.cancer.org/forum/127'

    def get_posts(self, url):
        url_soup = Community.get_soup(url)
        posts = url_soup.find_all('td', {'class', 'title'})
        url_list = []
        for post in posts:
            if post.div.a.string != '*Abbreviations* for all our Newbies':
                post_url = 'https://csn.cancer.org' + post.div.a.get('href')
                url_list.append(post_url)
        return url_list

    def get_next_page(self, url, delimiter):
        delimiter = '='
        url = url.split(delimiter)
        if len(url) == 1:
            url = url[0] + '?page=1'
        else:
            url[-1] = str(int(url[-1]) + 1)
            url = delimiter.join(url)
        return url

    def reach_last_page(self, url):
        url_soup = Community.get_soup(url)
        if url_soup.find(class_='pager-current last') is None:
            return False

    def get_title(self, url):
        url_soup = Community.get_soup(url)
        return url_soup.find(class_='with-tabs').string

    def get_body(self, url):
        url_soup = Community.get_soup(url)
        return str(url_soup.find('p').contents)

class HealthBoards(Community):
    def __init__(self, url):
        super().__init__(url)
        self.url = 'https://www.healthboards.com/boards/cancer-breast/index1.html'

    def get_posts(self, url):
        url_soup = Community.get_soup(url)
        posts = url_soup.find_all(class_='alt1')
        url_list = []
        for post in posts:
            if post.find('div') is not None:
                if post.find('div').find('a') is not None:
                    post_url = post.div.a.get('href')
                    if 'cancer-breast' in post_url:
                        url_list.append(post_url)
        del url_list[-1]
        return url_list

    def get_next_page(self, url, delimiter):
        # no delimiter is needed
        url_soup = Community.get_soup(url)
        return url_soup.find(rel='next').get('href')

    def reach_last_page(self, url):
        url_soup = Community.get_soup(url)
        if url_soup.find(rel='next') is not None:
            return False

    def get_title(self, url):
        url_soup = Community.get_soup(url)
        tags = url_soup.find_all(style='display: inline-block;')
        for tag in tags:
            if tag.find('strong') is not None:
                return tag.find('strong').string

    def get_body(self, url):
        url_soup = Community.get_soup(url)
        return str(url_soup.find(class_='alt1'))


from pyspark.sql import SparkSession
import pandas as pd
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.feature import StringIndexer
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
import time
start1 = time.process_time()
spark = SparkSession.builder.appName('rf').getOrCreate()
df = spark.read.csv('abalone.csv', header=True, inferSchema=True)
print("Read Time: ",time.process_time() - start1)
pd.DataFrame(df.take(5), columns=df.columns).transpose()
numeric_features = [t[0] for t in df.dtypes if t[1] == 'double']
df.select(numeric_features).describe().toPandas().transpose()

numericCols = ['Length', 'Diameter', 'Height', 'WholeWeight', 'ShuckedWeight', 'VisceraWeight', 'ShellWeight', 'Rings']
assembler = VectorAssembler(inputCols=numericCols, outputCol="features")
df = assembler.transform(df)

label_stringIdx = StringIndexer(inputCol='Sex', outputCol='labelIndex')
df = label_stringIdx.fit(df).transform(df)
pd.DataFrame(df.take(110), columns=df.columns).transpose()
train, test = df.randomSplit([0.7, 0.3], seed=42)

start2 = time.process_time()
rf = RandomForestClassifier(featuresCol='features', labelCol='labelIndex')
rfModel = rf.fit(train)
print("Model Time: ",time.process_time() - start2)

start3 = time.process_time()
predictions = rfModel.transform(test)
predictions.select('Sex', 'Length', 'Diameter', 'Height', 'WholeWeight', 'ShuckedWeight', 'VisceraWeight',
                   'ShellWeight', 'Rings', 'labelIndex', 'rawPrediction', 'prediction', 'probability').show(25)
predictions.select("labelIndex", "prediction").show(10)

evaluator = MulticlassClassificationEvaluator(labelCol="labelIndex", predictionCol="prediction")
accuracy = evaluator.evaluate(predictions)
print("Accuracy = %s" % accuracy)
print("Test Error = %s" % (1.0 - accuracy))
print("Predict Time: ",time.process_time() - start3)


import numpy as np
import datetime as dt
class Sjf:
    # initialize some needed variable
    delivered_processes = []
    ended_processes = []
    current_process = []
    processes = []
    finished_processes = 0

# init object, copy the table, estimate needed time to run
    def __init__(self, table_of_processes):
        self.processes = table_of_processes

# check if at the current time any process arrived
    def check_for_processes(self, time):
        for el in range(len(self.processes)):
            if int(self.processes[el][2]) == time:
                self.delivered_processes.append(self.processes[el])

# update waiting time for all processes that has come, and are not currently running
    def update_waiting_time(self):
        for i in range(len(self.delivered_processes)-1):
            if i != 0:
                self.delivered_processes[i][3] = int(self.delivered_processes[i][3]) + 1

# update current process run time, by decreasing it by 1, and check if it has finished
    def current_process_update(self):
        self.delivered_processes.sort(key=lambda x: x[1])
        if len(self.delivered_processes) != 0:
            self.current_process = self.delivered_processes[0]
            x = int(self.current_process[1])
            if x == 0:
                self.ended_processes.append(self.delivered_processes[0])
                del self.delivered_processes[0]
                self.finished_processes += 1
                Sjf.current_process_update(self)
            else:
                self.current_process[1] = str(x - 1)
                Sjf.update_waiting_time(self)

# sum waiting time for all processes at the end
    def count_wait_time(self):
        overall_waiting_time = 0
        for i in range(len(self.ended_processes)):
            overall_waiting_time += int(self.ended_processes[i][3])
        return overall_waiting_time

# using all above functions in correct order to get the finall result
    def main_loop(self):
        czas_s = dt.datetime.now()
        x, overall, avr = 0, 0, 0
        while len(self.processes) != self.finished_processes:
            Sjf.check_for_processes(self, x)
            Sjf.current_process_update(self)
            x += 1
        print(f"SJF\teverything finished\n\toverall time used: {x}")
        print("SJF\t", len(self.ended_processes))
        overall = Sjf.count_wait_time(self)
        avr = round(overall / len(self.ended_processes), 2)
        print(f"SJF\tOverall waiting time: {overall}s")
        print(f"SJF\tAverage waiting time: {avr}s")
        print("\tSFJ:\t",dt.datetime.now() - czas_s, "\n")


######################################################################
# Verification of UAV track planning model based on DQN
#---------------------------------------------------------------
# author by younghow
# email: younghowkg@gmail.com
# --------------------------------------------------------------
#将训练好的DQN模型放入仿真模拟环境中进行测试，
#可以使用env类中的reset_test函数对测试环境进行设置，
#生成测试环境的UAV的起点与终点，并根据难度等级生成城市环境的建筑数目与风况。
#----------------------------------------------------------------
# Put the trained DQN model into the simulation environment for testing.
# You can use the reset_test function in the env class to set the test environment,
# generate the starting point and end point of the UAV of the test environment,
# and generate the number of buildings,the number of buildings and wind conditions
#  in the urban environment according to the difficulty level.
##############################################################################
import numpy as np
import torch
import matplotlib
import matplotlib.pyplot as plt
import time
from env import *
import torch
LEARNING_RATE = 0.00033   #学习率
num_episodes = 80000  #训练周期长度
space_dim = 42 # n_spaces   状态空间维度
action_dim = 27 # n_actions   动作空间维度
threshold = 200
env = Env(space_dim,action_dim,LEARNING_RATE)

if __name__ == '__main__':
    check_point_Qlocal=torch.load('Qlocal.pth')
    check_point_Qtarget=torch.load('Qtarget.pth')
    env.q_target.load_state_dict(check_point_Qtarget['model'])
    env.q_local.load_state_dict(check_point_Qlocal['model'])
    env.optim.load_state_dict(check_point_Qlocal['optimizer'])
    epoch=check_point_Qlocal['epoch']
    #真实场景运行
    env.level= 8 #环境难度等级
    state = env.reset_test()  #环境重置1
    total_reward = 0
    env.render(1)
    n_done=0
    count=0

    n_test=1  #测试次数
    n_creash=0   #坠毁数目
    for i in range(n_test):
        while(1):
            if env.uavs[0].done:
                #无人机已结束任务，跳过
                break
            action = env.get_action(FloatTensor(np.array([state[0]])) , 0.01)   #根据Q值选取动作

            next_state, reward, uav_done, info= env.step(action.item(),0)  #根据选取的动作改变状态，获取收益

            total_reward += reward  #求总收益
            #交互显示
            print(action)
            env.render()
            plt.pause(0.01)
            if uav_done:
                break
            if info==1:
                success_count=success_count+1

            state[0] = next_state  #状态变更
        print(env.uavs[0].step)
        env.ax.scatter(env.target[0].x, env.target[0].y, env.target[0].z,c='red')
        plt.pause(100)


import cv2

img = cv2.imread("lena.jpg",-1) #flag: 1-color image;0-gray image;-1-image as such including alpha channel
# print(img) #pixels of the image

cv2.imshow('image',img) #flash the image
k = cv2.waitKey(0)&0xFF #cv2.waitKey(0) to wait for a key event
                        # 5000: 5 seconds, 0:forever stays
                        # 0xFF: if using 64 system machine

if k == 27: #escape key
    cv2.destroyAllWindows() #close the windows
elif k == ord('s'): #if s is pressed
    cv2.imwrite('lena_copy.png',img)


# stdlib Imports
import datetime
import math
import time

# Twisted Imports
from twisted.internet import defer

# PythonCollector Imports
from ZenPacks.zenoss.PythonCollector.datasources.PythonDataSource import PythonDataSourcePlugin

class FakeData(PythonDataSourcePlugin):
    @classmethod
    def config_key(cls, datasource, context):
        return (
            context.device().id,
            datasource.getCycleTime(context),
            context.id,
            datasource.id,
            )

    def collect(self, config):
        data = self.new_data()
        now = time.time()

        for datasource in config.datasources:
            for point in datasource.points:
                dpname = "{}_{}".format(datasource.datasource, point.id)
                data["values"][datasource.component][dpname] = {
                    "ten": 10,
                    "twenty": 20,
                    "fifty": 50,
                    "hundred": 100,
                    "epoch": now,
                    "pct1day": getpercentofday(),
                    "cos1min": getcos(now / 60),
                    "cos5min": getcos(now / 300),
                    "cos10min": getcos(now / 600),
                    "cos1hour": getcos(now / 3600),
                    }.get(point.id, None)

        return defer.succeed(data)

def getcos(n):
    """Return cosine of n between 0-10 instead of 0.0 and 1.0."""
    return (math.cos(n * math.pi) * 10) + 10

def getpercentofday():
    """Return the percent of the way through the day (0-100)."""
    now = datetime.datetime.now()
    return (now.hour * 3600 + now.minute * 60 + now.second) / 864.0


import sys
import json
from kapacitor.udf.agent import Agent, Handler, Server
from kapacitor.udf import udf_pb2
import signal
import logging
import re

logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s: %(message)s')
logger = logging.getLogger()

# Mirrors all points it receives back to Kapacitor
class Handler(Handler):
    def __init__(self, agent):
        self._agent = agent
        self._path = 'file:///data/.kapacitor/load/sideloadFiles'
        self._source = None
        self._formula = None
        self._eval = ''
        self._levels = []
        self._messages = []
        self._vars = []

    def parse_formula(self):
        _var = list(filter(None, re.split('[()+/\-* ]', self._formula)))
        _op = re.findall('[()+\-/* ]+', self._formula)

        for i in range(len(_var)):
            if re.fullmatch('\d+', _var[i]) is None:
                _var[i] = 'point.fieldsDouble[\'{}\']'.format(_var[i])

        if _op[0] == '(':
            for i in range(len(_var)):
                self._eval = '{}{}{}'.format(self._eval, _op[i], _var[i])
        else:
            for i in range(len(_var) - 1):
                self._eval = '{}{}{}'.format(self._eval, _var[i], _op[i])
            self._eval = '{}{}'.format(self._eval, _var[len(_var) - 1])

        if _op[len(_op) - 1] == ')':
            self._eval = '{})'.format(self._eval)

        self._vars = _var

    def parse_file(self):
        self._levels = []
        self._messages = []
        data = open("/data/.kapacitor/load/sideloadFiles/{}".format(self._source), 'r')
        for line in data:
            level, message = line.split('\t')
            self._levels.append(int(level))
            self._messages.append(message)

        data.close()

    def info(self):
        response = udf_pb2.Response()
        response.info.wants = udf_pb2.STREAM
        response.info.provides = udf_pb2.STREAM
        response.info.options['source'].valueTypes.append(udf_pb2.STRING)
        response.info.options['formula'].valueTypes.append(udf_pb2.STRING)
        return response

    def init(self, init_req):                   #init response
#logger.info("Init Response")
        response = udf_pb2.Response()
        success = True
        msg = ''

        for opt in init_req.options:
            if opt.name == 'source':
                self._source = opt.values[0].stringValue
            elif opt.name == 'formula':
                self._formula = opt.values[0].stringValue

        if self._source is None:
            success = False
            msg += ' must supply source'
            logger.info('No ok')
        if self._formula is None:
            success = False
            msg += ' must supply formula'
            logger.info('No ok')

        response = udf_pb2.Response()
        response.init.success = success
        response.init.error = msg[1:]

        if success:
            self.parse_formula()
            self.parse_file()

        return response

    def snapshot(self):                         #snapshor state
#logger.info("Snapshot Response")
        response = udf_pb2.Response()
        return response

    def restore(self, restore_req):             #restore ?
#logger.info("Restore Response")
        return response

    def begin_batch(self, begin_req):           #batch?
#logger.info("BBatch Response")
        raise Exception("not supported")

    def point(self, point):                     #single row
#logger.info("Point Response")
        response = udf_pb2.Response()
        response.point.CopyFrom(point)

        point.fieldsDouble.update(point.fieldsInt)

        response.point.fieldsDouble['formula result'] = eval(self._eval)
#logger.info(response.point.fieldsDouble['formula result'])

        for i in range(len(self._levels)):
            response.point.fieldsDouble['critical_level'] = self._levels[i]
            response.point.fieldsString['message'] = self._messages[i]
            self._agent.write_response(response, True) #response / flush

    def end_batch(self, end_req):               #batch end?
#logger.info("EBatch Response")
        raise Exception("not supported")

class accepter(object):                     #class for server config
    _count = 0
    def accept(self, conn, addr):           #new connection
        self._count += 1                    #count of connection
        a = Agent(conn, conn)               #create agent (communicate with kapacitor) in/out
        h = Handler(a)                      #create handler
        a.handler = h                       #save handler to agent

        logger.info("Starting Agent for connection %d", self._count) #start log
        a.start()                           #start agent
        a.wait()                            #wait request
        logger.info("Agent finished connection %d",self._count)      #end log

if __name__ == '__main__':
    path = "/tmp/alert.sock"           #path to socket
    if len(sys.argv) == 2:              #or we use argv for path
        path = sys.argv[1]
    server = Server(path, accepter())   #create server
    logger.info("Started server")       #log the start
    server.serve()                      #run server


import sys
import json

# Simple WSGI application.
# Derived from https://peps.python.org/pep-3333/#error-handling
def simple_app(environ, start_response):
    try:
        path_info = environ.get("PATH_INFO", "")
        if path_info == "/error":
            # When "/error" is requested, force an exception to be thrown,
            # which then falls into the except clause below.
            1 / 0

        # regular application code here
        status = "200 Froody"
        response_headers = [("content-type", "application/json")]
        start_response(status, response_headers)
        data = json.dumps({"msg": "Hello, Simple WSGI!"})
        return data.encode("UTF-8")
    except:
        # XXX should trap runtime issues like MemoryError, KeyboardInterrupt
        #     in a separate handler before this bare 'except:'...
        status = "500 Oops"
        response_headers = [("content-type", "application/json")]
        start_response(status, response_headers, sys.exc_info())
        data = json.dumps({"msg": "Oh, no! What have you done, Simple WSGI?"})
        return data.encode("UTF-8")

app = simple_app


import nntplib

from modules import *
import torch.nn.functional as F

class ST_encoder(nn.Module):
    def __init__(self, n_features, d_model, window):
        super(ST_encoder, self).__init__()
        self.linear = nn.Linear(n_features, d_model)
        self.tcna = TemporalConvNet(num_inputs=d_model, num_channels=[d_model], d_model=d_model, kernel_size=2,
                                   attention=True)
        self.tcnb = TemporalConvNet(num_inputs=d_model, num_channels=[d_model], d_model=d_model, kernel_size=2,
                                   attention=True)
        self.sgnn = SGNN(d_model)
        self.position = PositionalEncoding(d_model=d_model, max_len=window)
        encoder_layers = nn.TransformerEncoderLayer(d_model=d_model, nhead=1, dim_feedforward=256, dropout=0.3)
        self.transformer_encoder = nn.TransformerEncoder(encoder_layers, 2)

        decoder_layers1 = nn.TransformerDecoderLayer(d_model=d_model, nhead=1, dim_feedforward=256, dropout=0.3)
        self.transformer_decoder1 = nn.TransformerDecoder(decoder_layers1, 1)
        decoder_layers2 = nn.TransformerDecoderLayer(d_model=d_model, nhead=1, dim_feedforward=256, dropout=0.3)
        self.transformer_decoder2 = nn.TransformerDecoder(decoder_layers2, 1)

    def forward(self, x):
        x = F.relu(self.linear(x))
        x_a = self.tcna(x.permute(0, 2, 1)).permute(0, 2, 1)
        x_b = self.tcna(x.permute(0, 2, 1)).permute(0, 2, 1)
        x = torch.tanh(x_b)*torch.sigmoid(x_a)
        x = self.sgnn(x)
        x = self.position(x)
        x = self.transformer_encoder(x.permute(1, 0, 2))
        mu = self.transformer_decoder1(x[-1, :, :].unsqueeze(0), x)
        logvar = self.transformer_decoder1(x[-1, :, :].unsqueeze(0), x)
        return mu.squeeze(), logvar.squeeze()

class STAD(nn.Module):
    def __init__(self, n_features, window, d_model, n_latent=16):
        super(STAD, self).__init__()
        self.window = window
        self.d_model = d_model
        self.n_latent = n_latent
        self.n_hidden = n_latent * 4
        self.n_features = n_features
        self.linear = nn.Linear(n_features, d_model)
        self.ST_encoder = ST_encoder(d_model, d_model, window)
        self.linear2 = nn.Linear(d_model,n_features)
        self.linear3 = nn.Linear(d_model,n_features)
        self.end = nn.Sequential(

            # nn.Linear(d_model, d_model), nn.ReLU(),
            # nn.Linear(d_model, n_features), nn.ReLU(),
            nn.Linear(n_features, n_features), nn.Sigmoid(),
        )

    def forward(self, x):
        x = F.relu(self.linear(x))
        mu, logvar = self.ST_encoder(x)
        mu = self.linear2(mu)
        logvar = self.linear3(logvar)
        std = torch.exp(0.5 * logvar)
        eps = torch.randn_like(std)
        z = mu + eps * std
        x = self.end(z.squeeze())
        return x, mu, logvar


from MeDIT.ImageProcess import ResizeNiiFile, ResizeROINiiFile
import SimpleITK as sitk
import os
import numpy as np
from tkinter import _flatten
import shutil

def PathAcquire(path):
    pathlist = list(_flatten([[os.path.join(root, file) for file in files] for root, dirs, files in os.walk(path)
                              if len(files) > 0]))
    return pathlist

def GetResolution(root_pathlist):
    if type(root_pathlist) == str:
        root_pathlist = [root_pathlist]
    resolution = [list(sitk.ReadImage(path).GetSpacing()) for path in root_pathlist]
    return resolution

def ExpectedResolution(root_pathlist):
    resolution_list = GetResolution(root_pathlist)
    resolution_array = np.array(resolution_list)
    resolution_median = np.median(resolution_array, axis=0)
    resolution_var = np.var(resolution_array, axis=0)
    resolution_down = resolution_median - 10 * resolution_var
    resolution_up = resolution_median + 10 * resolution_var
    print(resolution_up)
    print(resolution_down)
    count = 0
    resolution_sum = np.zeros((1, 3))
    for path in root_pathlist:
        resolution = list(_flatten(GetResolution(path)))
        if resolution_down[0] < resolution[0] < resolution_up[0] and resolution_down[2] < resolution[2] < resolution_up[2]:
            resolution_sum += resolution
            count += 1
    resolution_range = resolution_sum / count
    return resolution_range.flatten()

# def RemoveWrongNiiShapeFile(root_pathlist):
#     for path in root_pathlist:
#         if GetResolution(path)[0][2] >= 2.0:
#             os.remove(path)

def GenerateDir(rootpath, storepath):
    for root, dirs, files in os.walk(rootpath):
        for file in files:
            filename, ext = os.path.splitext(file)
            foldername, _ = os.path.splitext(filename)
            if len(files) > 0:
                rootpath_old = os.path.join(root, file)
                storepath_new = os.path.join(storepath, foldername)
                if not os.path.isdir(storepath_new):
                    os.makedirs(storepath_new)
                shutil.copy(rootpath_old, storepath_new)

# def ResizeFile(store_pathlist, resolution):
#     for store_path in store_pathlist:
#         if os.path.splitext(store_path)[1] == '.gz':
#             ResizeNiiFile(store_path, expected_resolution=resolution)
#         elif os.path.splitext(store_path)[1] == '.nii':
#             ResizeROINiiFile(store_path, expected_resolution=resolution)
#         else:
#             print('the input file should be suffix .nii or .nii.gz')
#         os.remove(store_path)

def ResizeFile(store_pathlist):
    for store_path in store_pathlist:
        if os.path.splitext(store_path)[1] == '.gz':
            ResizeNiiFile(store_path, expected_resolution=[0.6834329, 0.6834329, 1.3852941])
        elif os.path.splitext(store_path)[1] == '.nii':
            ResizeROINiiFile(store_path, expected_resolution=[0.6834329, 0.6834329, 1.3852941])
        else:
            print('the input file should be suffix .nii or .nii.gz')
        os.remove(store_path)

rootpath = 'D:/JSYB'
storepath = 'Z:/JSYB_Resize'
# rootpath = r'D:\ZJ_assignment'
# storepath = r'D:\ZJ_assignment_resize'

# root_pathlist = PathAcquire(rootpath)
# RemoveWrongNiiShapeFile(root_pathlist)

# root_pathlist_new = PathAcquire(rootpath)
# resolution = ExpectedResolution(root_pathlist_new)

GenerateDir(rootpath, storepath)

store_pathlist = PathAcquire(storepath)
ResizeFile(store_pathlist)


#!/usr/bin/env python

import sys
import tempfile
import subprocess

def execute_shell(command):
    '''Execute shell line and return output'''
    splitted_command = command.split(' ')
    if sys.version_info[0] == 2:
        # two opens are required otherwise the read get an error
        fd_tmp_out, tmp_output = tempfile.mkstemp()
        with open(tmp_output, "wb") as f:
            ret = subprocess.call(splitted_command, stdout=f, stderr=subprocess.STDOUT)
        if ret != 0:
            raise RuntimeError('Bad shell command execution: ' + command)
        with open(tmp_output, "r") as f:
            output = ("".join(f.readlines())).strip().split('\n')
        return output
    else:
        refs = subprocess.check_output(splitted_command).decode('UTF-8').split('\n')
        refs.remove('')
        return refs


#!/usr/bin/env python

import time
import json
import serial
import sqlite3
import requests
import subprocess
import adafruit_gps
import Adafruit_DHT

#initial data
kilometraje = 0
latitud = 0.0
longitud = 0.0
altitud = 0.0
velocidad = 0
tanqueConductor = 0.0
tanquePasajero = 0.0
fecha = ""
hora = ""
enviado = 0
timePost = 60
timeUpdate = 600
outPost = 15
outUpdate = 60
ipTest = "190.143.75.85"
headers = {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'}
url = "https://pruebas.extranet.com.co/coreiot/api/public/EYT.Tracker/Registro/Registro/Nuevo"
dataBase = "/home/pi/coreiot/itemdb"
idItem = "e67d85d2-cf20-4027-be3e-c488e2d64a80"
sensor = Adafruit_DHT.DHT11
gpioSensor = 17
countGps = 0

time.sleep(5)
print('-' * 5+'START'+'-' * 5, flush=True)

def dict_factory(cursor, row):
    d = {}
    for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
    return d

def post_request(body, tOut):
    success = False
    try:
        r = requests.post(url, data=json.dumps(body), headers=headers, timeout=tOut)
        r.raise_for_status()
        if(r.json()['Error']):
            print('Error', flush=True)
        else:
            flag = 1
            success = True
    except requests.exceptions.HTTPError as errh:
        print ("Http Error:",errh, flush=True)
    except requests.exceptions.ConnectionError as errc:
        print ("Error Connecting:",errc, flush=True)
    except requests.exceptions.Timeout as errt:
        print ("Timeout Error:",errt, flush=True)
    except requests.exceptions.RequestException as err:
        print ("OOps: Something Else",err, flush=True)
    finally:
        if not success:
            flag = 0
    return flag

def watcher():
    connection = False
    try:
        pingRes = subprocess.check_output("ping 190.143.75.85 -c3 -q | grep received | awk -F ',' '{print $2}' | awk '{print $1}'", shell=True)
        if int(pingRes) > 0:
            connection = True
        else:
            print("Network down", flush=True)
    except:
        print("Network Test Error", flush=True)
    if connection:
        cursor.execute("SELECT id, Kilometraje, Latitud, Longitud, Altitud, TanqueConductor, TanquePasajero, Velocidad, Fecha, Hora FROM item WHERE Enviado = 0")
        results = cursor.fetchall()
        if len(results) > 0:
            idList = []
            jsondict = {}
            for val in results:
                idList.append(val['id'])
                del val['id']
            jsondict['Registros'] = results
            jsondict['idItem'] = 'e67d85d2-cf20-4027-be3e-c488e2d64a80'
            #Update
            updated = post_request(jsondict, outUpdate)
            if updated==1:
                sql="UPDATE item SET Enviado = 1 WHERE id IN ({seq})".format(seq=','.join(['?']*len(idList)))
                cursor.execute(sql, idList)
                dbconnect.commit()
                print('{} Updated'.format(len(idList)), flush=True)
            else:
                print('Error POST', flush=True)
        else:
            print('Up to date', flush=True)

# serial uart access
uart = serial.Serial("/dev/ttyS0", baudrate=9600, timeout=3000)
# Create a GPS module instance.
gps = adafruit_gps.GPS(uart, debug=False)
# Turn on the basic GGA and RMC info
gps.send_command(b'PMTK314,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0')
# Set update rate to once a second (1hz)
gps.send_command(b'PMTK220,1000')

#database set up
dbconnect = sqlite3.connect(dataBase, check_same_thread=False)
dbconnect.row_factory = dict_factory
cursor = dbconnect.cursor()

lastPost = time.monotonic()
lastUpdate = time.monotonic()
while True:
    # Every 60s post, 600s update
    gps.update()
    if not gps.has_fix:
        countGps += 1
        time.sleep(1)
        if countGps == 100:
            raise Exception('Error GPS timeout')
        continue
    countGps = 0
    current = time.monotonic()
    currentUpdate = time.monotonic()
    if current - lastPost >= timePost:
        lastPost = current
        #Fetch data from sensor
        tanqueConductor, tanquePasajero = Adafruit_DHT.read_retry(sensor, gpioSensor)
        #Fetch data from gps
        latitud = gps.latitude
        longitud = gps.longitude
        if gps.speed_knots is not None:
            velocidad = int(gps.speed_knots * 1.852)
        if gps.altitude_m is not None:
            altitud = gps.altitude_m
        fecha = '{}/{}/{}'.format(
            gps.timestamp_utc.tm_mday,
            gps.timestamp_utc.tm_mon,
            gps.timestamp_utc.tm_year)
        hora = '{:02}:{:02}:{:02}'.format(
            gps.timestamp_utc.tm_hour,
            gps.timestamp_utc.tm_min,
            gps.timestamp_utc.tm_sec)

        payload = {'IdItem':idItem,
         'Registros':[{
             'Kilometraje':kilometraje,
             'Latitud':latitud,
             'Longitud':longitud,
             'Altitud':altitud,
             'TanqueConductor':tanqueConductor,
             'TanquePasajero':tanquePasajero,
             'Velocidad':velocidad,
             'Fecha':fecha,
             'Hora':hora
             }]
         }

        #POST
        enviado = post_request(payload, outPost)
        if enviado == 0:
            print('Enviado: {}'.format(enviado), flush=True)

        #execute insert statement
        cursor.execute("INSERT INTO item (Kilometraje, Latitud, Longitud, TanqueConductor, TanquePasajero, Velocidad, Altitud, Fecha, Hora, Enviado)VALUES(?,?,?,?,?,?,?,?,?,?)",
                       (kilometraje, latitud, longitud, tanqueConductor, tanquePasajero, velocidad, altitud, fecha, hora, enviado))
        dbconnect.commit()

        #print('Id: {}'.format(idItem), flush=True)
        #print('Kilometraje: {}'.format(kilometraje), flush=True)
        #print('Latitud: {0:.6f}'.format(latitud), flush=True)
        #print('Longitud: {0:.6f}'.format(longitud), flush=True)
        #print('Tanque Conductor: {}'.format(tanqueConductor), flush=True)
        #print('Tanque Pasajero: {}'.format(tanquePasajero), flush=True)
        #print('Velocidad: {} km/h'.format(velocidad), flush=True)
        #print('Altitud: {} metros'.format(altitud), flush=True)
        #print('Fecha: {}'.format(fecha), flush=True)
        #print('Hora: {}'.format(hora), flush=True)

    if currentUpdate - lastUpdate >= timeUpdate:
        lastUpdate = currentUpdate
        watcher()

#close the connection
dbconnect.close()


# -*- coding: utf-8 -*-
from odoo import api
from odoo import fields
from odoo import models
from copy import deepcopy

import collections

class BaseSublist(models.Model):
    _name = 'base.sublist'
    _description = 'Base Sublist Model'
    _ps_sublist_field = ''
    _ps_sublist_model = ''

    @api.model
    def fields_get(self, allfields=None, attributes=None):
        """
        重写以支持tree试图sublist列表
        """
        res = super(BaseSublist, self).fields_get(allfields, attributes)

        # 导入的时候不获取sublist字段
        if not self.env.context.get('base_import') and not allfields \
                and self._ps_sublist_field and self._ps_sublist_model:
            line_fields = self.env[self._ps_sublist_model].fields_get()
            for line_field in line_fields:
                line_fields[line_field].update({'sortable': False})
                if 'currency_field' in line_fields[line_field]:
                    line_fields[line_field]['currency_field'] = self._ps_sublist_field + '.' + \
                                                                line_fields[line_field]['currency_field']
            res.update({
                self._ps_sublist_field + '.' + key: value for key, value in list(line_fields.items())
            })
        return res

    @api.model
    def search_read(self, domain=None, fields=None, offset=0, limit=None, order=None):
        """
        重写，一并读出子表中需要展示的字段数据
        """
        if fields:

            # sublist 字段的前缀
            pre_fix = self._ps_sublist_field + '.'

            # 需要展示的sublist字段名
            sublist_fields = [sublist_field for sublist_field in fields if sublist_field.startswith(pre_fix)]

            # 主表读数据之前先清掉sublist字段
            fields = list(set(fields) - set(sublist_fields))

            # 读主表数据
            reads = super(BaseSublist, self).search_read(domain, fields, offset, limit, order)

            res = []
            if sublist_fields:
                # 要读取的子表的字段名（去掉pre_fix前缀）
                line_fields = [line_field.replace(pre_fix, '') for line_field in sublist_fields]

                if line_fields:
                    # 取出要读取的所有子表数据的id
                    line_ids = [line_id for record in reads for line_id in record[self._ps_sublist_field]]

                    # 读子表数据
                    line_records = self.env[self._ps_sublist_model].search_read([('id', 'in', line_ids)], line_fields)

                    # 转成id为键值的字典，方便处理
                    records_dict = {line_record['id']: line_record for line_record in line_records}

                    # 组装数据
                    for res_record in reads:
                        line_records = [{
                            sublist_field: records_dict[line_id][sublist_field.replace(pre_fix, '')] for sublist_field in sublist_fields
                        } for line_id in res_record[self._ps_sublist_field]]

                        # line_records 所有的sublist 数据
                        for line_index, line_record in enumerate(line_records):
                            res_record = deepcopy(res_record)
                            res_record.update(line_record)
                            res_record.update({
                                'record_nrows': len(line_records),
                                'record_rowno': line_index,
                            })
                            res.append(res_record)

                        if len(res_record[self._ps_sublist_field]) == 0:
                            for sublist_field in sublist_fields:
                                res_record[sublist_field] = 0
                            res.append(res_record)
            # 获取相同的ID
            ids = [item for item, count in collections.Counter([record.get('id') for record in res]).items() if count > 1]
            for id in ids:
                i = 0
                for res_dict in res:
                    if id == res_dict.get('id'):
                        if i == 0:
                            i += 1
                        else:
                            # 更新此字典
                            res_dict.update({
                                'amount_untaxed_signed': 0,
                                'residual': 0,
                                'ps_residual_signed': 0,
                            })
            return res or reads
        else:
            return super(BaseSublist, self).search_read(domain, fields, offset, limit, order)


import re
import requests as rq
import pprint as pp

def getJSON(address):
    YandexApi = rq.get("http://geocode-maps.yandex.ru/1.x/"+"?format=json&geocode=%s" % (address))
    if YandexApi.status_code == 200:
        return YandexApi.json()
    else:
        print('Error: {} from server'.format(result.status_code))

def listGeoObject(address):
    response = getJSON(address)
    #data = response['response']['GeoObjectCollection']
    if response['response']['GeoObjectCollection']['metaDataProperty']['GeocoderResponseMetaData']['found']:
        return [item['GeoObject']['Point']['pos'] for item in response['response']['GeoObjectCollection']['featureMember']]#['GeoObject']['metaDataProperty']]
        #['Point']#['pos'])

if __name__ == '__main__':
    address = "Москва, Кутузовский проспект, 32"
    #response = getJSON(address)
    #pp.pprint(response)
    coord = []
    response = listGeoObject(address)
    for i in response:
        coord.append(i)
    print(coord)


# coding : utf-8
import re
import os
import sys
import getpass
import urllib3
import requests

from mutagen.easyid3 import EasyID3
from mutagen.mp3 import MP3
from mutagen.id3 import ID3, APIC, TIT2, TPE1, TALB, USLT

# ID3 info:
tagMap = {'cover': APIC, 'title': TIT2,
          'artist': TPE1, 'album': TALB, 'lyric': USLT}
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
headers = {'User-agent': 'Mozilla/5.0'}
MSCDIR = './mp3'
# print(repr(s)[1:-1])  # deal with invalid encoded filename

class netease_music:
    def __init__(self, path):
        '''path: direcotory that contains cache files'''
        self.path = os.path.abspath(path)
        self.id_name = {i[:i.find('-')]: i for i in os.listdir(path)
                        if i.endswith('.uc') or i.endswith('.uc!')}
        if self.id_name:
            if not os.path.exists(MSCDIR):
                os.mkdir(MSCDIR)
            print('Input :', path)
            print('Output:', MSCDIR)
        else:
            print('No cache file found in "{}"'.format(path))

    def getInfoFromWeb(self, musicId):
        # xpath for name and lrc:
        # self.nameXpath ='//div[@class="tit"]/em[@class="f-ff2"]/text()'
        # self.lrcSentencePt=re.compile(r'\[\d+:\d+\.\d+\](.*?)\\n')         # wrong  (r'\[\d+,\d+\](\(\d+,\d+\)(\w))+\n')

        # api :
        # type=song, lyric, comments, detail, artist, album, search
        # eg  API = 'https://api.imjad.cn/cloudmusic/?type=song&id=1234132'    download music

        dic = {}
        url = 'http://music.163.com/api/song/detail/?ids=[' + musicId + ']'
        res = requests.get(url, headers=headers).json()
        info = res['songs'][0]
        dic['artist'] = [info['artists'][0]['name']]
        dic['title'] = [info['name']]
        dic['cover'] = [info['album']['picUrl']]
        dic['album'] = [info['album']['name']]
        return dic

    def getPath(self, dic, musicId):
        '''get the name of music from info dict'''
        title = dic['title'][0]
        artist = dic['artist'][0]
        name = title + '(' + artist+')'
        for i in '>?*/\:"|<':
            name = name.replace(i, '-')  # convert to valid chars for file name
        name = re.sub('\s', '_', name)
        self.id_name[musicId] = name
        return os.path.join(MSCDIR, name + '.mp3')

    def decrypt(self, musicId, name):
        def _decrypt(cachePath):
            with open(cachePath, 'rb') as f:
                btay = bytearray(f.read())
            for i, j in enumerate(btay):
                btay[i] = j ^ 0xa3
            return btay
        cachePath = os.path.join(self.path, name)
        idpath = os.path.join(MSCDIR, musicId + '.mp3')
        info = self.getInfoFromWeb(musicId)
        path = self.getPath(info, musicId)
        if not os.path.exists(path):
            with open(path, 'wb') as f:
                f.write(bytes(_decrypt(cachePath)))

        '''  get info from index file
        if not os.path.exists(idpath):
            with open(idpath, 'wb') as f:
                f.write(bytes(_decrypt(cachePath)))
        try:
            info = dict(MP3(idpath, ID3=EasyID3))
        except:
            info = {}
        if info != {}:
            path = self.getPath(info, musicId)
            if os.path.exists(path):
                os.remove(idpath)
            else:
                os.rename(idpath, path)
        else:
            os.remove(idpath)
        '''
        return info, path

    def getLyric(self, musicId):
        url = 'http://music.163.com/api/song/lyric?id=' + musicId + '&lv=1&tv=-1'
        lrc = ''
        try:
            lyric = requests.get(url, headers=headers).json()
            lrc = lyric['lrc']['lyric']
            tlrc = lyric['tlyric']['lyric']
            # merge multi-lang lyrics
            dic = {}
            for i in lrc.splitlines():
                a = i.replace('[', ']').strip().split("]")
                dic[a[1].strip()+' '] = a[-1].strip()
            tdic = {}
            for m in tlrc.splitlines():
                n = m.replace('[', ']').strip().split(']')
                tdic[n[1].strip()] = n[-1].strip()
            dicCopy = dic.copy()
            dicCopy.update(tdic)
            lines = []
            for k, v in sorted(dicCopy.items(), key=lambda item: item[0]):
                lines.append("[%s]%s" % (k.strip(), v))
            lrc = "\n".join(lines)
        except Exception as e:
            pass
        return lrc

    def setID3(self, lrc, info, path):
        tags = ID3(path)
        # remove old unsychronized lyrics
        len(tags.getall("USLT")) != 0 and tags.delall("USLT")
        for t in ['album', 'title', 'artist']:
            t in info and tags.add(
                tagMap[t](encoding=3, lang='', desc='', text=info[t][0]))
        'cover' in info and tags.add(APIC(
            encoding=3,
            mime='image/png',
            type=3,
            desc='cover',
            data=requests.get(info['cover'][0], stream=True,
                              headers=headers).raw.read()
        ))
        tags.add(USLT(encoding=3, lang='eng', desc='aaa', text=lrc))
        tags.save()

    def getMusic(self):
        ct = 0  # count converted files
        for musicId, name in self.id_name.items():
            try:
                info, path = self.decrypt(musicId, name)
                ct += 1
                print('[{}]'.format(ct).ljust(6) + self.id_name[musicId])
                self.setID3(self.getLyric(musicId), info, path)
            except Exception as e:
                pass

def main(path=''):
    if not path:
        pre = '/'.join(os.getcwd().split(os.sep)[:3])
        if os.sys.platform.lower().startswith('win'):  # windows
            path = pre + '/AppData/Local/Netease/CloudMusic/Cache/Cache'
        else:  # mac or linux
            path = pre + '/Library/Containers/com.netease.163music/Data/Caches/online_play_cache'
    if os.path.isdir(path):
        netease_music(path).getMusic()
    else:
        print('Directory "{}" does not exist, specify cache files directory instead'.format(path))

if __name__ == '__main__':
    main() if len(sys.argv) < 2 else main(sys.argv[1])


#%%
from config import *

import torch
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
                              TensorDataset)
from torch.nn import CrossEntropyLoss, MSELoss
from tqdm import tqdm, trange
import os
from pytorch_transformers import BertTokenizer, BertModel, BertForMaskedLM
from classifiers import CLASSIFIER_CLASSES
from pytorch_transformers.optimization import AdamW, WarmupLinearSchedule
from multiprocessing import Pool, cpu_count
# from BertForMultipleChoice import BertForMultipleChoice
from utils import *
from convert_example_to_features import *
from visualization import * # for live plotting

from report_utils import *

# FIXME: save this in config

do_reports_analysis = True
if do_visualization:
    vis = Visualizations(visdom_env_name)

import logging
logging.basicConfig(level=logging.INFO)

classifier_class = CLASSIFIER_CLASSES[classifier_type]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

import random
import numpy as np
def set_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if device.type == 'cuda':
        torch.cuda.manual_seed_all(seed)

set_seed(42) # For reproductivity

#%%

def evaluate(eval_task_name, model, tokenizer):
    """
    eval_task_name: task name without file type

    return:
        accuracy: question accuracy

        preds: logits for all QC pair

        out_label_ids: label for all QC pair 0, 1
    """
    processor = BinaryClassificationProcessor()
    eval_examples = processor.get_dev_examples(data_dir, f'{eval_task_name}.tsv',q_type)
    label_list = processor.get_labels()
    eval_features = convert_examples_to_features(eval_examples,
                                                    label_list,
                                                    max_seq_length,
                                                    tokenizer,
                                                    output_mode)
    all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
    all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
    all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
    all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)

    #%%
    # FIXME: is there a need to use multi gpu here
    # if n_gpus > 1:
    #     model = torch.nn.DataParallel(model)
    # let's just use one for now
    eval_batch_size = per_gpu_eval_size

    eval_dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
    eval_sampler = SequentialSampler(eval_dataset)
    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=eval_batch_size)

    logger.info("***** Running evaluation *****")
    logger.info("  Num examples = %d", len(eval_dataset))
    logger.info("  Total batch size = %d", eval_batch_size)
    #%%

    eval_loss = 0.0
    nb_eval_steps = 0
    preds = None
    out_label_ids = None
    for batch in tqdm(eval_dataloader, desc="Evaluating"):
        model.eval()
        batch = tuple(t.to(device) for t in batch)

        with torch.no_grad():
            inputs = {'input_ids':      batch[0],
                        'attention_mask': batch[1],
                        'token_type_ids': batch[2],  # change this if using xlnet
                        'labels':         batch[3]}
            outputs = model(**inputs)
            tmp_eval_loss, logits = outputs[:2]

            # reshaped_logits = logits.view(-1, 5) # 5: num of choices
            # _, labels = torch.max(batch[3].view(-1, 5), 1)
            # loss_fct = CrossEntropyLoss()
            # tmp_eval_loss = loss_fct(reshaped_logits, labels)

            eval_loss += tmp_eval_loss.mean().item()
        nb_eval_steps += 1
        if preds is None:
            preds = logits.detach().cpu().numpy()
            out_label_ids = batch[3].detach().cpu().numpy()
        else:
            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
            out_label_ids = np.append(out_label_ids, batch[3].detach().cpu().numpy(), axis=0)

    eval_loss = eval_loss / nb_eval_steps

    #%%
    # for original classifier
    # preds_list = np.argmax(preds.reshape(-1,5), 1)
    # labels_list = np.argmax(out_label_ids.reshape(-1,5), 1)

    # for baseline classifier
    #preds_list = np.argmax(preds.reshape(-1,25), 1)//5
    #labels_list = np.argmax(out_label_ids.reshape(-1,25), 1)//5

    # for hidden nodes classifier
    pred_difference_array = []
    for items in preds:
        pred_difference_array.append(items[1] - items[0])
    pred_difference_array = np.array(pred_difference_array)
    preds_list = np.argmax(pred_difference_array.reshape(-1,5), 1)
    labels_list = np.argmax(out_label_ids.reshape(-1,25), 1)//5

    #%%
    eval_accuracy = sum([pred == label for pred, label in zip(preds_list, labels_list)]) / len(preds_list)
    print(preds_list)
    print(labels_list)
    print('-------------##########################-------------')
    results = {
        'file': eval_task_name,
        'eval_loss': eval_loss,
        'eval_accuracy': eval_accuracy,
        'param_stats': stats,
        'preds': preds_list,
        'labels': labels_list
        }
    return results, preds, out_label_ids

def evaluate_iterator(test_file_list, epoch_list):
    assert test_file_list != [] and epoch_list != []
    for epoch in epoch_list:
        cache_path = f'{cache_dir}epoch_{epoch}/'
        tokenizer = BertTokenizer.from_pretrained(cache_path, do_lower_case=True)
        # output only one logit
        model = classifier_class.from_pretrained(cache_path, num_labels=1)

        model.to(device)
        for file in test_file_list:
            results, preds_raw, labels_raw = evaluate(file, model, tokenizer)
            results['epoch'] = epoch
            print(results)

            # write acc to a txt file
            write_results(results)
            # write_results_to_parent_folder(results)

            # draw accuracy plot
            accuracy_plot_helper(results['eval_accuracy'], epoch, file)

            # visualization: visdom localhost 8097
            if do_visualization:
                visualize_helper(results['eval_accuracy'], epoch, file)

            # get reports analysis: qtype-iscorrect figure, output file
            # if do_reports_analysis:
            #     get_reports_analysis(epoch, file,
            #                         preds_raw, labels_raw,
            #                         data_dir=data_dir)

def write_results(results):
    output_eval_file = os.path.join(output_dir, "eval_results.txt")
    with open(output_eval_file, "a") as writer:
        logger.info("***** Eval results *****")
        for key in sorted(results.keys()):
            logger.info("  %s = %s", key, str(results[key]))
            writer.write("%s = %s\n" % (key, str(results[key])))
        writer.write('\n\n')

def write_results_to_parent_folder(results):
    output_eval_file = os.path.join(parent_output_dir, f"{train_type}.txt")
    with open(output_eval_file, "a") as writer:
        writer.write("%s = %s\n" % ('file', str(results['file'])))
        writer.write("%s = %s\n" % ('eval_loss', str(results['eval_loss'])))
        writer.write("%s = %s\n" % ('eval_accuracy', str(results['eval_accuracy'])))
        writer.write("%s = %s\n" % ('preds', str(results['preds'])))
        writer.write("%s = %s\n" % ('labels', str(results['labels'])))
        writer.write('\n\n')

# visualization
def visualize_helper(acc, epoch, file):
    if '17' in file:
        vis.plot_accuracy1(acc, epoch)
    if '18' in file:
        vis.plot_accuracy2(acc, epoch)
    if '3000' in file:
        vis.plot_accuracy3(acc, epoch)

def accuracy_plot_helper(acc, epoch, file):
    if '17' in file:
        acc_17.append(acc)
    if '18' in file:
        acc_18.append(acc)
    if '3000' in file:
        acc_3000.append(acc)
#%%

acc_17 = []
acc_18 = []
acc_3000 = []

evaluate_iterator(eval_files, eval_epochs) # set in config.py
# plot and save acc_epoch line figure

acc_plot(acc_17, eval_epochs, '17')
acc_plot(acc_18, eval_epochs, '18')
#acc_plot(acc_3000, epochs, '3000')


# -*- coding: utf-8 -*-
import pytest

from intertwine.geos.models import Geo, GeoData, GeoID, GeoLevel
from intertwine.utils.space import GeoLocation

@pytest.mark.unit
@pytest.mark.parametrize("parent_name, parent_abbrev, child_name", [
    ('Parent Test Republic', 'PTR', 'Child Test Geo'),
])
def test_geo_model_create(session, parent_name, parent_abbrev, child_name):
    """Test simple geo model interaction"""
    parent = Geo(name=parent_name, abbrev=parent_abbrev)
    child = Geo(name=child_name, path_parent=parent, parents=[parent])

    session.add(parent)
    session.add(child)
    session.commit()

    assert Geo[parent.derive_key()] is parent
    assert Geo[child.derive_key()] is child
    assert parent[child_name] is child

    parent_from_db = (
        session
        .query(Geo)
        .filter(Geo.human_id == parent.human_id)
        .first()
    )

    child_from_db = (
        session
        .query(Geo)
        .filter(Geo.human_id == child.human_id)
        .first()
    )

    assert parent_from_db is parent
    assert parent_from_db.name == parent_name
    assert parent_from_db.abbrev == parent_abbrev
    assert parent_from_db.uses_the is True
    assert parent_from_db.human_id == parent_abbrev.lower()
    assert parent_from_db.path_children.all()[0] is child_from_db
    assert parent_from_db.children.all()[0] is child_from_db

    assert child_from_db is child
    assert child_from_db.name == child_name
    assert child_from_db.abbrev is None
    assert child_from_db.uses_the is False
    assert child_from_db.human_id == Geo.PATH_DELIMITER.join(
                [child.path_parent.human_id,
                 child.name.lower().replace(' ', '_')])

    assert child_from_db.path_parent is parent_from_db
    assert child_from_db.parents.all()[0] is parent_from_db

@pytest.mark.unit
@pytest.mark.smoke
def test_geo_data_model(session):
    """Test simple geo data model interaction"""
    total_pop, urban_pop = 1000, 800
    latitude, longitude = 30.0, -97.0
    land_area, water_area = 4321, 1234

    geo = Geo(name='Test Geo')
    GeoData(geo=geo,
            total_pop=total_pop,
            urban_pop=urban_pop,
            latitude=latitude,
            longitude=longitude,
            land_area=land_area,
            water_area=water_area)

    session.add(geo)
    session.commit()

    assert geo.data.total_pop == total_pop
    assert geo.data.urban_pop == urban_pop
    assert geo.data.latitude.value == latitude
    assert geo.data.longitude.value == longitude
    assert geo.data.land_area == land_area
    assert geo.data.water_area == water_area

    assert GeoData[geo.data.derive_key()] is geo.data

    geo_data_from_db = session.query(GeoData).filter(
                            GeoData.geo == geo).first()

    assert geo_data_from_db is geo.data
    assert geo_data_from_db.total_pop == geo.data.total_pop
    assert geo_data_from_db.urban_pop == geo.data.urban_pop
    assert geo_data_from_db.latitude == geo.data.latitude
    assert geo_data_from_db.longitude == geo.data.longitude
    assert geo_data_from_db.land_area == geo.data.land_area
    assert geo_data_from_db.water_area == geo.data.water_area

@pytest.mark.unit
@pytest.mark.smoke
def test_geo_level_model(session):
    """Test simple geo level model interaction"""
    level = 'place'
    designation = 'city'
    geo = Geo(name='Test Geo Place')
    glvl = GeoLevel(geo=geo, level=level, designation=designation)

    session.add(geo)
    session.add(glvl)
    session.commit()

    assert geo.levels[level] is glvl
    assert glvl.geo is geo
    assert glvl.level == level
    assert glvl.designation == designation

    assert GeoLevel[glvl.derive_key()] is glvl

    glvl_from_db = session.query(GeoLevel).filter(
                    GeoLevel.geo == geo, GeoLevel.level == level).first()

    assert glvl_from_db is glvl
    assert glvl_from_db.geo is geo
    assert glvl_from_db.level == level
    assert glvl_from_db.designation == designation

@pytest.mark.unit
@pytest.mark.smoke
def test_geo_id_model(session):
    """Test simple geo id model interaction"""
    TEST_STANDARD = 'Test Standard'

    GeoID.STANDARDS.add(TEST_STANDARD)
    standard = TEST_STANDARD
    code = '12345'

    geo = Geo(name='Test Geo Place')
    glvl = GeoLevel(geo=geo, level='place', designation='city')
    gid = GeoID(level=glvl, standard=standard, code=code)

    session.add(geo)
    session.add(glvl)
    session.add(gid)
    session.commit()

    assert gid.level is glvl
    assert gid.standard == standard
    assert gid.code == code

    assert GeoID[gid.derive_key()] is gid
    assert GeoID[(standard, code)] is gid

    gid_from_db = session.query(GeoID).filter(
                    GeoID.standard == standard, GeoID.code == code).first()

    assert gid_from_db is gid
    assert gid_from_db.level is glvl
    assert gid_from_db.standard == standard
    assert gid_from_db.code == code

@pytest.mark.unit
@pytest.mark.smoke
def test_form_aggregate_geo(session):
    """Test geo creation that aggregates children data at a geo level"""
    data_level = 'place'

    child_a_dict = {'total_pop': 100,
                    'urban_pop': 80,
                    'latitude': 42.0,
                    'longitude': -71.0,
                    'land_area': 321,
                    'water_area': 123}

    child_a_geo = Geo(name='Child Test Geo A')
    GeoData(geo=child_a_geo, **child_a_dict)
    GeoLevel(geo=child_a_geo, level='place', designation='village')

    child_b_dict = {'total_pop': 300,
                    'urban_pop': 240,
                    'latitude': 44.0,
                    'longitude': -73.0,
                    'land_area': 645,
                    'water_area': 21}

    child_b_geo = Geo(name='Child Test Geo B')
    GeoData(geo=child_b_geo, **child_b_dict)
    GeoLevel(geo=child_b_geo, level='place', designation='town')

    child_c_dict = {'total_pop': 1000,
                    'urban_pop': 800,
                    'latitude': 30.0,
                    'longitude': -97.0,
                    'land_area': 4321,
                    'water_area': 1234}

    child_c_geo = Geo(name='Child Test Geo C')
    GeoData(geo=child_c_geo, **child_c_dict)
    GeoLevel(geo=child_c_geo, level='subdivision2', designation='county')

    child_a_area = child_a_dict['land_area'] + child_a_dict['water_area']
    child_b_area = child_b_dict['land_area'] + child_b_dict['water_area']

    geo_location_a = GeoLocation(child_a_dict['latitude'],
                                 child_a_dict['longitude'])
    geo_location_b = GeoLocation(child_b_dict['latitude'],
                                 child_b_dict['longitude'])

    geo_location = GeoLocation.combine_locations(
        (geo_location_a, child_a_area),
        (geo_location_b, child_b_area))

    sum_fields = ('total_pop', 'urban_pop', 'land_area', 'water_area')

    aggregate_dict = {f: child_a_dict[f] + child_b_dict[f] for f in sum_fields}

    aggregate_dict['latitude'], aggregate_dict['longitude'] = (
        geo_location.coordinates)

    parent_geo = Geo(name='Parent Test Geo',
                     children=[child_a_geo, child_b_geo, child_c_geo],
                     child_data_level=data_level)  # aggregate places only

    session.add(parent_geo)
    session.add(child_a_geo)
    session.add(child_b_geo)
    session.add(child_c_geo)
    session.commit()

    parent_data = parent_geo.data
    assert parent_data is not None
    assert parent_data.total_pop == aggregate_dict['total_pop']
    assert parent_data.urban_pop == aggregate_dict['urban_pop']
    assert parent_data.latitude == aggregate_dict['latitude']
    assert parent_data.longitude == aggregate_dict['longitude']
    assert parent_data.land_area == aggregate_dict['land_area']
    assert parent_data.water_area == aggregate_dict['water_area']

    parent_geo_data_from_db = session.query(GeoData).filter(
        GeoData.geo == parent_geo).first()

    assert parent_geo_data_from_db is not None
    assert parent_geo_data_from_db is parent_geo.data
    assert parent_geo_data_from_db.total_pop == aggregate_dict['total_pop']
    assert parent_geo_data_from_db.urban_pop == aggregate_dict['urban_pop']
    assert parent_geo_data_from_db.latitude == aggregate_dict['latitude']
    assert parent_geo_data_from_db.longitude == aggregate_dict['longitude']
    assert parent_geo_data_from_db.land_area == aggregate_dict['land_area']
    assert parent_geo_data_from_db.water_area == aggregate_dict['water_area']

@pytest.mark.unit
@pytest.mark.smoke
def test_geo_aliases(session):
    """Test creation of geo aliases and promoting an alias"""
    geo_data_dict = {'total_pop': 100,
                     'urban_pop': 80,
                     'latitude': 42,
                     'longitude': -71,
                     'land_area': 321,
                     'water_area': 123}

    level = 'place'
    geo = Geo(name='Test Geo')
    gdata = GeoData(geo=geo, **geo_data_dict)
    glvl = GeoLevel(geo=geo, level=level, designation='city')

    parent_geo = Geo(name='Test Parent Geo', abbrev='TPG')
    geo.path_parent = parent_geo
    geo.parents = [parent_geo]

    sibling_geo = Geo(name='Test Sibling Geo')
    sibling_geo.path_parent = parent_geo
    sibling_geo.parents = [parent_geo]

    child_geo = Geo(name='Test Child Geo')
    child_geo.path_parent = geo
    child_geo.parents = [geo]

    geo_alias_1 = Geo(name='Test Geo Alias 1', alias_targets=[geo],
                      path_parent=geo)
    geo_alias_2 = Geo(name='Test Geo Alias 2', alias_targets=[geo],
                      path_parent=parent_geo)
    geo_alias_3 = Geo(name='Test Geo Alias 3', alias_targets=[geo],
                      path_parent=parent_geo)

    session.add(geo)
    session.commit()

    assert geo.data is gdata
    assert geo.levels[level] is glvl
    assert parent_geo in geo.parents
    assert geo in child_geo.parents

    assert geo.path_parent is parent_geo
    assert child_geo.path_parent is geo
    assert geo_alias_1.path_parent is geo
    assert geo_alias_2.path_parent is parent_geo
    assert geo_alias_3.path_parent is parent_geo

    assert geo_alias_1.alias_targets[0] is geo
    assert geo_alias_2.alias_targets[0] is geo
    assert geo_alias_3.alias_targets[0] is geo

    aliases = geo.aliases.all()
    assert geo_alias_1 in aliases
    assert geo_alias_2 in aliases
    assert geo_alias_3 in aliases

    geo_from_db = session.query(Geo).filter_by(
        human_id=geo.human_id).first()

    aliases = geo_from_db.aliases.all()
    assert geo_alias_1 in aliases
    assert geo_alias_2 in aliases
    assert geo_alias_3 in aliases

    geo_alias_1.promote_to_alias_target()
    assert not geo_alias_1.alias_targets
    assert geo_alias_2.alias_targets[0] is geo_alias_1
    assert geo_alias_3.alias_targets[0] is geo_alias_1
    assert geo.alias_targets[0] is geo_alias_1

    assert geo_alias_1.data is gdata
    assert geo_alias_1.levels[level] is glvl
    assert parent_geo in geo_alias_1.parents
    assert geo_alias_1 in child_geo.parents

    # paths are unchanged by design
    assert geo.path_parent is parent_geo
    assert child_geo.path_parent is geo
    assert geo_alias_1.path_parent is geo
    assert geo_alias_2.path_parent is parent_geo
    assert geo_alias_3.path_parent is parent_geo


import numpy as np
import cv2 as cv
import torch
from models.experimental import attempt_load
from utils.general import non_max_suppression
import visualize
from multiprocessing import Process
import rclpy
from cv_bridge import CvBridge

class ObjectDetection:

	model = attempt_load("yolov7.pt", map_location="cpu")  # load FP32 model
	names = model.module.names if hasattr(model, 'module') else model.names
	colors = [[np.random.randint(0, 255) for _ in range(3)] for _ in names]

	def callback(msg):

		image=CvBridge().compressed_imgmsg_to_cv2(msg)
		mtx=np.array([[310.05107829,0.,293.37649276],[  0.,309.51905673,254.25816031],[0.,0.,1.]])

		dist=np.array([[-4.67303364e-01,2.26729559e-01,-3.79303222e-04,1.20996005e-04,-5.25339796e-02]])
		image= cv.undistort(image, mtx, dist, None, None)

		list1=[image]
		img=list1
		#process=Process(target=ObjectDetection.object_detection,args=(img,list1))
		#process.start()
		return ObjectDetection.object_detection(img,list1)

	def object_detection(img,list1):

		img = np.stack(img, 0)

		img = img[:, :, :, ::-1].transpose(0, 3, 1, 2)  # BGR to RGB, to bsx3x416x416
		img = np.ascontiguousarray(img)
		img = torch.from_numpy(img).to("cpu")
		img = img.float()  # uint8 to fp16/32
		img /= 255.0  # 0 - 255 to 0.0 - 1.0
		if img.ndimension() == 3:
			img = img.unsqueeze(0)

		#Inference
		with torch.no_grad():
			pred = ObjectDetection.model(img, augment=False)[0]

		pred = non_max_suppression(pred, 0.5, 0.45, classes=None, agnostic=False)

		return visualize.visualize(list1,pred,img,ObjectDetection.names,ObjectDetection.colors)