#TODO:10)avaldada browseri addon, mis laseks tabis avatud lehekohta käivaid kommentaare lugeda ja kommentaari lisada
#TODO:6)Teha nee, et savestaks  kommentaarid samasse faili olenemata protokollist (http, https, ftp jne.).
#TODO:8)Määrama kommentaaride kuvamisele jäjekorra.
#TODO:9)Panna oma lehel reklaame ja sellega raha teenida.
#TODO:14)teha nii, et kui ühelt iplt on palju kommentaare, siis nende nägemiseks peab klikkama, et näita kõiki.
#TODO:15)tausta värvid õigeks teha.
from flask import Flask, request, redirect, Response,render_template
from datetime import datetime
from re import match
import os
from random import choice

app = Flask(__name__)

HTMLi_algus=["""

        <title>Universaalne kommentaarium</title>

Universaalne Kommentaarium2
english
            <form method="GET">
                See veebiteenus võimaldab teil kommenteerida mistahes uudist mistahes veebiväljaandes või mistahes muud internetilehekülge.
Sisestage lehekülje, mida kommenteerida soovite, URL:<input type="text" name="kommenteeritava_URL">.
                Kuva all kommenteeritav lehekülg<input type="checkbox" name="kuva" checked>.
                <input type="submit" value="kommenteerima">
            </form>""","""

        <title>Universal commentary</title>

Universal commentary
eesti keel
            <form amethod="GET">
                This page offers you opportunity to comment any news in web or any other webpage.
Enter URL of the page you want to comment:<input type="text" name="URL_being_commented">.
                Show the page below<input type="checkbox" name="show" value="show" checked>.
                <input type="submit" value="to commenting">
            </form>"""]
parameeter_kommenteeriava_URL=["kommenteeritava_URL", "URL_being_commented"]
parameeter_kommentaar=["kommentaar","comment"]
parameeter_sisu_hinnang=["sisu","sisu"]
parameeter_kajastuse_hinnang=["kajastus","kajastus"]
kommentaare_ei_ole=["SELLE LEHE KOHTA KOMMENTAARE VEEL EI OLE","HERE ARE NO COMMENTS ABOUT THIS PAGE YET"]
browser_ei_toeta=["TEIE BROWSER EI TOETA HTML TAGi iframe´i","YOUR WEB BROWSER DOES NOT SUPORT HTML iframe TAG"]
HTMLi_postitamise_osa_1=['''
            <form method="POST">
                <textarea style="width:100%;height:120;resize:vertical" name="kommentaar" placeholder="Sinu kommentaar"></textarea>
                Meeldib uudise sisu? <input type="radio" name="sisu" value="meeldib"> Meeldib<input type="radio" name="sisu" value="ei meeldi"> Ei meeldi

                Meeldib kajastus ja juuresolevad kommentaarid? <input type="radio" name="kajastus" value="meeldib"> Meeldib<input type="radio" name="kajastus" value="ei meeldi"> Ei meeldi

                Postitad kommentaari ja annad hinnangu lehe ''','''
            <form method="POST">
                <textarea style="width:100%;height:120;resize:vertical" name="comment" placeholder="Your comment"></textarea>
                Do you like the news? <input type="radio" name="sisu" value="meeldib">Like<input type="radio" name="sisu" value="ei meeldi">Dislike

                Do you like presentation of the news? <input type="radio" name="kajastus" value="meeldib">Like<input type="radio" name="kajastus" value="ei meeldi">Dislike

                You are posting a comment and giving rating about page ''']
HTMLi_postitamise_osa_2=[''' kohta. <input type="submit" value="Postita''',''' . <input type="submit" value="Post''']
aeg=["aeg:","time:"]
hiljuti_kommenteeritud_lehed_tekst=["hiljuti kommenteeritud lehed","recently commented pages"]
parameeter_meeldib=["meeldib","liked"]
parameeter_ei_meeldi=["ei meeldi","disliked"]
parameeter_sisu=["sisu","content"]
parameeter_kajastus=["kajastus","presentation"]
parameeter_lehe=["Lehe ","There are no comments about page "]
parameeter_kohta=[' kohta pole kommentaare. Kuvatud kommentaarid on lehe ',"sown comments are about "]
parameeter_kohta_2=[" kohta.
","
"]

viited={"static/reklaamid/ekre":"https://ekre.ee/",
        "static/reklaamid/uueduudised":"https://uueduudised.ee/",
        "static/reklaamid/SÄ":"https://ekre.ee/noored/",
        "static/reklaamid/SÄ2":"https://www.facebook.com/Sinine2ratus/",
        "static/reklaamid/SÄ3":"https://steamcommunity.com/groups/SinineAratus#members",
        "static/reklaamid/SÄ4":"https://sininearatus.ee/yritused/"}
kuva_reklaami=True
betale_lubatud_ipd=set()

def vii_URL_oigele_kujule(url):
    if not match(r"^(?:f|ht)tps?://", url):
        url="http://"+url
    www_asukoht=match(r"^(?:f|ht)tps?://www.", url)
    if www_asukoht:#kui aadressis on www. algus
        return ((url[:www_asukoht.end()-4]+url[www_asukoht.end():]).lower())#eemaldab URList "www."i
    return url.lower()

def pole_faile(url):
    for kommentaari_fail in sorted(os.listdir(url)):#kui ei sisalda ühtegi faili.
        if os.path.isfile(url+"/"+kommentaari_fail):
            return False
    return True

def vali_suvaline_fail(p="static/reklaamid"):
    try:
        p+="/"+(choice(os.listdir(os.path.join(os.path.dirname(__file__), p))))
        if os.path.isfile(os.path.join(os.path.dirname(__file__), p)):
            return (p,viited[p[:p.rfind("/")]])
        else:
            return vali_suvaline_fail(p)
    except IndexError:
        return vali_suvaline_fail()

@app.route('/')#suunab eestikeelsele pealehele
def ainult_domeen():
    return(redirect("/et"))

@app.route('/et', methods=['GET', 'POST'])
def pealeht_et():
    return pealeht(0)

@app.route('/eng', methods=['GET', 'POST'])
def pealeht_eng():
    return pealeht(1)

def pealeht(keel):
    if parameeter_kommenteeriava_URL[keel] in request.args:#Juhul kui päringus on sees URL, mida klient kommenteerida tahab. näiteks "http://objektiiv.ee/"
        kommentaaride_kaust=vii_URL_oigele_kujule(request.args[parameeter_kommenteeriava_URL[keel]])
        if parameeter_kommentaar[keel] in request.form:#juhul kui päringus on kommentaar, mida klient postitada tahab.
            hiljuti_kommenteeritud_lehtede_fail=open("hiljuti kommenteeritud lehed")
            hiljuti_kommenteeritud_lehed="".join(hiljuti_kommenteeritud_lehtede_fail.readlines()[1:])
            hiljuti_kommenteeritud_lehtede_fail.close()
            hiljuti_kommenteeritud_lehtede_fail=open("hiljuti kommenteeritud lehed","w")
            hiljuti_kommenteeritud_lehtede_fail.write(hiljuti_kommenteeritud_lehed+"\n"+kommentaaride_kaust.replace("<","&lt").replace("\r","").replace("\n"," "))#paneb lehe hiljuti kommenteeritud lehtede hulka
            hiljuti_kommenteeritud_lehtede_fail.close()
            hinnang=0# kui sisu hindamata: 0,1,2 ;kui sisu meeldib: 6,7,8; kui sisu ei meeldi 3,4,5
            # kui kajastus hindamata: 0,3,6 ;kui kajastus meeldib: 2,5,8; kui kajastus ei meeldi 1,4,7
            if parameeter_sisu_hinnang[keel] in request.form:
                if request.form[parameeter_sisu_hinnang[keel]]=="meeldib":
                    hinnang=6
                elif request.form[parameeter_sisu_hinnang[keel]]=="ei meeldi":
                    hinnang=3
                    # print(request.form[parameeter_sisu_hinnang[keel]])
            if parameeter_kajastuse_hinnang[keel] in request.form:
                if request.form[parameeter_kajastuse_hinnang[keel]]=="meeldib":
                    hinnang+=2
                elif request.form[parameeter_kajastuse_hinnang[keel]]=="ei meeldi":
                    hinnang+=1
                    #print(request.form[parameeter_kajastuse_hinnang[keel]])
            if os.path.exists(kommentaaride_kaust):
                if os.path.isfile(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP'))):#on vaja faili enne kommentaari reavahetus vaja panna.
                    if hinnang:#!=0
                        vana_hinnang=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP'))).readline()#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                        if vana_hinnang==hinnang:
                            #print("ei muuda hinnangut.")
                            f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                            f.write("\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
                        else:
                            #print("uus hinnang on",hinnang,"vana sisu:",vana_hinnang)
                            vana_sisu=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP'))).read()[1:]
                            f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"w")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                            f.write(str(hinnang)+"\n"+vana_sisu[1:]+"\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
                            del vana_sisu
                    else:
                        f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                        f.write("\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
                    #print("fail olemas")
                else:
                    f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                    f.write(str(hinnang)+"\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
                    #print("faili pole")
            else:#kui selle lehe kohta veel kommentaare ei ole.#ei ole vaja faili algusesse reavahetust.
                os.makedirs(kommentaaride_kaust)#teeb vastava kausta
                f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                f.write(str(hinnang)+"\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
            f.close()
        html=HTMLi_algus[keel]+HTMLi_postitamise_osa_1[keel]+kommentaaride_kaust+HTMLi_postitamise_osa_2[keel]+"""!">
            </form>"""
        vähendatud_URL=kommentaaride_kaust
        on_algne_url=True
        while not os.path.exists(vähendatud_URL) or pole_faile(vähendatud_URL):#alamprogramm pole_faile on juhuks kui kaust olemas, aga selle sees on on ainult kaustad mitte failid(kommentaarid)
            #print("url:",vähendatud_URL)
            if vähendatud_URL.count("/")<3:
                html+='
'+"
"+kommentaare_ei_ole[keel]+"
"
                html_2=""
                break
            vähendatud_URL=vähendatud_URL[:vähendatud_URL.rfind("/")]
            on_algne_url=False
        else:
            if on_algne_url:
                html+='
'
            else:
                html+='
\n\t\t\t
'+parameeter_lehe[keel]+kommentaaride_kaust+parameeter_kohta[keel]+vähendatud_URL+parameeter_kohta_2[keel]#Lisab lehele märke, et kommentaarid pole täpse URLi kohta.
            sisu_meeldimisi=0
            sisu_mitte_meeldimisi=0
            kajastuse_meeldimisi=0
            kajastuse_mitte_meeldimisi=0
            html_2=""
            for kommentaari_fail in sorted(os.listdir(vähendatud_URL)):#loeb failidest kommentaare, et need html'i panna, mis kliendile saadetakse.
                if os.path.isfile(vähendatud_URL+"/"+kommentaari_fail):
                    html_2+="""

ip:"""+kommentaari_fail+"
"
                    fail=open(vähendatud_URL+"/"+kommentaari_fail)
                    #LIKEDE LOENDAMINE
                    hinnang=int(next(fail))
                    if hinnang//3==2:
                        sisu_meeldimisi+=1
                        html_2+='
'+parameeter_sisu[keel]+':

'+parameeter_meeldib[keel]+'
'
                    elif hinnang//3==1:
                        sisu_mitte_meeldimisi+=1
                        html_2+='
'+parameeter_sisu[keel]+':

'+parameeter_ei_meeldi[keel]+'
'
                    if hinnang%3==2:
                        kajastuse_meeldimisi+=1
                        html_2+='
'+parameeter_kajastus[keel]+':

'+parameeter_meeldib[keel]+'
'
                    elif hinnang%3==1:
                        kajastuse_mitte_meeldimisi+=1
                        html_2+='
'+parameeter_kajastus[keel]+':

'+parameeter_ei_meeldi[keel]+'
'
                    html_2+="\n\t\t\t\t\t
"
                    for kommentaar in fail:#paneb tühjade postituste ajad samale reale.
                        html_2+=aeg[keel]+kommentaar[:19]
                        if kommentaar[26:].strip("\n"):
                            html_2+="""
\n\t\t\t\t\t
"""+kommentaar[26:].strip("\n")+"
\n\t\t\t\t\t
"#paneb tarbetu realõpu kui [26:]. jätab viimase kommentaari viimase tähe ära kui [26:-1]
                        else:
                            html_2+=" ; "
                    fail.close()
                    html_2=html_2[:-5]+"

\n"
            if on_algne_url:
                try:
                    html_2+='\t\t\t\t
'+str(sisu_mitte_meeldimisi)+'
'
                    html+='\n\n'+str(sisu_meeldimisi)+'
'
                    #print("sisu keskmine hinnang:",sisu_meeldimisi)
                except ZeroDivisionError:
                    #print("sisu pole hinnatud")
                    try:
                        #print("kajastuse keskmine hinnang:",kajastuse_meeldimisi/(kajastuse_meeldimisi+kajastuse_mitte_meeldimisi))
                        html_2+='\t\t\t\t
'+str(kajastuse_mitte_meeldimisi)+'
'
                        html += '\n\n'+str(kajastuse_meeldimisi) + '
'
                    except ZeroDivisionError:
                        #print("kajastust pole hinnatud")
                        html_2+='\t\t\t\t

'
        html_2+='''

'''
        if "kuva" in request.args:
            html_2+='
'+browser_ei_toeta[keel]+'
'#TODO:4)panna iframes´is tehtud clickid muutma kommenteeritavat URLi
        return (html+html_2+'''

''')
    html=HTMLi_algus[keel]+"""

"""+hiljuti_kommenteeritud_lehed_tekst[keel]+":
"
    for i in open("hiljuti kommenteeritud lehed"):
        i=i.strip()
        html+="\n
"+i+"
"
    if False:#request.headers.get('X-Real-IP') in betale_lubatud_ipd:#kuva_reklaami:
        path,url=vali_suvaline_fail()
        html+="""

reklaam

        """
    else:
        html+="""

        """#pealeht
    print(html)
    return html+"""

"""

@app.route("/et/info")
def info_et():
    return """

            <title>Info universaalse kommentaariumi kohta</title>

Info universaalse kommentaariumi kohta
english

Lehe www.kommentaarium.ee loomise eesmärk on pakkuda võimalust kommenteerida teisi internetilehekülgi(eelkõige mõeldud uudiste kommenteerimiseks). Teatavasti mitmed internetiportaalid ei paku endapoolt võimalust enda uudiste kommenteerimiseks, mõnedes neist peab kommenteerimiseks sisse logima, mõnedes kustutatakse anonüümsed kommentaarid automaatselt mingi aja möödudes(postimehes näiteks) ja mõnedes väidetavalt kustutakse kommentaare lähtuvalt nendes väljendatavast poliitilisest vaatest.

        """
@app.route("/eng/info")
def info_eng():
    return """

            <title>About Universal commentary</title>

About Universal commentary
eesti keel

Purpous of creating webpage www.kommentaarium.ee is to offer oppurtunity to comment other webpages(especialy news). Many news portals do not offer commentary on theyr own page and on some you have to login to comment.

        """

@app.route("/et/plugin/info")#/info võib URLi lopust ära võtta.
def plugina_info_et():
    return """

        <title>Universaalse kommentaariumi plugina kohta</title>

Universaalse kommentaariumi plugina kohta
english

plugin võimaldab www.kommentaarid.ee teenust mugavamini kasutada.

"""
@app.route("/en/plugin/info")#/info võib URLi lopust ära võtta.
def plugina_info_en():
    return """

        <title>About add-on of www.kommentaarid.ee</title>

About add-on of www.kommentaarid.ee
eesti keel

The add-on offers you a interface to post and read comments about pages, that you are visiting.

"""

@app.route('/reklaam')
def miski():
    return render_template('reklaam/reklaam.html')

@app.route('/en/derivation')
def suvaline_pilt():
    return "\n"

@app.route('/portfelio')
def portfelio():
    return render_template('portfoolio/e-portfoolio.html')

"""@app.route('/betale',methods=["GET","POST"])
def render_static():
    global betale_lubatud_ipd
    if request.headers.get('X-Real-IP') in betale_lubatud_ipd:
        return(redirect("/"))
    if "salasona" in request.form:
        if request.form["salasona"]=="cdlijwdedwq":
            betale_lubatud_ipd.add(request.headers.get('X-Real-IP'))
            return (redirect("/"))
        else:
            return "vale salasõna"
    return render_template('/betale vorm.html')"""

@app.route("/plugin",methods=["GET"])
def plugina_data():
    if b"\n" in request.data:
        kommentaaride_kaust=vii_URL_oigele_kujule(request.data.split(b"\n",1)[0].decode("utf-8"))
        if os.path.exists(kommentaaride_kaust):
            f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serveris on vaja str funktsiooni kasutatda?
            if os.path.isfile(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP'))):#on vaja faili enne kommentaari reavahetus vaja panna.
                f.write("\n"+str(datetime.now())+request.data.split(b"\n",1)[1].decode("utf-8").replace("<", "&lt").replace("\r","").replace("\n"," "))
            else:
                f.write(str(datetime.now())+request.data.split(b"\n",1)[1].decode("utf-8").replace("<", "&lt").replace("\r","").replace("\n"," "))
        else:#kui selle lehe kohta veel kommentaare ei ole.#ei ole vaja faili algusesse reavahetust.
            os.makedirs(kommentaaride_kaust)#teeb vastava kausta
            f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')))#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
            f.write(str(datetime.now())+request.data.split(b"\n",1)[1].decode("utf-8").replace("<", "&lt").replace("\r","").replace("\n"," "))
        f.close()
        return "1"
    else:
        kommentaarid=""
        kommentaaride_kaust=vii_URL_oigele_kujule(request.data.decode("utf-8"))
        if os.path.exists(kommentaaride_kaust):#kui selle lehe kohta on kommentaare.
            for kommentaari_fail in os.listdir(kommentaaride_kaust):#loeb failidest kommentaare, et need html'i panna, mis kliendile saadetakse.
                if os.path.isfile(kommentaaride_kaust+"/"+kommentaari_fail):
                    fail=open(kommentaaride_kaust+"/"+kommentaari_fail)
                    kommentaarid+=kommentaari_fail+"\n"+fail.read()+"\n\n"
                    fail.close()
        return Response(kommentaarid[:-1], mimetype='text/plain')

@app.route("/sissejuhatuse-projekt",methods=["GET","POST"])
def galerii():
    return render_template('index.html')
@app.route("/sissejuhatuse-projekt2",methods=["GET","POST"])
def galerii2():
    return render_template('galerii2.html')

if __name__ == '__main__':
    app.run()


def get_list_top(string, symbol_count=5, quantity=3):
    result = ""
    list_big_word = {}
    for word in string.split(" "):
        if len(word) > symbol_count:
            list_big_word.update({word: list_big_word.get(word, 0) + 1})
    list_for_sort = list(list_big_word.items())
    list_for_sort.sort(key=lambda i: i[1], reverse=True)
    for item, _ in zip(list_for_sort, range(quantity)):
        result = result + " " + item[0]
    return result

import json

with open('data.json', 'r', encoding='utf-8') as fh:
    data = json.load(fh)
    words_array = ""
    for item in data["rss"]["channel"]["items"]:
        for word in item["description"].split(" "):
            words_array = words_array + " " + word
print(get_list_top(words_array, 6, 10))


import csv
import os
import requests
import sys
import re

def pripravi_imenik(ime_datoteke):
    '''Ce se ne obstaja, pripravi prazen imenik za dano datoteko.'''
    imenik = os.path.dirname(ime_datoteke)
    if imenik:
        os.makedirs(imenik, exist_ok=True)

def shrani(url, ime_datoteke, vsili_prenos=False):
    '''Vsebino strani na danem naslovu shrani v datoteko z danim imenom.'''
    try:
        print('Shranjujem {}...'.format(url))
        sys.stdout.flush()
        if os.path.isfile(ime_datoteke) and not vsili_prenos:
            print('shranjeno ze od prej!')
            return
        r = requests.get(url, headers={'Accept-Language': 'en'})
    except requests.exceptions.ConnectionError:
        print('stran ne obstaja!')
    else:
        pripravi_imenik(ime_datoteke)
        with open(ime_datoteke, 'w') as datoteka:
            datoteka.write(r.text)
            print('shranjeno!')

def vsebina_datoteke(ime_datoteke):
    '''Vrne niz z vsebino datoteke z danim imenom.'''
    with open(ime_datoteke) as datoteka:
        vsebina = datoteka.read()
    return vsebina

def datoteke(imenik):
    '''Vrne imena vseh datotek v danem imeniku skupaj z imenom imenika.'''
    return [os.path.join(imenik, datoteka) for datoteka in os.listdir(imenik)]

strani = ["1-100", "101-200", "201-300", "301-400", "401-500", "501-600", "601-700", "701-800", "801-900", "901-1000"]

for stran in strani:
    osnovni_naslov = 'http://www.atpworldtour.com/en/rankings/singles/'
    parametri = 'rankDate=2017-1-2&countryCode=all'
    naslov = '{}?{}&rankRange={}'.format(osnovni_naslov, parametri, stran)
    datoteka = 'tenis/{}.txt'.format(stran)
    shrani(naslov, datoteka)

moj_regex = r'td class="rank-cell">.*?(\d{1,4}).*?	.*?alt="(\w{1,3})".*?data-ga-label="(.*?)".*?<td class="age-cell".*?(\d{2}).*?player-activity.*?matchType=singles" data-ga-label="(\d{1,3})".*?'

def ustvari_slovarje (ime_datoteke,regex_izraz):
    igralci = []

    for mapa in datoteke(ime_datoteke):
        with open(mapa) as f:
            vsebina = f.read()
            for (i, j, k, l, m) in re.findall(regex_izraz, vsebina, flags=re.DOTALL):
                igralec = {"id":(int(i)), "Ranking": (int(i)), "Drzava": (j.strip()), "Ime": (k.strip()), "Starost": (int(l)),
                           "Turnirji": (int(m))}
                igralci.append(igralec)

    return igralci

def zapisi_tabelo(slovarji, imena_polj, ime_datoteke):
    pripravi_imenik(ime_datoteke)
    with open(ime_datoteke, 'w') as csv_dat:
        writer = csv.DictWriter(csv_dat, fieldnames=imena_polj)
        writer.writeheader()
        for slovar in slovarji:
            writer.writerow(slovar)

igralci = ustvari_slovarje("tenis", moj_regex)

zapisi_tabelo(igralci, ["id","Ranking", "Drzava", "Ime", "Starost", "Turnirji"], "igralci.csv")


"""""
Ex. 17

Create a linear regression model with the given points on a scatter plot. To do this get the data points slope, intercept, r, p,and std_err from stats (import scipy).linregress(x, y). Then graph it with list(map())
"""

import numpy as np
import random
import matplotlib.pyplot as plt
from scipy import stats

speed = np.array([])

for i in range(13):
  speed = np.append(speed,random.randint(60,120))

mean = np.mean(speed)
std = np.std(speed)
pct_75 = np.percentile(speed,75)
pct_99 = np.percentile(speed,99)

print(speed)
print(mean)
print(std)
print(pct_75)
print(pct_99)

y = np.random.triangular (1,50, 100, 250000)

x_val = [5,7,8,7,2,17,2,9,4,11,12,9,6]
y_val = [99,86,87,88,111,86,103,87,94,78,77,85,86]

slope, intercept, r, p, std_err = stats.linregress(x_val, y_val)

def myfunc(x):
  return slope * x + intercept

mymodel = list(map(myfunc, x_val))

plt.scatter(x_val, y_val)
plt.plot(x_val, mymodel)
plt.show()


# coding: utf-8
from flask.ext.mail import Mail
from flask.ext.security import Security as _Security
from flask.ext.security import MongoEngineUserDatastore

from dealer.contrib.flask import Dealer
from quokka.core.db import db
from quokka.core.cache import cache
from quokka.core.admin import configure_admin
from quokka.core.templates import render_template
from quokka.modules.accounts.models import Role, User

from . import (generic, babel, blueprints, error_handlers, context_processors,
               template_filters, before_request, views, themes, fixtures,
               oauthlib)

class Security(_Security):
    def render_template(self, *args, **kwargs):
        return render_template(*args, **kwargs)

def configure_extensions(app, admin):
    cache.init_app(app)
    babel.configure(app)
    generic.configure(app)
    Mail(app)
    Dealer(app)
    error_handlers.configure(app)
    db.init_app(app)
    fixtures.configure(app, db)
    themes.configure(app, db)  # Themes should be configured after db

    context_processors.configure(app)
    template_filters.configure(app)

    app.security = Security(app, MongoEngineUserDatastore(db, User, Role))

    blueprints.load_from_packages(app)
    blueprints.load_from_folder(app)

    configure_admin(app, admin) #配置管理参数

    if app.config.get('DEBUG_TOOLBAR_ENABLED'):
        try:
            from flask_debugtoolbar import DebugToolbarExtension
            DebugToolbarExtension(app)
        except:
            pass

    before_request.configure(app)
    views.configure(app)

    oauthlib.configure(app)

    if app.config.get('SENTRY_ENABLED', False):
        from .sentry import configure
        configure(app)

    return app


from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import classification_report
from math import sqrt
import jsonlines

def read_data(data_file):
	data = []
	with jsonlines.open(data_file) as reader:
		for obj in reader:
			data.append(obj)
	return data

def evaluate(data):
	score_af = []
	score_rt = []

	cleaned_data = []
	for line in data:
		if line[0] != None:
			cleaned_data.append(line)
	data = cleaned_data
	print ("size clean data %d" % len(data))

	for line in data:
		score_rt.append(int(line[0]))
		score_af.append(round(int(line[1])))

	tomatoes_rt = ['F' if x >= 60 else 'R' for x in score_rt]
	tomatoes_af = ['F' if x >= 60 else 'R' for x in score_af]
	print (classification_report(tomatoes_rt, tomatoes_af))

	rmse = sqrt(mean_squared_error(score_rt, score_af))
	print ("original rmse %f" % rmse)

	mae = mean_absolute_error(score_rt, score_af)
	print ("original mae %f" % mae)

	score_rt = [int(x/10) if x!=100 else 9 for x in score_rt]
	score_af = [int(x/10) if x!=100 else 9 for x in score_af]

	rmse = sqrt(mean_squared_error(score_rt, score_af))
	print ("point rmse %f" % rmse)

	mae = mean_absolute_error(score_rt, score_af)
	print ("point mae %f" % mae)

	score_af = []
	score_rt = []
	for line in data:
		if int(line[0]) != 100 and int(line[0]) != 0:
			score_rt.append(int(line[0]))
			score_af.append(round(int(line[1])))

	print ("size removed 0/100RT original rmse %d" % len(score_af))

	rmse = sqrt(mean_squared_error(score_rt, score_af))
	print ("removed 0/100RT original rmse %f" % rmse)

	mae = mean_absolute_error(score_rt, score_af)
	print ("removed 0/100RT original mae %f" % mae)

	score_rt = [int(x/10) if x!=100 else 9 for x in score_rt]
	score_af = [int(x/10) if x!=100 else 9 for x in score_af]

	rmse = sqrt(mean_squared_error(score_rt, score_af))
	print ("removed 0/100RT point rmse %f" % rmse)

	mae = mean_absolute_error(score_rt, score_af)
	print ("removed 0/100RT point mae %f" % mae)

if __name__ == "__main__":
	semantics = input("semantics? dfquad/quad/euler ")
	method = input("method? sent/nlp ")
	if semantics == "dfquad":
		if method == "sent":
			data = read_data('all_sent_dfquad_sent.jsonl')
		elif method == "nlp":
			data = read_data('all_nlp_dfquad_sent.jsonl')
	elif semantics == "quad":
		if method == "sent":
			data = read_data('all_sent_quad_sent.jsonl')
		elif method == "nlp":
			data = read_data('all_nlp_quad_sent.jsonl')
	elif semantics == "euler":
		if method == "sent":
			data = read_data('all_sent_euler_sent.jsonl')
		elif method == "nlp":
			data = read_data('all_nlp_euler_sent.jsonl')

	evaluate(data)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2013-2016 Paulo Freitas
# MIT License (see LICENSE file)
'''
Brazilian territorial distribution data exporter package
'''
# Package metadata

__version__ = '1.0-dev'
__author__ = 'Paulo Freitas <me@paulofreitas.me>'
__copyright__ = 'Copyright (c) 2013-2016 Paulo Freitas'
__license__ = 'MIT License'


import logging
import requests

logger = logging.getLogger(__name__)

def encoded_dict(in_dict):
    out_dict = {}
    for k, v in in_dict.iteritems():
        if isinstance(v, unicode):
            v = v.encode('utf8')
        elif isinstance(v, str):
            v.decode('utf8')
        out_dict[k] = v
    return out_dict

class VKApiConnector(object):
    __base_url = "https://api.vk.com/method/"
    __v = 5.68
    __resolve_screen_name_method = "utils.resolveScreenName"
    __wall_get_method = "wall.get"
    __token = NotImplemented
    __client_id = NotImplemented
    __sleep_time = NotImplemented

    @classmethod
    def config(cls, version, client_id, token, sleep_time=1):
        cls.version = version
        cls.__sleep_time = sleep_time
        cls.__client_id = client_id
        cls.__token = token

    @classmethod
    def __get_base_params(cls):
        return {
            'v': cls.__v,
            'client_id': cls.__client_id,
            'access_token': cls.__token
        }

    @classmethod
    def resolve_screen_name(cls, screen_name):
        try:
            logger.info("Access {} method".format(cls.__resolve_screen_name_method))
            request_params = cls.__get_base_params()
            request_params['screen_name'] = screen_name

            url = '{}{}'.format(cls.__base_url, cls.__resolve_screen_name_method)
            response = requests.post(url, encoded_dict(request_params) if request_params else None)

            if not response.ok:
                logger.error(response.text)
                return

            return response.json()['response']
        except Exception as ex:
            logger.exception(ex)

    @classmethod
    def get_wall(cls, owner_id):
        try:
            logger.info("Access {} method".format(cls.__wall_get_method))
            request_params = cls.__get_base_params()
            request_params['owner_id'] = owner_id
            request_params['count'] = 100

            url = '{}{}'.format(cls.__base_url, cls.__wall_get_method)
            response = requests.post(url, encoded_dict(request_params) if request_params else None)

            if not response.ok:
                logger.error(response.text)
                return

            return response.json()['response']
        except Exception as ex:
            logger.exception(ex)


import time
import urllib

# from .models import *
from django.utils import timezone
from datetime import datetime
from django.shortcuts import render, redirect
from .models import Message, ViewsCounter
from .functions import client_ip, count_views

def index(request):
    ip = client_ip(request)
    messages = Message.objects.all()
    context = {}
    context['messages'] = []
    for message in messages:
        message_counter = count_views(message, ip)
        context['messages'].append({'message': message, 'counter': message_counter})

    return render(request, 'pages/index.html', context)

def add(request):
    if not request.user.is_authenticated:
        return redirect('/')

    context = {}
    if request.method == 'POST':
        message = str(request.POST.get('message', ''))
        if len(message) > 160:
            context['error'] = 'Message can\'t contain more than 160 char'
            return render(request, 'pages/add.html', context)

        new_message = Message.objects.create(
            user=request.user, message=message, created=datetime.now(tz=timezone.utc))
        ViewsCounter.objects.create(message=new_message, ip=client_ip(request))
        context['success'] = 'Post added!'

    return render(request, 'pages/add.html', context)

def delete(request):
    if not request.user.is_authenticated:
        return redirect('/')

    if request.method == 'POST':
        message_id = str(request.POST.get('id', ''))
        Message.objects.get(user=request.user, id=message_id).delete()

    return redirect('/')

def edit(request, pk=0):
    if not request.user.is_authenticated:
        return redirect('/')

    context = {}
    if request.method == 'GET':
        context['message'] = Message.objects.get(id=pk, user=request.user)

        return render(request, 'pages/edit.html', context)

    if request.method == 'POST':
        edited_message = str(request.POST.get('message', ''))
        mes = Message.objects.get(id=pk, user=request.user)
        mes.message = edited_message
        mes.save()

    return redirect('/')

def me(request):
    return render(request, 'pages/me.html')


from keras.layers.pooling import Layer
import keras.backend as K

class GlobalAveragePooling1D_Mask0(Layer):
    """
    Global average pooling operation for temporal data.
    Masking out 0-padded input.
    """

    def __init__(self, data_format='channels_last', **kwargs):
        super(GlobalAveragePooling1D_Mask0, self).__init__(**kwargs)
        self.data_format = K.normalize_data_format(data_format)

    def compute_output_shape(self, input_shape):
        input_shape = input_shape[0]
        if self.data_format == 'channels_first':
            return (input_shape[0], input_shape[1])
        else:
            return (input_shape[0], input_shape[2])

    def call(self, inputs):
        inputs, model_inputs = inputs
        steps_axis = 1 if self.data_format == 'channels_last' else 2
        mask = K.max(model_inputs, axis=2, keepdims=True)
        inputs *= mask
        return K.sum(inputs, axis=steps_axis) / K.maximum(
            K.sum(mask, axis=steps_axis), K.epsilon())


from application import Hangman

def validateWord(string):
    """
    Returns true is the word is just a alphabetic string. Will return false if string contains numbers or other symbols
    """
    if string.isalpha() and " " not in string:
        return True
    print("That's not valid word, try again!")
    return False

if __name__ == "__main__":
    print(
    """Welcome to hangman!!

    """)
    hostName = str(input("Enter the host's name here: ")).lower()

    challengerName = str(input("Enter the challenger's name here: ")).lower()

    print(
    f"""Hey {hostName}!, enter your word below. Make sure {challengerName} isn't looking!!
    """)

    # Validate input string
    while(True):
        secret_word = input("Enter your word: ")
        if(validateWord(secret_word)):
            secret_word = secret_word.lower()
            break

    # Initiate the board
    board = Hangman(challengerName, hostName, secret_word)

    first=True
    # Start the game
    try:
        while board.challenger.guesses != 0:

            while(True):
                if first == False:
                    letter = input("Enter a letter or enter '!' to make a guess at the word: ")
                else:
                    letter = input("Enter a letter: ")

                if letter == "!" and first == False:
                    full_guess = input("ok, enter your full guess: ")
                    if validateWord(full_guess) and board.makeFullGuess(full_guess.lower()):
                        print(f"""
                        THATS CORRECT {board.challenger.name}!

                        {full_guess} is the correct answer!!, YOU'VE WON!!
                        """)
                        raise StopIteration
                elif (board.validateLetter(letter)):
                    letter = letter.lower()
                    break

            print("""

            """)
            # Check if letter is in word
            if board.host.checkLetter(letter):
                print("Well done that was a correct guess!")
                board.addToCorrectGuesses(letter)
            else:
                print("That letter was not there!")
                board.addToWrongGuesses(letter)
                board.challenger.oneLess()

            board.hangmanState()
            if first:
                first = False
    except StopIteration as e:
        pass


def binarySearch(lst, n):
    low, high = 0, len(lst) - 1     # high value is len(lst) -1
    while low <= high:                # <=
        mid = (low + high) / 2
        if lst[mid] > n:
            high = mid - 1          # +-1
        elif lst[mid] < n:
            low = mid + 1
        else:
            return mid
    return None


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/vagrant/envs/conversocial/lib/python2.7/site-packages/shardmonster/tests/test_sharder.py
# Compiled at: 2016-06-16 06:14:05
from mock import Mock
from shardmonster import api, sharder
from shardmonster.tests.base import ShardingTestCase

class TestSharder(ShardingTestCase):

    def setUp(self):
        api.activate_caching(0.5)
        super(TestSharder, self).setUp()

    def tearDown(self):
        api.activate_caching(0)
        super(TestSharder, self).tearDown()

    def test_basic_copy(self):
        api.set_shard_at_rest('dummy', 1, 'dest1/test_sharding')
        doc1 = {'x': 1, 'y': 1}
        doc1['_id'] = self.db1.dummy.insert(doc1)
        api.start_migration('dummy', 1, 'dest2/test_sharding')
        manager = Mock(insert_throttle=None)
        sharder._do_copy('dummy', 1, manager)
        doc2, = self.db2.dummy.find({})
        self.assertEquals(doc1, doc2)
        return

    def test_sync_after_copy(self):
        api.set_shard_at_rest('dummy', 1, 'dest1/test_sharding')
        api.start_migration('dummy', 1, 'dest2/test_sharding')
        doc1 = {'x': 1, 'y': 1}
        doc1['_id'] = self.db1.dummy.insert(doc1)
        self.db2.dummy.insert(doc1)
        initial_oplog_pos = sharder._get_oplog_pos('dummy', 1)
        self.db1.dummy.update({'x': 1}, {'$inc': {'y': 1}})
        api.set_shard_to_migration_status('dummy', 1, api.ShardStatus.MIGRATING_SYNC)
        sharder._sync_from_oplog('dummy', 1, initial_oplog_pos)
        doc2, = self.db2.dummy.find({})
        self.assertEquals(2, doc2['y'])

    def test_delete_after_migration(self):
        api.set_shard_at_rest('dummy', 1, 'dest1/test_sharding')
        api.start_migration('dummy', 1, 'dest2/test_sharding')
        doc1 = {'x': 1, 'y': 1}
        doc1['_id'] = self.db1.dummy.insert(doc1)
        self.db2.dummy.insert(doc1)
        api.set_shard_to_migration_status('dummy', 1, api.ShardStatus.POST_MIGRATION_DELETE)
        manager = Mock(delete_throttle=None)
        sharder._delete_source_data('dummy', 1, manager)
        self.assertEquals(0, self.db1.dummy.find({}).count())
        doc1_actual, = self.db2.dummy.find({})
        self.assertEquals(doc1, doc1_actual)
        return

    def test_sync_ignores_other_collection(self):
        api.set_shard_at_rest('dummy', 1, 'dest1/test_sharding')
        api.start_migration('dummy', 1, 'dest2/test_sharding')
        doc1 = {'x': 1, 'y': 1}
        doc1['_id'] = self.db1.dummy.insert(doc1)
        self.db2.dummy.insert(doc1)
        initial_oplog_pos = sharder._get_oplog_pos('dummy', 1)
        self.db1.other_coll.insert(doc1)
        self.db1.other_coll.update({'x': 1}, {'$inc': {'y': 1}})
        api.set_shard_to_migration_status('dummy', 1, api.ShardStatus.MIGRATING_SYNC)
        sharder._sync_from_oplog('dummy', 1, initial_oplog_pos)
        doc2, = self.db2.dummy.find({})
        self.assertEquals(1, doc2['y'])

    def test_sync_uses_correct_connection(self):
        """This tests for a bug found during a rollout. The connection for the
        metadata was assumed to be the same connection as the source data was
        going to be coming from. This is *not* always the case.
        """
        api.set_shard_at_rest('dummy', 1, 'dest2/test_sharding')
        api.start_migration('dummy', 1, 'dest1/test_sharding')
        doc1 = {'x': 1, 'y': 1}
        doc1['_id'] = self.db1.dummy.insert(doc1)
        self.db2.dummy.insert(doc1)
        initial_oplog_pos = sharder._get_oplog_pos('dummy', 1)
        self.db2.dummy.update({'x': 1}, {'$inc': {'y': 1}})
        api.set_shard_to_migration_status('dummy', 1, api.ShardStatus.MIGRATING_SYNC)
        sharder._sync_from_oplog('dummy', 1, initial_oplog_pos)
        doc2, = self.db1.dummy.find({})
        self.assertEquals(2, doc2['y'])


import logging
import boto3
from botocore.exceptions import ClientError
from eks_config import  keypair_name, instance_type, Linux_image_id, Linux_subnet, Linux_sg

def create_ec2_instance( Linux_image_id, instance_type, keypair_name, Linux_subnet, Linux_sg):

    ec2_client = boto3.client('ec2')
    try:
        response = ec2_client.run_instances(
                                            ImageId=Linux_image_id,
                                            InstanceType=instance_type,
                                            KeyName=keypair_name,
                                            SubnetId=Linux_subnet,
                                            SecurityGroupIds=Linux_sg,
                                            MinCount=1,
                                            MaxCount=1)
    except ClientError as e:
        logging.error(e)
        return None
    return response['Instances'][0]

def main():

    # Set up logging
    logging.basicConfig(level=logging.DEBUG,
                        format='%(levelname)s: %(asctime)s: %(message)s')

    # Provision and launch the EC2 instance
    instance_info = create_ec2_instance(Linux_image_id, instance_type, keypair_name,Linux_subnet, Linux_sg)
    if instance_info is not None:
        logging.info(f'Launched EC2 Instance {instance_info["InstanceId"]}')
        logging.info(f'    VPC ID: {instance_info["VpcId"]}')
        logging.info(f'    Private IP Address: {instance_info["PrivateIpAddress"]}')
        logging.info(f'    Current State: {instance_info["State"]["Name"]}')

if __name__ == '__main__':
    main()


import os
import numpy as np
from PIL import Image

from . import matrixGame

class TextScroll(matrixGame.MatrixGame):

    def __init__(self, text, text_colour=[255, 255, 255], back_colour=[0, 0, 0], scroll_speed=.1, font='tahoma.fnt'):
        super(TextScroll, self).__init__()
        self._load_font('fonts', font)

        self._coloured_pixels = []
        self.i = 0
        self.scroll_speed = scroll_speed
        self.passed_time = 0

        self._show_message(text, text_colour, back_colour)

    ####
    # Text assets
    ####

    def _load_font(self, path, filename):
        atlas = self._load_glyph_atlas(os.path.join(path, filename))
        image_file = atlas['page'][0]['file']
        img = Image.open(os.path.join(path, image_file)).convert('RGB')
        pix = img.load()
        self._text_dict = {}
        for c in atlas['char']:
            char_info = atlas['char'][c]
            char = []

            for y in range(int(char_info['y']), int(char_info['y'] + char_info['height'])):
                line = []
                for x in range(int(char_info['x']), int(char_info['x'] + char_info['width'])):
                    line.append(pix[x, y])
                char.append(line)
            self._text_dict[c] = np.rot90(char, -1).reshape(-1, 3)

        return img

    # loads a BMFont Text format glyph atlas into a dictionary
    # see https://71squared.com/blog/bitmap-font-file-format for more info
    @staticmethod
    def _load_glyph_atlas(filename):
        atlas = {}
        for line in open(filename):
            attributes = line.split(" ")
            attributes = [x for x in attributes if x != '' and x != '\n']
            dictkey = attributes[0]
            if dictkey in atlas:
                attribdict = atlas[dictkey]
            else:
                attribdict = atlas[dictkey] = {}
            if dictkey in ['char', 'page']:
                c = int(attributes[1].split("=")[1])
                entry = {}
                for attrib in attributes[2:]:
                    key, value = attrib.split("=")
                    try:
                        entry[key] = float(value)
                    except:
                        entry[key] = value.strip('\"\n')
                    attribdict[c] = entry
            else:
                for attrib in attributes[1:]:
                    key, value = attrib.split("=")
                    try:
                        attribdict[key] = float(value)
                    except ValueError:
                        strval = value.strip('\"\n')
                        if ',' in strval:
                            arry = strval.split(',')
                            try:
                                arry = map(float, arry)
                            finally:
                                attribdict[key] = arry
                        else:
                            attribdict[key] = strval
        return atlas

    def _get_char_pixels(self, s):
        """
        Internal. Safeguards the character indexed dictionary for the
        show_message function below
        """

        if len(s) == 1 and ord(s) in self._text_dict.keys():
            return list(self._text_dict[ord(s)])
        else:
            return list(self._text_dict[-1])

    def _show_message(
            self,
            text_string,
            text_colour,
            back_colour
    ):
        """
        Scrolls a string of text across the LED matrix using the specified
        speed and colours
        """
        dummy_colour = [0, 0, 0]
        string_padding = [dummy_colour] * self.PIXELS * self.PIXELS
        letter_padding = [dummy_colour] * self.PIXELS
        # Build pixels from dictionary
        scroll_pixels = []
        scroll_pixels.extend(string_padding)
        for s in text_string:
            #scroll_pixels.extend(self._trim_whitespace(self._get_char_pixels(s)))
            scroll_pixels.extend(self._get_char_pixels(s))
            scroll_pixels.extend(letter_padding)
        scroll_pixels.extend(string_padding)
        # Recolour pixels as necessary
        self._coloured_pixels = [
            [(text_colour[0] * pixel[0] / 255) + (back_colour[0] * (255-pixel[0]) / 255),
             (text_colour[1] * pixel[1] / 255) + (back_colour[1] * (255-pixel[1]) / 255),
             (text_colour[2] * pixel[2] / 255) + (back_colour[2] * (255-pixel[2]) / 255)]
            for pixel in scroll_pixels
            ]

    def update(self, passed_time):
        self.passed_time += passed_time
        if self.passed_time > self.scroll_speed * 1000:
            self.i += 1
            self.passed_time -= self.scroll_speed * 1000

        previous_rotation = self.rotation
        self.rotation -= 90

        # Shift right by 16 pixels per frame to scroll
        scroll_length = len(self._coloured_pixels) // self.PIXELS
        if self.i > scroll_length - self.PIXELS:
            self.i = 0

        start = self.i * self.PIXELS
        end = start + (self.PIXELS * self.PIXELS)
        self.set_pixels(self._coloured_pixels[start:end])
        self._rotation = previous_rotation


from django.conf.urls import patterns, include, url
from django.contrib.auth.models import User, Group
from django.contrib import admin

from .views import ApiEndpoint
admin.autodiscover()

from rest_framework import viewsets, routers
from rest_framework import permissions
from oauth2_provider.ext.rest_framework import TokenHasReadWriteScope, TokenHasScope

# from: http://django-oauth-toolkit.readthedocs.org/en/latest/rest-framework/getting_started.html
# ViewSets define the view behavior.
class UserViewSet(viewsets.ModelViewSet):
    permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]
    model = User

class GroupViewSet(viewsets.ModelViewSet):
    permission_classes = [permissions.IsAuthenticated, TokenHasScope]
    required_scopes = ['groups']
    model = Group

# Routers provide an easy way of automatically determining the URL conf
router = routers.DefaultRouter()
router.register(r'users', UserViewSet)
router.register(r'groups', GroupViewSet)

urlpatterns = patterns('',
    # Login/logout
    url(r'^accounts/login/$', 'django.contrib.auth.views.login'),
    url(r'^accounts/logout/$', 'django.contrib.auth.views.logout'),

    # default pages
    url(r'^$', 'bbp.views.home', name='home'),
    url(r'^about/$', 'bbp.views.about', name='about'),
    url(r'^faq/$', 'bbp.views.faq', name='faq'),
    url(r'^contact/$', 'bbp.views.contact', name='contact'),
    url(r'^terms/$', 'bbp.views.terms', name='terms'),
    url(r'^privacy/$', 'bbp.views.privacy', name='privacy'),

    # Link in subsidiary modules
    url(r'^device/', include('device.urls', namespace='device')),
    url(r'^member/', include('bbp.member.urls', namespace='member')),

    # API test -(login_required)
    url(r'^secret$', 'bbp.views.secret_page', name='secret'),
    # http://django-oauth-toolkit.readthedocs.org/en/latest/tutorial/tutorial_03.html

    # OAuth2 Provider
    # url(r'^oauth2/', include('provider.oauth2.urls', namespace = 'oauth2')),

    # OAuth Provider
    # http://django-oauth-toolkit.readthedocs.org/en/latest/tutorial/tutorial_01.html
    url(r'^o/', include('oauth2_provider.urls', namespace='oauth2_provider')), # look ma, I'm a provider!
    url(r'^api/hello', ApiEndpoint.as_view()),  # and also a resource server!

    # API
    url(r'^1.0/api/', include(router.urls)),

    # url(r'^blog/', include('blog.urls')),

    url(r'^admin/', include(admin.site.urls)),
)


from django.contrib import admin
from django.contrib.sites.shortcuts import get_current_site

from documents.models import Document

class DocumentAdmin(admin.ModelAdmin):
	def save_model(self, request, obj, form, change):
		obj.site = get_current_site(request)
		obj.author = request.user
		obj.save()

	def queryset(self, request):
		qs = super(DocumentAdmin, self).queryset(request)
		return qs.filter(site=get_current_site(request))

admin.site.register(Document, DocumentAdmin)


# No. This is not efficient.
# No. I don't care.
# Yes. I am leaving it as it is.
# The answer is right.

row = 10
col = 10
count = 0

target = 2000000

while True:
    col += 1
    # arbitrary value
    if col > 100:
        col = 10
        row += 1
    # arbitrary end condition
    if row > 100:
        break
    count = 0

    # determine the number of shapes per row/col based on initialized values
    # basically determines smaller "sub-grids" of the full grid that provides
    #  the number of the row x col shapes that fit
    for i in range(1, row + 1):
        r = 0
        for ii in range(1, row + 1):
            if ii + i <= row + 1:
                r += 1
        for j in range(1, col + 1):
            c = 0
            for jj in range(1, col + 1):
                if jj + j <= col + 1:
                    c += 1
            count += r*c

    # through testing, found the minimum possible score, used this as condition to catch first instance of it
    if target - count == 2:
        break

print(count)
print(row, col, row * col)


from ops.forms import ServiceForm, AffectedForm, ServiceVehicleForm, ArrestForm, ArrestPaymentForm
from django.shortcuts import render_to_response, redirect
from django.template.context import RequestContext
from django.contrib.auth.decorators import login_required
from django.forms.formsets import formset_factory
from ops.models import ServiceVehicle, ServiceAffected, Service, ArrestPayment
from personal.models import Firefighter
from common.models import BasePerson, TelephoneNumber, PersonTelephoneNumber
from django.contrib import messages
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.exceptions import ValidationError
from django.db import transaction
import json
from datetime import datetime

@login_required
def list_services(request):
    services_qs = Service.objects.all()
    paginator = Paginator(services_qs, 15)
    page = request.GET.get('page')
    try:
        services = paginator.page(page)
    except PageNotAnInteger:
        services = paginator.page(1)
    except EmptyPage:
        services = paginator.page(paginator.num_pages)

    return render_to_response("list_services.html", RequestContext(request, {"services": services, "paginator": paginator}))

@login_required
def view_service(request, service_id):
    service = Service.objects.get(id=service_id)
    return render_to_response("view_service.html", RequestContext(request, {"service": service}))

@login_required
@transaction.commit_on_success
def insert_service(request):
    params = {}
    AffectedFormSet = formset_factory(AffectedForm, extra=0)
    VehiclesFormSet = formset_factory(ServiceVehicleForm, extra=1)
    firefighter = request.user.get_profile()
    service_form = ServiceForm()
    crew_dict = {}

    if request.method == 'POST':
        data = request.POST.copy()
        data['time'] = data['time'][0:-2]+":"+data['time'][-2:]

        service_form = ServiceForm(data)
        affected_formset = AffectedFormSet(data, prefix='affected')
        vehicles_formset = VehiclesFormSet(data, prefix='vehicles')
        if service_form.is_valid() and affected_formset.is_valid() and vehicles_formset.is_valid():
            service = service_form.save()
            service.created_by = firefighter
            service.save()

            for vf in vehicles_formset.forms:
                if "lead" in vf.cleaned_data:
                    s_vehicle = ServiceVehicle()
                    s_vehicle.service = service
                    s_vehicle.lead = Firefighter.objects.get(id=vf.cleaned_data['lead'])
                    s_vehicle.vehicle = vf.cleaned_data['vehicle']
                    if vf.cleaned_data['driver']:
                        s_vehicle.driver = Firefighter.objects.get(id=vf.cleaned_data['driver'])

                    s_vehicle.save()

                    for crew_id in vf.cleaned_data['crew_ids'].split(","):
                        if crew_id != "":
                            s_vehicle.crew.add(Firefighter.objects.get(id=crew_id))

            for af in affected_formset.forms:
                if 'first_name' in af.cleaned_data:
                    if af.cleaned_data["id_document"]:
                        person, _ = BasePerson.objects.get_or_create(id_document=af.cleaned_data["id_document"])
                    else:
                        person = BasePerson()
                    person.first_name = af.cleaned_data['first_name']
                    person.first_name_2 = af.cleaned_data['first_name_2']
                    person.last_name = af.cleaned_data['last_name']
                    person.last_name_2 = af.cleaned_data['last_name_2']
                    person.gender = af.cleaned_data['gender']
                    if af.cleaned_data["primary_email"]:
                        person.primary_email = af.cleaned_data["primary_email"]

                    person.save()
                    if af.cleaned_data["phone_code"] and af.cleaned_data["phone_number"]:
                        telephone = TelephoneNumber(code=af.cleaned_data["phone_code"],
                                                    number=af.cleaned_data["phone_number"])
                        telephone.save()
                        PersonTelephoneNumber(person=person,type='O',
                                              telephone_number=telephone).save()

                    s_affected = ServiceAffected(person_affected=person,
                                                 notes=af.cleaned_data["notes"],
                                                 type=af.cleaned_data["type"])
                    s_affected.save()
                    service.affected.add(s_affected)
            messages.success(request, u'El servicio fue guardado exitosamente')
            return redirect(list_services)
        else:
            crew_ids_str = ""
            for k, v in data.iteritems():
                if "crew_ids" in k and v!="":
                    crew_ids_str = crew_ids_str+","+v
            crew_ids = [x for x in crew_ids_str.split(",") if x!='']

#            import ipdb;ipdb.set_trace()
            crew = Firefighter.objects.filter(id__in=crew_ids)
            for member in crew:
                crew_dict[member.id] = str(member)
    else:
        affected_formset = AffectedFormSet(prefix='affected')
        vehicles_formset = VehiclesFormSet(prefix='vehicles')

    params['form'] = service_form
    params['affected'] = affected_formset
    params['vehicles'] =  vehicles_formset
    params['media'] = service_form.media
    params['ff'] = firefighter
    params['crew_data'] = json.dumps(crew_dict)

    return render_to_response("insert_service.html", RequestContext(request, params))

@login_required
@transaction.commit_on_success
def insert_arrest(request):
    params = {}
    arrest_form = ArrestForm()
    firefighter = request.user.get_profile()

    if request.method == 'POST':
        data = request.POST.copy()
        arrest_form = ArrestForm(data)
        if arrest_form.is_valid():
            arrest = arrest_form.save(commit=False)
            arrest.created_by = firefighter
            arrest.arrested = Firefighter.objects.get(id=arrest_form.cleaned_data['arrested'])
            arrest.save()
            messages.success(request, u'El arresto fue guardado exitosamente')
            return redirect(insert_arrest)

    params['arrest_form'] = arrest_form
    return render_to_response("insert_arrest.html", RequestContext(request, params))

@login_required
@transaction.commit_on_success
def insert_arrest_payment(request):
    params = {}
    arrest_payment_form = ArrestPaymentForm()
    firefighter = request.user.get_profile()

    if request.method == 'POST':
        data = request.POST.copy()
        data['start_time_time'] = data['start_time_time'][0:-2]+":"+data['start_time_time'][-2:]
        data['end_time_time'] = data['end_time_time'][0:-2]+":"+data['end_time_time'][-2:]

        arrest_payment_form = ArrestPaymentForm(data)
        if arrest_payment_form.is_valid():
            arrest_payment = ArrestPayment()
            arrest_payment.created_by = firefighter
            arrest_payment.start_time = datetime.combine(arrest_payment_form.cleaned_data['start_time_date'], arrest_payment_form.cleaned_data['start_time_time'])
            arrest_payment.end_time = datetime.combine(arrest_payment_form.cleaned_data['end_time_date'], arrest_payment_form.cleaned_data['end_time_time'])
            arrest_payment.payer = Firefighter.objects.get(id=arrest_payment_form.cleaned_data['payer'])
            try:
                arrest_payment.full_clean()
                arrest_payment.save()
                messages.success(request, u'El pago arresto fue guardado exitosamente')
                return redirect(insert_arrest_payment)
            except ValidationError as e:
                messages.error(request, e.message_dict["__all__"][0])

    params['arrest_payment_form'] = arrest_payment_form
    return render_to_response("insert_arrest_payment.html", RequestContext(request, params))


##########
# This script is created by Huilei Xu, 4/2016
# Used to convert conventional SAM/BAM read alignment to Novoalign Native format for utilizing CIMS downstream analysis scripts
# Novoalign Native format:
#1. read header: >xx_xxx_xxx
#2. S/L/R: S means single ended, L means read from first file, R means read from second file
#3. read Sequence
#4. Base qualities
#5. U/R/QC/NM/QL: U means unique alignment, R means multiple alignments, QC means failure by QC, NM means no alignment, QL means failure by mapQ
#6. (assuming #5 is U/R) mapping score: Phred format
#7. (assuming #5 is U/R) mapping quality: Phred format
#8. aligned sequence: fasta header, truncated at first space
#9. aligned offset: 1-based position of the alignment in the sequence
#10. strand
#11. pair sequence
#12. pair offset
#13. pair strand
#14. mismatches
import pysam
import re
ifname = "/home/hxu/test/BrainD.Aligned.out.sam"
samfile = pysam.AlignmentFile(ifname,"r")
output = open(ifname.replace('sam','novoalign'),'w')
for read in samfile.fetch():
	if read.is_read1:
		pairFlag = 'L'
	elif read.is_read2:
		pairFlag = 'R'
	else:
		pairFlag = 'S'

	if read.is_reverse:
		strand = 'R'
	else:
		strand = 'F'

	if str(read.qual) == 'None':
		readqual = '.'
	else:
		readqual = read.qual

	mapn = read.get_tag('NH')
	mutations = []
	if mapn == 1:
		status = 'U'
		dels = []
		snps = []
		ins = []
		mutations = []
		### parsing CIGAR to get insertion base (MD not include insertion info)
		cigar = read.cigar
		idx_read = 0
		offset = 0
		read_clipped = ''
		for (code,num) in cigar:
			offset = idx_read + 1
			if code == 4:
				idx_read += num
				read_clipped += read.seq[idx_read:]
				continue
			elif code == 2: #del
				continue
			elif code == 1: #ins
				ins.append(str(offset)+"+"+read.seq[idx_read:idx_read+num])
				read_clipped += read.seq[idx_read:idx_read+num]
				idx_read += num
				continue
			read_clipped += read.seq[idx_read:idx_read+num]
			idx_read += num

		### parsing MD tag to get deletion and mismatch base (CIGAR not include these info)
		mdstr = read.get_tag('MD')
		mdSub = re.sub(r'([\\^]*[ACGT]+)[0]*', ' \\1 ', mdstr)
		mdSplit = re.split('[ ]+', mdSub)
		startpos = 0
		for md in mdSplit:
			if md.isdigit():
				startpos += int(md)
			elif md.startswith('^'):
				delpos = startpos
				for base in md[1:]:
					delpos +=1
					dels.append('%d-%s'%(delpos,base))
			elif md.isalpha():
				for base in md:
					startpos +=1
					snps.append('%d%s>%s'%(startpos,base,read_clipped[startpos-1]))

		mutations = dels + ins + snps

		if len(mutations)>0:
			output.write("\t".join(['@'+read.qname,pairFlag,read.seq,readqual,status,str(read.mapq),str(read.mapq),\
		'>'+str(read.reference_name),str(read.pos+1),strand,'.','.','.'," ".join(mutations)])+"\n")
		else:
			output.write("\t".join(['@'+read.qname,pairFlag,read.seq,readqual,status])+"\n")

	else:
		if  mapn > 1:
			status = 'R'
		elif mapn == 0:
			status = 'NM'
		output.write("\t".join(['@'+read.qname,pairFlag,read.seq,readqual,status])+"\n")

output.close()


import torch
from torch import nn
import torch.nn.functional as F
from torch import optim
from torchvision import datasets, transforms
from torch.autograd import Variable
from torchvision import models

# model = models.densenet201(pretrained=True)

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50)
        self.fc2 = nn.Linear(50, 10)

        # Initialize params.
        for name, param in self.named_parameters():
            if 'bias' not in name:
                nn.init.kaiming_normal(param)
            else:
                nn.init.constant(param, 0)

    def forward(self, x):
        x = x.view(-1, 1, 28, 28)
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x)

class Q(nn.Module):
    def __init__(self):
        super(Q, self).__init__()
        self.fc1 = nn.Linear(20, 10)
        self.fc2 = nn.Linear(10, 5)
        self.fc3 = nn.Linear(5, 1)

        # Initialize params.
        for name, param in self.named_parameters():
            if 'bias' not in name:
                nn.init.kaiming_normal(param)
            else:
                nn.init.normal(param, 0, 1)

    def forward(self, x):
        # print(x)
        x = x.view(-1, 20)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return F.relu(x)


__author__ = 'AVIRAM'

#sys.path.insert(0, 'lib')  #we need this line in order to make libraries imported from lib folder work properly
import requests  #Used for http requests
import symbol

symbol2=["SP","NSDQ","DAX","GOLD","C.OIL"]
priceForIndex=[50,20,25,500,100]

def calculatingRisk(symbolInput,enterPri,stopPri,vol,type):#(type=1-long, type=0-short)
    if (type==1):#long
        sumPoint=enterPri-stopPri
    else:#short
        sumPoint=stopPri-enterPri

    priceForPoint=symbol.Symbol.getPrice(symbolInput)#get the price from date store

    '''
    for x in  range(len(symbol2)):
        if (symbol2[x]==symbolInput):
            risk=sumPoint*vol*priceForIndex[x]
    '''
    risk=sumPoint*vol*priceForPoint

    return risk

#print(calculatingRisk("SP",100,90,1,1))


"""
Given a word, you need to judge whether the usage of capitals in it is right or not.

We define the usage of capitals in a word to be right when one of the following cases holds:

All letters in this word are capitals, like "USA".
All letters in this word are not capitals, like "leetcode".
Only the first letter in this word is capital if it has more than one letter, like "Google".
Otherwise, we define that this word doesn't use capitals in a right way.
Example 1:
Input: "USA"
Output: True
Example 2:
Input: "FlaG"
Output: False
Note: The input will be a non-empty word consisting of uppercase and lowercase latin letters.
"""
class Solution:
    def detectCapitalUse(self, word):
        """
        :type word: str
        :rtype: bool
        """
        if not word:
            return True
        is_capital = word[-1].isupper()
        if is_capital and not word[0].isupper():
            return False
        for w in word[1:]:
            if is_capital != w.isupper():
                return False
        return True

    def detectCapitalUse2(self, word):
        capital_count = 0
        for w in word:
            if w.isupper():
                capital_count += 1
        return capital_count == len(word) or capital_count == 0 \
            or (capital_count == 1 and word[0].isupper())


import os
import csv
import datetime

now = datetime.datetime.now()

def parseing(fileName):
    debugCounter = 0
    sectionTemplate = """
:doc:`{subject}{catNumber}`{SpTopic} {Term}
    | Section {section} ({classNumber}) Credits: {units}; {mixture}; {component}
    | Instructor: {Instructor}
    |{Building}:{Room} {Location} {Days} {Time}

	{description}
"""

    sectionTemplateMultiRoom = """
:doc:`{subject}{catNumber}`{SpTopic} {Term}
    | Section {section} ({classNumber}) Credits: {units}; {mixture}; {component}
    | Instructor: {Instructor}
    {multiRoom}

	{description}
"""

    sectionTemplateLab = """
:doc:`{subject}{catNumber}`{SpTopic} {Term}
    | Section {section}/{labSection} ({classNumber}) Credits: {units}; {mixture}; {component}
    | Instructor: {Instructor}
    |{Building}:{Room} {Location} {Days} {Time}
    |{labBuilding}: {labRoom} ({labLocation}) {labDay} {labTime} (lab)

	{description}
"""

    comp314_315Template = """
	{subject}{catNumber} {Term} (Description: :doc:`comp314-315`)
		| Section {section} ({classNumber}) Credits: {units}; {mixture}; {component}
		| Instructor: {Instructor}
	        |{Building}:{Room} {Location} {Days} {Time}

	{description}
	"""

    topicsSectionTemplate = """
{subject}{catNumber} Topic{topics} {Term}
	| Section {section} ({classNumber}) Credits: {units}; {mixture}; {component}
	| Instructor: {Instructor}
	|{Building}:{Room} {Location} {Days} {Time}
	| Description similar to: :doc:`{docName}`

{description}
"""

    headerTemplate = """
{semester} Schedule {txtURLline} {where}
==========================================================================
{created}

The following courses will (tentatively) be held during the {semester} semester.

For open/full status and latest changes, see
`LOCUS <http://www.luc.edu/locus>`_.

**In case of conflict, information on LOCUS should be considered authoritative.**

See `Textbook Information {textBookURLline}`_.

Section titles lines link to the course description page,
except for some labeled special topics courses related to an existing course.

The 4-digit number in parentheses after the section is the LOCUS registration code.

Be sure to look at the section's notes or LOCUS for an 8-week courses with more than one schedule line:
Friday line(s) are likely to be isolated makeup days, not every week.

{graduateLink}

**View Campus Specific Courses below :**{pages}

.. _{season}_undergraduate_courses_list:

{udergradeTxt}
~~~~~~~~~~~~~~~~~~~~~

"""

    gradHeadingTemplate = """

.. _{0}_graduate_courses_list_{1}:

Graduate Courses
~~~~~~~~~~~~~~~~~~~~~

"""

    indepStudyTemplate = """
:doc:`{}` 1-6 credits
	You cannot register
	yourself for an independenst study course!
	You must find a faculty member who
	agrees to supervisor the work that you outline and schedule together.  This
	*supervisor arranges to get you registered*.  Possible supervisors are: full-time department faculty
	"""
    classes = []
    headerObject = {
        "semester": "",
        "textBookURLline": "<https://docs.google.com/spreadsheets/d/138_JN8WEP8Pv5uqFiPEO_Ftp0mzesnEF5IFU1685w3I/edit?usp=sharing>",
        "created": "",
        "graduateLink": "",
        "campusURLTemplateCuneo": "",
        "season": "",
        "udergradeTxt": "Undergraduate Courses",
        "where": "",
        "pages": "",
        "txtURLline": "",
    }
    object = {
        "subject": "",
        "catNumber": "",
        "section": "",
        "classNumber": "",
        "title": "",
        "component": "",
        "units": "",
        "topics": "",
        "Building": "",
        "Location": "",
        "Room": "",
        "Days": "",
        "Time": "",
        "Instructor": "",
        "classCap": "",
        "totalStudents": "",
        "waitCap": "",
        "waitTotal": "",
        "minEnroll": "",
        "Attributes": "",
        "roomCharicteristics": "",
        "CombinedSID": "",
        "ClassEquiv": "",
        "SpTopic": "",
        "Term": "",
        "mixture": "",
        "description": "",
        "hasLab": False,
        "labBuilding": "",
        "labRoom": "",
        "labTime": "",
        "labDay": "",
        "labLocation": "",
        "Term": "",
        "isStudy": False,
        "isMultiRoom": False,
        "docName": "",
        "multiRoom": "",
    }
    file = open(fileName, "r")
    reader = csv.reader(file, delimiter=",")
    LSBuildings = ["Cuneo", "Mundelein", "Crown", "Sullivan", "Life Science", "Dumbach"]
    checker = 0
    string = ""
    appendToList = False
    hasLab = False
    semester = ""
    season = ""
    for row in reader:
        # print(row)

        for i in range(0, len(row)):
            if "COMP" == row[0]:
                object["subject"] = row[i].lower().strip()
                object["catNumber"] = str(int(row[i + 1]))
                if object["catNumber"] == "398":
                    object["isStudy"] = True
                object["section"] = row[i + 2]
                if "L" in row[i + 2]:
                    for m in range(0, len(classes)):
                        if (
                            classes[m]["catNumber"] == object["catNumber"]
                            and classes[m]["section"] in object["section"]
                        ):
                            classes[m]["hasLab"] = True
                            hasLab = True
                            classes[m]["labSection"] = row[i + 2]
                else:
                    object["section"] = row[i + 2][1:]

                object["classNumber"] = row[i + 3]
                object["title"] = row[i + 4]
                object["component"] = row[i + 5]
                object["units"] = row[i + 6]
                if i + 7 <= len(row):
                    object["topics"] = row[i + 7]
                    break
                break
            elif " Fall " in row[0]:
                season = "Fall"
                semester = "Fall " + getYear(row[0].split(" "), season)
                headerObject["season"] = season
                headerObject["semester"] = semester
            elif " Spring " in row[0]:
                season = "Spring"
                semester = "Spring " + getYear(row[0].split(" "), season)
                headerObject["season"] = season
                headerObject["semester"] = semester
            elif " Summer " in row[0]:
                season = "Summer"
                semester = "Summer " + getYear(row[0].split(" "), season)
                headerObject["season"] = season
                headerObject["semester"] = semester
            elif "Week" in row[0]:
                object["Term"] = "[" + str(row[0]) + "]"
            elif row[0] == "Bldg:" and not hasLab:
                if object["Building"] != "":
                    object["Building"] = object["Building"] + " +" + str(row[i + 1])
                    object["Room"] = object["Room"] + " + " + row[i + 3]
                    object["Days"] = (
                        str(object["Days"]) + " + " + str(convertDays(row[i + 5]))
                    )
                    object["Time"] = object["Time"] + " + " + row[i + 7]
                    for j in range(0, len(LSBuildings)):
                        if LSBuildings[j] in row[i + 1]:
                            object["Location"] = (
                                "(Lake Shore)" + " +" + object["Location"]
                            )
                            break
                        if row[i + 1] == "TBA":
                            object["Location"] = "" + " + " + object["Location"]
                            break
                        if "Online" in row[i + 1]:
                            object["Location"] = "(Online)" + " +" + object["Location"]
                            break
                        if j == len(LSBuildings) - 1:
                            object["Location"] = (
                                "(Water Tower)" + " +" + object["Location"]
                            )

                    object["isMultiRoom"] = True
                else:
                    object["Building"] = row[i + 1]
                    object["Room"] = row[i + 3]
                    object["Days"] = convertDays(row[i + 5])
                    object["Time"] = row[i + 7]
                    if 9 < len(row):
                        object["Instructor"] = row[9]
                    for j in range(0, len(LSBuildings)):
                        if LSBuildings[j] in object["Building"]:
                            object["Location"] = "(Lake Shore)"
                            break
                        elif object["Building"] == "TBA":
                            object["Location"] = ""
                            break
                        elif "Online" in object["Building"]:
                            object["Location"] = "(Online)"
                            break
                        else:
                            object["Location"] = "(Water Tower)"
                if object["Instructor"] == "":
                    object["Instructor"] = "N/A"
                break
            elif row[0] == "Bldg:" and hasLab:
                for m in range(0, len(classes)):
                    if (
                        classes[m]["catNumber"] == object["catNumber"]
                        and classes[m]["section"] in object["section"]
                    ):
                        classes[m]["labBuilding"] = row[i + 1]
                        waterTower = True
                        for j in range(0, len(LSBuildings)):
                            if LSBuildings[j] in classes[m]["labBuilding"]:
                                classes[m]["labLocation"] = "Lake Shore"
                                waterTower = False
                            elif classes[m]["labBuilding"] == "TBA":
                                classes[m]["labLocation"] = ""
                                waterTower = False
                            elif classes[m]["labBuilding"] == "Online":
                                classes[m]["labLocation"] = "Online"
                                waterTower = False
                        if waterTower:
                            classes[m]["labLocation"] = "Water Tower"

                        classes[m]["labRoom"] = row[i + 3]
                        classes[m]["labDay"] = convertDays(row[i + 5])
                        classes[m]["labTime"] = row[i + 7]
                break
            elif row[0] == "Class Enrl Cap:":
                object["classCap"] = row[i + 1]
                object["totalStudents"] = row[i + 3]
                object["waitCap"] = row[i + 5]
                object["waitTotal"] = row[i + 7]
                if i + 9 < len(row[i]):
                    object["minEnroll"] = row[i + 9]
                    break
                break
            elif row[0] == "Attributes:":
                object["Attributes"] = row[1]
            elif row[0] == "Room Characteristics:":
                object["roomCharicteristics"] = row[i + 1]
                break
            elif row[0] == "Class Equivalents:":
                if len(row) > i + 1:
                    object["ClassEquiv"] = row[i + 1]
            elif "Combined with" in row[0]:
                tempList = row[0].split()
                for q in range(0, len(tempList)):
                    if "COMP" == tempList[q]:
                        tempSplit = tempList[q + 1].split("-")
                        classNum = tempSplit[0]
                        object["docName"] = tempList[q].lower() + classNum
            elif "_______" in row[0]:
                if string != "":
                    object["description"] = "**Notes**\n        " + string
                string = ""
                if hasLab == False:
                    appendToList = True
                hasLab = False
                break
            elif row[0] == "Combined Section ID:":
                object["CombinedSID"] == row[1] + row[2] + row[3]
                break
            elif row[0] == "":
                for q in range(0, len(row)):
                    if row[q] != "":
                        object["mixture"] = row[q]
                        if object["mixture"] == "(Online)":
                            object["Location"] = "(Online)"
                            object["Building"] = "Online"

            else:
                if row[i] != "":
                    string += row[i] + "\n        "
        if appendToList:
            if "Report ID:" not in object["description"]:
                classes.append(object)
                # print(object)
                # print(len(classes))
            elif object["isStudy"]:
                continue

            object = object.fromkeys(object, "")
            appendToList = False
            object["Building"] = ""
            object["Days"] = ""
            object["Room"] = ""
            object["Days"] = ""
            object["Time"] = ""
            object["Location"] = ""
    mainRST = open("./checkFolder/" + season.lower() + ".rst", "w")
    onlineRST = open("./checkFolder/online" + season.lower() + ".rst", "w")
    lakeRST = open("./checkFolder/lakeshore" + season.lower() + ".rst", "w")
    waterRST = open("./checkFolder/watertower" + season.lower() + ".rst", "w")
    headerObject[
        "pages"
    ] = """
	* :doc:`lakeshore{0}`
	* :doc:`watertower{0}`
	* :doc:`online{0}`""".format(
        season.lower()
    )
    mainRST.write(headerTemplate.format(**headerObject))
    headerObject["where"] = "(Lake Shore)"
    headerObject[
        "pages"
    ] = """
	* :doc:`{0}`
	* :doc:`watetower{0}`
	* :doc:`online{0}`""".format(
        season.lower()
    )
    lakeRST.write(headerTemplate.format(**headerObject))
    headerObject["where"] = "(Online)"
    headerObject[
        "pages"
    ] = """
	* :doc:`lakeshore{0}`
	* :doc:`watertower{0}`
	* :doc:`{0}`""".format(
        season.lower()
    )
    onlineRST.write(headerTemplate.format(**headerObject))
    headerObject["where"] = "(Water Tower)"
    headerObject[
        "pages"
    ] = """
	* :doc:`lakeshore{0}`
	* :doc:`{0}`
	* :doc:`online{0}`""".format(
        season.lower()
    )
    waterRST.write(headerTemplate.format(**headerObject))
    Check398 = True
    Check499 = True
    Check490 = True
    for k in range(0, len(classes)):
        currentLine = ""
        if classes[k]["hasLab"]:
            currentLine = sectionTemplateLab.format(**classes[k])
        elif classes[k]["isMultiRoom"]:
            multiRoom = ""
            bList = classes[k]["Building"].split("+")
            dList = classes[k]["Days"].split("+")
            rList = classes[k]["Room"].split("+")
            lList = classes[k]["Location"].split("+")
            tList = classes[k]["Time"].split("+")
            for times in range(0, len(bList)):
                multiRoom = multiRoom + "| {0}: {1} {2} {3} {4} \n    ".format(
                    bList[times], rList[times], lList[times], dList[times], tList[times]
                )
            classes[k]["multiRoom"] = multiRoom
            currentLine = sectionTemplateMultiRoom.format(**classes[k])
        elif classes[k]["catNumber"] == "314" or classes[k]["catNumber"] == "315":
            currentLine = comp314_315Template.format(**classes[k])
        elif classes[k]["catNumber"] == "388" or classes[k]["catNumber"] == "488":
            currentLine = topicsSectionTemplate.format(**classes[k])
        elif (
            "398" in classes[k]["catNumber"]
            or "499" in classes[k]["catNumber"]
            or "490" in classes[k]["catNumber"]
        ):
            if "398" in classes[k]["catNumber"]:
                if Check398:
                    currentLine = indepStudyTemplate.format("398")
                    Check398 = False
                else:
                    currentLine = 0
            if "490" in classes[k]["catNumber"]:
                if Check499:
                    currentLine = indepStudyTemplate.format("499")
                    Check499 = False
                else:
                    currentLine = 0
            if "499" in classes[k]["catNumber"]:
                if Check490:
                    currentLine = indepStudyTemplate.format("490")
                    Check490 = False
                else:
                    currentLine = 0
        else:
            currentLine = sectionTemplate.format(**classes[k])
        createHeading = False
        if (
            int(classes[k - 1]["catNumber"]) < 400
            and int(classes[k]["catNumber"]) >= 400
        ):
            createHeading = True

        if currentLine != 0:
            if "Lake" in classes[k]["Location"]:
                if createHeading:
                    lcurrentLine = (
                        gradHeadingTemplate.format(season, "Lake Shore")
                        + "\n"
                        + currentLine
                    )
                    lakeRST.write(lcurrentLine + "\n")
                else:
                    lakeRST.write(currentLine + "\n")
            if "Water" in classes[k]["Location"]:
                if createHeading:
                    wcurrentLine = (
                        gradHeadingTemplate.format(season, "Water Tower")
                        + "\n"
                        + currentLine
                    )
                    waterRST.write(wcurrentLine + "\n")
                else:
                    waterRST.write(currentLine + "\n")
            if "Online" in classes[k]["Location"]:
                if createHeading:
                    ocurrentLine = (
                        gradHeadingTemplate.format(season, "Online")
                        + "\n"
                        + currentLine
                    )
                    onlineRST.write(ocurrentLine + "\n")
                else:
                    onlineRST.write(currentLine + "\n")
            if createHeading:
                fcurrentLine = (
                    gradHeadingTemplate.format(season, "Fall") + "\n" + currentLine
                )
                mainRST.write(fcurrentLine + "\n")
            else:
                mainRST.write(currentLine + "\n")

    mainRST.close()
    onlineRST.close()
    lakeRST.close()
    waterRST.close()

def convertDays(days):
    if days == "M":
        return "Monday"
    if days == "Tu":
        return "Tuesday"
    if days == "W":
        return "Wednesday"
    if days == "Th":
        return "Thursday"
    if days == "F":
        return "Friday"
    if days == "Sa":
        return "Saturday"
    if days == "MWF":
        return "Monday, Wednesday, Friday"
    if days == "TuTh":
        return "Tuesday, Thursday"
    if days == "MW":
        return "Monday, Wednesday"

# the first few lines of the csv have the semester and yearself.
# once the parser gets to that row it parses it looking for the year.
def getYear(words, season):
    for i in range(0, len(words)):
        if words[i] == season:
            return words[i + 1]
    return str(now.year)

name = input("Enter file name.")
parseing(name)


from pymongo import MongoClient
import re
import matplotlib.pyplot as plt
import numpy as np

connection = MongoClient("mongodb://localhost:27017/")
db = connection.fake_news                     # name of db (it should be changed to real_news or fake_news
collections = db.collection_names()

# get controversial words from bias lexicon
file1 = open('input/bias-lexicon.txt', 'r')
text = file1.read()

words = text.split("\n")

bias_counter = 0
all_replies = 0

for collection in collections:
    if collection == "25073877":
        print("Skip realDonaldTrump")
        continue
    tweets = db[collection].find().sort("replies_count", -1).batch_size(10)
    for tweet in tweets:
        replies_count = db[collection].find_one({'id': tweet["id"]})["replies_count"]
        if replies_count > 20:
            replies = db[collection].find_one({'id': tweet["id"]})["replies"]
            # count replies with at least one controversy word
            for reply in replies:
                bias_word_exists = False
                reply_text = reply["text"].lower()

                all_replies += 1

                # check if at least one controversy word is included
                for word in words:
                    if re.search(r"\b" + re.escape(word) + r"\b", reply_text):
                        bias_word_exists = True
                if bias_word_exists:
                    bias_counter += 1

file1.close()
print("All replies: ", all_replies)
print("Bias word replies: ", bias_counter)

# get controversial words from Mejova controversy lexicon
file2 = open('input/controversial_words.txt', 'r')
text = file2.read()

words = text.split(", ")

contr_counter = 0

for collection in collections:
    if collection == "25073877":
        print("Skip realDonaldTrump")
        continue
    tweets = db[collection].find().sort("replies_count", -1).batch_size(10)
    for tweet in tweets:
        replies_count = db[collection].find_one({'id': tweet["id"]})["replies_count"]
        if replies_count > 20:
            replies = db[collection].find_one({'id': tweet["id"]})["replies"]
            # count replies with at least one controversy word
            for reply in replies:
                contr_word_exists = False
                reply_text = reply["text"].lower()

                # check if at least one controversy word is included
                for word in words:
                    if re.search(r"\b" + re.escape(word) + r"\b", reply_text):
                        contr_word_exists = True
                if contr_word_exists:
                    contr_counter += 1

file2.close()
print("All replies: ", all_replies)
print("Controversy word replies: ", contr_counter)

# get skepticism words from skepticism lexicon
file3 = open('input/skepticism_words.txt', 'r')
text = file3.read()

words = text.split(", ")

skepticism_counter = 0

for collection in collections:
    if collection == "25073877":
        print("Skip realDonaldTrump")
        continue
    tweets = db[collection].find().sort("replies_count", -1).batch_size(10)
    for tweet in tweets:
        replies_count = db[collection].find_one({'id': tweet["id"]})["replies_count"]
        if replies_count > 20:
            replies = db[collection].find_one({'id': tweet["id"]})["replies"]
            # count replies with at least one controversy word
            for reply in replies:
                skepticism_word_exists = False
                reply_text = reply["text"].lower()

                # check if at least one controversy word is included
                for word in words:
                    if re.search(r"\b" + re.escape(word) + r"\b", reply_text):
                        skepticism_word_exists = True
                if skepticism_word_exists:
                    skepticism_counter += 1

file3.close()
print("All replies: ", all_replies)
print("Controversy word replies: ", skepticism_counter)

# write results to file
file4 = open('input/Fake news - words_results.txt', 'w')
file4.write("%d replies with bias words out of %d replies \n" % (bias_counter,all_replies))
file4.write("%d replies with controversy words out of %d replies \n" % (contr_counter,all_replies))
file4.write("%d replies with skepticism words out of %d replies \n" % (skepticism_counter,all_replies))
file4.close()

# create plot
n_groups = 3
controversy_replies = (bias_counter, contr_counter, skepticism_counter)
no_controversy_replies = (all_replies - bias_counter, all_replies - contr_counter, all_replies - skepticism_counter)

# create plot
fig, ax = plt.subplots()
index = np.arange(n_groups)
bar_width = 0.25
opacity = 0.8

rects1 = plt.bar(index, controversy_replies, bar_width,
                 alpha=opacity,
                 color=(0.2, 0.4, 0.6, 0.9),
                 label='With',
                 zorder=3)

rects2 = plt.bar(index + bar_width, no_controversy_replies, bar_width,
                 alpha=opacity,
                 color=(0.6, 0.3, 0.5, 0.9),
                 label='Without',
                 zorder=3)

plt.legend(loc=4)
plt.ylabel('Number of replies')
plt.title('Fake news - percentage of replies with and without controversial words')
plt.xticks(index + bar_width/2, ('Bias lexicon', 'Controversy lexicon', 'Skepticism lexicon'),
           rotation=45, ha="right")

def autolabel(rects):
    """
    Attach a text label above each bar displaying its height
    """
    for rect in rects:
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width()/2., 0.99*height,
                '%d %%' % int(height*100/all_replies),
                ha='center', va='bottom')

autolabel(rects1)
autolabel(rects2)

ax.yaxis.grid(linestyle=':', zorder=0)
plt.tight_layout()

plt.savefig("plots/Fake news - Replies with controversy words.svg")
plt.show()


number=int(input())
for i in range(number):
    tag=0
    l=[]
    l.append(int(input()))
    l.append(list(map(int,input().split(" "))))
    l.append(list(map(int, input().split(" "))))
    minus=[]
    for j in range(l[0]):
        minus.append(l[2][j]-l[1][j])
        if minus[-1]<0:
            tag=1
            break
    head=0
    tail=0
    for j in minus:
        if j!=0:
            head=j
            break
    for j in range(len(minus)-1,-1,-1):
        if minus[j]!=0:
            tail=j
            break
    for j in range(head,tail):
        if minus[j]!=minus[j+1]:
            tag=1
    if tag==1:
        print("NO")
    else:
        print("YES")


import heapq
import collections
import numpy as np
import matplotlib.pyplot as plt
import os
from sklearn.model_selection import train_test_split
from natsort import natsorted
from PIL import Image

traindata_img = os.listdir(r'001.Black_footed_Albatross_img')
traindata_img2 = natsorted(traindata_img)
print(traindata_img2)
trainlen = []
trainlen6 = []

"""
for i in traindata2:
    datafile = '0/' + i
    num_lines = sum(1 for line in open(datafile))
    if num_lines == 6:
        root, ext = os.path.splitext(i)
        trainlen6.append(root)
        print("aaaaaaaaa")

"""

train = []
count = 0
#  訓練とテストのデータセット用意
for f in traindata_img2:
    #root, ext = os.path.splitext(f)
    img = np.array(Image.open(
        '001.Black_footed_Albatross_img/' + f), dtype='float32')
    img = img / 255
    img = img.astype('float32')
    img = img.transpose(2, 0, 1)
    print(img.shape)
    for i in range(10):
        train.append(img)

    """
    if root in trainlen6:
        for i in range(6):
            train.append(img)
    else:
        for i in range(5):
            train.append(img)
    """
    print(count)
    count = count + 1

    #img = img.transpose(2,0,1)
train = np.array(train,dtype = 'float32')
print(train.shape)
print(train[0])
np.save("birds_img",train)


class PacketAdditionalInfoTestCase(TestCase):

    # Atributos internos de la clase
    timestamp = timezone.now()
    timestamp_insertion = timezone.now()

    def setUp(self):
        ip_source = Ips.objects.create(Ip="127.0.0.2", Hostname="localhost", Tag="localhost")
        ip_dest = Ips.objects.create(Ip="127.0.0.3", Hostname="localhost", Tag="localhost")
        port_source = Ports.objects.create(Tag="ftp")
        port_dest = Ports.objects.create(Tag="ssh")
        mac_source = Macs.objects.create(
            MAC="00:00:00:00:00:00:00:00:00:00:00:00:08:00",
            TAG="Mac local1")
        mac_dest = Macs.objects.create(
            MAC="00:00:00:00:00:00:00:00:00:00:00:00:08:00",
            TAG="Mac local2")
        log_sources = LogSources.objects.create(
            Description="Firewall of gnu/linux kernel",
            Type="Iptables",
            Model="iptables v1.4.21",
            Active=1,
            Software_Class="Firewall",
            Path="iptables",
        )
        event = Events.objects.create(
            Timestamp=self.timestamp,
            Timestamp_Insertion=self.timestamp_insertion,
            ID_Source=log_sources,
            Comment="Iptables Events",
        )
        packet = PacketEventsInformation.objects.create(
            ID_IP_Source=ip_source,
            ID_IP_Dest=ip_dest,
            ID_Source_Port=port_source,
            ID_Dest_Port=port_dest,
            Protocol="ICMP",
            ID_Source_MAC=mac_source,
            ID_Dest_MAC=mac_dest,
            RAW_Info="LOG RAW INFO",
            TAG="Connection ICMP",
            id=event
        )
        tag = Tags.objects.create(Description="Packet ID", Tag="ID")
        PacketAdditionalInfo.objects.create(
            ID_Tag=tag,
            ID_Packet_Events=packet,
            Value="32731 DF"
        )


from datetime import datetime
import json
import uuid
import requests  # http://docs.python-requests.org/en/master/
from django.conf import settings
from tworaven_apps.utils.view_helper import \
    (get_request_body_as_json,
     get_json_error,
     get_json_success)
from tworaven_apps.utils.basic_response import (ok_resp,
                                                err_resp,
                                                err_resp_with_data)

class GetDataSetFileInfo(object):

    def __init__(self):
        """ to get the JSON representation of the dataset"""
        self.status_code = None
        self.res = None
        dataverse_server = settings.DATAVERSE_SERVER  # no trailing slash
        persistentId = settings.DATASET_PERSISTENT_ID  # doi or hdl of the dataset

        # Get JSON Representation of a Dataset
        publish_url = '%s/api/datasets/:persistentId/?persistentId=%s'% (dataverse_server,
                                                                        persistentId)

        print('-' * 40)
        print('making request: %s' % publish_url)
        r = requests.get(publish_url)

        # -------------------
        # Print the response
        # -------------------
        print('-' * 40)
        # print(r.json())
        # print(r.status_code)
        self.status_code = r.status_code
        if r.status_code == 200:
            self.res = r.json()
        else:
            self.res = None

    def return_status(self):
        if self.res is None:
            return err_resp(self.res)
        else:
            # print("The response we are looking for ", self.res)
            return ok_resp(self.res)

    def get_dataset_id(self):
        """ return dataset Id from the Info"""
        if self.status_code == 200:
            dataset_id = self.res['data']['id']
            print("dataset ID ", dataset_id)
            return ok_resp(dataset_id)
        else:
            return err_resp(self.res)

    def get_version_id(self):
        """ return version ID"""
        if self.status_code == 200:
            # print("The dataset file info :  ", self.res)
            version_id = self.res['data']['latestVersion']['id']
            # print("version ID", version_id)
            return ok_resp(version_id)
        else:
            return err_resp(self.res)

    def get_version_number(self):
        """return latest version number"""
        if self.status_code == 200:
            # print("The dataset file info for version number :  ", self.res)
            version_number = self.res['data']['latestVersion']['versionNumber']
            print("version number", version_number)
            return ok_resp(version_number)
        else:
            return err_resp(self.res)


import os

path = '/Users/zhangsicai/Desktop/Taiger/Projects/Tender/SkillsCSVs/'
files = os.listdir(path)
s = []
for file in files:
    with open(path+"/"+file, encoding='utf-8') as f:
        filecontent=f.read()
        s.append(filecontent)

#for scontent in s:
 #   skill = scontent.split('|')
  #  s1.append(skill)
s1=str(s)
print(str(s))

skillpath = '/Users/zhangsicai/Desktop/Extractedskills/'
matchedskillsfiles = os.listdir(skillpath)
matchedskills = []

allfiles_allskill_count =0
allfiles_allmatch_count = 0
file_count = 0

for matchedskillsfile in matchedskillsfiles:
    with open(skillpath+"/"+matchedskillsfile, "r") as f:
        popdatas = json.load(f)
        #popdatas['matchedSkills']
        allskills = popdatas['matchedSkills']

        allskill_count = len(allskills)
        allfiles_allskill_count += allskill_count

        ##count number of matched skills if it can be found in the database
        count_match = 0
        for allskill in allskills:
            if allskill in s1:
                count_match +=1
                #print(allskill + 'is in')

            else:
                print(allskill + "is not in")
        allfiles_allmatch_count += count_match
        file_count += 1
        try:
            fileaccuracy = count_match / allskill_count
        except ZeroDivisionError:
            print('matched skills set list is empty')

        print('file'+str(file_count)+'matched skills accuracy is:', '%.2f%%' % (fileaccuracy*100))

overallaccuracy = allfiles_allmatch_count/allfiles_allskill_count
print('overall accuracy is:', '%.2f%%' % (overallaccuracy*100))