#!/usr/bin/env python
# -*- coding: utf-8 -*-

from collections import defaultdict
import os
import math
import struct
from numpy import corrcoef as wsp_r
from numpy import add
from collections import Counter

def wczytaj_fh(arg):
    '''
    Funkcja wczytujaca dane pomiarow dendrochronologicznych zapisanych w plikach
    fh. Przyjmuje atrybuty:
    arg - krotka z pelnymi sciezkami do pliku
    Zwraca słownik z wczytanymi danymi
    '''

    partOfSample = 0  # w ktorej czesci proby jestesmy:
                        # 0-pierwsza proba, 1-HEADER, 2-DATA
    # definicje linijki od ktorej zaczynaja sie pomiary
    dataTypeLine = {
        "DATA:HalfChrono": 2, "DATA:Tree": 1, "DATA:Chrono": 3,
        "DATA:Quadro": 3, "DATA:Single": 1, "DATA:Double": 2}

    MeasureType = ''  # Typ zapisu pomiarow w pliku, leg patrz dataTypeLine
    partOfSample = 0    # W ktorej czesci sekwencji jestesmy:
                        #  0 - poza sekwencja;
                        #  1 - odczyt naglowka;
                        #  2 - odczyt pomiarow;
    measTab = []  # tablica z pomiarami sekwencji
    seqDict = {}  # slownik z sekwencjami wczytanymi z pliku
    beginTrig = 0  # trigger do pomijania pierwszej linijki z def

    for fn in arg:
        plik = open(fn, 'r')

        for l in plik:
            l = l.rstrip('\r\n')

            if len(l) > 0:
                # w jakiej czesci pliku jestesmy i w jaki tryb odczytu
                # powinnismy wejsc
                if l == 'HEADER:' and (partOfSample == 0 or partOfSample == 1):
                    # '0' nowa proba - zerujemy wszystkie dane
                    # '1' proba bez pomiarow - pomijam wczytane metadane
                    sample = Sequence()
                    sample.source = fn  # dopisz nazwe pliku skad pochodzi
                    partOfSample = 1
                    beginTrig = 1

                if l == 'HEADER:' and partOfSample == 2:
                    # zaczyna sie nowa proba - stara dorzucamy do slownika

                    partOfSample = 1
                    sample.uaktualnijPom(measTab)  # dopisujemy pomiary
                    measTab = []  # zerujemy tablice pomiarow
                    if sample.KeyCode() not in seqDict.keys():
                        seqDict[sample.KeyCode()] = sample
                    else:
                        unikat = False
                        i = 0
                        while unikat is False:
                            i += 1
                            nowy_KeyCode = sample.KeyCode() + "(" + str(i) + ")"
                            if nowy_KeyCode not in seqDict.keys():
                                # uaktualnij nazwe proby i wpisz do slownika
                                print("Nadano nowa nazwe zdublowanej probie: " +
                                      nowy_KeyCode)
                                sample.wpiszMetadana("KeyCode", nowy_KeyCode)
                                seqDict[str(sample.KeyCode())] = sample
                                unikat = True

                    sample = Sequence()
                    sample.source = fn  # dopisz nazwe pliku skad pochodzi
                    beginTrig = 1

                if l in dataTypeLine.keys() and partOfSample == 1:
                    # wchodzimy w odczyt pomiarow
                    MeasureType = dataTypeLine[l]  # typ zapisu danych
                    measTab = []  # tablica z pomiarami
                    partOfSample = 2  # tryb odczytu danych
                    beginTrig = 1

                #  tryby odczytu pliku
                if partOfSample == 1 and beginTrig == 0:
                    # tryb wczytywania metadanych do naglowka
                    lin = l.split('=')
                    if unicode(lin[0]) == "DateEnd":
                        sample.ustawDateEnd(int(lin[1]))
                    else:
                        sample.wpiszMetadana(str(lin[0]).encode('cp1250'),
                                             str(lin[1]).encode('cp1250'))

                if partOfSample == 2 and beginTrig == 0:
                    # Jezeli mniej niz 5 znakow to znaczy ze jeden odczyt w l
                    if len(l) < 5:
                        if int(l) != 0 and l.isdigit():
                            measTab.append(int(l))
                    else:
                        ml = l.split(' ')
                        ml = filter(lambda a: a != '', ml)

                        if MeasureType == 1:
                            for measure in ml:
                                if int(measure) != 0 and measure.isdigit():
                                    measTab.append(int(measure))

                        elif MeasureType == 2:
                            for m in xrange(len(ml)):
                                if m % 2 == 0 and int(ml[m]) != 0 and ml[m].isdigit():
                                    measTab.append(int(ml[m]))

                        elif MeasureType == 3:
                            for m in xrange(len(ml)):
                                if m % 4 == 0 and int(ml[m]) != 0 and ml[m].isdigit():
                                    measTab.append(int(ml[m]))
                beginTrig = 0
        plik.close()
        # seqDict[str(sample.KeyCode())] = sample

    sample.uaktualnijPom(measTab)  # dopisujemy pomiary
    seqDict[str(sample.KeyCode())] = sample

    return seqDict

def zapisz_fh(fn, arg):
    '''
    funkcja zapisująca ciagi przekazane slowniku jako obiekty Sequence do pliku
    o nazwie podanej w fn
    '''

    wypis = ''
    for ciag in arg.values():
        wypis += 'HEADER:\n'
        wypis += ciag.wypiszMetadaneAll()
        wypis += 'DATA:Tree\n'

        # wypisujemy pomiary w odpowiednim formatowaniu do pliku
        for i, pom in enumerate(ciag.wypiszPomiary()):
            wypis += (6 - len(str(pom))) * " " + str(pom)
            if i > 0 and (i + 1) % 10 == 0:
                wypis += '\n'

        # uzupełniamy zerami ostatni rząd pomiarów o ile ich liczba nie jest
        # wielokrotnością dziesięciu.
        if len(ciag.wypiszPomiary()) % 10 != 0:
            wypis += (10 - len(ciag.wypiszPomiary()) % 10) * '     0'
            wypis += '\n'

        p = open(fn, 'w')
        p.write(wypis.encode('cp1250'))
        p.close()

def wczytaj_r(fn):
    '''
    Funkcja wczytuje pomiary zapisane w formie binarnej z pliku zapisanego w
    standardzie Cracow [avr, avs, r0, av0, itd.]
    jednorazowo do funkcji przekazuje sie sciezke dostepu do pliku binarnego,
    ktory jest przetwarzany i zwracany jest słownik z pomiarami i nazwa proby.
    W sekwencji nie bedzie zadnych innych metadanych - nie sa obslugiwane przez
    ten format
    '''

    plik = open(fn, "rb")
    proba = Sequence()
    kc = str(os.path.basename(fn).split('.')[0]).encode('cp1250')  # sprawdzic czy dziala
    proba.wpiszMetadana("KeyCode", kc)

    # ile bajtow sklada sie na plik
    stat = os.stat(fn)
    dl = int(stat.st_size)

    i = 0
    tablica_pom = []  # tablica z pomiarami
    while i < (dl/2):
        plik.seek(i*2)
        t = struct.unpack('bb', plik.read(2))

        t = (t[0]*100)+t[1]

        if i > 2:
            tablica_pom.append(t)
        elif i == 1:
            # zapisujemy poczatek bielu
            if t != 0:
                pocz = t
        elif i == 2:
            # dodajemy biel do proby
            if t != 0:
                t = t-pocz
                proba.wpiszMetadana("SapWood", t)
        i += 1

    return proba

def zapisz_r(fn, arg):
    """
    Funkcja zapisuje podany obiekt Sequence w postaci pliku binarnego w formacie
    Cracow [avr, avs, r0, av0, itd.]
    """
    # zmienne funkcji
    ciag = ""
    i = 0

    """
    Tutaj niestety coś jest nie tak, trzeba pogrzebac w opisie specyfikacji
    pliku r, narazie biel nie bedzie zapisywany
    if a[3]!="---":
        koniec_b=len(a)-31
        pocz_b=koniec_b-int(a[3])+1
        ciag=struct.pack("bbbbbb", (pocz_b/100),(pocz_b-((pocz_b/100)*100)),
                         (koniec_b/100),(koniec_b-(koniec_b/100)*100),0,0)
    else:
    """
    # przygotowujemy naglowek pliku binarnego, narazie biel nie jest zapisywany
    ciag = struct.pack("bbbbbb", 0, 0, 0, 0, 0, 0)

    # przypisujemy tablice z pomiarami do zmiennej
    a = arg.wypiszPomiary()

    while i < len(a):
        skladowa_1 = int(a[i]) / 100
        skladowa_2 = int(a[i]) - (skladowa_1 * 100)
        ciag += struct.pack('bb', skladowa_1, skladowa_2)
        i += 1

    plik = open(fn, 'wb')
    plik.write(ciag)
    plik.close()

def wczytaj_rwl(fn):
    """
    Funkcja wczytująca pomiary z plikow rwl do standardowej struktury Sequence
    Wczytane dane podobnie jak Cracow format nie będą przechowywać żadnych
    metadanych.
    obslugiwane sa pliki rwl zapisane w formacie tucson.
    """

    pierwsza = 0  # trigger dla pierwszej proby
    proba = Sequence({})  # obiekt proby
    measTab = []  # tablica z pomiarami dla proby
    nazwa = ''  # zmienna przetrzymujaca aktualna nazwe proby ktora wczytujemy
    seqDict = {}  # slownik z wczytanymi sekwencjami zwracany na koncu

    plik = open(fn, 'r')

    for line in plik:
        line = line.rstrip('\n')

        # filtrujemy wszystkie wartości None
        wartosc = filter(None, line.split(" "))
        print(wartosc)

        if len(wartosc[0]) < 9:  # jeżeli nazwa sekwencji jest ponizej 8 znakow
            ww = wartosc    # zakladamy ze wszystko ladnie sie podzielilo
        else:  # jesli nie jest to dzielimy na twardo po znakach w lacuchu
            ww = []
            ww.append(line[:8])  # nazwa proby
            ww.append(str(int(line[8:12])))  # data pocz
            i = 12
            while i < len(line):
                ww.append(str(int(line[i:i+6])))
                i = i+6

        if ww[0] != nazwa and len(ww) > 0:
            #  przy pierwszej probie nic nie jest jeszcze wczytane
            if pierwsza == 1:
                proba.uaktualnijPom(measTab[:-1])  # wpisujemy pomiary bez ostatniego 999
                seqDict[str(proba.KeyCode())] = proba
            pierwsza = 1

            # ustawiamy nowa probe
            proba = Sequence({"KeyCode": ww[0], "DateBegin": ww[1]})
            measTab = []
            measTab += ww[2:]  # dodajemy pomiary z aktualnego wiersza
        else:
            measTab += ww[2:]  # dodajemy pomiary z nastepnych wierszow do tej pr

    proba.uaktualnijPom(measTab[:-1])  # wpisujemy pomiary bez ostatniego 999
    seqDict[str(proba.KeyCode())] = proba

    plik.close()

    return seqDict

def zapisz_rwl(fn, arg):
    '''
    Funkcja zapisuje w podanej nazwie pliku, wszystkie przekazane w slowniku
    obiekty sekwencji w formataowaniu zgodnym z rwl (tucson)
    '''

    zapis = ''  # ciag znakow do zapisu do pliku
    powt = set()  # zbior z nazwami prob - ulatwienie sprawdzania duplikatow

    for a in arg.values():
        # ustalamy pierwsza kolumne z kodem proby
        if len(a.KeyCode()) < 9:
            # jezeli dl nazwy jest krotsza od 9 - dokladamy brakujace spacje
            nazwa = a.KeyCode() + ((8 - len(a.KeyCode())) * " ")
            powt.add(nazwa)  # dodajemy nazwe do zbioru unikatow
        else:
            # jezeli nazwa jest dluzsza od 8 - skracamy nazwe
            n = a.KeyCode()
            while True:
                if n[:9] not in powt:
                    nazwa = n[:9]
                    break
                else:
                    j = 0
                    while True:
                        if n[:(8 - j // 10)] + str(j) not in powt:
                            nazwa = n[:(8 - j // 10)] + str(j)
                            break
                        else:
                            j = j + 1
                # dodaj nazwe do zbioru unikatow
                powt.add(nazwa)
                break

        pom = a.wypiszPomiary()
        for i, pomiar in enumerate(pom):
            if i == 0:
                zapis += (nazwa + ((4 - len(str(a.DateBegin()))) * ' ') +
                          str(a.DateBegin()) + (6 - len(str(pomiar))) * ' ' +
                          str(pomiar))
            else:
                if (int(a.DateBegin()) + i) % 10 == 0:

                    zapis += ("\n" + nazwa +
                              ((4-len(str((a.DateBegin()/10 + int(i / 10) + 1) * 10))) * " ") +
                              str((a.DateBegin() / 10 + int(i / 10) + 1) * 10) +
                              ((6 - len(str(pomiar))) * " ") + str(pomiar))
                else:
                    zapis = zapis + ((6 - len(str(pomiar))) * " ") + str(pomiar)

        # dopisujemy 999 jako koniec sekwencji
        if a.DateEnd() % 10 == 9:
            zapis += "\n" + nazwa + ((4 - len(str(a.DateEnd()))) *
                                     " ") + str(a.DateEnd() + 1) + "   999\n"
        else:
            zapis += "   999\n"

    plik = open(fn, "w")
    plik.write(str(zapis).encode('cp1250'))
    plik.close
    zapis = ""

def zapisz_txt(fn, a):
    zapis = unicode(a.KeyCode()) + "\n"

    for i in a.wypiszPomiary():
        zapis = zapis + str(i) + "\n"

    plik = open(fn, "w")
    # print zapis
    plik.write(zapis.encode('cp1250'))
    plik.close

def przygotujDoWykresu(a):
    a = math.log((int(a)))
    return a

def wsp_T(n, r):
    # n-dlugosc porownywanych sekwencji
    # r-wsp r dla danej sekwencji

    t = (float(r)*math.sqrt(float(n)-2))/(math.sqrt(1-(float(r)*float(r))))
    return t

def cdi(wartosc_glk, nakladka, dl_proby, TBP, TH):
    t = (float(TBP)+float(TH))/2
    c = (((float(wartosc_glk)-50)+(50*math.sqrt(float(nakladka)/float(dl_proby))))*t)/10
    return c

def korelacja(sample, reference, ile_kor):
    # a-proba
    # b-referencja
    wyniki = []  # lista z wynikiami
    pocz_a = 0
    pocz_b = 0
    kon_a = 0
    kon_b = 0
    ovl = 0
    # tablice dla poszczegolnych standaryzacji
    standBPa = [0, 0]
    standBPb = [0, 0]
    standHa = []
    standHb = []
    glka = []
    glkb = []
    # przepisujemy dane sekwencji do odpowiedniej struktury
    a = [sample.KeyCode(), sample.wypiszPomiary()]
    b = [reference.KeyCode(), reference.wypiszPomiary()]

    max_nakladka = int(len(a[1]))  # max nakladke ustawiamy na dlugosc proby

    # jezeli referencja jest krotsza to wtedy jej dlugosc jest max nakladka
    if int(len(b[1])) < int(len(a[1])):
        max_nakladka = int(len(b[1]))

    # liczymy standaryzacje
    dluzsza = len(b[1])
    if len(a[1]) > dluzsza:
        dluzsza = len(a[1])

    for i in xrange(dluzsza):
        # standaryzacja BP
        if i < len(a[1]) - 2 and i > 1:
            standBPa.append((math.log(
                                 (5 * float(a[1][i])) /
                                 (float(a[1][i - 2]) + float(a[1][i - 1]) +
                                  float(a[1][i]) + float(a[1][i + 1]) +
                                  float(a[1][i + 2])))))
        if i < len(b[1]) - 2 and i > 1:
            standBPb.append((math.log(
                                 (5 * float(b[1][i])) /
                                 (float(b[1][i - 2]) + float(b[1][i - 1]) +
                                  float(b[1][i]) + float(b[1][i + 1]) +
                                  float(b[1][i + 2])))))

        # standaryzacja H
        if i < (len(a[1]) - 1):
            standHa.append((math.log((float(a[1][i]) / float(a[1][i + 1])), 10)))
        if i < (len(b[1]) - 1):
            standHb.append((math.log((float(b[1][i]) / float(b[1][i + 1])), 10)))

        # GLK
        if i < len(a[1]) - 1:
            if (float(a[1][i + 1]) - float(a[1][i]) >= 0):
                glka.append(1)
            else:
                glka.append(0)
        if i < len(b[1]) - 1:
            if (float(b[1][i + 1]) - float(b[1][i]) >= 0):
                glkb.append(1)
            else:
                glkb.append(0)

    i = 25 - len(a[1])
    while i < len(b[1]) - 25:
        # przypadek pierwszy: proba wystaje z lewej strony i jest krotsza od
        # standardu
        if i <= 0 and (len(a[1]) + i - 1) < len(b[1]):
            pocz_b = 0
            kon_b = i + len(a[1])
            pocz_a = len(a[1]) - (len(a[1]) + i)
            kon_a = len(a[1])
            ovl = i + len(a[1])
            glk_sklad = list(add(glka[pocz_a:kon_a - 1], glkb[pocz_b:kon_b - 1]))
            # liczymy zgodnosc miedzy dwoma seriami
            glk = Counter(glk_sklad)[0] + Counter(glk_sklad)[2]

            # opc = 1
            wynik_p = korelacja_poj(
                a[1][pocz_a: kon_a],
                b[1][pocz_b: kon_b],
                max_nakladka, standBPa[pocz_a + 2: kon_a - 2],
                standBPb[pocz_b + 2: kon_b - 2],
                standHa[pocz_a: kon_a - 1],
                standHb[pocz_b: kon_b - 1],
                glk)

        # przypadek drugi: proba wystaje z lewej strony i jest dluzsza od
        # standardu
        if i <= 0 and (len(a[1]) + i - 1) >= len(b[1]):
            pocz_b = 0
            kon_b = len(b[1])
            pocz_a = len(a[1]) - (len(a[1]) + i)
            kon_a = pocz_a + len(b[1])
            ovl = len(b[1])
            glk_sklad = list(add(glka[pocz_a:kon_a - 1], glkb[pocz_b:kon_b - 1]))
            # liczymy zgodnosc miedzy dwoma seriami
            # glk=glk_sklad.count(0)+glk_sklad.count(2)
            glk = Counter(glk_sklad)[0] + Counter(glk_sklad)[2]

            # opc = 2
            wynik_p = korelacja_poj(
                a[1][pocz_a: kon_a],
                b[1][pocz_b: kon_b],
                max_nakladka, standBPa[pocz_a + 2: kon_a - 2],
                standBPb[pocz_b + 2: kon_b - 2],
                standHa[pocz_a: kon_a - 1],
                standHb[pocz_b: kon_b - 1],
                glk)

        # przypadek trzeci: proba poczatek ma w standardzie i jest krotsza od
        # standardu
        if i > 0 and (len(a[1]) + i - 1) < len(b[1]):
            pocz_b = i
            kon_b = len(a[1]) + i
            pocz_a = 0
            kon_a = len(b[1])
            ovl = len(a[1]) - 1
            glk_sklad = list(add(glka[pocz_a:kon_a - 1], glkb[pocz_b:kon_b - 1]))
            # liczymy zgodnosc miedzy dwoma seriami
            # glk=glk_sklad.count(0)+glk_sklad.count(2)
            glk = Counter(glk_sklad)[0] + Counter(glk_sklad)[2]

            # opc = 3
            wynik_p = korelacja_poj(
                a[1][pocz_a: kon_a],
                b[1][pocz_b: kon_b],
                max_nakladka, standBPa[pocz_a + 2: kon_a - 2],
                standBPb[pocz_b + 2: kon_b - 2],
                standHa[pocz_a: kon_a - 1],
                standHb[pocz_b: kon_b - 1],
                glk)

        # przypadek czwarty: proba poczatek ma w standardzie i jest dluzsza od
        # standardu
        if i > 0 and (len(a[1]) + i - 1) >= len(b[1]):
            pocz_b = i
            kon_b = len(b[1])
            pocz_a = 0
            kon_a = len(b[1]) - i
            ovl = len(b[1]) - i
            glk_sklad = list(add(glka[pocz_a:kon_a - 1], glkb[pocz_b:kon_b - 1]))
            # liczymy zgodnosc miedzy dwoma seriami
            # glk=glk_sklad.count(0)+glk_sklad.count(2)
            glk = Counter(glk_sklad)[0] + Counter(glk_sklad)[2]

            # opc = 4
            wynik_p = korelacja_poj(
                a[1][pocz_a: kon_a],
                b[1][pocz_b: kon_b],
                max_nakladka, standBPa[pocz_a + 2: kon_a - 2],
                standBPb[pocz_b + 2: kon_b - 2],
                standHa[pocz_a: kon_a - 1],
                standHb[pocz_b: kon_b - 1],
                glk)

        # przesuniecie wzgledem referencji
        wynik_p.append(i)  # poczatek
        wynik_p.append(int(ovl))  # ovl

        # dopisujemy nazwy:
        wynik_p.append(str(a[0]))  # proby
        wynik_p.append(str(b[0]))  # referencji

        # print len(a[1][pocz_a:kon_a]), len(b[1][pocz_b:kon_b]), opc, i,
        # wynik_p
        j = 0
        if len(wyniki) == 0:
            wyniki.append(wynik_p)

        else:
            while j < len(wyniki) and j < ile_kor:
                if wyniki[j][6] < wynik_p[6]:
                    wyniki.insert(j, wynik_p)
                    break
                j += 1
        if len(wyniki) > ile_kor:
            wyniki.pop()
        i += 1

    return wyniki

def korelacja_poj(a, b, dl_a, standBPa, standBPb, standHa, standHb, glk):
    # a,b musza miec taka sama dlugosc
    '''
    standBPa=[]
    standBPb=[]
    standHa=[]
    standHb=[]
    glk=0
    '''
    wynik = []
    # zmieniamy pomiary ze str na float
    a = map(float, a)
    b = map(float, b)

    # obliczanie wyniku korelacji:
    # wsp r dla czystych pomiarow:
    wynik.append(round(float(wsp_r(a, b)[1][:1]), 2))

    # wsp T dla stand BP
    wynik.append(
        round(
            wsp_T(
                len(standBPa), float(wsp_r(standBPa, standBPb)[1][:1])), 1))

    # wsp T dla stand H
    wynik.append(
        round(
            wsp_T(
                len(standHa), float(wsp_r(standHa, standHb)[1][:1])), 1))

    # wsp T bez standaryzacji
    wynik.append(round(wsp_T(len(a), float(wsp_r(a, b)[1][:1])), 1))

    if glk != 0:
        # GLK
        wynik.append(round((float(glk)/(len(a)-1))*100))
        # GSL
        if ((float(glk)/(len(a)-1))*100 >= (50+(82.7/math.sqrt(len(a))) and
           (float(glk)/(len(a)-1))*100 < (50+(116.3/math.sqrt(len(a)))))):
            wynik.append("*")
        elif ((float(glk)/(len(a)-1))*100 >= (50+(116.3/math.sqrt(len(a)))) and
              (float(glk)/(len(a)-1))*100 < (50+(154/math.sqrt(len(a))))):
            wynik.append("**")
        elif (float(glk)/(len(a)-1))*100 >= (50+(154/math.sqrt(len(a)))):
            wynik.append("***")
        else:
            wynik.append("")

    else:
        wynik.append(0.00001)
        wynik.append("")

    # CDI
    wynik.append(int(cdi(wynik[4], len(a), dl_a, wynik[1], wynik[2])))

    return wynik

def koreluj_polozenie(a, b):
    '''
    funkcja oblicza wskazniki korelacji dla dwoch promieni o ustalonej odgornie
    pozycji i zwraca tablice wynikow dla danej sytuacji.
    przyjmuje jako argumenty 2 tablice jako obkiety Sequence
    '''

    przes = 0  # offset w probie o starszej dacie poczatku
    tab = [[], []]  # tablica z danymi do porownania
    # tablice do obliczenia standaryzacji
    standBPa = [0, 0]
    standBPb = [0, 0]
    standHa = []
    standHb = []
    glka = []
    glkb = []
    # Sprawdzamy ktora proba ma pozniejsza date poczatku i to ona bedzie data
    # poczatku porownywanej sekwencji, dodatkowo ustalamy dlugosc porownywanej
    # sekwencji
    mlodsza = a
    starsza = b
    if a.DateBegin() < b.DateBegin():  # proba a jest starsza
        mlodsza = b
        starsza = a

    # ustalamy ktora proba zostala scieta pozniej
    data_kon = int(mlodsza.DateEnd())
    if data_kon > int(starsza.DateEnd()):
        data_kon = int(starsza.DateEnd())

    # ustalamy jakie przesuniecie miedzy probami na poczatku
    # ustalamy ile lat trzeba wyciac z przodu proby a
    przes = abs(starsza.DateBegin() - mlodsza.DateBegin())

    # przygotowujemy tablice z danymi do liczenia statystk
    i = 0
    while i < data_kon - mlodsza.DateBegin() + 1:
        # mlodsza proba - z przesunieciem
        tab[0].append(mlodsza.wypiszPomiary()[i])
        tab[1].append(starsza.wypiszPomiary()[i+przes])
        i += 1
    if len(tab[0]) < 30:
        wynik = ['xxx', 'xxx', 'xxx', 'xxx', 'xxx', 'xxx', 'xxx']
        return wynik
    else:
        for i in xrange(len(tab[0])):
            # standaryzacja BP
            if i < len(tab[1])-2 and i > 1:
                standBPa.append((math.log(
                    (5 * float(tab[0][i])) /
                    (float(tab[0][i - 2]) + float(tab[0][i - 1]) +
                     float(tab[0][i]) + float(tab[0][i + 1]) +
                     float(tab[0][i + 2])))))
            if i < len(tab[1])-2 and i > 1:
                standBPb.append((math.log(
                    (5 * float(tab[1][i])) /
                    (float(tab[1][i - 2]) + float(tab[1][i - 1]) +
                    float(tab[1][i]) + float(tab[1][i + 1]) +
                    float(tab[1][i + 2])))))

            # standaryzacja H
            if i < (len(tab[0])-2):
                standHa.append(
                    (math.log((float(tab[0][i]) / float(tab[0][i+1])), 10)))
            if i < (len(tab[1])-2):
                standHb.append(
                    (math.log((float(tab[1][i]) / float(tab[1][i+1])), 10)))

            # GLK
            if i < len(tab[0])-1:
                if (float(tab[0][i+1])-float(tab[0][i]) >= 0):
                    glka.append(1)
                else:
                    glka.append(0)
            if i < len(tab[1])-1:
                if (float(tab[1][i+1])-float(tab[1][i]) >= 0):
                    glkb.append(1)
                else:
                    glkb.append(0)

        glk = list(add(glka[:], glkb[:]))
        glk = Counter(glk)[0]+Counter(glk)[2]

        wynik = korelacja_poj(tab[0],
                              tab[1],
                              len(tab[0]),
                              standBPa[2:],
                              standBPb[2:],
                              standHa[1:],
                              standHb[1:],
                              glk)
        return wynik

class Sequence:
    '''
    klasa przetrzymujaca dane naglowkowe i pomiary dla jednego ciagu
    dendrochronologicznego w postaci slownika.
    Przy inicjowaniu mozna podac slownik z danymi, nie jest to konieczne.
    W przypadku braku jakichkolwiek danych zainicjowany zostanie pusty slownik
    z mozliwoscia dodawania danych
    '''

    def __init__(self, tablica=None):

        # importujemy dane do struktury
        if isinstance(tablica, dict):
            self.sample = tablica
        else:
            self.sample = {"KeyCode": "Unknown",
                           "DateBegin": 1,
                           "measurements": []}

        if "DateBegin" not in self.sample.keys():
            self.sample["DateBegin"] = 1

        self.forb_keys = {'measurements', 'DateEnd', 'Length'}
        self._edytowany = 0  # czy proba byla edytowana: 0-Nie, 1-Tak

    # metody zwracajace warotsci klasy
    def KeyCode(self):
        if 'KeyCode' in self.sample.keys():
            return self.sample['KeyCode']
        else:
            print("Brak metadanej: KeyCode")

    def DateBegin(self):
        if "DateEnd" in self.sample.keys() and "measurements" in self.sample.keys():
            if len(self.sample['measurements']) > 0:
                self.ustawDateEnd(self.sample['DateEnd'])
            else:
                print("No measurements in sample, DateBegin set to 1")

        return int(self.sample['DateBegin'])

    def DateEnd(self):
        return int(self.sample['DateBegin']) - 1 + len(self.sample['measurements'])

    def Length(self):
        return len(self.sample['measurements'])

    def SapWood(self):
        # nie sprawdzamy typu to zostalo zrobione przy wczytaniu lub edycji
        if 'SapWood' in self.sample.keys():
            return int(self.sample['SapWood'])
        else:
            return("")

    def Bark(self):
        if 'Bark' in self.sample.keys():
            return self.sample['Bark']
        else:
            return("")

    def Podkor(self):
        if 'Podkor' in self.sample.keys():
            return self.sample['Podkor']
        else:
            return("")

    def wypiszPomiary(self):
        return self.sample['measurements']

    def wypiszMetadana(self, nagl):
        # zwracamy wartosc dla podanej definicji pola o ile istnieje
        if isinstance(nagl, str) and nagl in self.sample.keys():
            return self.sample[nagl]

    def wypiszMetadaneAll(self):
        # zwraca cala tablice z danymi opisowymi
        wypis = ''
        for k in self.sample.keys():
            if k is not 'measurements':
                wypis += unicode(k) + "=" + unicode(self.sample[k]) + "\n"
        wypis += "DateEnd=" + unicode(self.DateEnd()) + "\n"
        wypis += "Length=" + unicode(self.Length()) + "\n"
        return wypis

    # metody aktualizujace
    def wpiszMetadana(self, nagl, wartosc):
        # aktualizujemy wartosc w metadanych, sprawdzenie typu danych w
        # zaleznosci do czego bedzie uzywane pozniej, ta metoda nie wolno
        # zmieniac pomiarow (pola measurements)
        if nagl not in self.forb_keys:
            wartosc = wartosc
            self.sample[nagl] = wartosc
            self._edytowany = 1

        if nagl == "DateEnd":
            self.ustawDateEnd(wartosc)

    def ustawDateBegin(self, wart):
        # ustawiamy date poczatku, przyjmuje tylko wartosci int
        try:
            self.sample['DateBegin'] = int(wart)
            self._edytowany = 1
        except:
            print("Only integer values are respected as DateBegin")

    def ustawDateEnd(self, wart):
        # ustawiamy date konca, przyjmuje tylko wartosci int
        try:
            # uaktualniamy date konca
            if 'measurements' in self.sample.keys():
                self.sample['DateBegin'] = (
                    int(wart) + 1 - len(self.sample['measurements']))
                self._edytowany = 1
                if 'DateEnd' in self.sample.keys():
                    del self.sample['DateEnd']
            else:
                self.sample['DateEnd'] = int(wart)
        except:
            print("Only integer values are respected as DateEnd")

    def uaktualnijPom(self, wart):
        # nadpisujemy tabele pomiarow i aktualizujemy dlugosc ciagu
        self.sample['measurements'] = wart
        self._edytowany = 1

    def dodajPomiar(self, wart):
        if str(wart).isdigit():
            measTab = self.sample['measurements']
            measTab.append(wart)
            self.sample['measurements'] = measTab
        else:
            print("Only numerical values are resepected as measurements!")

    def usunOstatniPomiar(self):
        measTab = self.sample['measurements']
        measTab.pop()
        self.sample['measurements'] = measTab

class recursivedefaultdict(defaultdict):
    def __init__(self):
        self.default_factory = type(self)

class Baza:
    '''
    klasa grupujaca wszystkie proby pochodzace z jednego drzewa. Ulozenie zalezy
    pol grupujacych, dostepne sa 2 mozliwosci - sekcje i ilosc pomiarow w sekcji
    Przy definiowaniu trzeba podac nazwy pol zgodne z defpol ktore definiuja kod
    drzewa, sekcji i kierunku(numery walka)
    '''

    def __init__(self):
        # kontener przechowujacy proby dendro
        self.baza = recursivedefaultdict()
        self.stos = ''
        self.drzewo = ''

    def __getitem__(self, it):
        return self.baza[it]

    def dodajCiagi(self, ciagi, stos):
        """
        metoda przyjmuje slownik z sekwencjami, sprawdza czy w podanym stosie nie
        wystepuja powtorzenia nazwy i dodaje sekwencje do stosu.
        """
        for ciag in ciagi.values():
            if ciag.KeyCode() not in self.baza[stos].keys():
                self.baza[stos][ciag.KeyCode()] = ciag
            else:
                val = 1  # wartosc dodawana do nazwy proby w nawiasie - unikalnosc
                while True:
                    if ciag.KeyCode() + "(" + str(val) + ")" not in self.baza[stos].keys():
                        ciag.wpiszMetadana("KeyCode", ciag.KeyCode() +
                                           "(" + str(val) + ")")
                        break
                    else:
                        val += 1
            self.baza[stos][ciag.KeyCode()] = ciag

    def usunCiag(self, ciagNazwa, stos):
        if stos in self.baza.keys():
            if ciagNazwa in self.baza[stos].keys():
                del self.baza[stos][ciagNazwa]
            else:
                print("No sequence by that name")
        else:
            print("No stack by that name")

    def probyzStosu(self, stos, wybrane):
        '''
        zwraca slownik z obiektami Ciagow zapisanych we wskazanym stosie
        jeżeli w zmiennej wybrane przekazano tablice za nazwami prob, tylko one
        zostaną zwrócone w słowniku wynikowym
        '''

        # jeżeli nie podano żadnych prób zaznaczamy wszystkie:
        if len(wybrane) == 0:
            wybrane = self.baza[stos].keys()

        # wybieramy próby ze stosu
        tab = {k: self.baza[stos][k] for k in wybrane}

        return tab

    def ileDrzew(self, stos):
        ileD = len(sorted(self.baza[stos].keys()))
        return ileD


from mrecsys.utils.deploy import SocketServer
from mrecsys.utils.deploy import InferenceUnit
import mrecsys.factorization.service
import numpy as np
import os

os.chdir(os.path.dirname(__file__))
service = SocketServer(mrecsys.factorization.service.__config_path__)
inference_unit = InferenceUnit(mrecsys.factorization.__model_path__, 'als')

def handle_factorization_result(result, selected_items=None):
    result = np.array(result)
    result = np.apply_along_axis(lambda x: x.astype(int)[0], axis=1, arr=result)
    if selected_items is not None:
        result = np.intersect1d(result, selected_items)
    return inference_unit.item_indexer.multi_deindex(result)

def handler(revc_json):
    service_name = revc_json['service_name']
    service_token = revc_json['service_token']
    response_json = {
        "status": "Unsupported Request",
        "code": 415
    }
    if service_name == service.name and service_token == service.token:
        data = revc_json['data']
        request = data['request']

        if request == 'update_latest':
            inference_unit.update_latest()
            response_json = {
                "status": "ok",
                "code": 200,
            }
            return response_json

        inputs = data['inputs']
        selected_items = inputs['selected_items']

        if request == 'similar_items':
            item_id = inputs['item_id']
            item_id = inference_unit.item_indexer.index(item_id)
            selected_items = inference_unit.item_indexer.multi_index(selected_items)
            selected_items = selected_items[selected_items != -1]
            if item_id == -1 or selected_items.shape[0] < 1:
                response_json = {
                    "status": "Invalid Item ID",
                    "code": 415
                }
                return response_json

            result = inference_unit.model.similar_items(item_id, inference_unit.interactions.tocsr())
            result = handle_factorization_result(result)

            response_json = {
                "status": "ok",
                "code": 200,
                "predicts": list(result.astype(str)),
            }
            return response_json

        user_id = inputs['user_id']
        user_id = inference_unit.user_indexer.index(user_id)
        if user_id == -1:
            response_json = {
                "status": "Invalid User ID",
                "code": 415,
            }
            return response_json

        selected_items = inference_unit.item_indexer.multi_index(selected_items)
        try:
            filter_items = inputs['filter_items']
            filter_items = inference_unit.item_indexer.multi_index(filter_items)
            selected_items = np.setdiff1d(selected_items, filter_items)
        except:
            pass
        selected_items = selected_items[selected_items != -1]
        if selected_items.shape[0] < 1:
            response_json = {
                "status": "Invalid Item ID",
                "code": 415,
            }
            return response_json

        if request == 'rank_items':
            result = inference_unit.model.rank_items(userid=user_id,
                                                     user_items=inference_unit.interactions.tocsr(),
                                                     selected_items=list(selected_items))
        if request == 'recommend':
            result = inference_unit.model.recommend(userid=user_id,
                                                    user_items=inference_unit.interactions.tocsr(),
                                                    filter_items=list(filter_items))
        result = handle_factorization_result(np.array(result))
        response_json = {
            "status": "ok",
            "code": 200,
            "predicts": list(result.astype(str)),
        }
    return response_json

def run():
    service.start(request_handler=handler)

if __name__ == '__main__':
    run()


from card import Card
from deck import Deck

class Card_set:
    def __init__(self):
        self._cards=[]
        self._digit_list = []
        self._face_card_list = []
        self._ace_card_list = []

    def segregate(self):
        # 2 - 10, J, Q, K, A
        self._digit_list = [digit_card.get_card_no() for digit_card in self._cards if str(digit_card.get_card_no()).isdigit()]
        self._face_card_list = [face_card.get_card_no() for face_card in self._cards if face_card.get_card_no() in ('Jack', 'Queen', 'King')]
        self._ace_card_list = [ace_card.get_card_no() for ace_card in self._cards if ace_card.get_card_no() == 'Ace']

    def get_first_2_cards(self, game_deck):
        self._cards.append(game_deck.pick_next_card())
        self._cards.append(game_deck.pick_next_card())

    def add_to_set(self, game_deck, player):
        self._cards.append(game_deck.pick_next_card())
        print(f'{player} draw {self._cards[len(self._cards)-1].display_card()}')

    def display_set(self, player_name):
        if player_name == 'Player': print(f'\nYou are currently at: {self.get_score()}')
        else: print(f'\nDealer is currently at: {self.get_score()}')
        print(f'with the hand [', end=' ')
        for key,card in enumerate(self._cards):
            if key==len(self._cards)-1:
                print(f'{card.display_card()}', end='')
            else:
                print(f'{card.display_card()}, ', end='')
        print(f']\n')
        #if self.get_score() == 'Bust!':
        #    raise ValueError('Busted')

    def is_black_jack(self):
        if self.get_score()=='Black Jack!':
            return True
        else:
            return False

    def get_score(self):
        res=0
        #self._digit_list = self._face_card_list = self._ace_card_list=[]
        self.segregate()
        while True:
            res = sum(self._digit_list) + (len(self._face_card_list)*10) + (len(self._ace_card_list)*11)
            if res > 21:
                if len(self._ace_card_list):
                    self.fix_ace_card_list()
                else:
                    return 'Bust!'
            elif res==21:
                return 'Black Jack!'
            else:
                return res

    def fix_ace_card_list(self):
        self._ace_card_list.pop()
        self._digit_list.append(1)


# -*- coding: utf-8 -*-
"""
Created on 2018.8.9

@author: zhangjun
"""

import os
import tensorflow as tf
import argparse
import utils.utils as utils
from model_collections.deep_wide_model import deep_wide_model
from model_collections.word2vec_model import word2vec_model

parser = argparse.ArgumentParser()

parser.add_argument('-gpu_num',         default='0', type=str, help='witch gpu to use')
parser.add_argument('-batch_size',      default=2048, type=int, help='batch size')
parser.add_argument('-learning_rate',   default=0.001, type=float, help='learning rate')
parser.add_argument('-opt_algo',        default='rmsprop', type=str, help='optimize algorithm')
parser.add_argument('-epochs',          default=10, type=int, help='epochs num')
parser.add_argument('-regular_rate',    default=0, type=float, help='regular rate')
parser.add_argument('-dropout_rate',    default=0, type=float, help='dropout rate')
parser.add_argument('-skip_step',       default=100, type=float, help='skip step')
parser.add_argument('-layer_size_list', default='64,32,16', type=str, help='layer size list for MLP')

parser.add_argument('-train_data_name', default='train_data', type=str, help='train data name')
parser.add_argument('-test_data_name', default='test_data', type=str, help='test data name')
parser.add_argument('-pred_data_name', default='pred_data', type=str, help='pred data name')
parser.add_argument('-data_path', default='../data/', type=str, help='data path')

parser.add_argument('-save', default='False', type=str, help='save or not')
parser.add_argument('-restore', default='False', type=str, help='restore or not')

parser.add_argument('-num_sampled', default=3, type=int, help='negative num_ ampled')
parser.add_argument('-vocab_embed_size', default=128, type=int, help='vocab embedding size')

parser.add_argument('-low_freq_threshold', default=0, type=int, help='cate low freq threshold')

parser.add_argument('-model', default='deep_wide', type=str, help='restore or not')

parser.add_argument('-csv_sep', default=' ', type=str, help='csv sep')

args = parser.parse_args()

os.environ['CUDA_VISIBLE_DEVICES']= args.gpu_num

def main():

    sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)))

    if args.model == 'deep_wide':

        #---------deep_wide start-------------------------------------------------------------

        model = deep_wide_model(sess = sess,
                                        batch_size = args.batch_size,
                                        learning_rate = args.learning_rate,
                                        opt_algo = args.opt_algo,
                                        epochs = args.epochs,
                                        regular_rate = args.regular_rate,
                                        dropout_rate = args.dropout_rate,
                                        skip_step = args.skip_step,
                                        layer_size_list = list(map(lambda x : int(x) , args.layer_size_list.split(","))))

        model.get_data(data_path = args.data_path,
                   train_data_name = args.train_data_name,
                   test_data_name = args.test_data_name,
                   pred_data_name = args.pred_data_name,
                   sep = args.csv_sep,
                   low_freq_threshold = args.low_freq_threshold)

        model.bulid()

        model.train(save = utils.str2bool(args.save),
                restore = utils.str2bool(args.restore))

        if len(args.pred_data_name) >0:
            model.predict(data_path = args.data_path)

        #---------deep_wide end-------------------------------------------------------------

    elif args.model == 'word2vec':

        #---------word2vec start-------------------------------------------------------------

        model = word2vec_model(sess = sess,
                                        batch_size = args.batch_size,
                                        learning_rate = args.learning_rate,
                                        opt_algo = args.opt_algo,
                                        epochs = args.epochs,
                                        regular_rate = args.regular_rate,
                                        dropout_rate = args.dropout_rate,
                                        skip_step = args.skip_step,
                                        num_sampled = args.num_sampled,
                                        embed_size = args.vocab_embed_size)

        model.get_data(data_path = args.data_path,
                   train_data_name = args.train_data_name,
                   test_data_name = '',
                   pred_data_name = '',
                   sep = ' ',
                   low_freq_threshold = args.low_freq_threshold)

        model.bulid()

        model.train(save = utils.str2bool(args.save),
                    restore = utils.str2bool(args.restore))

        model.write_embedding_matrix(sess = sess,
                                     data_path = args.data_path,
                                     file_name = 'vocab_embedding_matrix')

        #---------word2vec end-------------------------------------------------------------

    else:
        pass

if __name__ == '__main__':
    main()


import warnings
import datetime
import os
import pickle
import pandas as pd
import numpy as np
from sklearn.metrics import accuracy_score, explained_variance_score
import seaborn as sns
import matplotlib.pyplot as plt

def save_obj(obj, path):
    with open(path, "wb") as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def plot_summary(summary, output_dir=None, filename="shap_plot", plot_top_n_shap=16):
    plt.clf()
    plt.figure(figsize=(8, 12))
    # plot without all bootstrapping values
    summary = summary[["mean", "std", "min", "max"]]
    num_features = len(list(summary.index))
    if (plot_top_n_shap != 1 and type(plot_top_n_shap) == float) or type(
        plot_top_n_shap
    ) == int:
        # if plot_top_n_shap != 1.0 but includes 1 (int)
        if plot_top_n_shap <= 0:
            raise ValueError(
                "plot_top_n_shap should be a float between 0 and 1.0 or an integer >= 1. You set to zero or negative."
            )
        elif plot_top_n_shap < 1:
            plot_top_n_shap = int(np.round(plot_top_n_shap * num_features))
        summary = summary.iloc[:plot_top_n_shap, :]
        filename += f"_top_{plot_top_n_shap}"

    hm = sns.heatmap(
        summary.round(3), annot=True, xticklabels=True, yticklabels=True, cbar=False
    )
    hm.set_xticklabels(summary.columns, rotation=45)
    hm.set_yticklabels(summary.index, rotation=0)
    plt.ylabel("Features")
    plt.tight_layout()
    plt.show(block=False)
    plt.savefig(output_dir + f"summary_{filename}.png", dpi=100)
    return

def shaps_to_summary(
    shaps_n_splits,
    feature_names=None,
    output_dir=None,
    filename="shap_summary",
    plot_top_n_shap=16,
):
    shaps_n_splits.columns = [
        "split_{}".format(n) for n in range(shaps_n_splits.shape[1])
    ]
    if feature_names:
        shaps_n_splits.index = feature_names
    # else:
    # 	shaps_n_splits.index = [str(n) for n in shaps_n_splits.index]
    # add summary stats
    shaps_n_splits["mean"] = shaps_n_splits.mean(axis=1)
    shaps_n_splits["std"] = shaps_n_splits.std(axis=1)
    shaps_n_splits["min"] = shaps_n_splits.min(axis=1)
    shaps_n_splits["max"] = shaps_n_splits.max(axis=1)
    shaps_n_splits_sorted = shaps_n_splits.sort_values("mean")[::-1]
    shaps_n_splits_sorted.to_csv(f"{output_dir}summary_values_{filename}.csv")

    plot_summary(
        shaps_n_splits_sorted,
        output_dir=output_dir,
        filename=filename,
        plot_top_n_shap=plot_top_n_shap,
    )
    return

def gen_report_shap_class(results, output_dir="./", plot_top_n_shap=16):
    # Create shap_dir
    timestamp = datetime.datetime.utcnow().isoformat()
    timestamp = timestamp.replace(":", "").replace("-", "")
    shap_dir = output_dir + f"shap-{timestamp}/"
    os.mkdir(shap_dir)

    feature_names = results[0][1].output.feature_names
    # save all TP, TN, FP, FN indexes
    indexes_all = {}

    for model_results in results:
        model_name = model_results[0].get("ml_wf.clf_info")[1]
        indexes_all[model_name] = []
        shaps = model_results[
            1
        ].output.shaps  # this is (N, P, F) N splits, P predictions, F feature_names
        # make sure there are shap values (the
        if np.array(shaps[0]).size == 0:
            continue

        y_true_and_preds = model_results[1].output.output
        n_splits = len(y_true_and_preds)

        shaps_n_splits = {
            "all": [],
            "tp": [],
            "tn": [],
            "fp": [],
            "fn": [],
        }  # this is key with shape (F, N) where F is feature_names, N is mean shap values across splits
        # Obtain values for each bootstrapping split, then append summary statistics to shaps_n_splits
        for split_i in range(n_splits):
            shaps_i = shaps[split_i]  # all shap values for this bootstrapping split
            y_true = y_true_and_preds[split_i][0]
            y_pred = y_true_and_preds[split_i][1]
            #split_performance = accuracy_score(y_true, y_pred)
            split_performance = explained_variance_score(y_true, y_pred)

            # split prediction indexes into TP, TN, FP, FN, good for error auditing
            indexes = {"tp": [], "tn": [], "fp": [], "fn": []}
            for i in range(len(y_true)):
                if y_true[i] == y_pred[i] and y_pred[i] == 1:
                    indexes["tp"].append(i)
                elif y_true[i] == y_pred[i] and y_pred[i] == 0:
                    indexes["tn"].append(i)
                elif y_true[i] != y_pred[i] and y_pred[i] == 1:
                    indexes["fp"].append(i)
                elif y_true[i] != y_pred[i] and y_pred[i] == 0:
                    indexes["fn"].append(i)
            indexes_all[model_name].append(indexes)

            #  For each quadrant, obtain F shap values for P predictions, take the absolute mean weighted by performance across all predictions
            for quadrant in ["tp", "tn", "fp", "fn"]:
                if len(indexes.get(quadrant)) == 0:
                    warnings.warn(
                        f"There were no {quadrant.upper()}s, this will output NaNs in the csv and figure for this split column"
                    )
                shaps_i_quadrant = shaps_i[
                    indexes.get(quadrant)
                ]  # shape (P, F) P prediction x F feature_names
                abs_weighted_shap_values = np.abs(shaps_i_quadrant) * split_performance
                shaps_n_splits[quadrant].append(
                    np.mean(abs_weighted_shap_values, axis=0)
                )
            #  obtain F shap values for P predictions, take the absolute mean weighted by performance across all predictions
            abs_weighted_shap_values = np.abs(shaps_i) * split_performance
            shaps_n_splits["all"].append(np.mean(abs_weighted_shap_values, axis=0))

        # Build df for summary statistics for each quadrant
        for quadrant in ["tp", "tn", "fp", "fn"]:
            shaps_n_splits_quadrant = pd.DataFrame(shaps_n_splits.get(quadrant)).T
            shaps_to_summary(
                shaps_n_splits_quadrant,
                feature_names,
                output_dir=shap_dir,
                filename=f"shap_{model_name}_{quadrant}",
                plot_top_n_shap=plot_top_n_shap,
            )

        # Single csv for all predictions
        shaps_n_splits_all = pd.DataFrame(shaps_n_splits.get("all")).T
        shaps_to_summary(
            shaps_n_splits_all,
            feature_names,
            output_dir=shap_dir,
            filename=f"shap_{model_name}_all_predictions",
            plot_top_n_shap=plot_top_n_shap,
        )
    save_obj(indexes_all, shap_dir + "indexes_quadrant.pkl")
    return

def gen_report_shap_regres(results, output_dir="./", plot_top_n_shap=16):
    # Create shap_dir
    timestamp = datetime.datetime.utcnow().isoformat()
    timestamp = timestamp.replace(":", "").replace("-", "")
    shap_dir = output_dir + f"shap-{timestamp}/"
    os.mkdir(shap_dir)

    feature_names = results[0][1].output.feature_names
    # save all TP, TN, FP, FN indexes
    indexes_all = {}

    for model_results in results:
        model_name = model_results[0].get("ml_wf.clf_info")[1]
        indexes_all[model_name] = []
        shaps = model_results[
            1
        ].output.shaps  # this is (N, P, F) N splits, P predictions, F feature_names
        # make sure there are shap values (the
        if np.array(shaps[0]).size == 0:
            continue

        y_true_and_preds = model_results[1].output.output
        n_splits = len(y_true_and_preds)

        shaps_n_splits = {
            "all": [],
            "lp": [],
            "lm": [],
            "um": [],
            "up": []
        }  # this is key with shape (F, N) where F is feature_names, N is mean shap values across splits
        # Obtain values for each bootstrapping split, then append summary statistics to shaps_n_splits
        for split_i in range(n_splits):
            shaps_i = shaps[split_i]  # all shap values for this bootstrapping split
            y_true = y_true_and_preds[split_i][0]
            y_pred = y_true_and_preds[split_i][1]
            split_performance = explained_variance_score(y_true, y_pred)

            # split prediction indexes into upper, median, lower, good for error auditing
            indexes = {"lp": [], "lm": [], "um": [], "up": []}
            q=np.array([25,50,75])
            prc=np.percentile(y_true,q)
            for i in range(len(y_true)):
                if prc[0] >= y_pred[i]:
                    indexes["lp"].append(i)
                elif prc[0] < y_pred[i] and prc[1] >= y_pred[i]:
                    indexes["lm"].append(i)
                elif prc[1] < y_pred[i] and prc[2] >= y_pred[i]:
                    indexes["um"].append(i)
                elif prc[2] < y_pred[i]:
                    indexes["up"].append(i)
            indexes_all[model_name].append(indexes)

            #  For each quadrant, obtain F shap values for P predictions, take the absolute mean weighted by performance across all predictions
            for quadrant in ["lp", "lm", "um", "up"]:
                if len(indexes.get(quadrant)) == 0:
                    warnings.warn(
                        f"There were no {quadrant.upper()}s, this will output NaNs in the csv and figure for this split column"
                    )
                shaps_i_quadrant = shaps_i[
                    indexes.get(quadrant)
                ]  # shape (P, F) P prediction x F feature_names
                abs_weighted_shap_values = np.abs(shaps_i_quadrant) * split_performance
                shaps_n_splits[quadrant].append(
                    np.mean(abs_weighted_shap_values, axis=0)
                )
            #  obtain F shap values for P predictions, take the absolute mean weighted by performance across all predictions
            abs_weighted_shap_values = np.abs(shaps_i) * split_performance
            shaps_n_splits["all"].append(np.mean(abs_weighted_shap_values, axis=0))

        # Build df for summary statistics for each quadrant
        for quadrant in ["lp", "lm", "um", "up"]:
            shaps_n_splits_quadrant = pd.DataFrame(shaps_n_splits.get(quadrant)).T
            shaps_to_summary(
                shaps_n_splits_quadrant,
                feature_names,
                output_dir=shap_dir,
                filename=f"shap_{model_name}_{quadrant}",
                plot_top_n_shap=plot_top_n_shap,
            )

        # Single csv for all predictions
        shaps_n_splits_all = pd.DataFrame(shaps_n_splits.get("all")).T
        shaps_to_summary(
            shaps_n_splits_all,
            feature_names,
            output_dir=shap_dir,
            filename=f"shap_{model_name}_all_predictions",
            plot_top_n_shap=plot_top_n_shap,
        )
    save_obj(indexes_all, shap_dir + "indexes_quadrant.pkl")
    return

def gen_report(
    results, prefix, metrics, gen_shap=True, output_dir="./", plot_top_n_shap=16
):
    if len(results) == 0:
        raise ValueError("results is empty")
    df = pd.DataFrame(columns=["metric", "score", "Classifier", "type"])
    for val in results:
        score = val[1].output.score
        if not isinstance(score, list):
            score = [score]
        name = val[0][prefix + ".clf_info"][1].split("Classifier")[0]
        permute = val[0][prefix + ".permute"]
        for scoreval in score:
            for idx, metric in enumerate(metrics):
                df = df.append(
                    {
                        "Classifier": name,
                        "type": "null" if permute else "data",
                        "metric": metrics[idx],
                        "score": scoreval[idx] if scoreval[idx] is not None else np.nan,
                    },
                    ignore_index=True,
                )
    for name, subdf in df.groupby("metric"):
        sns.set(style="whitegrid", palette="pastel", color_codes=True)
        sns.set_context("talk")
        plt.figure(figsize=(12, 6))
        ax = sns.violinplot(
            x="Classifier",
            y="score",
            hue="type",
            data=subdf,
            split=True,
            inner="quartile",
            hue_order=["data", "null"],
            order=[group[0] for group in df.groupby("Classifier")],
        )
        ax.set_ylabel(name)
        sns.despine(left=True)
        import datetime

        timestamp = datetime.datetime.utcnow().isoformat()
        timestamp = timestamp.replace(":", "").replace("-", "")
        plt.savefig(f"test-{name}-{timestamp}.png")

    # create SHAP summary csv and figures
    if gen_shap:
        reg_metrics=["explained_variance_score","max_error",
            "mean_absolute_error","mean_squared_error",
            "mean_squared_log_error","median_absolute_error",
            "r2_score","mean_poisson_deviance",
            "mean_gamma_deviance"
            ]
        if any([True for x in metrics if x in reg_metrics]):
            gen_report_shap_regres(results, output_dir=output_dir, plot_top_n_shap=plot_top_n_shap)
        else:
            gen_report_shap_class(results, output_dir=output_dir, plot_top_n_shap=plot_top_n_shap)


def work(hours, rate):

    pay = hours * rate
    str_work = str(pay)
    print("I have made " + str_work + " this week.")

hours_worked = 40
pay_earned = 8.25

work(hours_worked, pay_earned)

work(40, 8.25)


from SPARQLWrapper import SPARQLWrapper, JSON

uri = "http://data.org/PLE/resource/jorgeGlas"

class Semantico():
    sbcEndpoint = SPARQLWrapper("http://localhost:8890/sparql/")

    def consultaVirutoso(self, texto):
        consulta = """
         SELECT ?s ?p ?o ?i
            WHERE
                {
                    ?s ?p ?o .FILTER regex(str(?s), "%s") .
                }
                """ % (texto)
        self.sbcEndpoint.setQuery(consulta)
        self.sbcEndpoint.setReturnFormat(JSON)
        results = self.sbcEndpoint.query().convert()
        return results["results"]["bindings"]

    def consultaPorUri(self, uri):
        consulta = """
                    SELECT ?p ?o
                        WHERE
                        {
                            <%s> ?p  ?o
                        }
                """ % (uri)
        self.sbcEndpoint.setQuery(consulta)
        self.sbcEndpoint.setReturnFormat(JSON)
        results = self.sbcEndpoint.query().convert()
        return results["results"]["bindings"]

texto = "Odebrecht"
semantico = Semantico()
# semantico.consultaPorUri(uri)
print(semantico.consultaVirutoso(texto))


__author__ = 'vantani'
# -*- coding: utf-8 -*-

import logging
import logging.config
from hashlib import md5
from lxml.html import parse,tostring

from pyquery import PyQuery as pq
from titlecase import titlecase
import uuid
import memcache
import string_utils
from similarity import string_similarity
import re

mc = None
BASE_URL="http://www.flipkart.com/search?q="
logging.config.fileConfig('../logging.conf')
logger = logging.getLogger("root")

# Helper functions to clean up price values to a number
def sanitize_price(price_value):
    # Remove all Rupee symbols and commas
    cleaned_price = price_value.strip().replace(',','').replace(u'\u20b9','')
    # Remove all characters except a .
    non_decimal = re.compile(r'[^\d.]+')
    price_digs = non_decimal.sub('', cleaned_price)
    # After this there might be a leading . , remove it
    if price_digs =='':
        return 0

    if price_digs[0] == '.':
        price = price_digs[1:]
    else:
        price = price_digs
    return price

def getkey(str):
    key = str.encode('utf8')
    key = md5(key).hexdigest()
    logger.debug( key)
    return key

class FlipkartParser:

    def __init__(self):
        #mc = pylibmc.Client(["127.0.0.1"], binary=True,behaviors={"tcp_nodelay": True,"ketama": True})
        self.mc = memcache.Client(['localhost:11211'], debug=1)
        #self.mc.flush_all()

    def get_page(self,search_term,search_type="Rest"):
        val=self.mc.get(getkey("%s%s%s" % ("flipkart",search_term,search_type)))
        if val is None:
            val = parse(BASE_URL+search_term).getroot()
            val.make_links_absolute()
            self.mc.set(getkey("%s%s%s" % ("flipkart",search_term,search_type)),tostring(val))
        return val

    def flipkart_book_parser(self,search_term,search_type):

        d = pq(self.get_page(search_term,"book"))
        price_d = d('div.pu-final').map(lambda i, e: pq(e).text())
        for p in price_d:
            logger.debug( p)

        name_d = d('a.lu-title').map(lambda i, e: pq(e).text())
        for p in name_d:
            logger.debug( p)

        author_d = d('div.lu-title-wrapper span.fk-font-11').map(lambda i, e: pq(e).text())
        for p in author_d:
            logger.debug( p)

        discount_d = d('div.pu-discount').remove('span.pu-old').map(lambda i, e: pq(e).text())
        for p in discount_d:
            logger.debug( p)

        img_d = d('div.lu-image a.lu-image-link img').map(lambda i, e: pq(e).attr('src'))
        for p in img_d:
            logger.debug( p)

        url_d = d('a.lu-title').map(lambda i, e: pq(e).attr('href'))
        for p in url_d:
            logger.debug( p)

        prices=[]
        for price, name, img, url, author, discount in map(None, price_d,name_d,img_d,url_d,author_d,discount_d ):
            if price:
                if name:
                    weight = string_similarity(string_utils.clean_words(search_term), string_utils.clean_words(name))
                else:
                    weight = 0.0

                price = {'uuid':str(uuid.uuid4()),'source':'http://localhost/static/cache/images/stores/Flipkart.png', 'price':float(sanitize_price(price)),'name':(name),
                         'img':img if string_utils.is_url(img) else 'http://google.com', 'url':url,'author':author,
                         'discount':' '.join(discount.split()) if discount else None,
                         'weight':weight,'type':search_type}
                if price:
                    self.mc.set(price['uuid'],price,time=84000)
                    prices.append(price)

        logger.debug( prices)
        return prices

    def flipkart_rest_parser(self, search_term,search_type):
        d = pq(self.get_page(search_term))
        price_d = d('div.pu-final').map(lambda i, e: pq(e).text())
        for p in price_d:
            logger.debug( p)

        name_d = d('div.pu-title a').map(lambda i, e: pq(e).text())
        for p in name_d:
            logger.debug( p)

        img_d = d('a.pu-image img').map(lambda i, e: pq(e).attr('src'))
        for p in img_d:
            logger.debug( p)

        url_d = d('div.pu-title a').map(lambda i, e: pq(e).attr('href'))
        for p in url_d:
            logger.debug( p)

        usp_d = d('ul.pu-usp').map(lambda i, e: pq(e).text())
        for p in usp_d:
            logger.debug( p)

        discount_d = d('div.pu-discount').remove('span.pu-old').map(lambda i, e: pq(e).text())
        for p in discount_d:
            logger.debug( p)

        prices=[]
        for price, name, img, url, usp, discount in map(None, price_d,name_d,img_d,url_d,usp_d,discount_d ):
            if price:
                if name:
                    weight = string_similarity(string_utils.clean_words(search_term), string_utils.clean_words(name))
                else:
                    weight = 0.0

                price={'uuid':str(uuid.uuid4()),'source':'http://localhost/static/cache/images/stores/Flipkart.png', 'price':float(sanitize_price(price)),'name':name,
                       'img':img if string_utils.is_url(img) else 'http://google.com', 'url':url,'usp':usp,
                       'discount':discount,'type':search_type,
                       'weight':weight}

                if price:
                    self.mc.set(price['uuid'],price,time=84000)
                    prices.append(price)

        logger.debug(prices)
        return prices


from django.test import TestCase
from django.core.urlresolvers import reverse
from django.test.client import Client

from wagtail.wagtailsearch.backends import get_search_backend

from molo.core.models import ArticlePage, SiteLanguage
from molo.core.tests.base import MoloTestCaseMixin

class TestSearch(TestCase, MoloTestCaseMixin):

    def setUp(self):
        self.client = Client()
        # Creates Main language
        self.english = SiteLanguage.objects.create(
            locale='en',
        )
        # Creates translation Language
        self.french = SiteLanguage.objects.create(
            locale='fr',
        )
        self.mk_main()

        # Creates a section under the index page
        self.english_section = self.mk_section(
            self.section_index, title='English section')

    def test_search(self):
        self.backend = get_search_backend('default')
        self.backend.reset_index()

        for a in range(0, 20):
            ArticlePage.objects.create(
                title='article %s' % (a,), depth=a,
                subtitle='article %s subtitle' % (a,),
                slug='article-%s' % (a,), path=[a])

        self.backend.refresh_index()

        response = self.client.get(reverse('search'), {
            'q': 'article'
        })
        self.assertContains(response, 'Page 1 of 2')
        self.assertContains(response, '→')
        self.assertNotContains(response, '←')

        response = self.client.get(reverse('search'), {
            'q': 'article',
            'p': '2',
        })
        self.assertContains(response, 'Page 2 of 2')
        self.assertNotContains(response, '→')
        self.assertContains(response, '←')

        response = self.client.get(reverse('search'), {
            'q': 'article',
            'p': 'foo',
        })
        self.assertContains(response, 'Page 1 of 2')

        response = self.client.get(reverse('search'), {
            'q': 'article',
            'p': '4',
        })
        self.assertContains(response, 'Page 2 of 2')

        response = self.client.get(reverse('search'), {
            'q': 'magic'
        })
        self.assertContains(response, 'No search results for magic')

        response = self.client.get(reverse('search'))
        self.assertContains(response, 'No search results for None')

    def test_search_works_with_multilanguages(self):
        self.backend = get_search_backend('default')
        self.backend.reset_index()
        eng_article = self.mk_article(
            self.english_section, title="English article")

        self.mk_article_translation(
            eng_article, self.french, title='French article')

        self.backend.refresh_index()

        self.client.get('/locale/en/')
        response = self.client.get(reverse('search'), {
            'q': 'article'
        })
        self.assertContains(response, 'English article')
        self.assertNotContains(response, 'French article')

        self.client.get('/locale/fr/')
        response = self.client.get(reverse('search'), {
            'q': 'article'
        })
        self.assertContains(response, 'French article')
        self.assertNotContains(response, 'English article')


import random
import torch
from torch import nn
import numpy as np
from extract import read_file
from utils import move_to_device
from pprint import pprint

PAD, UNK = '<PAD>', '<UNK>'
CLS = '<CLS>'
STR, END = '<STR>', '<END>'
SEL, rCLS, TL = '<SELF>', '<rCLS>', '<TL>'

class Vocab(object):
    def __init__(self, filename, min_occur_cnt, specials = None):
        idx2token = [PAD, UNK] + (specials if specials is not None else [])
        self._priority = dict()
        num_tot_tokens = 0
        num_vocab_tokens = 0
        for line in open(filename).readlines():
            try:
                token, cnt = line.rstrip('\n').split('\t')
                cnt = int(cnt)
                num_tot_tokens += cnt
            except:
                print(line)
            if cnt >= min_occur_cnt:
                idx2token.append(token)
                num_vocab_tokens += cnt
            self._priority[token] = int(cnt)
        self.coverage = num_vocab_tokens/num_tot_tokens
        self._token2idx = dict(zip(idx2token, range(len(idx2token))))
        self._idx2token = idx2token
        self._padding_idx = self._token2idx[PAD]
        self._unk_idx = self._token2idx[UNK]

    def priority(self, x):
        return self._priority.get(x, 0)

    @property
    def size(self):
        return len(self._idx2token)

    @property
    def unk_idx(self):
        return self._unk_idx

    @property
    def padding_idx(self):
        return self._padding_idx

    def idx2token(self, x):
        if isinstance(x, list):
            return [self.idx2token(i) for i in x]
        return self._idx2token[x]

    def token2idx(self, x):
        if isinstance(x, list):
            return [self.token2idx(i) for i in x]
        return self._token2idx.get(x, self.unk_idx)

def _back_to_txt_for_check(tensor, vocab, local_idx2token=None):
    for bid, xs in enumerate(tensor.t().tolist()):
        txt = []
        for x in xs:
            if x == vocab.padding_idx:
                break
            if x >= vocab.size:
                assert local_idx2token is not None
                assert local_idx2token[bid] is not None
                tok = local_idx2token[bid][x]
            else:
                tok = vocab.idx2token(x)
            txt.append(tok)
        txt = ' '.join(txt)
        print (txt)

def ListsToTensor(xs, vocab=None, local_vocabs=None, unk_rate=0.):
    if 0:
        print ('xs:')
        print (xs)
        print (len(xs))
        assert False

    if 0:
        print('local_vocabs:')
        print(local_vocabs)
        assert False

    if 0:
        if local_vocabs is not None:
            print('local_vocabs')
            print (len(local_vocabs))
            assert False

    pad = vocab.padding_idx if vocab else 0
    if 0:
        print ('pad:')
        print (pad)
        assert False

    def toIdx(w, i):
        if vocab is None:
            return w
        if isinstance(w, list):
            if 0:
                if local_vocabs is not None:
                    print ('w is list')
                    print (w)
            return [toIdx(_, i) for _ in w]
        if random.random() < unk_rate:
            return vocab.unk_idx
        if local_vocabs is not None:
            local_vocab = local_vocabs[i]
            if (local_vocab is not None) and (w in local_vocab):
                if 0:
                    print ('local_vocab:')
                    print (local_vocab)
                    print (w)
                    print (local_vocab[w])
                    #assert False
                return local_vocab[w]
        return vocab.token2idx(w)

    max_len = max(len(x) for x in xs)
    ys = []
    for i, x in enumerate(xs):
        y = toIdx(x, i) + [pad]*(max_len-len(x))
        if local_vocabs is not None and not isinstance(x, list):
            assert False
        if 0:
            print ('x:')
            print (x)
            if vocab is not None:
                print('[vocab.token2idx(_) for _ in x]:')
                print ([vocab.token2idx(_) for _ in x])
                print('vocab.unk_idx:')
                print(vocab.unk_idx)
            print ('y:')
            print (y)
            assert False
        ys.append(y)
    data = np.transpose(np.array(ys))
    return data

def _ListsToTensor(xs, vocab=None, local_vocabs=None, unk_rate=0.):
    if 0:
        print ('xs:')
        print (xs)
        print (len(xs))
        assert False

    if 0:
        print('local_vocabs:')
        print(local_vocabs)
        assert False

    if 0:
        if local_vocabs is not None:
            print('local_vocabs')
            print (len(local_vocabs))
            assert False

    pad = vocab.padding_idx if vocab else 0
    if 0:
        print ('pad:')
        print (pad)
        assert False

    def toIdx(w, i):
        if vocab is None:
            return w
        if isinstance(w, list):
            if 0:
                if local_vocabs is not None:
                    print ('w is list')
                    print (w)
            return [toIdx(_, i) for _ in w]
        if random.random() < unk_rate:
            return vocab.unk_idx
        if local_vocabs is not None:
            local_vocab = local_vocabs[i]
            if (local_vocab is not None) and (w in local_vocab):
                if 0:
                    print ('local_vocab:')
                    print (local_vocab)
                    print (w)
                    print (local_vocab[w])
                    #assert False
                return local_vocab[w]
        return vocab.token2idx(w)

    max_len = max(len(x) for x in xs)
    ys = []
    for i, x in enumerate(xs):
        y = toIdx(x, i) + [pad]*(max_len-len(x))
        if 0:
            print ('x:')
            print (x)
            print ('y:')
            print (y)
            assert False

        if local_vocabs is not None and not isinstance(x, list):
            assert False
        if 0:
            print ('x:')
            print (x)
            if vocab is not None:
                print('[vocab.token2idx(_) for _ in x]:')
                print ([vocab.token2idx(_) for _ in x])
                print('vocab.unk_idx:')
                print(vocab.unk_idx)
            print ('y:')
            print (y)
            assert False
        ys.append(y)
    data = np.transpose(np.array(ys))
    return data

def ListsofStringToTensor(xs, vocab, max_string_len=20):
    max_len = max(len(x) for x in xs)
    ys = []
    for x in xs:
        y = x + [PAD]*(max_len -len(x))
        if 0:
            print ('x:')
            print (x)
            print ('y:')
            print (y)
            assert False

        zs = []
        for z in y:
            if 0:
                print ('z:')
                print (z)
            z = list(z[:max_string_len])
            if 0:
                print('z:')
                print(z)
                assert False
            zs.append(vocab.token2idx([STR]+z+[END]) +
                      [vocab.padding_idx]*(max_string_len - len(z)))
        ys.append(zs)
        if 0:
            print (ys)
            assert False

    data = np.transpose(np.array(ys), (1, 0, 2))
    return data

def ArraysToTensor(xs):
    "list of numpy array, each has the same demonsionality"
    x = np.array([ list(x.shape) for x in xs])
    if 0:
        print ('x')
        print (x)
        assert False
    shape = [len(xs)] + list(x.max(axis = 0))
    if 0:
        print ('shape:')
        print (shape)

    data = np.zeros(shape, dtype=np.int)
    for i, x in enumerate(xs):
        slicing_shape = list(x.shape)
        slices = tuple([slice(i, i+1)]+[slice(0, x) for x in slicing_shape])
        if 0:
            print ('shape:')
            print (shape)
            print('slicing_shape:')
            print(slicing_shape)
            print ('slices:')
            print (slices)
            assert False

        data[slices] = x
        #tensor = torch.from_numpy(data).long()
    if 0:
        print ('data:')
        print (data.shape)
        #assert False

    return data

def _ArraysToTensor(xs):
    "list of numpy array, each has the same demonsionality"
    x = np.array([list(x.shape) for x in xs])
    if 0:
        print('x')
        print(x)
    shape = [len(xs)] + list(x.max(axis=0))
    if  0:
        print('shape:')
        print(shape)
    #assert False

    data = np.zeros(shape, dtype=np.int)
    for i, x in enumerate(xs):
        slicing_shape = list(x.shape)
        slices = tuple([slice(i, i+1)]+[slice(0, x) for x in slicing_shape])
        if 1:
            print('shape:')
            print(shape)
            print('slicing_shape:')
            print(slicing_shape)
            print('slices:')
            print(slices)
            assert False

        data[slices] = x
        #tensor = torch.from_numpy(data).long()
    if 0:
        print('data:')
        print(data.shape)
        #assert False

    return data

def batchify(data, vocabs, unk_rate=0.):
    if 0:
        print('len(data):')
        print (len(data))
        #assert False
    if 1:
        _conc = ListsToTensor(
            [[CLS]+x['concept'] for x in data],
            vocabs['concept'], unk_rate=unk_rate)
    if 0:
        print ('_conc')
        print (_conc)
        print(_conc.shape)
        assert False

    _conc_char = ListsofStringToTensor(
        [[CLS]+x['concept'] for x in data],
        vocabs['concept_char'])
    if 0:
        print('_conc_char')
        print(_conc_char)
        print(_conc_char.shape)
        assert False

    _depth = ListsToTensor([ [0]+x['depth'] for x in data])

    if 0:
        print('_conc')
        print(_conc.shape)
        print('_conc_char')
        print(_conc_char.shape)
        print('_depth')
        print(_depth.shape)
        #assert False

    all_relations = dict()
    cls_idx = vocabs['relation'].token2idx(CLS)
    rcls_idx = vocabs['relation'].token2idx(rCLS)
    self_idx = vocabs['relation'].token2idx(SEL)
    all_relations[tuple([cls_idx])] = 0 # CLS to other nodes
    all_relations[tuple([rcls_idx])] = 1 # other nodes to CLS
    all_relations[tuple([self_idx])] = 2 # Self loop

    _relation_type = []
    for bidx, x in enumerate(data):
        n = len(x['concept'])
        if 0:
            print ("x['concept]:")
            print (x['concept'])
            print("len(x['concept']):")
            print (n)
            #assert False

        brs = [ [2]+[0]*(n) ] # from [CLS] to [CLS] (SEL) & other nodes (CLS)
        #print ('n:', n)
        for i in range(n):
            rs = [1] # from the node to CLS
            for j in range(n):
                all_path = x['relation'][i][j]
                if 0:
                    print ('all_path:')
                    print (all_path)
                    #assert False

                path = random.choice(all_path)['edge']
                if 0:
                    print ('path:')
                    print (path)
                    #assert False

                if len(path) == 0: # self loop
                    path = [SEL]
                if len(path) > 8: # too long distance
                    path = [TL]
                path = tuple(vocabs['relation'].token2idx(path))
                if 0:
                    print ('path:')
                    print (path)
                    print ('----------')
                    #assert False

                rtype = all_relations.get(path, len(all_relations))
                if 0:
                    print ('rtype:')
                    print (rtype)
                    print('len(all_relations):')
                    print(len(all_relations))

                if rtype == len(all_relations):
                    all_relations[path] = len(all_relations)
                rs.append(rtype)
            rs = np.array(rs, dtype=np.int)
            if 0:
                print ('i:', i)
                print ('rs:', rs)
                print (len(rs))
                print ('n:', n)
                #assert False
            brs.append(rs)
        if 0:
            print ('brs:')
            print (brs)
            print (len(brs))
        brs = np.stack(brs)
        if 0:
            print ('brs:')
            print (brs)
            print (brs.shape)
        #assert False
        if 0:
            print (brs.shape)
            assert False

        _relation_type.append(brs)

    if 0:
        for i, brs in enumerate(_relation_type):
            print (i, brs.shape)
        assert False

    _relation_type = np.transpose(ArraysToTensor(_relation_type), (2, 1, 0))
    # _relation_bank[_relation_type[i][j][b]] => from j to i go through what
    if 0:
        print('_relation_type:')
        print(_relation_type.shape)
        assert False

    B = len(all_relations)
    _relation_bank = dict()
    _relation_length = dict()
    for k, v in all_relations.items():
        if 0:
            print ('k')
            print (k)
            print ('v')
            print (v)
            assert False
        _relation_bank[v] = np.array(k, dtype=np.int)
        _relation_length[v] = len(k)
        if 0:
            print('_relation_bank[v]')
            print(_relation_bank[v])
            print('_relation_length[v]')
            print(_relation_length[v])
    if 0:
        assert False
    _relation_bank = [_relation_bank[i] for i in range(len(all_relations))]
    if 0:
        for relation in _relation_bank:
            print(relation.shape)
        assert False

    _relation_length = [_relation_length[i] for i in range(len(all_relations))]
    if 0:
        print('_relation_length')
        print(_relation_length)
        assert False

    #_relation_bank = np.transpose(_ArraysToTensor(_relation_bank))
    _relation_bank = np.transpose(ArraysToTensor(_relation_bank))
    _relation_length = np.array(_relation_length)
    if 0:
        print('_relation_bank')
        #print(_relation_bank)
        print(_relation_bank.shape)
        print('_relation_length')
        #print(_relation_length)
        print(_relation_length.shape)
        print('all_relations')
        print(len(all_relations))
        assert False

    local_token2idx = [x['token2idx'] for x in data]
    local_idx2token = [x['idx2token'] for x in data]
    if 0:
        print('local_token2idx:')
        print(local_token2idx)
        #print('local_idx2token:')
        #print(local_idx2token)
        assert False

    augmented_token = [[STR]+x['token']+[END] for x in data]
    if 0:
        print ('augmented_token:')
        print (augmented_token)
        assert False

    _token_in = ListsToTensor(augmented_token, vocabs['token'], unk_rate=unk_rate)[:-1]
    _token_char_in = ListsofStringToTensor(augmented_token, vocabs['token_char'])[:-1]

    if 0:
        print ('augmented_token:')
        print (augmented_token)
        print ("[ x['cp_seq'] for x in data]:")
        print ([ x['cp_seq'] for x in data])
        assert False

    _token_out = _ListsToTensor(augmented_token, vocabs['predictable_token'], local_token2idx)[1:]
    _cp_seq = _ListsToTensor([ x['cp_seq'] for x in data], vocabs['predictable_token'], local_token2idx)
    if 0:
        print ('_token_out:')
        print (_token_out)
        print ('_cp_seq:')
        print (_cp_seq)
        assert False
    if 0:
        print ('_token_in:')
        print(_token_in)
        print('_token_out:')
        print(_token_out)
        assert False

    if 0:
        print ('_conc:')
        print (_conc.shape)
        print('_relation_type')
        print(_relation_type.shape)
        print('_relation_bank')
        print(_relation_bank.shape)
        print('_relation_length')
        print(_relation_length.shape)
        print('all_relations')
        print(len(all_relations))
        assert False

    ret = {
        'concept': _conc,
        'concept_char': _conc_char,
        'concept_depth': _depth,
        'relation': _relation_type,
        'relation_bank': _relation_bank,
        'relation_length': _relation_length,
        'local_idx2token': local_idx2token,
        'local_token2idx': local_token2idx,
        'token_in':_token_in,
        'token_char_in':_token_char_in,
        'token_out':_token_out,
        'cp_seq': _cp_seq
    }
    return ret

class DataLoader(object):
    def __init__(self, vocabs, lex_map, filename, batch_size, for_train):
        self.data = read_file(filename)
        self.vocabs = vocabs
        self.lex_map = lex_map
        self.batch_size = batch_size
        self.train = for_train
        self.unk_rate = 0.
        self.nprocessors = 8
        self.record_flag = False

        if 0:
            print (self.batch_size)
            assert False

    def set_unk_rate(self, x):
        self.unk_rate = x

    def record(self):
        self.record_flag = True

    def __iter__(self):
        idx = list(range(len(self.data)))
        if 0:
            print ('idx')
            print (idx)
            assert False

        if self.train:
            random.shuffle(idx)
            idx.sort(key = lambda x: len(self.data[x]))
            if 0:
                print (idx[:100])
                print([len(self.data[x]) for x in idx][:100])
                assert False

        batches = []
        num_tokens, batch = 0, []
        for i in idx:
            if 0:
                print (i)
                print (self.data[i].graph.nodes)
                print (self.data[i].graph.edges)
                print(self.data[i].name2concept)
                print(len(self.data[i].name2concept))
                print(self.data[i].target)
                print(len(self.data[i].target))
                print (len(self.data[i]))
                print ('---------')
            num_tokens += len(self.data[i])
            batch.append(self.data[i])
            if num_tokens >= self.batch_size or len(batch)>256:
                batches.append(batch)
                num_tokens, batch = 0, []

        if not self.train or num_tokens > self.batch_size/2:
            batches.append(batch)

        if self.train:
            random.shuffle(batches)

        def work(graph):
            concept, depth, relation, ok = graph.collect_concepts_and_relations()
            assert ok, "not connected"
            tok = graph.target
            if 0:
                print ('tok:')
                print (tok)
                assert False

            cp_seq, token2idx, idx2token = self.lex_map.get(concept, self.vocabs['predictable_token'])
            if 0:
                print ('cp_seq:')
                print (cp_seq)
                print ('concept:')
                print (concept)
                assert False

            if 0:
                print ('vocab:concept:')
                print (
                    np.amin(
                        list(
                            self.vocabs['concept']._token2idx.values())),
                    np.amax(
                        list(
                            self.vocabs['concept']._token2idx.values())))

                print ('vocab:token:')
                print (
                    np.amin(
                        list(
                            self.vocabs['token']._token2idx.values())),
                    np.amax(
                        list(
                            self.vocabs['token']._token2idx.values())))

                print ('vocab:predictable_token:')
                print (
                    np.amin(
                        list(
                            self.vocabs['predictable_token']._token2idx.values())),
                    np.amax(
                        list(
                            self.vocabs['predictable_token']._token2idx.values())))

                print ('token2idx:')
                print (
                    np.amin(
                        list(
                            token2idx.values())),
                    np.amax(
                        list(
                            token2idx.values())))
                print ('-----------------')

                print('concept:')
                print (concept)
                print ('-----------------')
                print ('tok:')
                print (tok)
                print ('-----------------')
                print ('cp_seq:')
                print (cp_seq)
                print ('-----------------')
                for tk in cp_seq:

                    if tk in concept:
                        print (tk, 'in concept')
                    if tk in self.vocabs['predictable_token']._token2idx:
                        print (tk, 'in vocab:predictable_token')
                    if tk in self.vocabs['concept']._token2idx:
                        print (tk, 'in vocab:concept')
                    if tk in self.vocabs['token']._token2idx:
                        print (tk, 'in vocab:token')
                    if tk in token2idx:
                        print (tk, 'in token2idx')
                    print ('===')
                assert False

            item = {'concept': concept,
                    'depth': depth,
                    'relation': relation,
                    'token': tok,
                    'cp_seq': cp_seq,
                    'token2idx': token2idx,
                    'idx2token': idx2token
            }
            return item

        for batch in batches:
            res = [work(g) for g in batch]
            if not self.record_flag:
                yield batchify(res, self.vocabs, self.unk_rate)
            else:
                yield batchify(res, self.vocabs, self.unk_rate), res

def parse_config():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--token_vocab', type=str, default='../data/cs/token_vocab')
    parser.add_argument('--concept_vocab', type=str, default='../data/cs/concept_vocab')
    parser.add_argument('--predictable_token_vocab', type=str, default='../data/cs/predictable_token_vocab')
    parser.add_argument('--token_char_vocab', type=str, default='../data/cs/token_char_vocab')
    parser.add_argument('--concept_char_vocab', type=str, default='../data/cs/concept_char_vocab')
    parser.add_argument('--relation_vocab', type=str, default='../data/cs/relation_vocab')

    parser.add_argument('--train_data', type=str, default='../data/cs/train.txt')
    parser.add_argument('--train_batch_size', type=int, default=88888)

    return parser.parse_args()

if __name__ == '__main__':
    from extract import LexicalMap
    import time
    args = parse_config()
    vocabs = dict()
    vocabs['concept'] = Vocab(args.concept_vocab, 5, [CLS])
    vocabs['token'] = Vocab(args.token_vocab, 5, [STR, END])
    vocabs['predictable_token'] = Vocab(args.predictable_token_vocab, 5, [END])
    vocabs['token_char'] = Vocab(args.token_char_vocab, 100, [STR, END])
    vocabs['concept_char'] = Vocab(args.concept_char_vocab, 100, [STR, END])
    vocabs['relation'] = Vocab(args.relation_vocab, 5, [CLS, rCLS, SEL, TL])
    lexical_mapping = LexicalMap()

    if 0:
        pprint (vocabs)
        for key in vocabs:
            print (vocabs[key]._token2idx)
            print (key)
            assert False
        assert False

    train_data = DataLoader(vocabs, lexical_mapping, args.train_data, args.train_batch_size, for_train=True)
    epoch_idx = 0
    batch_idx = 0
    last = 0
    while True:
        st = time.time()
        for d in train_data:
            d = move_to_device(d, torch.device('cpu'))
            batch_idx += 1
            #if d['concept'].size(0) > 5:
            #    continue
            print (epoch_idx, batch_idx, d['concept'].size(), d['token_in'].size())
            c_len, bsz = d['concept'].size()
            t_len, bsz = d['token_in'].size()
            print (bsz, c_len*bsz, t_len * bsz)
            #print (d['relation_bank'].size())
            #print (d['relation'].size())

            #_back_to_txt_for_check(d['concept'], vocabs['concept'])
            #for x in d['concept_depth'].t().tolist():
            #    print (x)
            #_back_to_txt_for_check(d['token_in'], vocabs['token'])
            #_back_to_txt_for_check(d['token_out'], vocabs['predictable_token'], d['local_idx2token'])
            #print ('===================')
            #_back_to_txt_for_check(d['cp_seq'], vocabs['predictable_token'], d['local_idx2token'])
            #_back_to_txt_for_check(d['relation_bank'], vocabs['relation'])


# -*- coding:utf-8  -*-

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import unittest

class PythonOrgSearch(unittest.TestCase):

    def setUp(self):
        path = '/Users/y/chromedriver'
        self.driver = webdriver.Chrome(path)

    def test_search_in_python_org(self):
        driver = self.driver
        driver.get("https://www.baidu.com")
        self.assertIn("百度一下", driver.title)
        elem = driver.find_element_by_id("kw")
        elem.send_keys("pp100")
        elem.send_keys(Keys.RETURN)
        driver.implicitly_wait(10)
        assert "No results found." not in driver.page_source

    def tearDown(self):
        self.driver.close()

if __name__ == "__main__":
    unittest.main()