import logging
import os
from BlorgMusicData.models import DataSource, SongPost
from BlorgMusicHelper.stringhelper import *
from google.appengine.ext import db

__author__ = 'abeaupre'

class Dao():
    @classmethod
    def fetchSourceList(cls):
        return DataSource.objects.all()

    @classmethod
    def fetchSongPostList(cls, sourceKey = None, page = 1, size = 50):
        dataSource = None
        if sourceKey:
            dataSource = DataSource.objects.get(pk=sourceKey)

        songList = Dao.GetSongPostList(dataSource, size, page, 'CreateTime', True)

        if dataSource:
            dataLength = SongPostDataSourceList.objects.filter(DataSource_id = dataSource.id).count()
        else:
            dataLength = SongPost.objects.count()

        result = dict()
        result['songList'] = songList
        result['dataLength'] = dataLength
        return result

    @classmethod
    def ArtistItem_GetOrAdd(cls, name, hasCover=False, hasFeaturing=False, hasMashup=False, hasRemix=False):
        return ArtistItem.objects.get_or_create(Name=name, defaults=
            {'HasRemix':hasRemix, 'HasMashup':hasMashup, 'HasCover':hasCover, 'HasFeaturing':hasFeaturing})[0]

    @classmethod
    def GetArtistItem(cls, key):
        return ArtistItem.objects.get(key)

    @classmethod
    def GetArtistItemList(cls, pageSize, pageOffset, orderBy, orderDesc):
        itemOffset = (pageOffset-1)*pageSize
        orderString = ('-' if orderDesc else '') + orderBy
        return ArtistItem.objects.order_by(orderString)[itemOffset:pageSize]

    @classmethod
    def SongItem_GetList(cls, pageSize, pageOffset, orderBy, orderDesc):
        itemOffset = (pageOffset-1)*pageSize
        orderString = ('-' if orderDesc else '') + orderBy
        itemList = SongItem.objects.order_by(orderString)[itemOffset:pageSize]
        for item in itemList:
            item.ArtistRoleList = Dao.SongArtistRoleList_GetAll(item)

    @classmethod
    def GetParseItemList(cls, pageSize, pageOffset, orderBy, orderDesc):
        itemOffset = (pageOffset-1)*pageSize
        orderString = ('-' if orderDesc else '') + orderBy
        return ParseItem.objects.order_by(orderString)[itemOffset:pageSize]

    @classmethod
    def SongItem_Get(cls, key):
        songItem = SongItem.objects.get(pk=key)
        songItem.ArtistRoleList = Dao.SongArtistRoleList_GetAll(songItem)
        return songItem

    @classmethod
    def GetParseItem(cls, key):
        return ParseItem.objects.get(pk=key)

    @classmethod
    def GetSongPost(cls, key):
        return SongPost.objects.get(pk=key)

    @classmethod
    def GetBlogPost(cls, key):
        return BlogPost.objects.get(pk=key)

    @classmethod
    def GetSongPostList(cls, dataSource, pageSize, pageOffset, orderBy, orderDesc):
        itemOffset = (pageOffset-1)*pageSize
        orderString = ('-' if orderDesc else '') + orderBy
        if dataSource:
            #return SongPostDataSourceList.objects.filter(DataSource_id = dataSourceId).order_by(orderString)[itemOffset:pageSize]
            #TODO:  determine the best way to perform this query.  Denormalization or get list of SongPostIDs and iterate or investigate django-nonrel Indexing module (supports joins)
            songPostSourceList = SongPostDataSourceList.objects.filter(DataSource_id = dataSource.id).order_by(orderString)[itemOffset:pageSize]
            songPostList = []
            for item in songPostSourceList:
                songPost = SongPost.objects.get(pk=item.SongPost_id)
                songPostList.append(songPost)
            return songPostList
        else:
            return SongPost.objects.order_by(orderString)[itemOffset:pageSize]

    @classmethod
    def SongPost_AddFromParseData(cls, parsedText, songUrl, blogPostItem):
        if parsedText['successful']:

            songTitle = parsedText['songTitle']
            primaryArtistName = parsedText['primaryArtist']
            artistRoleList = parsedText['artistRoleList']
            songNotes = parsedText['songNote']
            inputString = parsedText['inputString']

            #for each of the listed artists, getoradd
            songKeyArtists = []
            primaryArtist = None
            for artistRole in artistRoleList:
                roleType = artistRole['role']
                artistName = artistRole['name']
                artistItem = cls.ArtistItem_GetOrAdd(artistName)
                if artistItem:
                    songKeyArtists.append({'artistItem':artistItem, 'roleType': roleType})
                    if roleType == 'P':
                        primaryArtist = artistItem

            #generate the SongKey
            songKey = cls.generateSongKey(songTitle, songKeyArtists)
            formattedSongTitle = formatSongTitle(songTitle, songNotes, artistRoleList)

            #look for the existing song
            songItemDefaults = {
                'SongTitle':songTitle,
                'FormattedSongTitle':formattedSongTitle,
                'SongNotes':songNotes,
                'PrimaryArtist': primaryArtist
            }

            songItem =  SongItem.objects.get_or_create(SongKey=songKey, defaults=songItemDefaults)[0]
            songPostDefaults = {
                'CanonicalSong' : songItem,
                'Blog' : blogPostItem,
                'OriginalText': inputString
            }
            return SongPost.objects.get_or_create(SongUrl = songUrl, defaults=songPostDefaults)[0]

    @classmethod
    def AddParseItem(cls, parsedText):
        inputString = parsedText['inputString']
        return ParseItem.objects.get_or_create(InputString=inputString)[0]

    @classmethod
    def AddParseItemSourceItemList(cls, parsedSong, dataSource):
        logging.info(parsedSong)
        return ParseItemSourceList.objects.get_or_create(Parse = parsedSong, SourceItem = dataSource)[0]

    @classmethod
    def GetDataSource(cls, dataSourceName):
        try:
            dataSource = DataSource.objects.get(SourceName=dataSourceName)
        except DataSource.DoesNotExist:
            dataSource = None
        return dataSource

    @classmethod
    def GetOrAddParseItem(cls, inputString, mp3href, blogPostItem):
        try:
            parseItem = ParseItem.objects.get(InputString=inputString)
        except ParseItem.DoesNotExist:
            parseItem = ParseItem(InputString=inputString, Url=mp3href, SourceUrl=blogPostItem.PostUrl)
            parseItem.save()
        return parseItem

    @classmethod
    def BlogSource_GetOrInsert(cls, sourceDomain, blogName):
        return BlogSource.objects.get_or_create(BlogUrl=sourceDomain, defaults={'BlogName':blogName})[0]

    @classmethod
    def BlogPost_GetOrInsert(cls, sourceUrl, blogPostSummary, blogSource):
        return BlogPost.objects.get_or_create(PostUrl = sourceUrl, defaults={'Summary':blogPostSummary, 'Blog':blogSource})[0]

    @classmethod
    def BlogPost_Get(cls, sourceUrl):
        return BlogPost.objects.get(PostUrl = sourceUrl)

    @classmethod
    def DeleteArtistItem(cls, artistItem):
        artistItem.delete()

    @classmethod
    def DeleteSongItem(cls, songItem):
        songItem.delete()

    @classmethod
    def DeleteParseItem(cls, parseItem):
        parseItem.delete()

    @classmethod
    def DataSource_GetOrAdd(cls, dataSourceName, dataSourceUrl):
        return DataSource.objects.get_or_create(SourceName = dataSourceName, defaults={'SourceUrl': dataSourceUrl})[0]

    @classmethod
    def FormatSongKey(cls, songTitle):
        return toLowerAndStripPunct(songTitle)

    # Key is defined as:
    #  formatted title + a sorted list of artistIds in CSV
    @classmethod
    def generateSongKey(cls, songTitle, artistRoleList):
        artistKey =''
        comma = ''
        for artistRoleItem in sorted(artistRoleList, key=lambda roleItem: roleItem['artistItem'].id):
            if artistRoleItem['roleType'] in['P', 'R', 'M']:
                artistKey += comma + str(artistRoleItem['artistItem'].id)
                comma =','
        return cls.FormatSongKey(songTitle) + '_' + artistKey

    @classmethod
    def DataSource_GetListAll(cls):
        dataSourceList = DataSource.objects.all()
        if not dataSourceList:
            Dao.DataSource_GetOrAdd('twitter', 'http://api.twitter.com/1/evilnohear/lists/all-music/statuses.xml')
            Dao.DataSource_GetOrAdd('twitter-electronic', 'http://api.twitter.com/1/evilnohear/lists/electronic-blogs/statuses.xml')
            Dao.DataSource_GetOrAdd('twitter-indie', 'http://api.twitter.com/1/evilnohear/lists/indie-blogs/statuses.xml')

        return dataSourceList

    @classmethod
    def ParseItem_GetListAll(cls):
        return ParseItem.objects.all()

    @classmethod
    def SongArtistRoleList_GetAll(cls, songItem):
        SongArtistRoleList.objects.filter(Song=songItem)


import urllib
from datetime import datetime
from urllib.parse import urlunparse, urlencode

import requests
from django.conf import settings

from apps.authorization.models import HabrUserProfile, HabrUser

def save_user_profile(backend, user, response, *args, **kwargs):
    if backend.name == 'vk-oauth2':
        api_url = urlunparse(('https',
                              'api.vk.com',
                              '/method/users.get',
                              None,
                              urlencode(dict(fields=','.join(('first_name', 'last_name', 'career', 'email',
                                                              'sex', 'bdate', 'country', 'city', 'photo_max_orig')),
                                             access_token=response['access_token'],
                                             v='5.92')),
                              None
                              ))
        response = requests.get(api_url)
        if response.status_code != 200:
            return
        data = response.json()['response'][0]
        user.is_confirmed = True
        if data.get('first_name'):
            user.habruserprofile.first_name = data["first_name"]
        if data.get('last_name'):
            user.habruserprofile.last_name = data["last_name"]
        if data.get('career'):
            user.habruserprofile.place_of_work = data['career'][-1]['company']
            user.habruserprofile.specialization = data['career'][-1]['position']
        if data.get('sex'):
            user.habruserprofile.gender = HabrUserProfile.MALE if data['sex'] == 2 else HabrUserProfile.FEMALE
        if data.get('bdate'):
            user.habruserprofile.birth_date = datetime.strptime(data['bdate'], '%d.%m.%Y').date()
        if data.get('country'):
            user.habruserprofile.country = data['country']['title']
        if data.get('city'):
            user.habruserprofile.city = data['city']['title']
        if data.get('photo_max_orig'):
            urllib.request.urlretrieve(data['photo_max_orig'], f'{settings.MEDIA_ROOT}/avatars/{user.pk}.jpg')
            user.habruserprofile.avatar = f'avatars/{user.pk}.jpg'
        user.save()
    elif backend.name == 'google-oauth2':
        user.is_confirmed = True
        if response.get('given_name'):
            user.habruserprofile.first_name = response['given_name']
        if response.get('family_name'):
            user.habruserprofile.last_name = response['family_name']
        if response.get('picture'):
            urllib.request.urlretrieve(response['picture'], f'{settings.MEDIA_ROOT}/avatars/{user.pk}.jpg')
            user.habruserprofile.avatar = f'avatars/{user.pk}.jpg'
        user.save()
    elif backend.name == 'github':
        user.is_confirmed = True
        if response.get('name'):
            user.habruserprofile.first_name = response['name']
        if response.get('avatar_url'):
            urllib.request.urlretrieve(response['avatar_url'], f'{settings.MEDIA_ROOT}/avatars/{user.pk}.jpg')
            user.habruserprofile.avatar = f'avatars/{user.pk}.jpg'
        user.save()
    else:
        return


__author__ = 'paulpatterson'

from unittest import TestCase
import xml.etree.ElementTree as ET
from pathlib import Path
import shutil
from collections import namedtuple
import tempfile

from jack_compiler.AbstractSyntaxTree import stringify_xml
from jack_compiler.JackCompiler import JackCompiler
from tests.globals import NAND_2_TETRIS, ACTUAL_COMPARE, EXPECTED_COMPARE

TestStrings = namedtuple("TestStrings", "actual expected")

TESTFILE = NAND_2_TETRIS / "JackCompiler" / "tests" / "CompilationTests.xml"
PARSE_TREE = NAND_2_TETRIS / "JackCompiler" / "tests" / "ParseTree.xml"

class CompilerVMGeneration(TestCase):

    @classmethod
    def setUpClass(cls):
        cls.tests = ET.parse(TESTFILE.as_posix()).getroot()

        cls.jack_project_directory = None
        cls.jack_file_path = None
        cls.vm_file_path = None

    def tearDown(self):
        if self.jack_project_directory is not None:
            parent_directory = self.jack_project_directory.parent
            shutil.rmtree(parent_directory.as_posix())
            self.jack_project_directory = None

    def _get_test_components(self, test_id):
        test_tree = self.tests.find("*[@id='{}']".format(test_id))
        expected_vm = test_tree.find("expected_vm").text.strip()
        jack_code = test_tree.find("jack_class").text
        return jack_code, expected_vm

    def make_temporary_test_directory(self, dir_name):
        tmp_dir = Path(tempfile.mkdtemp())
        proj_dir = tmp_dir / dir_name
        proj_dir.mkdir(parents=False)
        return proj_dir

    def set_up_compilable_jack_project(self, test_id, jack_code):
        self.jack_project_directory= self.make_temporary_test_directory(test_id)
        self.jack_file_path = self.jack_project_directory / "Main.jack"
        self.vm_file_path = self.jack_file_path.with_name("Main.vm")
        with open(self.jack_file_path.as_posix(), "w+") as jack_file:
            jack_file.write(jack_code)

    def _write_results_to_compare_files(self, actual_vm, expected_vm):
        with open(ACTUAL_COMPARE.as_posix(), 'w') as compareOne:
            with open(EXPECTED_COMPARE.as_posix(), 'w') as compareTwo:
                compareOne.write(actual_vm)
                compareTwo.write(expected_vm)

    def _write_abstract_syntax_tree(self, ast):
        with open(PARSE_TREE.as_posix(), "w") as astfile:
            astfile.write(str(ast))

    def _vm_strings_for_test(self, test_id):
        jack_code, expected_vm = self._get_test_components(test_id)

        # the next step sets the value of self.jack_project_directory, self.jack_file_path, and self.vm_file_path
        self.set_up_compilable_jack_project(test_id, jack_code)

        try:
            compiler = JackCompiler(path=self.jack_file_path)
            compiler.compile()
        except Exception as e:
            print(e)
            return None

        self._write_abstract_syntax_tree(compiler.abstract_syntax_trees[0])
        with open(self.vm_file_path.as_posix()) as vmfile:
            actual_vm = vmfile.read().strip()
            test_strings = TestStrings(actual_vm, expected_vm)

        self._write_results_to_compare_files(test_strings.actual, test_strings.expected)

        return test_strings

    def test_numbers(self):
        test_strings = self._vm_strings_for_test("Numbers")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_seven(self):
        test_strings = self._vm_strings_for_test("Seven")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_c2b_main(self):
        test_strings = self._vm_strings_for_test("C2BMain")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_if(self):
        test_strings = self._vm_strings_for_test("simple-if")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_if_else(self):
        test_strings = self._vm_strings_for_test("simple-if-else")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_nested_if_else(self):
        test_strings = self._vm_strings_for_test("simple-if-else")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_while(self):
        test_strings = self._vm_strings_for_test("simple-while")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_convert_to_binary(self):
        test_strings = self._vm_strings_for_test("ConvertToBin")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_simple_array(self):
        test_strings = self._vm_strings_for_test("simple-array")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_two_arrays(self):
        test_strings = self._vm_strings_for_test("two-simple-arrays")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_index_assignment(self):
        test_strings = self._vm_strings_for_test("indexed-assignment")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_complex_arrays_fragment(self):
        test_strings = self._vm_strings_for_test("complex-arrays-fragment")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_average(self):
        test_strings = self._vm_strings_for_test("Average")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_pong_main(self):
        test_strings = self._vm_strings_for_test("Pong-Main")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_pong_ponggame(self):
        test_strings = self._vm_strings_for_test("Pong-PongGame")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_pong_bat(self):
        test_strings = self._vm_strings_for_test("Pong-Bat")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_pong_ball(self):
        test_strings = self._vm_strings_for_test("Pong-Ball")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_square_main(self):
        test_strings = self._vm_strings_for_test("Square-Main")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)
    #
    def test_square_squaregame(self):
        test_strings = self._vm_strings_for_test("Square-SquareGame")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_square_square(self):
        test_strings = self._vm_strings_for_test("Square-Square")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)

    def test_complex_arrays(self):
        test_strings = self._vm_strings_for_test("ComplexArrays")
        self.assertIsNotNone(test_strings, "failed to generate either expected vm code, actual vm code, or both.")
        self.assertMultiLineEqual(test_strings.actual, test_strings.expected)


from problem2 import *
import sys
import math

'''
    Unit test 2:
    This file includes unit tests for problem2.py.
'''

#-------------------------------------------------------------------------
def test_python_version():
    ''' ----------- Problem 2 (50 points in total)---------------------'''
    assert sys.version_info[0]==3 # require python 3.6 or above
    assert sys.version_info[1]>=6

#---------------------------------------------------
def test_sample_features():
    ''' (10 points) sample_features'''
    # a dataset of 3 features, 4 samples
    X = np.array([[1.,2.,3.,4. ],
                  [2.,4.,6.,8. ],
                  [3.,6.,9.,12.]])
    count= 0
    for i in range(5):
        X1,fid = sample_features(X,2) # sample 2 features from 3
        if i>0 and np.allclose(fid,fid_old): # each time we should have a different sample of features
            count+=1
        assert type(X1) == np.ndarray
        assert type(fid) == np.ndarray
        assert X1.shape == (2,4)
        assert fid.shape == (2,)
        x = np.array([1,2,3,4])
        assert np.allclose(X1[0],x*(fid[0]+1))
        assert np.allclose(X1[1],x*(fid[1]+1))
        assert (X1[0]-X1[1]).sum()!=0 # the same feature cannot be sampled twice
        X1_old, fid_old = X1,fid
    assert count<3
    for _ in range(5):
        n = np.random.randint(2,5)
        p1 = np.random.randint(100,200)
        p2 = np.random.randint(100,200)
        X = np.bmat([[np.zeros((p1,n))],[np.ones((p2,n))]])
        X1, fid = sample_features(X,100)
        assert X1.shape == (100,n)
        assert fid.shape == (100,)
        assert np.allclose(X1.sum()/n/100, float(p2)/(p1+p2),atol = 0.1)
#---------------------------------------------------
def test_random_forest():
    ''' (10 points) random_forest'''
    # 3 features, 2 training samples
    X = np.array([[1.,1.],
                  [1.,2.],
                  [3.,3.]])
    Y = np.array(['good','bad'])
    count =0
    for _ in range(20):
        T,Fs = random_forest(X,Y,1,2)  # create an ensemble of only one tree, each tree uses 2 features
        assert len(T) == 1
        assert len(Fs) == 1
        if Fs[0][0]==0 or Fs[0,1]==0:
            f0=True
        if Fs[0][0]==1 or Fs[0,1]==1:
            f1=True
        if Fs[0,0]==2 or Fs[0,1]==2:
            f2=True
        if T[0].root.isleaf:
            count+=1
    assert count>5
    assert count<20
    assert f0 and f1 and f2 # make sure both features have been sampled at least once
    # 3 features, 4 training samples
    X = np.array([[1,2,4,5],
                  [2,2,3,4],
                  [3,3,4,5]])
    Y = np.array(['good','good','bad','good'])
    T,Fs = random_forest(X,Y,20,2) # create an ensemble of 20 decision trees and 2 features
    assert len(T) == 20
    x = np.array([4,3,4])
    for t in T:
        p=t.predict_1(x)
        if p=='good':
            g = True
            count+=1
        if p=='bad':
            b = True
    assert g and b # different tree should be able to make different predictions
    # 3 features, 4 training samples
    X = np.array([[1,2,4,5],
                  [2,2,2,2],
                  [3,3,3,3]])
    Y = np.array(['good','good','bad','good'])
    T,Fs = random_forest(X,Y,20,2) # create an ensemble of 10 decision trees
    assert len(T) == 20
    x = np.array([4,2,3])
    for t in T:
        p=t.predict_1(x)
        if p=='good':
            g = True
        if p=='bad':
            b = True
    assert g and b # different tree should be able to make different predictions
#---------------------------------------------------
def test_predict_1():
    ''' (10 points) predict_1'''
    # 3 features, 4 training samples
    X = np.array([[1,2,4,5],
                  [2,2,2,2],
                  [3,3,3,3]])
    Y = np.array(['good','good','bad','good'])
    count =0
    for _ in range(3):
        T,F = random_forest(X,Y,11,2) # create an ensemble of 11 decision trees
        assert len(T) == 11
        x = np.array([4,2,3])
        p=predict_1(T,F,x)
        if p=='bad':
            count+=1
    assert count>0
#---------------------------------------------------
def test_predict():
    ''' (20 points) predict'''
    # load a dataset from file
    filename = 'data2.csv'
    X = np.loadtxt(filename, dtype=float, delimiter=',',skiprows=1,usecols=range(1,17), unpack=True)
    Y = np.loadtxt(filename, dtype=int, delimiter=',',skiprows=1,usecols=0, unpack=True)
    assert X.shape == (16,400)
    assert Y.shape == (400,)
    n = float(len(Y))
    for _ in range(3):
        # train a decision tree over half of the dataset
        t = DecisionTree(X[:,::2],Y[::2])
        # test on the other half
        Y_predict = t.predict(X[:,1::2])
        accuracy0 = sum(Y[1::2]==Y_predict)/float(n)*2.
        print('test accuracy of a decision tree:', accuracy0)
        # train bagging over half of the dataset
        T = p1.bagging(X[:,::2],Y[::2],11)
        # test on the other half
        Y_predict = p1.predict(T,X[:,1::2])
        accuracy1 = sum(Y[1::2]==Y_predict)/n*2.
        print('test accuracy of bagging:', accuracy1)
        assert accuracy1 >= .75
        # train over half of the dataset
        T,F = random_forest(X[:,::2],Y[::2],11,10)
        # test on the other half
        Y_predict = predict(T,F,X[:,1::2])
        accuracy2 = sum(Y[1::2]==Y_predict)/n*2.
        print('test accuracy of random forest:', accuracy2)
        assert accuracy2 >= accuracy0
        assert accuracy2 >= accuracy1-.05


from basic_objects import *

#defining one cell's size
CELL_WIDTH = 50
CELL_HEIGHT = 50

#defining basic colors
BACKGROUND_COLOR = "#550000"
WALL_COLOR = "#777777"
PLAYER_COLOR = "#dddddd";
EXIT_COLOR = "#00aa00";

#defining a level
class level(sprite.Sprite):
    def __init__(self, input_file_name):
        sprite.Sprite.__init__(self)
        #the map is stored in a .txt file
        #in the following format
        # X is player's initial position
        # . is an empty cel;
        # # is a wall
        # E is an exit cell
        self.input_file = open(input_file_name, "r")
        self.level_map = []
        for line in self.input_file:
            cur_row = line
            if line[len(line) - 1] == '\n':
                cur_row = line[0: len(line) - 1]
            self.level_map.append(cur_row)
        #calculating total width and height
        self.width = len(self.level_map[0]) * CELL_WIDTH
        self.height = len(self.level_map) * CELL_HEIGHT
        #player's initial position
        self.player_row = -1
        self.player_col = -1
        for row_counter in range(len(self.level_map)):
            for col_counter in range(len(self.level_map[row_counter])):
                if self.level_map[row_counter][col_counter] == 'X':
                    self.player_row = row_counter
                    self.player_col = col_counter
        #exit position
        self.exit_row = -1
        self.exit_col = -1
        for row_counter in range(len(self.level_map)):
            for col_counter in range(len(self.level_map[row_counter])):
                if self.level_map[row_counter][col_counter] == 'E':
                    self.exit_row = row_counter
                    self.exit_col = col_counter

        self.hero = player(CELL_WIDTH, CELL_HEIGHT, PLAYER_COLOR, BACKGROUND_COLOR)
        self.wall_block = wall(CELL_WIDTH, CELL_HEIGHT, WALL_COLOR)
        self.exit_block = map_exit(CELL_WIDTH, CELL_HEIGHT, EXIT_COLOR)

    #checking whether the (row, col) cell is empty and inside the field
    def is_valid(self, row, col):
        good = True
        if row < 0 or col < 0:
            good = False
        if row >= len(self.level_map) or col >= len(self.level_map[0]):
            good = False
        if good:
            if self.level_map[row][col] != '#':
                return True
            else:
                return False
        else:
            return False

    #checking if the user has found an exit
    def finished(self):
        return self.exit_row == self.player_row and self.exit_col == self.player_col

    #making a movement if possible
    def update(self, left, right, up, down):
        #only one move is allowed
        #so the others will be ignored
        done = False
        #moving in chosen direction if possible
        if left and self.is_valid(self.player_row, self.player_col - 1):
            done = True
            self.player_col -= 1
        if not done and right and self.is_valid(self.player_row, \
                                                self.player_col + 1):
            done = True
            self.player_col += 1
        if not done and up and self.is_valid(self.player_row - 1, \
                                             self.player_col):
            done = True
            self.player_row -= 1
        if not done and down and self.is_valid(self.player_row + 1, \
                                               self.player_col):
            done = True
            self.player_row += 1

    #drawing the current state of the map
    def draw(self, screen):
        #drawing a player
        screen.blit(self.hero.background, (self.player_col * CELL_WIDTH, \
                                      self.player_row * CELL_HEIGHT))
        #drawing the walls and exit block
        for row in range(len(self.level_map)):
            for col in range(len(self.level_map[row])):
                #a wall
                if self.level_map[row][col] == '#':
                    screen.blit(self.wall_block.background, (col * CELL_WIDTH, \
                                                  row * CELL_HEIGHT))
                # an exit block
                if self.level_map[row][col] == 'E':
                    screen.blit(self.exit_block.background, (col * CELL_WIDTH, \
                                                        row * CELL_HEIGHT))


import datetime
import queue
import random
import re
import time
import urllib
import urllib.robotparser

import requests
from bs4 import BeautifulSoup

USER_AGENT = [{'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'},\
              {'User-Agent':'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11'},\
              {'User-Agent':'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'},\
              {'User-Agent':'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0'},\
              {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/44.0.2403.89 Chrome/44.0.2403.89 Safari/537.36'},\
              {'User-Agent':'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'},\
              {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'},\
              {'User-Agent':'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0'},\
              {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'},\
              {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'},\
              {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'},\
              {'User-Agent':'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11'},\
              {'User-Agent':'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'}]

DELAY_SEC = 5
SUCCESS_CODE = 200

def download_page(url, num_retries=2, headers=None, data=None):
    """下载网页内容，如果状态码不匹配，返回空字符串。否则返回utf-8编码的字符串
    """
    print(time.ctime(), ' : ', 'Downloading...:', url)
    # headers = header.update(USER_AGENT[random.randrange(0, len(USER_AGENT))])
    headers = headers or {}
    try:
        response = requests.get(url, headers=headers, data=data)
        code = response.status_code
        page_content = str(response.content, encoding='utf-8')
    except requests.RequestException as e:
        print(time.ctime(), ' : Download error ', e.args)
        page_content = ''
        if num_retries > 0:
            time.sleep(DELAY_SEC)
            return download_page(url, num_retries-1, data=data)

    if code != SUCCESS_CODE:
        print(time.ctime(), ' : The status code is:', code, '\nPlease check!')
        page_content = ''
    if page_content:
        print(time.ctime(), ' : Response string is not None')
        # print(page_content)
    return page_content

def link_crawler(seed_url, link_regex=None, delay=DELAY_SEC, max_depth=-1, max_urls=-1, **kw):
    """通过匹配链接关键字从种子链接开始爬取网页
    """
    # 待爬取队列
    crawl_queue = queue.deque([seed_url])
    # 口路url是否已被爬取，以及所在深度
    seen = {seed_url: 0}
    # 记录爬取的总页面数
    num_urls = 0
    # 记录被禁止访问的页面
    blocked_url = []
    # 记录抓取数据
    crawled_data = []

    robot_parse = get_robot(seed_url)
    throttles = throttle(delay)
    headers = kw.get('headers', {})

    while crawl_queue:
        url = crawl_queue.pop()
        # 检查url是否通过网站robots.txt的限制
        user_agent = USER_AGENT[random.randrange(0, len(USER_AGENT))]
        if robot_parse.can_fetch(user_agent, url):
            throttles.wait(url)
            headers.update(user_agent)
            html = download_page(url, headers=headers)
            links = []

            depth = seen.get(url, -1)
            if depth != max_depth:
                # 深度不匹配， 还可以继续抓取页面
                if link_regex:
                    # 搜寻匹配的网址
                    links.extend(link for link in get_links(html) if re.search(link_regex, link))

                for link in links:
                    link = normalize(seed_url, link)
                    # 检查是否已经爬取过该地址
                    if link not in seen:
                        seen[link] = depth + 1
                        # 检查网页是否在本网站之内
                        if same_domain(seed_url, link):
                            # 是就加入爬取队列之中
                            crawl_queue.append(link)

            # 检查是否已经达到最大下载数量
            num_urls = num_urls + 1
            if num_urls == max_urls:
                break

        else:
            blocked_url.append(url)
            print('Blocked by robots.txt: ', url)

class throttle(object):
    """通过time.sleep()控制对相同网站下载网页的间隔
    """
    def __init__(self, delay):
        # 两次下载的间隔时间
        self.delay = delay
        # 记录上次访问网站时的时间戳
        self.domains_time_stamp = {}
        # 记录睡眠睡眠次数
        self.times = 0

    def wait(self, url):
        domain = urllib.parse.urlparse(url).netloc
        last_accessed_time = self.domains_time_stamp.get(domain)
        now_time = datetime.datetime.now()
        if self.delay > 0 and last_accessed_time is not None:
            sleep_secs = self.delay - (now_time - last_accessed_time).seconds
            if sleep_secs > 0:
                time.sleep(sleep_secs)
                self.times = self.times + 1
                print(time.ctime(), ': Sleeped : ', sleep_secs, self.times)
        self.domains_time_stamp[domain] = now_time

def get_robot(seed_url):
    """返回robots.txt的解析对象
    """
    # robot_parse = urllib.robotparser.RobotFileParser()
    robot_parse = urllib.robotparser.RobotFileParser()
    robot_parse.set_url(urllib.parse.urljoin(seed_url, '/robots.txt'))
    robot_parse.read()
    return robot_parse

def normalize(seed_url, link):
    """去除链接干扰项
    """
    parse_result = urllib.parse.urlparse(link)
    return urllib.parse.urljoin(seed_url, parse_result.path)

def same_domain(url1, url2):
    """如果两个网址属于同一网站，返回True
    """
    return urllib.parse.urlparse(url1).netloc == urllib.parse.urlparse(url2).netloc

def get_links(html):
    """通过正则匹配获取链接
    """
    webpage_regex = re.compile(r'.*?', re.S)
    return webpage_regex.findall(html)

if __name__ == '__main__':
    # link_crawler('http://example.webscraping.com', '/(index|view)', delay=0, num_retries=1, user_agent='BadCrawler')
    link_crawler('http://example.webscraping.com', '/(index|view)', delay=5, num_retries=1, max_depth=1, user_agent='GoodCrawler')


# Config file keys
TRANSACTION = "transaction"
PREPROCESSING = "preprocessing"
URL = "url"
CHUNK_SIZE = "chunk_size"
DATA_COLLECTION_TZ = "data_collection_tz"
LOOKBACK_PERIODS = "lookback_periods"
TIME_GAPS = "time_gaps"
TIMESTEPS_IN = "timesteps_in"
TIMESTEPS_OUT = "timesteps_out"
GT_VAR_SHIFT = "gt_var_shift"
GT_VAR_NAME = "gt_var_name"
TRAINING = "training"
HIDDEN_UNITS = "hidden_units"
DENSE_UNITS_SMALL = "dense_units_small"
DENSE_UNITS_LARGE = "dense_units_large"
N_CONV_FILTERS = "n_conv_filters"
KERNEL_SIZE = "kernel_size"
DROPOUT_RATE = "dropout_rate"
BATCH_SIZE = "batch_size"
EPOCHS = "epochs"
FILEPATHS = "filepaths"
RAW_INPUT = "raw_input"
CONVERTED_UNIX_TS = "converted_unix_ts"
OHLCV = "ohlcv"
FEATURES = "features"
SEQUENTIAL = "sequential"
TIMESTAMPS = "timestamps"
INPUT_SCALER = "input_scaler"
OUTPUT_SCALER = "output_scaler"
INPUT_SEQUENCES = "input_sequences"
OUTPUT_SEQUENCES = "output_sequences"
INPUT_ARRAY = "input_array"
OUTPUT_ARRAY = "output_array"
HOLDOUT_DF = "holdout_df"

TEXT = "text"
CONVERTED_TZ = "converted_tz"
TARGET_VOCAB_SIZE = "target_vocab_size"
MAX_DOC_LEN = "max_doc_len"
BUFFER_SIZE = "buffer_size"
LOGGING_FREQUENCY = "logging_frequency"
NUM_ENCODER_LAYERS = "num_encoder_layers"
NUM_ENCODER_HEADS = "num_encoder_heads"
ENCODER_FEED_FORWARD_UNITS = "feed_forward_units"
WORD_EMBEDDING_DIM = "word_embedding_dim"
ES_PATIENCE = "es_patience"
ES_MIN_DELTA = "es_min_delta"
HOURLY_NEWS = "hourly_news"
VECTORIZED_NEWS = "vectorized_news"

COMMON = "common"
RANDOM_STATE = "random_state"
TEST_SIZE = "test_size"
N_FOLDS = "n_folds"
TRAIN_START_DATE = "train_start_date"
TRAIN_END_DATE = "train_end_date"
MODEL_PATH = "model_path"

# Other constants
CONFIG_FILEPATH = "config_filepath"

# Preprocessing
PRICE = "price"
VOLUME = "volume"
YEAR = "year"
MONTH = "month"
DAY = "day"
HOUR = "hour"
OPEN = "open"
HIGH = "high"
LOW = "low"
CLOSE = "close"
WEEKDAY = "weekday"
TIMESTAMP = "timestamp"
TITLE = "title"
TITLE_TEXT = "title_text"
NONE = "none"

# kwargs keys
TRAINING_FLAG = "training_flag"
MODEL_NAME = "model_name"


import logging
import queue

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO, format='%(message)s')

QUEUE_TIMEOUT = 10 ** -2

class LambdaWorker(object):
    '''Simple worker that waits for jobs, runs them, and returns results.'''

    def __init__(self, job_queue, result_dict, worker_id):
        self.functions = {}
        self._job_queue = job_queue
        self._result_dict = result_dict
        self._id = worker_id
        self.await_jobs()

    def await_jobs(self):
        '''Waits for new jobs to come in via the job queue, runs the job, and
        puts the result into the results dict.'''

        logger.info('[WORKER {}] Ready!'.format(self._id))

        while True:
            try:
                job = self._job_queue.get(timeout=QUEUE_TIMEOUT)
            except queue.Empty:
                logger.debug('[WORKER {}] found empty queue'.format(self._id))
                continue

            logger.debug('[WORKER {}] Running job {}'
                         .format(self._id, job.job_id))
            result = job.run()

            logger.debug('[WORKER {}] Returning result {}'
                         .format(self._id, job.job_id))
            self._result_dict[job.job_id] = result


from __future__ import division, print_function, absolute_import
import dataSet_ts as dt
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
import datetime
import sys, time
import os, math
import kellyGetData as kgd
############### HYPERPARAMETERS ################

appliances = [ 'kettle','microwave', 'washing machine', 'dish washer' , 'fridge']#
windows = {1:("2013-02-27", "2014-02-27")}
#To specify more than one building, just add a key in the dict windows
#i.e. 2:("2013-02-27", "2013-04-27")

def main(args):

    lr = float(args['lr']) #0.0001
    nilmkt_fileName = args['nilmkt_fileName']       # Path to ukdale.h5
    n_steps = int(args['n_steps'])                  # Size of input sequence (length of instance)
    n_hidden_units = int(args['hiddenUnitsLayer1']) # First layer number of units
    training_iters = int(args['iterations'])        # Number of iterations or epochs
    stride_train = int(args['strideTrain'])         # Separation between instances of training set
    num_sequences_per_batch = int(args['numSequences']) #based on appliance
    stride_test =  n_steps                          # Separation between instances of test set
    applianceTest =  int(args['applianceToTest'])   # Apliance (order number in the list) of the appliance to evaluate
    period = 6                                      # It is the period that most of other works use
    loadParam = args['loadAsKelly']
    target_inclusion_prob = float(args['target_inclusion_prob'])
    loadAsKelly = True
    period = int(args['period'])
    data_path = args['data_path']
    if(loadParam == 'N' or loadParam == 'n' or loadParam == 'no' or loadParam == 'NO' or loadParam == 'No'):
        loadAsKelly = False
    # Setting the number of nodes in hidden states of the 3 layers
    n_inputs_0 = n_hidden_units   # Number of cells in input vector
    n_inputs_1 = int(n_hidden_units/2)
    n_inputs_2 = n_hidden_units
    n_inputs_3 = n_steps

    ########### FUNCTIONS TO DEFINE WEIGHTS AND BIAS ####################

    def weight_variable(shape, m=0, std=0.5):
        initial = tf.random_normal(shape,mean=m, stddev=std) #truncated_normal, stddev=0.1 / w['in'] = tf.random_normal
        return tf.Variable(initial)

    def bias_variable(shape, c=1.0):
        initial = tf.constant(c, shape=shape)
        return tf.Variable(initial)

    ############ UPLOADING DATA SET ############

    reader = dt.ReaderTS(windows, appliances, n_steps, stride_train, stride_test, period,
                            flgScaling=1, trainPer=0.5, valPer=0.25, testPer=0.25)

    XdataSet,YdataSet = reader.load_csvdata(period, data_path, target_inclusion_prob, nilmkt_fileName, applianceTest, loadAsKelly, num_sequences_per_batch)

    # Depending on the applianceTest (-1 for all): x.shape = [batch, seqLen]
    #                                y.shape = [batch, seqLen] or [batch, seqLen, numAppliances]
    x_train, x_test, x_val  = XdataSet['train'], XdataSet['test'],  XdataSet['val']
    y_train, y_test, y_val  = YdataSet['train'], YdataSet['test'],  YdataSet['val']

    x_train = x_train.reshape((-1, n_steps, 1))
    y_train = y_train.reshape((-1, n_steps, 1))
    x_val = x_val.reshape((-1, n_steps, 1))
    y_val = y_val.reshape((-1, n_steps, 1))
    x_test = x_test.reshape((-1, n_steps, 1))
    y_test = y_test.reshape((-1, n_steps, 1))
    ##############   STARTING TO CONSTRUCT GRAPH: INPUTS AND OPERATIONS #####################

    x = tf.placeholder(tf.float32, [None, n_steps, 1]) #batch_size x length x dimensions
    y = tf.placeholder(tf.float32, [None, n_steps, 1])

    # Size of the batch (it could change in validation and/or test runs)
    initBatch = tf.placeholder(tf.int32, shape=())

    layer_2_units = (n_steps-3)*8

    #layer 1
    conv1 = tf.layers.conv1d(inputs=x, filters=8, kernel_size=4, strides=1,
        padding='valid', activation = tf.nn.relu)
    conv1 = tf.reshape(conv1,[-1, layer_2_units])

    #layer 2
    Win_2 = weight_variable([layer_2_units,128])
    bin_2 = bias_variable([128])
    out_2 = tf.nn.relu(tf.matmul(conv1, Win_2) +  bin_2)

    #layer 3
    Win_3 = weight_variable([128,layer_2_units])
    bin_3 = bias_variable([layer_2_units])
    out_3 = tf.nn.relu(tf.matmul(out_2, Win_3) +  bin_3)

    #layer 4
    Win_4 = weight_variable([layer_2_units,n_steps])
    bin_4 = bias_variable([n_steps])
    out_4 = tf.nn.relu(tf.matmul(out_3, Win_4) +  bin_4)
    out_4 = tf.expand_dims(out_4, 2)

    #layer 5
    pred = tf.layers.conv1d(inputs=out_4, filters=1, kernel_size=4, strides=1,
        padding='same', activation = tf.nn.relu) #for now use same, the paper says valid

    #pred has too many predictions
    print("conv1.shape:",conv1.shape)
    print("out_2.shape:",out_2.shape)
    print("out_3.shape:",out_3.shape)
    print("out_4.shape:",out_4.shape)
    print("predictions.shape:",pred.shape)
    print("x.shape:",x.shape)
    print("y.shape:",y.shape)

    ## Indicating to tensorflow that save the values of the parameters/layers in each iteration##
    tf.summary.histogram('conv1', conv1)
    tf.summary.histogram('Win_2', Win_2)
    tf.summary.histogram('Win_3', Win_3)
    tf.summary.histogram('Win_4', Win_4)
    ### Flatten real Y and prediction to be able to measure the results as other models do
    yForMetric = tf.reshape(y,[-1])
    predForMetric = tf.reshape(pred,[-1])

    ### Calculating metrics (done just in validation/testing. Not part of the optimization)
    #Check what's the difference btw these different calculatinos. Meanwhile, staying with upMae0
    #tf.metrics.mean_absolute_error. =  tf.contrib.metrics.streaming_mean_absolute_error
    mae0, upMae0 = tf.metrics.mean_absolute_error(labels=yForMetric,predictions=predForMetric) # Some problem with initialization
    upMae1 = tf.reduce_mean(tf.abs(yForMetric-predForMetric))

    sum_output = tf.reduce_sum(pred)
    sum_target = tf.reduce_sum(y)
    relativeError = (sum_output - sum_target) / tf.maximum(sum_output, sum_target)

    #### Defining cost function and optimizator
    cost = tf.losses.mean_squared_error(yForMetric,predForMetric)
    tf.summary.scalar('cost', cost) #Track values of cost
    '''
    #Evaluate to add a regularizer
    l1_regularizer = tf.contrib.layers.l1_regularizer(scale=0.5)#scale=0.05, scope=None
    regularization_penalty = tf.contrib.layers.apply_regularization(l1_regularizer, [W_conv])
    '''
    train_op = tf.train.AdamOptimizer(lr, epsilon=0.1).minimize(cost)

    # Show the sizes of the weight matrices to know number of total paramters used
    for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
        print(v)

    ## Auxiliar structure to track cost and metrics.
    ## Could be done w/ summary.scalar but haven't figured out yet - Tensorboard
    tracking = {'step': [], 'cost': [], 'mae': [], 'relerr':[]}

    ############   START RUNNING   #############

    with tf.Session() as sess:
        init = tf.global_variables_initializer()
        sess.run(init)
        sess.run(tf.local_variables_initializer()) # for mean absolute error, that need local variables initialized
        merged = tf.summary.merge_all()
        # Write values of some variables to check them later in tensorboard
        train_writer = tf.summary.FileWriter('Logs/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M"),sess.graph)
        test_writer = tf.summary.FileWriter('Logs/LogsTestAE_SepApp/')
        # Print metrics on validation set every (1/10) of the iterations
        step = 0
        while (step < training_iters):
            [summary,op_train] = sess.run([merged, train_op], feed_dict={x: x_train,y: y_train, initBatch: x_train.shape[0]})
            train_writer.add_summary(summary, step)
            if (step % (training_iters/10) == 0):
                trncost, metric_upMae0, metric_upMae1, out_preds, out_re = sess.run(
	                    [cost,upMae0, upMae1, pred, relativeError], feed_dict={x: x_val, y: y_val, initBatch: x_val.shape[0]})
                tracking['step'].append(step)
                tracking['cost'].append(trncost)
                tracking['mae'].append(metric_upMae0)
                tracking['relerr'].append(out_re)
                print("Time {} Step {:5d} Cost {:10.2f} Metric {:6.2f}/{:6.2f} RelError {:3.2f}".format(
                    datetime.datetime.now().strftime("%y-%m-%d_%H-%M"),step,trncost,metric_upMae0, metric_upMae1 ,out_re))
            step+=1
        tstcost, out_preds, metric_upMae0, out_re  =sess.run([cost, pred,upMae0, relativeError], feed_dict={x: x_test, y: y_test,initBatch: x_test.shape[0]})
        print ("\nTEST Cost {} Metric {} Relat err {}".format(tstcost,metric_upMae0, out_re))

        savedFolder = 'Experiments/app{}_{}'.format(applianceTest,datetime.datetime.now().strftime("%y-%m-%d_%H-%M"))
        os.makedirs(savedFolder)
        yFlat = y_test.flatten()
        predFlat = out_preds.flatten()

        # Concatenate results horizontally to have again the one large sequence
        newTest =np.concatenate([y_test[i] for i in range(y_test.shape[0])],0) # tf.concat([y_test[i] for i in range(y_test.shape[0])],0)
        newPred = np.concatenate([out_preds[i]  for i in range(y_test.shape[0])],0) #tf.concat([preds[i] for i in range(preds.shape[0])],0)
        newInput = np.concatenate([x_test[i]  for i in range(y_test.shape[0])],0)

        assert newTest.shape == newPred.shape
        print ("Test size {}".format(newTest.shape))

        fLog = open('{}/{}_output'.format(savedFolder,applianceTest), 'w')
        fLog.write("Step,Cost,MAE, Rel error\n")
        for i , item in enumerate(tracking['step']):
          a = tracking['step'][i]
          b = tracking['cost'][i]
          c = tracking['mae'][i]
          d = tracking['relerr'][i]
          fLog.write("{},{},{},{}\n".format(a,b,c,d))
        fLog.write("Test,{},{},{}\n".format(tstcost,metric_upMae0,out_re))
        f, axarr = plt.subplots(3, 1, sharex=True)

        axarr[0].plot(newTest)
        axarr[0].set_title('Y-real')

        axarr[1].plot(newPred)
        axarr[1].set_title('Y-pred')

        axarr[2].plot(newInput)
        axarr[2].set_title('X-real')

        f.subplots_adjust(top=0.92, bottom=0.05, left=0.10, right=0.95, hspace=0.3, wspace=0.3)
        plt.savefig('{}/{}_plots'.format(savedFolder,applianceTest))#self.savedFolder+'/'+ch.name+str(numfig)
        plt.clf()

if __name__ == "__main__":

    import sys, time
    if len(sys.argv) > 1:
        config_file_name = sys.argv[-1]
    else:
        config_file_name = 'config_kelly.txt'

    f = open(config_file_name, 'r')
    lines = f.readlines()
    params = {}

    for line in lines:
        line = line.split('\n')[0]
        param_list = line.split(' ')
        param_name = param_list[0]
        param_value = param_list[1]
        params[param_name] = param_value

    main(params)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author "jhe"
@date 2018/6/1 21:02
"""

addr = {
    '安徽省': {
        '安庆市': ['太湖县','潜山县','宿松县','怀宁县','枞阳县','岳西县','桐城市','迎江区'],
        '合肥市': ['庐阳区','蜀山区'],
        '芜湖市': ['镜湖区 ','鸠江区 '],
    },
    '江苏省': {
        '苏州市':['吴中区','平江区'],
        '南京市':['玄武区','鼓楼区'],
        '无锡市':['崇安区','北塘区 ']
    }
}
# b返回上一级，
# provin_list = []
# city_list = []
exit_flag = False
back_flag = False
while not back_flag and not exit_flag:
    provin_list = []

    for province in addr:
        provin_list.append(province)
        print(provin_list.index(province),':',province)
    provin = input('请选择省份[q退出]：')
    if provin.isdigit():
        provin = int(provin)
        if provin < len(provin_list):
            print('您已选择',provin_list[provin])
            while not back_flag and not exit_flag:
                city_list = []

                for city in addr[provin_list[provin]]:
                    city_list.append(city)
                    print(city_list.index(city), ':', city)
                city_no = input('请选择市[b返回上一级,q退出]：')
                if city_no.isdigit():
                    city_no = int(city_no)
                    if city_no < len(city_list):
                        print('您已选择',city_list[city_no])
                        while not back_flag and not exit_flag:
                            conty_list = []
                            for conty in addr[provin_list[provin]][city_list[city_no]]:
                                conty_list.append(conty)
                                print(conty_list.index(conty), ':', conty)
                            conty_no = input('请选择县[b返回上一级,q退出]：')
                            if conty_no.isdigit():
                                conty_no = int(conty_no)
                                if conty_no < len(conty_list):
                                    print('您已选择',conty_list[conty_no])
                                    print('您选择的地址是', provin_list[provin],city_list[city_no],conty_list[conty_no])
                                    exit_flag = True
                                    break
                                else:
                                    print('no such choice! input again!')
                            elif conty_no == 'q':
                                exit_flag = True
                                break
                            elif conty_no == 'b':
                                back_flag = True
                            else:
                                print('invalid input! input again!')
                        else:
                            back_flag = False
                    else:
                        print('no such choice! input again!')
                elif city_no == 'q':
                    exit_flag = True
                    break
                elif city_no == 'b':
                    back_flag =  True
                else:
                    print('invalid input! input again!')
            else:
                back_flag = False
        else:
            print('no such choice! input again!')
    elif provin == 'q':
        break
    else:
        print('invalid input! input again!')


"""IMDB class file"""
import requests

class IMDB:
    """Class to retrieve movie info using apache64"""

    def __init__(self, title, field=""):
        """Instance variables"""
        self.title = title
        self._field = field

    @property
    def field(self):
        return self._field

    @field.setter
    def field(self, input_field):
        """Returns IMDB field for the instantiated movie"""
        try:
            title = self.title.strip().replace(' ', '+')
            imdb = 'http://theapache64.xyz:8080/movie_db/search?keyword={0}'.format(title)
            json = requests.Session().get(imdb).json()
            value = json['data'][input_field]
        except KeyError:
            value = "N/A"
        self._field = value


#python first
#django second
#your apps
#locals directory

from django.conf import settings
from django.contrib import messages
from django.core.mail import send_mail
from django.shortcuts import render, render_to_response, RequestContext, get_object_or_404, HttpResponseRedirect
from .models import Product

#MessagesCommands

#messages.debug(request, '' % count)
#messages.info(request, '')
#messages.success(request, '')
#messages.warning(request, '')
#messages.error(request, '')

# Create your views here.
from cart.forms import ProductQtyForm
from .forms import Product

def all_products(request):
	products = Product.objects.all()

	return render_to_response("products/all.html",
								locals(),
								context_instance=RequestContext(request))

def single_product(request, slug):
	add_product = ProductQtyForm()
	product = get_object_or_404(Product, slug=slug)

	return render_to_response("products/single.html",
								locals(),
								context_instance=RequestContext(request))

def home(request):
	products = Product.objects.all()

	return render_to_response("home.html",
								locals(),
								context_instance=RequestContext(request))

def thankyou(request):

	form = ProductFrom(request.POST or None)

	if form.is_valid():
		save_it = form.save(commit=False)
		save_it.save()

		#send_mail(subject, message, from_email, to_list, fail_silently=True)
		subject = 'You Have Succesfully Created a Product'

		return HttpResponseRedirect('/thank-you/')

	return render_to_response("thankyou.html",
								locals(),
								context_instance=RequestContext(request))

def aboutus(request):

	return render_to_response("aboutus.html",
								locals(),
								context_instance=RequestContext(request))

def contactus(request):

	return render_to_response("contactus.html",
								locals(),
								context_instance=RequestContext(request))

def search(request):
	try:
		q = request.GET.get('q', '')
	except:
		q = False

	if q:
		k = q.split()
		if len(k) >= 2:
			products = []
			for i in k:
				all_products = Product.objects.filter(name__icontains=i).distinct()
				for product in all_products:
					if product not in products:
						products.append(product)
		else:
			products = Product.objects.filter(name__icontains=q)

	return render_to_response("products/search.html",
								locals(),
								context_instance=RequestContext(request))


# coding=utf-8

import cv2
import numpy as np
from matplotlib import pyplot as plt

img = cv2.imread('F:/lane.jpg', 1)
rows, cols, ch = img.shape

pts1 = np.float32([[329, 244], [994, 244], [225, 372], [1099, 372]])
pts2 = np.float32([[300, 0], [600, 0], [300, 300], [600, 300]])

M = cv2.getPerspectiveTransform(pts1, pts2)

dst = cv2.warpPerspective(img, M, (900,800))

plt.subplot(131), plt.imshow(img), plt.title('Input')
plt.subplot(133), plt.imshow(dst), plt.title('Output')
#plt.show()

img1 = dst.copy()
img2 = dst.copy()

gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)

#gray = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 5, 7)
#retval, gray = cv2.threshold(gray, 50, 255, cv2.THRESH_BINARY)

img1 = cv2.GaussianBlur(gray, (3, 3), 0)
edges = cv2.Canny(img1, 50, 150, apertureSize=3)
lines2 = cv2.HoughLines(edges, 1, np.pi / 180, 118)
result = img1.copy()

# 经验参数
minLineLength = 500
maxLineGap = 30
lines = cv2.HoughLinesP(edges, 1, np.pi / 180, 80, minLineLength, maxLineGap)
for x1, y1, x2, y2 in lines[0]:
    cv2.line(img2, (x1, y1), (x2, y2), (255, 0, 0), 4)

for line in lines2[0]:
    rho = line[0] #第一个元素是距离rho
    theta = line[1] #第二个元素是角度theta
    print(rho)
    print(theta)
    if (theta < (np.pi/4.)) or (theta > (3.*np.pi/4.0)):  #垂直直线
                #该直线与第一行的交点
        pt1 = (int(rho/np.cos(theta)),0)
        #该直线与最后一行的焦点
        pt2 = (int((rho-edges.shape[0]*np.sin(theta))/np.cos(theta)),edges.shape[0])
        #绘制一条白线
        cv2.line( edges, pt1, pt2, (128),10)
    else: #水平直线
        # 该直线与第一列的交点
        pt1 = (0,int(rho/np.sin(theta)))
        #该直线与最后一列的交点
        pt2 = (edges.shape[1], int((rho-edges.shape[1]*np.cos(theta))/np.sin(theta)))
        #绘制一条直线
        cv2.line(edges, pt1, pt2, (128), 10)

plt.subplot(221), plt.imshow(img), plt.title('Input')
plt.subplot(222), plt.imshow(dst), plt.title('Output')
plt.subplot(223), plt.imshow(edges), plt.title('canny')
plt.subplot(224), plt.imshow(img2), plt.title('changed')

plt.show()

CNN/直线识别/透视变换.py


# Copyright (c) 2013, VHRS and contributors
# For license information, please see license.txt

from __future__ import unicode_literals

import frappe
from frappe import _, msgprint

def execute(filters=None):
    if not filters:
        filters = {}

    columns = get_columns()

    data = []
    row = []
    conditions, filters = get_conditions(filters)
    contractors = get_contractors(conditions, filters)
    for contractor in contractors:
        row = [contractor.name]
        salary_slips = get_salary_slips(contractor.name,filters)
        basic,da,oa,ot,wg,pf,esic,canteen,pt,otesic,sc,otsc,eesic,eotesic,epf =  get_ss(contractor.name,filters)
        for ss in salary_slips:
            if ss.md:
                row += [ss.md]
            else:
                row += [""]
            if ss.twh:
                row += [ss.twh]
            else:
                row += [""]
            if basic:
                row += [basic]
            else:
                row += [""]
            if da:
                row += [da]
            else:
                row += [""]
            if oa:
                row += [oa]
            else:
                row += [""]
            if ot:
                row += [ot]
            else:
                row += [""]
            if wg:
                row += [wg]
            else:
                row += [""]
            if ss.gp:
                row += [ss.gp]
            else:
                row += [""]
            if pf:
                row += [pf]
            else:
                row += [""]
            if esic:
                row += [esic]
            else:
                row += [""]
            if canteen:
                row += [canteen]
            else:
                row += [""]
            if pt:
                row += [pt]
            else:
                row += [""]
            if otesic:
                row += [otesic]
            else:
                row += [""]
            if ss.td:
                row += [ss.td]
            else:
                row += [""]
            if sc:
                row += [sc]
            else:
                row += [""]
            if otsc:
                row += [otsc]
            else:
                row += [""]
            if eesic:
                row += [eesic]
            else:
                row += [""]
            if eotesic:
                row += [eotesic]
            else:
                row += [""]
            if epf:
                row += [epf]
            else:
                row += [""]

            if ss.transport:
                row += [ss.transport]
            else:
                row += [""]

            data.append(row)

    return columns, data

def get_ss(contractor,filters):
    from_date = filters.get("from_date")
    to_date = filters.get("to_date")
    b_total = da_total = oa_total = ot_total = wg_total = pf_total = esic_total = canteen_total = pt_total = otesic_total = sc_total = otsc_total = eesic_total = eotesic_total = epf_total = 0
    ssp = frappe.get_all("Salary Slip",filters={'start_date':from_date,'end_date':to_date,'contractor':contractor})
    for ss in ssp:
        basics = frappe.db.sql("""select td.amount as basic from `tabSalary Detail` td where
            td.salary_component='Basic' and td.parent=%s""",(ss['name']),as_dict=1)
        for b in basics:
            b_total += b['basic']
        da = frappe.db.sql("""select td.amount as da from `tabSalary Detail` td where
            td.salary_component='Dearness Allowance' and td.parent=%s""",(ss['name']),as_dict=1)
        for d in da:
            da_total += d['da']
        oa = frappe.db.sql("""select td.amount as oa from `tabSalary Detail` td where
            td.salary_component='Other Allowances' and td.parent=%s""",(ss['name']),as_dict=1)
        for o in oa:
            oa_total += o['oa']
        ot = frappe.db.sql("""select td.amount as ot from `tabSalary Detail` td where
            td.salary_component='Overtime' and td.parent=%s""",(ss['name']),as_dict=1)
        for t in ot:
            ot_total += t['ot']
        wg = frappe.db.sql("""select td.amount as wg from `tabSalary Detail` td where
            td.salary_component='Wages' and td.parent=%s""",(ss['name']),as_dict=1)
        for w in wg:
            wg_total += w['wg']
        pf = frappe.db.sql("""select td.amount as pf from `tabSalary Detail` td where
            td.salary_component='Provident Fund' and td.parent=%s""",(ss['name']),as_dict=1)
        for p in pf:
            pf_total += p['pf']
        esic = frappe.db.sql("""select td.amount as esic from `tabSalary Detail` td where
            td.salary_component='ESIC' and td.parent=%s""",(ss['name']),as_dict=1)
        for e in esic:
            esic_total += e['esic']
        canteen = frappe.db.sql("""select td.amount as canteen from `tabSalary Detail` td where
            td.salary_component='Canteen' and td.parent=%s""",(ss['name']),as_dict=1)
        for c in canteen:
            canteen_total += c['canteen']
        ptax = frappe.db.sql("""select td.amount as pt from `tabSalary Detail` td where
            td.salary_component='Professional Tax' and td.parent=%s""",(ss['name']),as_dict=1)
        for pt in ptax:
            pt_total += pt['pt']
        otesic = frappe.db.sql("""select td.amount as otesic from `tabSalary Detail` td where
            td.salary_component='OT ESIC' and td.parent=%s""",(ss['name']),as_dict=1)
        for ote in otesic:
            otesic_total += ote['otesic']
        sc = frappe.db.sql("""select td.amount as sc from `tabSalary Detail` td where
            td.salary_component='Service Charge' and td.parent=%s""",(ss['name']),as_dict=1)
        for s in sc:
            sc_total += s['sc']
        otsc = frappe.db.sql("""select td.amount as otsc from `tabSalary Detail` td where
            td.salary_component='OT Service Charge' and td.parent=%s""",(ss['name']),as_dict=1)
        for os in otsc:
            otsc_total += os['otsc']
        eesic = frappe.db.sql("""select td.amount as eesic from `tabSalary Detail` td where
            td.salary_component='Employer ESIC' and td.parent=%s""",(ss['name']),as_dict=1)
        for ee in eesic:
            eesic_total += ee['eesic']
        eotesic = frappe.db.sql("""select td.amount as eotesic from `tabSalary Detail` td where
            td.salary_component='Employer OT ESIC' and td.parent=%s""",(ss['name']),as_dict=1)
        for eote in eotesic:
            eotesic_total += eote['eotesic']
        epf = frappe.db.sql("""select td.amount as epf from `tabSalary Detail` td where
            td.salary_component='Employer PF' and td.parent=%s""",(ss['name']),as_dict=1)
        for ep in epf:
            epf_total += ep['epf']

    return b_total,da_total,oa_total,ot_total,wg_total,pf_total,esic_total,canteen_total,pt_total,otesic_total,sc_total,otsc_total,eesic_total,eotesic_total,epf_total

def get_salary_slips(contractor,filters):
    from_date = filters.get("from_date")
    to_date = filters.get("to_date")
    salary_slips = frappe.db.sql("""select ss.name,sum(vrc.total) as transport,sum(ss.payment_days) as md,sum(ss.total_working_hours) as twh,sum(ss.gross_pay) as gp,sum(ss.total_deduction) as td from `tabSalary Slip` ss
    left join `tabVan Rate Calculation` vrc on vrc.employee = ss.employee
    where
    ss.start_date=%s and ss.end_date=%s group by ss.contractor having ss.contractor=%s """,(from_date,to_date,contractor),as_dict=1)
    return salary_slips

def get_columns():
    columns = [
        _("Contractor") + ":Data:180",
        _("Man Days") + ":Int:100",
        _("OT Hours") + ":Float:100",
        _("Basic") + ":Currency:120",
        _("DA") + ":Currency:120",
        _("Other Allowance") + ":Currency:120",
        _("Overtime") + ":Currency:100",
        _("Stipend") + ":Currency:100",
        _("Gross Pay") + ":Currency:100",
        _("Provident Fund") + ":Currency:100",
        _("ESIC") + ":Currency:100",
        _("Canteen") + ":Currency:100",
        _("Professional Tax") + ":Currency:100",
        _("OT ESIC") + ":Currency:100",
        _("Total Deduction") + ":Currency:100",
        _("Service Charge") + ":Currency:100",
        _("OT Service Charge") + ":Currency:100",
        _("Employer ESIC") + ":Currency:100",
        _("Employer OT ESIC") + ":Currency:100",
        _("Employer PF") + ":Currency:100",
        _("Transport") + ":Currency:100",

    ]
    return columns

def get_contractors(conditions, filters):
    from_date = filters.get("from_date")
    to_date = filters.get("to_date")
    active_contractors = frappe.db.sql(
        """select name from `tabContractor` %s order by name""" % conditions, filters, as_dict=1)
    return active_contractors

def get_conditions(filters):
    conditions = ""
    if filters.get("contractor"):
        conditions += "where name = %(contractor)s"
    return conditions, filters


import turtle
t = turtle.Pen()
turtle.speed(0)

def my_circle(red, blue, green):
 	t.color(red, blue, green)
 	t.begin_fill()
 	t.circle(50)
 	t.end_fill()

my_circle(0, 0, 0)
my_circle(1, 1, 1)
t.up()
t.forward(200)
t.down()
my_circle(0.9, 0.75, 0)
turtle.exitonclick()


from xai.brain.wordbase.verbs._asphyxiate import _ASPHYXIATE

#calss header
class _ASPHYXIATED(_ASPHYXIATE, ):
	def __init__(self,):
		_ASPHYXIATE.__init__(self)
		self.name = "ASPHYXIATED"
		self.specie = 'verbs'
		self.basic = "asphyxiate"
		self.jsondata = {}


import telegram
import logging
import requests
import datetime
from flask import Flask, request
from telegram.ext import (Updater, CommandHandler, MessageHandler, Filters, RegexHandler, ConversationHandler)

logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)

app = Flask(__name__)
logger = logging.getLogger(__name__)

bot_token = "579475538:AAFjEgtt1BFPIU_f4IjYyaix-heREgouX3g"

updater = Updater(token=bot_token)
dispatcher = updater.dispatcher

def get_url(method):
  return "https://api.telegram.org/bot{}/{}".format(bot_token,method)

def marca(bot, update):
    update.message.reply_text("Jogo marcado!")

def confirma(bot, update):
    update.message.reply_text("Confirmado.")

def desconfirma(bot, update):
    update.message.reply_text("Desconfirmado!")

def cancel(bot, update):
    update.message.reply_text('Sem futebol essa semana entao...', reply_markup=ReplyKeyboardRemove())
    return ConversationHandler.END

marca_handler = CommandHandler('marca', marca)
dispatcher.add_handler(marca_handler)

confirmar_handler = CommandHandler('confirma', confirma)
dispatcher.add_handler(confirmar_handler)

desconfirmar_handler = CommandHandler('desconfirma', desconfirma)
dispatcher.add_handler(desconfirmar_handler)

updater.start_polling()


import numpy as np
import basic_shapes as bs

def createCircle(N, r, g, b,radius):
    vertices = [0, 0, 0, r, g, b]  # Primer vertice
    indices = []

    dtheta = 2 * np.pi / N

    for i in range(N):
        theta = i * dtheta

        vertices += [
            # vertex coordinates
            radius * np.cos(theta), radius * np.sin(theta), 0,
            r, g, b]

        # Se crean triángulos respecto al centro para rellenar la circunferencia
        indices += [0, i, i + 1]
    # Conectado al segundo vertice
    indices += [0, N, 1]

    vertices = np.array(vertices, dtype=np.float32)
    indices = np.array(indices, dtype=np.uint32)

    return bs.Shape(vertices, indices)

def createTrail(radio, N):
    vertices = []
    indices = []

    dtheta = 2 * np.pi / N

    for i in range(N):
        theta = i * dtheta

        vertices += [
            radio * np.cos(theta), radio * np.sin(theta), 0,

            0.91, 0.93, 0.85]

        if i == 0:
            indices += [0, 0, 1]
        elif i == N - 1:
            pass
        else:
            indices += [i, i, i + 1]

    # Uniendo circunferencia
    indices += [N - 1, N - 1, 0]

    vertices = np.array(vertices, dtype=np.float32)
    indices = np.array(indices, dtype=np.uint32)

    return bs.Shape(vertices, indices)


import requests
import lxml.html
import rdflib

problems = ['Directed by', '\n', 'Produced by', '[a]', '[1]', '[2]', '[3]', '[4]', '[5]', '[6]', '[7]', '[8]',
                '[9]', ' (p.g.a.)', 'Executive Producer', ': ', ' ', ', ', '', 'Running time', 'Starring']

people_link_problems = ['#cite_note-1', '#cite_note-2', '#cite_note-Elliot-1', '#cite_note-comingsoon-1', '#cite_note-Fletcher-2']

def get_producers(doc):
    producers = []

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Produced by')]//@title"):
        producers.append(t)

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Produced by')]//text()"):
         t = t.strip()
         if t not in problems:
             skip = False
             for producer in producers:
                 if t in producer:
                     skip = True
                     break
             if(not skip):
                producers.append(t)

    producers.sort()
    return producers

def get_directors(doc):
    directors = []

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Directed by')]//@title"):
        directors.append(t)

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Directed by')]//text()"):
        t = t.strip()
        if t not in problems:
            skip = False
            for director in directors:
                if t in director:
                    skip = True
                    break
            if not skip:
                directors.append(t)

    directors.sort()
    return directors

def get_running_time(doc):
    runningtime = []

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Running time')]//text()"):
        if t not in problems:
            runningtime.append(t.strip())

    return runningtime

def get_starring(doc):
    starring = []

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Starring')]//@title"):
        starring.append(t)

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Starring')]//text()"):
        t = t.strip()
        if t not in problems:
            skip = False
            for star in starring:
                if t in star:
                    skip = True
                    break
            if not skip:
                starring.append(t)

    starring.sort()
    return starring

def check_octavia_spencer(films_list, films_names, starring_list):
    for i in range(len(films_list)):
        if "The Shape of Water" == films_names[i]:
            stars = starring_list[i]
            for star in stars:
                if "Octavia Spencer" == star:
                    print("Yes")
                    print("---------")
                    print(films_list[i])
                    print(films_names[i])
                    print(starring_list[i])
                    return
    print("No")

def check_meryl_streep(films_list, starring_list):
    count_meryl_strip = 0
    for i in range(len(films_list)):
        stars = starring_list[i]
        for star in stars:
            if "Meryl Streep" == star:
                print(films_list[i])
                count_meryl_strip += 1
    print("Number of films is " + str(count_meryl_strip))

def get_release_date(doc):
    release_dates = []

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Release date')]//span[@class = 'bday dtstart published updated']//text()"):
        release_dates.append(t)

    #TODO: CHECK IF TO SORT, CHECK MORE EXAMPLES OF MISSING DATES
    return release_dates

def get_based_on(doc):
    based_on = []

    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Based on')]"):
        based_on.append("Yes")
        return based_on

    based_on.append("No")
    return based_on

def get_people(people_set, doc):
    for t in doc.xpath("//table[@class = 'infobox vevent']//tr[contains(.//text(),'Directed by') or " +
                       "contains(.//text(),'Produced by') or contains(.//text(),'Starring')]//a//@href"):
        if t not in people_link_problems:
            people_set.add("https://en.wikipedia.org" + t)

def get_birthday(doc):
    bd = []
    for t in doc.xpath("//table[@class = 'infobox biography vcard' or @class = 'infobox vcard']//tr[contains(.//text(),'Born')]//span[@class = 'bday']//text()"):
        bd.append(t)

    return bd

def create():
    start_link = "https://en.wikipedia.org/wiki/List_of_Academy_Award-winning_films"

    ontology = rdflib.Graph()

    r = requests.get(start_link)
    doc = lxml.html.fromstring(r.content)

    films_entities = []
    films_list = []
    films_names = []
    producers_list = []
    directors_list = []
    running_time_list = []
    starring_list = []
    based_on_list = []
    release_date_list = []
    people_set = set()

    for t in doc.xpath("//table[@class = 'wikitable sortable']//tr[position() > 1 and td[position()=2]//text() > 2009]//td[position() = 1]//a//@href"):
        films_list.append("https://en.wikipedia.org" + t)

    for t in doc.xpath("//table[@class = 'wikitable sortable']//tr[position() > 1 and td[position()=2]//text() > 2009]//td[position() = 1]//a//@title"):
        films_names.append(t)

    for link in films_list:
        r = requests.get(link)
        doc = lxml.html.fromstring(r.content)

        print(link)
        producers_list.append(get_producers(doc))
        print(producers_list[-1])
        #directors_list.append(get_directors(doc))
        #running_time_list.append(get_running_time(doc))
        #starring_list.append(get_starring(doc))
        #release_date_list.append(get_release_date(doc))
        #based_on_list.append(get_based_on(doc))
        #print(based_on_list[-1])
        #get_people(people_set, doc)

    #people_list = list(people_set)
    #bdays = []
    #occupations = []
    # for people_link in people_list:
    #     r = requests.get(people_link)
    #     doc = lxml.html.fromstring(r.content)

        #bdays.append(get_birthday(doc))
        # occupations.append(get_occupation(doc))
        #
        # print(people_link)
        # print(occupations[-1])

    #     occ = []
    #     for t in doc.xpath("//table[@class = 'infobox biography vcard' or @class = 'infobox vcard']//tr[contains(.//text(),'Occupation')]//li//text()"):
    #         if t not in problems:
    #             occ.append(t)
    #
    #
    #     for t in doc.xpath("//table[@class = 'infobox biography vcard' or @class = 'infobox vcard']//tr[contains(.//text(),'Occupation')]//td[@class = 'infobox-data role' or @class='infobox-data']//text()"):
    #         if t not in problems:
    #             occ.append(t)
    #
    #
    #     occupations.append(sorted(list(set(occ))))
    #     #print(occupations[-1])

    # for i in range(len(films_list)):
    #     print(str(i) + " " + films_list[i])
    #     print(directors_list[i])
    #     print(producers_list[i])
    #     print(running_time_list[i])
    #     print(starring_list[i])
    #     print(release_date_list[i])
    #     print("--------------")

    # for i in range(len(people_list)):
    #     print(str(i) + " " + people_list[i])
    #     print(bdays[i])

    # print("------------------------")
    # check_meryl_streep(films_list, starring_list)
    # print("------------------------")
    # check_octavia_spencer(films_list, films_names, starring_list)

create()


import os
import numpy as np
import sys

if __name__ == '__main__':
	time_p = []
	for i in [-6, -5, -4, -3, -2, -1]:
		for j in [1, 2.5, 5, 7.5]:
			time_p.append(j*10**(i))
	print(time_p)
	tau = 0.777660157519
	for idx, val in enumerate(time_p):
		time_p[idx] = val / tau
	print(time_p)


import numpy as np
import json
import os
import pandas as pd

from tqdm.auto import tqdm
from multiprocessing import Pool
from pandarallel import pandarallel
pandarallel.initialize(nb_workers=50)

from xgboost import XGBClassifier

from model import Model
from utils import read_df

all_columns = ['item_domain_id', 'item_id', 'item_title', 'site_id', 'sku',
       'product_id', 'product_family_id', 'count', 'date_first', 'date_last',
       'date_diff', 'date_first_day', 'date_first_month', 'date_last_day',
       'date_last_month', 'sold_quantity_first', 'sold_quantity_last',
       'sold_quantity_sum', 'sold_quantity_mean', 'sold_quantity_std',
       'sold_quantity_min', 'sold_quantity_max', 'sold_quantity_mode',
       'sold_quantity_mode_tx', 'current_price_first', 'current_price_last',
       'current_price_sum', 'current_price_mean', 'current_price_std',
       'current_price_min', 'current_price_max', 'current_price_mode',
       'current_price_mode_tx', 'minutes_active_first', 'minutes_active_last',
       'minutes_active_sum', 'minutes_active_mean', 'minutes_active_std',
       'minutes_active_min', 'minutes_active_max', 'minutes_active_mode',
       'minutes_active_mode_tx', 'currency_first', 'currency_last',
       'currency_mode', 'currency_mode_tx', 'listing_type_first',
       'listing_type_last', 'listing_type_mode', 'listing_type_mode_tx',
       'shipping_logistic_type_first', 'shipping_logistic_type_last',
       'shipping_logistic_type_mode', 'shipping_logistic_type_mode_tx',
       'shipping_payment_first', 'shipping_payment_last',
       'shipping_payment_mode', 'shipping_payment_mode_tx',
       'minutes_active_series', 'current_price_series',
       'sold_quantity_series']

categorical_columns = ['site_id', 'item_id']
categorical_columns_with_nan = ['item_domain_id', 'product_id', 'product_family_id']
categorical_columns_composed = ['currency', 'listing_type', 'shipping_logistic_type', 'shipping_payment']
categorical_composed_suffix = ['_first', '_last', '_mode']

categorical_columns = categorical_columns + categorical_columns_with_nan
for category_name in categorical_columns_composed:
    for suffix in categorical_composed_suffix:
        categorical_columns.append(category_name+suffix)

numeric_columns = ['count', 'date_first_day', 'date_first_month', 'date_last_day', 'date_last_month']
numeric_columns_composed = ['sold_quantity', 'current_price', 'minutes_active']
numeric_composed_suffix = ['_first', '_last', '_mode', '_mode_tx', '_sum', '_mean', '_std', '_min', '_max']

for numeric_name in numeric_columns_composed:
    for suffix in numeric_composed_suffix:
        numeric_columns.append(numeric_name+suffix)

categorical_to_numeric_composed_suffix = ['_mode_tx']

for category_name in categorical_columns_composed:
    for suffix in categorical_to_numeric_composed_suffix:
        numeric_columns.append(category_name+suffix)

series_columns = ['sold_quantity_series', 'current_price_series', 'minutes_active_series']
series_columns_headers = {}

series_columns_max = 29
for series_name in series_columns:
    series_columns_headers[series_name] = [series_name+'_'+str(i) for i in range(series_columns_max)]
    numeric_columns = numeric_columns + series_columns_headers[series_name]

features = numeric_columns + categorical_columns

class XGBoostFeaturesV2(Model):
    model_name = 'xgboost_features_v2_classification'

    def __init__(self, dataset_path):
        Model.__init__(self, self.model_name, dataset_path)

    def prepare_data(self):
        print('Number of features:', len(features), 'Non-used features: ', set(all_columns) - set(features) - set(series_columns))

        df_train_x_features = read_df(os.path.join(self.dataset_path, self.default_paths['train_data_x_processed']))
        df_train_y_features = read_df(os.path.join(self.dataset_path, self.default_paths['train_data_y_processed']))

        #Randomize and order sync train data
        x_df = df_train_x_features.sample(frac=1, random_state=42).reset_index(drop=True).copy()
        y_df = df_train_y_features.set_index('sku').loc[x_df['sku']].reset_index().copy()

        for column in numeric_columns_composed:
            x_df[column+'_std'] = x_df[column+'_std'].fillna(0)
            y_df[column+'_std'] = y_df[column+'_std'].fillna(0)
        for column in categorical_columns_with_nan:
            x_df[column] = x_df[column].astype(str).astype("category")
            y_df[column] = y_df[column].astype(str).astype("category")

        categorical_pandas_order = {}

        for column in categorical_columns:
            x_df[column] = x_df[column].astype("category")
            categorical_pandas_order[column] = x_df[column].cat.categories
            y_df[column] = y_df[column].astype("category")
            y_df[column] = y_df[column].cat.set_categories(categorical_pandas_order[column], ordered=True)

        for column in categorical_columns:
            x_df[column] = x_df[column].cat.codes
            y_df[column] = y_df[column].cat.codes

        for column in series_columns:
            def series_to_df(data):
                result = np.ones(series_columns_max)*-1
                data = json.loads(data)
                index_limit = len(data) if len(data) <= series_columns_max else series_columns_max
                result[-index_limit:] = data[-index_limit:]
                #return result
                return pd.Series(result, index=series_columns_headers[column])

            series_x_df = x_df[column].parallel_apply(series_to_df)
            series_y_df = y_df[column].parallel_apply(series_to_df)

            x_df = x_df.join(series_x_df)
            y_df = y_df.join(series_y_df)

        X = x_df[features].values
        y = np.array(list(y_df['sold_quantity_series'].apply(json.loads).values))
        y = y.cumsum(axis=1)

        #y_target_stock and y_target_date_0
        y_ts = np.zeros(len(y))
        y_td = np.zeros(len(y))

        for i, t in enumerate(y):
            where_non_zero = np.where(t > 0)[0]
            if len(where_non_zero) == 0:
                y_ts[i] = 0
                y_td[i] = 0
                continue
            j = np.random.choice(where_non_zero)
            y_ts[i] = t[j]
            y_td[i] = np.where(t == t[j])[0][0]

        X = np.concatenate((X, np.reshape(y_ts, (-1, 1))), axis=1)
        y = y_td

        self.prepared_dataset = (X, y)
        self.categorical_pandas_order = categorical_pandas_order

    def train(self):
        X, y = self.prepared_dataset
        self.model = XGBClassifier(n_estimators=1000, max_depth=6, learning_rate=0.1,
                    random_state=0, tree_method='gpu_hist',
                    objective='multi:softprob', num_class=30, use_label_encoder=False,
                    gpu_id=3)
        self.model.fit(X, y)

    def predict(self, df_test):
        df_test = read_df(df_test)
        df_test_fromtrain_x_features = read_df(os.path.join(self.dataset_path, self.default_paths['test_fromtrain_data_x_processed']))

        test_x_df = df_test_fromtrain_x_features.set_index('sku').loc[df_test['sku']].reset_index().copy()
        for column in numeric_columns_composed:
            test_x_df[column+'_std'] = test_x_df[column+'_std'].fillna(0)
        for column in categorical_columns_with_nan:
            test_x_df[column] = test_x_df[column].astype(str).astype("category")
        for column in categorical_columns:
            test_x_df[column] = test_x_df[column].astype("category")
            categorical_order = list(self.categorical_pandas_order[column])
            if len(test_x_df[column].cat.categories) != len(categorical_order):
                for cat in set(test_x_df[column].cat.categories) - set(categorical_order):
                        categorical_order.append(cat)
            test_x_df[column] = test_x_df[column].cat.set_categories(categorical_order, ordered=True)
            test_x_df[column] = test_x_df[column].cat.codes
        for column in  series_columns:
            def series_to_df(data):
                result = np.ones(series_columns_max)*-1
                data = json.loads(data)
                index_limit = len(data) if len(data) <= series_columns_max else series_columns_max
                result[-index_limit:] = data[-index_limit:]
                #return result
                return pd.Series(result, index=series_columns_headers[column])

            series_x_test_df = test_x_df[column].parallel_apply(series_to_df)
            test_x_df = test_x_df.join(series_x_test_df)

        X_test = test_x_df[features].values
        X_test = np.concatenate((X_test, np.reshape(df_test['target_stock'].values, (-1, 1))), axis=1)

        preds = self.model.predict_proba(X_test)
        probabilities = (preds/preds.sum(axis=1)[:,None]).round(4)

        return probabilities


from scipy import constants as C
import numpy as np
import dispersion
import surface
import volume
import input_tools

# Example of separating Nd:glass wavelength and its second harmonic in a Pellin-Broca prism.

# Suggested plotter command
# python ray_plotter.py out/test o3d

# Units and scales

mks_length = 1.054e-6 / (2*np.pi)
bundle_scale = 1e-4
mm = 1000*mks_length
inch = 100*mks_length/2.54
fs = 1e15*mks_length/C.c
deg = 180/np.pi
helper = input_tools.InputHelper(mks_length)
mess = 'Processing input file...\n'

# Setup pulse parameters

theta = 0.0 # initial propagation angle
a00 = 1.0
w00 = 1.0
r00 = 0.1/mm # spot size of radiation
rb = r00*bundle_scale
t00 = 500/fs
band = (0.9,2.1)

# Setup prism

material = dispersion.BK7(mks_length)
nrefr = np.sqrt(1+material.chi(w00)[0])
prism_box = (1/inch,1/inch,0.75/inch)
refraction_angle = 30/deg
incidence_angle = np.arcsin(nrefr*np.sin(refraction_angle))
mess += 'TIR angle = ' + str(np.arcsin(1/nrefr)*deg) + ' deg\n'

# Set up dictionaries

sim = {}
wave = []
ray = []
optics = []
diagnostics = {}

sim['mks_length'] = mks_length
sim['mks_time'] = mks_length/C.c
sim['message'] = mess

ray.append({})
ray[-1]['number'] = (64,1,4,1)
ray[-1]['bundle radius'] = (rb,rb,rb,rb)
ray[-1]['loading coordinates'] = 'cylindrical'
# Ray box is always put at the origin
# It will be transformed appropriately by SeaRay to start in the wave
ray[-1]['box'] = band + (0.0,3*r00) + (0.0,2*np.pi) + (0.0,0.0)

wave.append({}) # fundamental
wave[-1]['a0'] = (0.0,a00*np.cos(theta),0.0,-a00*np.sin(theta)) # EM 4-potential (eA/mc^2) , component 0 not used
wave[-1]['r0'] = (t00,r00,r00,t00) # 4-vector of pulse metrics: duration,x,y,z 1/e spot sizes
wave[-1]['k0'] = (w00,w00*np.sin(theta),0.0,w00*np.cos(theta)) # 4-wavenumber: omega,kx,ky,kz
# 0-component of focus is time at which pulse reaches focal point.
# If time=0 use paraxial wave, otherwise use spherical wave.
# Thus in the paraxial case the pulse always starts at the waist.
wave[-1]['focus'] = (0.0,0.0,0.0,-31/mm)
wave[-1]['supergaussian exponent'] = 2

wave.append({}) # second harmonic
wave[-1]['a0'] = (0.0,a00*np.cos(theta),0.0,-a00*np.sin(theta)) # EM 4-potential (eA/mc^2) , component 0 not used
wave[-1]['r0'] = (t00,r00,r00,t00) # 4-vector of pulse metrics: duration,x,y,z 1/e spot sizes
wave[-1]['k0'] = (2*w00,2*w00*np.sin(theta),0.0,2*w00*np.cos(theta)) # 4-wavenumber: omega,kx,ky,kz
# 0-component of focus is time at which pulse reaches focal point.
# If time=0 use paraxial wave, otherwise use spherical wave.
# Thus in the paraxial case the pulse always starts at the waist.
wave[-1]['focus'] = (0.0,0.0,0.0,-31/mm)
wave[-1]['supergaussian exponent'] = 2

optics.append({})
optics[-1]['object'] = volume.PellinBroca('P1')
optics[-1]['dispersion outside'] = dispersion.Vacuum()
optics[-1]['dispersion inside'] = material
optics[-1]['size'] = prism_box
optics[-1]['angle'] = refraction_angle
optics[-1]['origin'] = (-10/mm,0.,0.)
optics[-1]['euler angles'] = helper.rot_zx(incidence_angle)

optics.append({})
optics[-1]['object'] = surface.EikonalProfiler('det')
optics[-1]['size'] = (1/inch,1/inch)
optics[-1]['origin'] = (100/mm,0.0,10/mm)
optics[-1]['euler angles'] = helper.rot_zx(-90/deg)

diagnostics['suppress details'] = False
diagnostics['clean old files'] = True
diagnostics['orbit rays'] = (16,1,4,1)
diagnostics['base filename'] = 'out/test'