# -*- coding: utf-8 -*-
"""
Matthew Dirks
10/21/2019
Module to scrape SBG data from Synergy Education Platform

Dependencies:
    selenium
    geckodriver (for Firefox)
    ChromeDriver (for Chrome)
"""

# import modules
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException, \
    ElementClickInterceptedException
import student_module as stu
import lt_module as ltm
import classperiod_module as cpm
from nitty_gritty_of_grading import DEFAULT_X_CODE

# Global variables (shudder)
# Keyword to look for in assignment name of 'overall grade' column
OVERALL_GRADE_KEYWORD = 'IGNORE'
SYNERGY_URL = 'https://wa-bsd405.edupoint.com/'
SYNERGY_TRAIN_MODE_URL = 'https://wa-bsd405.edupoint.com/train/login.aspx'

# Define functions

def count_assignments(driver):
    '''
    This function takes a selenium WebDriver object as an argument.
    The WebDriver should be pointed at a Grade Book Main page
    on the Synergy Education Platform.
    It returns the number of columns on that gradebook page.
    '''
    assignment_elts = driver.find_element_by_id(
            'ctl00_lowerFixedBarContainer_hf_AssignmentNameIndex')
    assignment_list_str = assignment_elts.get_attribute('value')
    assignments = assignment_list_str.split('║')
    return len(assignments)

def assignment_is_probably_lt(assignment_name):
    '''
    This function returns True if a given string is likely the name
    of an LT assignment.
    It's really stupid. It just checks whether the first two characters
    are 'LT'
    '''
    return assignment_name[0:2] == 'LT'

def get_classperiod_name(driver):
    '''
    This function takes a selenium WebDriver object as an argument.
    The WebDriver should be pointed at a Grade Book Main page
    on the Synergy Education Platform.
    The function returns a string describing the class period.
    All spaces and commas from Synergy are replaced by underscores.
    (e.g. 'Dirks_M__Int_Math_Top_1_S1(3)')
    '''
    '''
    Look for code like the following
    <span id="ctl00_lbl_ACC_ClassFocus" class="sr-only">
        Current Focus: (S1) Dirks, M  Int Math Top 1 S1(3)
    </span>
    '''
    span_elt = driver.find_element_by_id('ctl00_lbl_ACC_ClassFocus')
    class_name = str(span_elt.text)
    # Chop off leading string 'Current focus',
    # and also everything after 'SEC'.
    current_focus_str = 'Current_Focus: '
    current_focus_len = len(current_focus_str)
    sec_index = class_name.find(' SEC:')
    class_name = class_name[current_focus_len:sec_index].strip()
    return cpm.replace_punctuation_with_underscores(class_name)

def get_lt_list(driver):
    '''
    This function takes a selenium WebDriver object as an argument.
    The WebDriver should be pointed at a Grade Book Main page
    on the Synergy Education Platform.
    The function returns a list of LearningTarget objects
    corresponding to gradebook assignments whose titles begin with 'LT'.
    For example, if there is an assignment called 'LT1A Basket Weaving',
    the list will contain a corresponding LearningTarget object
    with lt_label '1A' and lt_brief 'Basket Weaving'.
    '''
    list_of_lts = []
    assignment_elts = driver.find_element_by_id(
            'ctl00_lowerFixedBarContainer_hf_AssignmentNameIndex')
    assignment_list_str = assignment_elts.get_attribute('value')
    assignments = assignment_list_str.split('║')
    for column, assignment in enumerate(assignments):
        assignment = assignment.strip()
        if assignment_is_probably_lt(assignment):
            first_space_index = assignment.find(' ')
            lt_label = assignment[0:first_space_index]
            lt_brief = assignment[first_space_index+1:]
            list_of_lts.append(ltm.LearningTarget(lt_label,
                                                  lt_brief,
                                                  gb_column=column))
    return(list_of_lts)

def get_student_list(driver):
    '''
    This function takes a selenium WebDriver object as an argument.
    The WebDriver should be pointed at a Grade Book Main page
    on the Synergy Education Platform.
    The function returns a list of Student objects
    corresponding to the students on the Grade Book page.
    '''
    list_of_students = []
    table_id = driver.find_element_by_id('ctl00_cphbody_gv_StudentsScores')
    students = table_id.find_elements_by_tag_name('tr')
    for index, record in enumerate(students[1:-1]):  # Skip heading and bottom
        # Figure out student name by splitting on newlines and then commas.
        # If for some reason a student name has multiple commas,
        # assume everything after the first comma is in the first name.
        student_name = str(record.text).split('\n')[0].split(', ')
        student_last = student_name[0]
        student_first = student_name[1]
        if len(student_name) > 2:  # Add on to first name if necessary
            for next_name in student_name[2:]:
                student_first += ', ' + next_name
        list_of_students.append(
                stu.Student(index, student_last, student_first))
    return(list_of_students)

def get_lt_score_matrix(driver, list_of_lts):
    '''
        This function takes two arguments:
        (i) a selenium WebDriver object pointed to a Synergy gradebook page
        (ii) a list of LTs whose gb_column values correspond to their
             position in the Synergy gradebook page
    The WebDriver should be pointed at a Grade Book Main page
    on the Synergy Education Platform.
    The function returns a tuple of two elements:
        (i) a list of lists, where each inner list contains
            one student's scores on LTs in the list.
            If the student does not have a score, it is stored as -1.
        (ii) a list of lists, where each inner list contains
            comments on one student's scores on LTs in the list
    KNOWN ISSUES:
        * TODO: Currently returns [[]] for comments matrix.
    '''
    score_matrix = [[]]
    comment_matrix = [[]]
    lt_count = 0
    student_count = len(get_student_list(driver))
    # Figure out which columns correspond to LT assignments
    lt_column_indices = []
    assignment_count = count_assignments(driver)
    for index in range(assignment_count):
        if bool(ltm.find_lt_by_column(index, list_of_lts)):
            lt_column_indices.append(index)
            lt_count += 1
    # Find Synergy table with scores
    table = driver.find_element_by_id('ctl00_cphbody_GV_Assignments')
    # Find boxes containing scores (skip header and footer)
    # Note that this includes both LTs and other stuff
    all_score_boxes = table.find_elements_by_class_name('SAI')
    # Build matrix of scores and comments
    # Iterate through all score boxes and save the ones from LT columns
    '''
    all_score_boxes[0].click()
    '''
    student_index = 0
    column_index = 0
    for box_index, box in enumerate(all_score_boxes):
        # print(f"Box index = {box_index}, student index = {student_index}")
        '''
        # Switch to current box
        clicked_box = driver.switch_to.active_element
        '''
        # If current box is in an LT column, read and store
        # score and comments
        if column_index in lt_column_indices:
            # print("This is an LT column.")
            # Read score from currently active box
            score = box.text
            # If score is a float, convert it to float.
            # Otherwise, store it as 'exempt' code
            # so that it doesn't figure into the grade
            try:
                score = float(score)
            except ValueError:
                score = DEFAULT_X_CODE
            score_matrix[student_index].append(score)
            '''
            # Read comment from comment box
            comment_box = driver.find_element_by_id('txt_NotesPublic')
            comment = comment_box.text
            comment_matrix.append(comment_box)
            '''
            # print(f"Read score '{score}' with comment '{comment}'")
        '''
        # Press right arrow to move to next score
        print("Moving right ->")
        clicked_box.send_keys(Keys.RIGHT)
        '''
        # Move to next student if we've read
        # all of the current student's scores
        if (box_index + 1) % assignment_count == 0:
            # print(f"Incrementing student index to {student_index+1}")
            student_index += 1
            if student_index < student_count:
                score_matrix.append([])
                comment_matrix.append([])
            '''
            # Click on first box of next row
            all_score_boxes[box_index + 1].click()
            '''
            # Set column index to -1; will increment shortly.
            column_index = -1
        # Increment column index
        column_index += 1
    return (score_matrix, comment_matrix)

def DEPRECATED_get_lt_score_matrix(driver, list_of_lts):
    '''
    This function is deprecated.
    TODO: write get_lt_score_matrix() that operates by reading boxes
    left to right. Should return a list of lists for scores
    and a list of lists for comments.

    This function takes two arguments:
        (i) a selenium WebDriver object pointed to a Synergy gradebook page
        (ii) a list of LTs whose gb_column values correspond to their
             position in the Synergy gradebook page
    The WebDriver should be pointed at a Grade Book Main page
    on the Synergy Education Platform.
    The function returns a list of lists, where each inner list
    contains one student's scores on LTs in the list.
    If the student does not have a score, the score is listed as -1.
    '''
    score_matrix = []
    # Find Synergy table with scores
    table_id = driver.find_element_by_id('ctl00_cphbody_GV_Assignments')
    # Find rows containing scores (skip header and footer)
    rowstrings = table_id.find_elements_by_tag_name('tr')[1:-1]
    # Build score matrix
    for student_index, rowstring in enumerate(rowstrings):
        score_matrix.append([])  # Initialize student row as empty list
        row = str(rowstring.text).split('\n')
        for index, score in enumerate(row):
            # Fix bug where blank boxes are lumped into next score
            if score[:2] == '  ':
                row.insert(index, '  ')
                row[index + 1] = score.strip()
                score = row[index]
            # If this column corresponds to an LT, add score to the matrix.
            lt = ltm.find_lt_by_column(index, list_of_lts)
            if bool(lt):  # If this column has a corresponding LT
                try:
                    score = float(score)
                except ValueError:
                    score = DEFAULT_X_CODE
                score_matrix[student_index].append(score)
    return score_matrix

def initialize_driver_with_user_input(synergy_url=SYNERGY_URL):
    '''
    This function opens a browser window and instructs a user to navigate
    to a gradebook page on Synergy.
    It creates and returns the relevant WebDriver object.
    On error, returns false
    '''
    # Prompt user to navigate to gradebook, and then grab source
    print("1: Wait for the browser to open.")
    print("2: Navigate to the gradebook page you want to update.")
    print("3: Update LT scores in Synergy if applicable.")
    print("4: Then come back here and press Enter.")
    # Launch Chrome and go to Synergy
    browser = webdriver.Chrome()
    browser.get(synergy_url)
    user_input = input("Press Enter to select current gradebook page " +
                       "or 0 to cancel>")
    if user_input.strip() == '0':
        return False
    # Focus on source for main frame in current browser window
    browser.switch_to.frame(browser.find_element_by_id('FRAME_CONTENT'))
    try:
        browser.find_element_by_id('ctl00_cphbody_GV_Assignments')
    except NoSuchElementException:
        print("Error: could not locate assignment grid.")
        return False
    return browser

def create_classperiod_from_synergy(browser):
    '''
    This function takes one argument, a WebDriver object pointed to a
    Synergy gradebook page.
    It returns a ClassPeriod object containing the students on that page,
    the learning targets, and student scores on learning targets.
    If a student score is blank, a letter, or anything other than
    a valid score (0-4), is is replaced with -1 (exempt).
    '''
    # Scrape data
    class_name = get_classperiod_name(browser)
    list_of_lts = get_lt_list(browser)  # Get list of LTs
    student_list = get_student_list(browser)  # Get list of students
    # Get scores and comments
    score_matrix, comment_matrix = get_lt_score_matrix(browser, list_of_lts)
    '''
    print("Scores:")
    print(score_matrix)
    print("Comments:")
    print(comment_matrix)
    '''
    # Create ClassPeriod object from the above data
    cp = cpm.ClassPeriod(class_name, student_list, list_of_lts)
    # Add student scores to the ClassPeriod
    for student_index, student in enumerate(cp.students_in_period):
        # Get the list of student scores across LTs.
        # This will be a list of floats such as [3.0, 4.0, 2.0]
        scores = score_matrix[student_index]
        for score_index, score in enumerate(scores):
            lt_label = list_of_lts[score_index].lt_label
            assert type(score) == float or type(score) == int
            student = stu.add_lt(student, lt_label, score)
    return cp

def assignment_exists_with_keyword(driver, keyword):
    '''
    Uses a selenium WebDriver pointed at a Synergy gradebook page
    to check whether an assignment exists whose title contains
    the specified keyword. Returns True if so, False if not.
    '''
    assignment_elts = driver.find_element_by_id(
            'ctl00_lowerFixedBarContainer_hf_AssignmentNameIndex')
    assignment_list_str = assignment_elts.get_attribute('value')
    assignments = assignment_list_str.split('║')
    for assignment in assignments:
        if keyword.upper() in assignment.upper():
            return True
    return False

def fill_overall_scores(driver,
                        scores,
                        comments=[],
                        keyword=OVERALL_GRADE_KEYWORD):
    '''
    This function takes three arguments:
        (i) driver, a selenium WebDriver argument pointed at a Synergy
            gradebook page;
        (ii) scores, a list of overall grades (ints),
             one per student;
        (iii) comments, a list of comments (strings), one per student;
                blank by default
        (iv) A keyword that is found only in the overall grade column.
             By default this should be something like "Overall".
    It returns True on success, False on failure.
    Note that the grades must be ints or errors will occur!
    '''
    student_count = len(get_student_list(driver))
    # Take care of a few conventions
    keyword = keyword.upper()
    # If no comments given, change it to a list of empty comments
    if comments == []:
        comments = ['']*student_count
    # Validate data
    # Make sure all scores are ints
    for score in scores:
        if type(score) != int:
            print("Error: fill_overall_scores expects a list of ints.")
            print(f"Non-int found: {score}")
            return False
    # Make sure specified column exists
    if not assignment_exists_with_keyword(driver, keyword):
        print("Error: could not find overall grade column.")
        print(f"Expected to find keyword {keyword}, but could not find it.")
        return False
    # Make sure number of scores == number of students
    if len(scores) != student_count:
        print("Error: score count does not match student count.")
        return False
    # Make sure number of comments == number of students
    if len(comments) != student_count:
        print("Error: comment count does not match student count.")
        return False
    # Find column that contains the overall scores
    assignment_elts = driver.find_element_by_id(
         'ctl00_lowerFixedBarContainer_hf_AssignmentNameIndex')
    assignment_list_str = assignment_elts.get_attribute('value')
    assignments = assignment_list_str.split('║')
    overall_grade_column = -1  # Initialize as nonsense
    for column, assignment in enumerate(assignments):
        if (keyword in assignment.upper()) and\
                (assignment[:2].upper() != 'LT'):
            overall_grade_column = column
            break
    if overall_grade_column == -1:
        print("Error: could not find column " +
              f"whose header contains {keyword}")
        return False
    # Create list containing all score boxes
    table = driver.find_element_by_id('ctl00_cphbody_GV_Assignments')
    score_boxes = table.find_elements_by_class_name('SAI')
    # Click on the first-row box of the overall grade column
    try:
        score_boxes[overall_grade_column].click()
    except ElementClickInterceptedException:
        print("Could not write scores. Try saving the page first.")
    # Write first score, press enter, and move down the list.
    for student_index in range(len(scores)):
        clicked_box = driver.switch_to.active_element
        clicked_box.send_keys(scores[student_index])
        # Write comment if applicable
        comment = comments[student_index]
        if comment != '':
            driver.switch_to.parent_frame
            comment_box = driver.find_element_by_id('txt_NotesPublic')
            comment_box.click()
            comment_box.send_keys(comments[student_index])
            # Press Enter to move to next score box
            comment_box.send_keys(Keys.ENTER)
        else:
            # Press Enter to move to the next score box
            clicked_box.send_keys(Keys.ENTER)
    # Switch focus back to grades frame
    driver.switch_to.default_content
    return True

def fill_username_on_home(driver, username):
    '''
    This function fills in a given username on the Synergy login page.
    '''
    input_elt = WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.ID, "Login")))
    input_elt = driver.find_element_by_id('Login')
    input_elt.send_keys(username)

# Main code
if __name__ == "__main__":
    # Prompt user to navigate to gradebook, and then grab source
    browser = initialize_driver_with_user_input()
    cp = create_classperiod_from_synergy(browser)
    print(cp)
    scores = cp.get_list_of_overall_grades()
    comments = [''] * len(get_student_list(browser))
    fill_overall_scores(browser, scores, comments, OVERALL_GRADE_KEYWORD)


import sjuman2_spate505 as part1
import os,sys
from nltk.parse import CoreNLPParser,CoreNLPDependencyParser
from nltk.stem.wordnet import WordNetLemmatizer
from sqlite3 import Error
import re

def return_deptree(qstn,parser):
 parse = next(parser.raw_parse(qstn))
 return parse

def lambda_noun(ner_tag,tag,l,from_clause,wh_query):
 if tag == 'COUNTRY':
  if from_clause == "":
   from_clause+=" countries co "
  else:
   from_clause+=", countries co "
  if wh_query == "":
   wh_query+=" lower(co.name) like '%"+ l.lower() + "%'"
  else:
   wh_query+=" and lower(co.name) like '%"+ l.lower() + "%'"
 if tag == 'LOCATION':
  if from_clause == "":
   from_clause+=" continents cs "
  else:
   from_clause+=", continents cs "
  if wh_query == "":
   wh_query+=" lower(cs.continent) like '%"+ l.lower() + "%'"
  else:
   wh_query+=" and lower(cs.continent) like '%"+ l.lower() + "%'"
 if tag == 'PERSON':
  if from_clause == "":
   from_clause+=" artist ar "
  else:
   from_clause+=", artist ar "
  if wh_query == "":
   wh_query+=" lower(ar.name) like '%"+ l.lower() + "%'"
  else:
   wh_query+=" and lower(ar.name) like '%"+ l.lower() + "%'"
 if tag == 'CITY':
  if from_clause == "":
   from_clause+=" cities ci "
  else:
   from_clause+=", cities ci "
  if wh_query == "":
   wh_query+=" lower(ci.name) like '%"+ l.lower() + "%'"
  else:
   wh_query+=" and lower(ci.name) like '%"+ l.lower() + "%'"

 return from_clause,wh_query

def ans_geog_yesno(qstn,dep_parse,parse,ner_tag):
 select_clause=""
 from_clause=""
 wh_query=""
 cit=[]
 moun=[]
 oce=[]
 list_tag=[]

 if 'SBAR' not in parse:
  select_clause="Select case when count(*)>=1 then 'Yes' else 'No' end "
 for l in list(dep_parse.triples()):
  if 'nsubj' in l[1] or 'nmod' in l[1]:
   if 'NNP' in l[2][1]:
    tag = [ner_tag[i][1] for i in range(0,len(ner_tag)) if ner_tag[i][0].lower()==l[2][0].lower()]
    if len(tag)==0:
     tag='X'
    else:
     tag=tag[0]
    if tag=='CITY':
     cit.append(l[2][0])
    if 'mountain' in qstn:
     moun.append(l[2][0])
    if 'ocean' in qstn:
     oce.append(l[2][0])
    if l[2][0].lower()=='everest':
     break
    list_tag.append(tag)
    from_clause,wh_query = lambda_noun(ner_tag,tag,l[2][0],from_clause,wh_query)

 if len(moun)!=0:
  moun=moun[0]
  from_clause+=" mountains m "
  wh_query+=" lower(m.name) like '%"+ moun.lower() + "%'"
 if len(oce)!=0:
  oce=oce[0]
  from_clause+=" seas s "
  wh_query+=" lower(s.ocean) like '%"+ oce.lower() + "%'"
 if 'LOCATION' in list_tag and 'COUNTRY' in list_tag:
  from_clause+=", countrycontinents cc "
  wh_query+=" and co.id=cc.countryid and cs.id=cc.continentid"
 if 'COUNTRY' in list_tag and len(cit)!=0:
  wh_query+=" and co.id=ci.id "

 query=select_clause + " from " + from_clause[1:] + " where " + wh_query
 print(query)
 return query

def ans_geog_wh(dep_parse,parse,ner_tag):
 select_clause=""
 from_clause=""
 wh_query=""
 list_tag=[]
 for l in list(dep_parse.triples()):
  if 'advmod' in l[1]:
   if l[2][0]=='Where':
    select_clause="Select co.name"
  if 'cop' in l[1]:
   if l[0][0]=='What':
    select_clause="Select ci.name"
  if 'nmod' in l[1] or 'nsubj' in l[1]:
   if 'NNP' in l[2][1]:
    tag = [ner_tag[i][1] for i in range(0,len(ner_tag)) if ner_tag[i][0].lower()==l[2][0].lower()]
    if len(tag)==0:
     tag='X'
    else:
     tag=tag[0]
    list_tag.append(tag)
    from_clause,wh_query = lambda_noun(ner_tag,tag,l[2][0],from_clause,wh_query)

 if 'CITY' in list_tag:
  from_clause+=" , countries co "
  wh_query+=" and co.id=ci.id"
 if 'COUNTRY' in list_tag:
  from_clause+=", cities ci "
  wh_query+=" and ci.id=co.id"

 query=select_clause + " from " + from_clause[1:] + " where " + wh_query
 print(query)
 return query

def ans_music_yesno(dep_parse,parse,ner_tag,qstn,albums,tracks):
 select_clause=""
 from_clause=""
 wh_query=""
 list_tag=[]
 # for album
 list1=[]
 # for track
 list2=[]
 # for country
 country_mapping={"Beyonce":"TX","Gaga":"NY","Jackson":"CA","Swift":"PA","Madonna":"MI"}
 artists=['Beyonce','Gaga','Jackson','Swift','Madonna']
 birth_places=['Houston','Manhattan','Holmby Hills','Reading','Bay City']
 list3=[]
 # for artist
 list4=[]

 if 'SBAR' not in parse[0:10]:

  select_clause="Select case when count(*)>=1 then 'Yes' else 'No' end "
 for l in list(dep_parse.triples()):
  if 'nsubj' in l[1] or 'dobj' in l[1] or 'nmod' in l[1]:
   if 'NNP' in l[2][1] or 'NN' in l[2][1]:
    tag = [ner_tag[i][1] for i in range(0,len(ner_tag)) if ner_tag[i][0]==l[2][0]]
    if len(tag)==0:
     tag='X'
    else:
     tag=tag[0]
     list_tag.append(tag)
    if 'nsubj' in l[1]:
     if tag=='PERSON':
      list4.append(l[2][0])
      list4[0]=re.sub("'","",list4[0])
      from_clause,wh_query = lambda_noun(ner_tag,tag,l[2][0],from_clause,wh_query)
     elif l[2][0] in albums:
      list1.append(l[2][0])
     elif l[2][0] in tracks:
      list2.append(l[2][0])
    if 'Papa' in qstn:
     list2.append('Papa')
    if tag=='COUNTRY':
     for w in artists:
      if w in qstn:
       list3.append(country_mapping[w])

    if 'dobj' in l[1]:
     list2.append(l[2][0])

 if len(list3)==0:
  if 'born' in qstn:
   for w in birth_places:
    if w in qstn:
     list3.append(w)

 if len(list4)==0:
  for w in artists:
   if w in qstn:
    from_clause+=" artist ar "
    wh_query+=" ar.name like '%"+ w.lower() +"%' "
 if len(list3)!=0:
  wh_query+=" and lower(ar.placeofbith) like '%"+ list3[0].lower() + "%' "
 if 'PERSON' in list_tag and len(list2)!=0:
  wh_query+=" and ar.id=al.artsitid "

 if list1:
  if wh_query == "":
   wh_query+=" lower(al.name) like '%"+ list1[0].lower() + "%' "
  else:
   wh_query+=" and lower(al.name) like '%"+ list1[0].lower() + "%' "

 if list2:
  if from_clause == "":
   from_clause+=" album al, track t "
  else:
   from_clause+=", album al, track t "
  if wh_query == "":
   wh_query+=" al.albumid=t.albumid and lower(t.name) like '%"+ list2[0].lower() + "%' "
  else:
   wh_query+=" and al.albumid=t.albumid and lower(t.name) like '%"+ list2[0].lower() + "%' "

 query=select_clause + " from " + from_clause + " where " + wh_query
 print(query)
 return query

def ans_music_wh(dep_parse,parse,ner_tag,qstn,albums,tracks):
 select_clause=""
 from_clause=""
 wh_query=""
 list_tag=[]
 artists=['Beyonce','Gaga','Jackson','Swift','Madonna']
 # for album
 list1=[]
 # for track
 list2=[]
 # for country
 list3=[]
 # for artist
 list4=[]

 for l in list(dep_parse.triples()):
  if 'advmod' in l[1]:
   if l[2][0]=='Where':
    select_clause="Select ar.placeofbith"
  if 'nsubj' in l[1]:
   if l[2][0]=='Who':
    select_clause="Select ar.name"
    from_clause=" artist ar"
  if 'det' in l[1]:
   if l[2][0]=='Which':
    select_clause="Select ar.name"
    from_clause=" artist ar"
  if 'nsubj' in l[1] or 'dobj' in l[1] or 'nmod' in l[1] or 'nsubjpass' in l[1]:
   if 'NNP' in l[2][1] or 'NN' in l[2][1]:
    tag = [ner_tag[i][1] for i in range(0,len(ner_tag)) if ner_tag[i][0]==l[2][0]]
    if len(tag)==0:
     tag='X'
    else:
     tag=tag[0]
    list_tag.append(tag)
    if tag=='PERSON':
     list4.append(l[2][0])
     list4[0]=re.sub("'","",list4[0])
     from_clause,wh_query = lambda_noun(ner_tag,tag,l[2][0],from_clause,wh_query)
    elif l[2][0] in albums:
     list1.append(l[2][0])
    elif l[2][0] in tracks:
     list2.append(l[2][0])
    if tag=='COUNTRY':
     list3.append(country_mapping[list4[0]])
    if 'dobj' in l[1]:
     list2.append(l[2][0])
 if 'born' in qstn:
  for w in artists:
   if w in qstn:
    from_clause+=" artist ar"
    wh_query+=" lower(ar.name) like '%"+ w.lower() +"%' "
 if len(list2)==0:
  for w in tracks:
   if w in qstn:
    list2.append(w)
 if len(list1)!=0:
  if from_clause == "":
   from_clause+=" album al "
  else:
   from_clause+=", album al "
  if wh_query == "":
   wh_query+=" lower(al.name) like '%"+ list1[0].lower() + "%' "
  else:
   wh_query+=" and lower(al.name) like '%"+ list1[0].lower() + "%' "
 if len(list2)!=0:
  if from_clause == "":
   from_clause+=" album al, track t "
  else:
   from_clause+=", album al, track t "
  if wh_query == "":
   wh_query+=" ar.id=al.artsitid and al.albumid=t.albumid and lower(t.name) like '%"+ list2[0].lower() + "%' "
  else:
   wh_query+=" and  ar.id=al.artsitid and al.albumid=t.albumid and lower(t.name) like '%"+ list2[0].lower() + "%' "

 query=select_clause + " from " + from_clause + " where " + wh_query
 print(query)
 return query

#--- Lambda function to return Noun and generate appropriate where clause for that root word
def returnNounReps(ner_tag,l):
 movie=""
 location=""
 person=""
 title=""
 tag=[ner_tag[i][1] for i in range(0,len(ner_tag)) if ner_tag[i][0]==l[0]]
 if len(tag)==0:
  tag='X'
 else:
  tag=tag[0]
 if tag=='PERSON':
  if person!="":
   person+=" and "
  person+=" lower(p.name) like '%" + str(l[0]).lower() + "%'"
 elif tag in ['LOCATION','COUNTRY','CITY','STATE']:
  location=" lower(p.pob) like '%" + str(l[0]).lower() + "%'"
 elif tag=='NATIONALITY':
  place=nationality_mapping[l[0].upper()]
  location=" lower(p.pob) like '%" + str(place).lower() + "%'"
 elif tag=='TITLE':
  title=l2[0].lower()
 else:
  if movie!="":
   movie+=" and "
  movie+=" lower(m.name) like '%" + str(l[0]).lower() + "%'"
 return person,location,title,movie

#-- Generate the clauses to be used in the query
def getClause(parse,ner_tag):
 nationality_mapping={"ITALIAN": "ITALY", "GERMAN": "GERMANY", "FRENCH":"FRANCE", "BRITISH": "UK", "AMERICAN": "USA" }
 acts=['act','star']
 person=""
 location=""
 movie=""
 title=""
 title1=""
 direct='N'
 act='N'
 oscar='N'
 osc=""
 from_clause=""
 year=""
 born=""
 movie_mapping={'movie': 'film', 'film' : 'film', 'picture' : 'film'}
 for l in list(parse.triples()):

  if 'amod' in l[1]:
   if 'JJ' in l[2][1]:
    try:
     pb=nationality_mapping[l[2][0].upper()]
    except KeyError:
     pb=""
    if pb!="":
     if person!="":
      person+=" and "
     person+=" lower(p.pob) like '%" + str(pb).lower() + "%'"
    if l[1][0].lower() in ['act','actor','actress']:
     act='Y'
    if l[1][0].lower() in ['direct','director']:
     direct='Y'

  # Handling the subject  and modifiers: Nouns associated with the VERB
  if 'nsubj' in l[1] or 'nmod' in l[1] or 'compound' in l[1]:
   if 'VB' in l[0][1]:
    lm=WordNetLemmatizer().lemmatize(l[0][0],'v')
    if lm=="win":
     oscar='Y'
    elif lm=='born':
     born='Y'
    elif lm in ['star','act']:
     act='Y'
    elif lm in ['direct']:
     direct='Y'
   elif 'NNP' in l[0][1]:
    per,loc,title,mov=returnNounReps(ner_tag,l[0])
    if person!="" and per!="":
     person+=" and "
    person+=per
    if loc!="" and person!="":
     person+=" and "
    person+=loc
    if movie!="" and mov!="":
     movie+=" and "
    movie+=mov
   if 'NNP' in l[2][1]:
    per,loc, title, mov=returnNounReps(ner_tag,l[2])
    if person!="" and per!="":
     person+=" and "
    person+=per
    if loc!="" and person!="":
     person+=" and "
    person+=loc
    if movie!="" and mov!="":
     movie+=" and "
    movie+=mov
   if 'NN' in l[0][1]:
    tag=[ner_tag[i][1] for i in range(0,len(ner_tag)) if ner_tag[i][0].lower()==l[0][0].lower()]
    if len(tag)==0:
     tag='X'
    else:
     tag=tag[0]
    if tag=='TITLE':

     if l[0][0].lower() in ['actor','actress','act']:
      act='Y'
     if l[0][0].lower() in ['director','directors']:
      direct='Y'
  # Handling adjectives: finding category for oscars
  if 'amod' in l[1] or ('nsubj' in l[1] and 'JJ' in l[2][1]) or ('dobj' in l[1]):
   if 'best' in l[2][0].lower():
    oscar='Y'
    word=l[0][0].lower()
    if word in ['movie','picture','cinema']:
     word="film"
    title1="best%" + word
   if 'dobj' in l[1] and 'best' in title1:
    title1+="%" +l[2][0].lower()

  # Handling case: by-direct and with-act
  if 'case' in l[1]:
   if l[2][1]=='IN':
    if l[2][0].lower()=='with':
     act='Y'
    if l[2][0].lower()=='by':
     direct='Y'
   if l[2][1]=='CD' or l[0][1]=='CD':
    if l[2][1]=='CD':
     d=l[2][0]
    else:
     d=l[0][0]
    if born=='Y':
     if person!="":
      person+=" and "
     person+=" p.dob like '" + d + "%'"
    if oscar=='Y':
     #if osc!="":
      #osc+=" and "
     osc+=" and o.year=" + d
    else:
     if movie!="":
      movie+=" and "
     movie+=" m.year=" + d

 if title1!="":
  if osc!="":
   osc+=" and "
  osc+="  lower(o.type) like '%" + title1 + "%'"
 from_clause=""
 if person!="":
  from_clause+=" person p "
 if movie!="":
  from_clause+=", movie m "
 if oscar=='Y':
  from_clause+=", oscar o "
 if act=='Y':
  from_clause+=", actor a "
 if direct=='Y':
  from_clause+=", director d "

 return from_clause,person,movie,location,oscar,osc,born

# generate the query + base conditions
def generate_query(dep_parse,ner_tag, parse):
 from_clause,person,movie,location,oscar,osc,born=getClause(dep_parse,ner_tag)
 select_clause=""
 if 'SBAR' not in parse:
  select_clause="Select case when count(*)>=1 then 'Yes' else 'No' end "
 # Setting base conditions for where clause

 wh_query=person + movie + osc + location
 if person!="":
  #wh_query=person
  if 'actor' in from_clause:
   wh_query+=" and p.id=a.actor_id"
  elif 'director' in from_clause:
   wh_query+= " and p.id=d.director_id"

 if 'oscar' in from_clause and 'movie' in from_clause:
  wh_query+=" and m.id=o.movie_id "

 if 'oscar' in from_clause and 'person' in from_clause:
  wh_query+=" and o.person_id=p.id  and " + osc

 if 'movie' in from_clause and 'actor' in from_clause:
  wh_query+=" and a.movie_id=m.id"

 query=select_clause + " from " + from_clause[1:] + " where " + wh_query
 print(query)
 return query

if __name__=='__main__':
 filename=""
 while True:
  pathname = os.path.dirname(sys.argv[0])
  if len(sys.argv)==2:
   filename=sys.argv[1]
   if os.path.isfile(os.path.join(pathname,filename)):
    filename=os.path.join(pathname,filename)
    break
  else:
   print("File name not provided or file does not exist, please enter the filename:")
   filename=input().strip()
   if os.path.isfile(os.path.join(pathname,filename)):
    filename=os.path.join(pathname,filename)
    break
 geog=['place','continent','region','mountain','ocean','river','country','capital','forest','desert','locate','border','city','state','valley','sea','hill','geography','waterfall','volcano']
 movies_list =['actor','actress','movie','direct','film','oscar','star','theatre','play','plot','drama','shoot','cinema','act']
 music_list = ['sing','record','album','song','artist','rap','track','record','music','compose','rock','pop','dance','singer','blues','indie']

 #Setting up music category
 conn=part1.create_connection(os.path.join("Database","music.sqlite"))
 if not(conn is None):
  mus_name,songs=part1.music(conn)
  cur = conn.cursor()
  cur.execute("SELECT name FROM album")
  rows = cur.fetchall()
  albums=set()
  for row in rows:
   r=str(rows[0]).lower().split(" ")
   albums.add(r[0])
   albums.add(r[1])
  cur.execute("SELECT name FROM track")
  rows = cur.fetchall()
  tracks=set()
  for row in rows:
   r=str(rows[0]).lower().split(" ")
   tracks.add(r[0])
   tracks.add(r[1])
   tracks.add(r[2])

 #Setting up geography category
 conn=part1.create_connection(os.path.join("Database","WorldGeography.sqlite"))
 if not(conn is None):
  geog_db=part1.geography_db(conn)

 #Getting the set of similar words from wordnet
 geog_set=part1.create_lists(geog)
 mov_set=part1.create_lists(movies_list)
 music_set=part1.create_lists(music_list)

 #Getting the tags
 qstn,ner,pos=part1.tagging(filename)
 parser = CoreNLPParser(url='http://localhost:9000')
 dep_parser = CoreNLPDependencyParser(url='http://localhost:9000')
 # Printing parse tree and output
 for i in range(0,len(qstn)):
  qtype="YesNo"
  print("Question: ", qstn[i].strip())
  parsed=part1.parse_tree(qstn[i],parser)
  dep_parsed=return_deptree(qstn[i],dep_parser)
  #print(list(dep_parsed.triples()),parsed,ner[i])
  #continue
  if 'SBAR' in parsed[0:15]:
   qtype="WH"
  category=part1.categorize(qstn[i],ner[i],pos[i],geog_set,mov_set,music_set, mus_name,songs,geog_db)
  print(category)

  if category=="Music":

   if qtype=="WH":

    sql_query=ans_music_wh(dep_parsed,parsed,ner[i],qstn[i],albums,tracks)
    try:
     conn=part1.create_connection(os.path.join("Database","music.sqlite"))
     cur = conn.cursor()
     cur.execute(sql_query)
     rows = cur.fetchall()
     if not rows:
      print("I don't know!")
      print("----------------------------------------")
     else:
      print(rows[0][0])
      print("----------------------------------------")
    except Error:
     print("I don't know!")
     print("----------------------------------------")

   else:
    sql_query=ans_music_yesno(dep_parsed,parsed,ner[i],qstn[i],albums,tracks)
    try:
     conn=part1.create_connection(os.path.join("Database","music.sqlite"))
     cur = conn.cursor()
     cur.execute(sql_query)
     rows = cur.fetchall()
     print(rows[0][0])
     print("----------------------------------------")
    except Error:
     print("I don't know!")
     print("----------------------------------------")

  elif category=="Movies":
   if qtype=="WH":
    print("E")
    p=parsed.split("\n")
    for p1 in p:
     print(p1.strip())
    answer_movies(qstn[i],ner[i],pos[i],parsed,geog_set)

   else:
    #print(parsed)
    #answer_movie_yesno(qstn[i],ner[i],pos[i],parsed,geog_set)
    generate_query(dep_parsed,ner[i],'S')
    print(ner[i])
    print("----------------------------------------")

  else:
   if qtype=="WH":
    sql_query=ans_geog_wh(dep_parsed,parsed,ner[i])
    try:
     conn=part1.create_connection(os.path.join("Database","WorldGeography.sqlite"))
     cur = conn.cursor()
     cur.execute(sql_query)
     rows = cur.fetchall()
     if not rows:
      print("I don't know!")
      print("----------------------------------------")
     else:
      print(rows[0][0])
      print("----------------------------------------")
    except Error:
     print("I don't know!")
     print("----------------------------------------")

   else:
    sql_query = ans_geog_yesno(qstn[i],dep_parsed,parsed,ner[i])
    try:
     conn=part1.create_connection(os.path.join("Database","WorldGeography.sqlite"))
     cur = conn.cursor()
     cur.execute(sql_query)
     rows = cur.fetchall()
     print(rows[0][0])
     print("----------------------------------------")
    except Error:
     print("I don't know!")
     print("----------------------------------------")


import scipy.misc
import numpy as np
import torch
from PIL import Image
from torch.utils.data import Dataset,DataLoader
import os
from torch.autograd import Variable
import torch.nn as nn
import argparse
import torch.optim as optim
import torchvision
import random , time , math
import torchvision.transforms as transforms
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
from tensorboardX import SummaryWriter

parser =  argparse.ArgumentParser(description='vae model')
parser.add_argument('-b',type=int,dest='batch_size',required=True)
parser.add_argument('-e',type=int,dest='epoch',required=True)
parser.add_argument('-tn',type=int,dest='train_num',required=True)
parser.add_argument('-lat',type=int,dest='latent_size',required=True)
parser.add_argument('-lam',type=float,dest='lambda_kl',required=True)
parser.add_argument('-tm',type=int,dest='test_num',required=True)
parser.add_argument('-exp',type=str,dest='exp',required=True)
parser.add_argument('-tsne',type=int,dest='tsne',required=True)
args = parser.parse_args()
writer = SummaryWriter('runs/exp_'+args.exp)

class imagedataset(Dataset):
    def __init__(self,img,mode):
        self.img = img
        self.mode = mode
    def __getitem__(self,i):
        if self.mode == 'train':
            x = self.img[i]
            y = self.img[i]
            return x,y,i
        elif self.mode == 'test':
            x = self.img[i]
            return x,i
        else: return ValueError('Wrong mode')
    def __len__(self):
        return self.img.shape[0]

def get_data(path,num,name,batch_size,shuffle=False):
    data = os.listdir(path)
    data.sort()
    data = data[:num]
    arr = []
    for x in data:
        print(os.path.join(path,x))
        #input()
        im = Image.open(os.path.join(path,x))
        im = np.array(im)
        #print(im)
        #input()
        im = (im/127.5)-1
        #print(im)
        #input()
        #print(im)
        #input()
        arr.append(im)
    arr = np.array(arr)
    print('saving data ...')
    np.save(name+'.npy',arr)
    arr = torch.FloatTensor(arr)
    dataset = imagedataset(arr,name)
    return DataLoader(dataset,batch_size=batch_size,shuffle=shuffle)
'''
data = os.listdir('test')
data.sort()
data = data[0]
im = Image.open('test/'+data)
print('test/'+data)
input()
im = (np.array(im)/127.5)-1
print(im.shape)
input()
print(im)
input()
#im = transforms.ToTensor()(im)
#im = torch.FloatTensor(im).view(64,3,64)
im = torch.FloatTensor(im).permute(2,0,1)
#im = transforms.ToTensor()(im)
print(im)
input()
torchvision.utils.save_image(im,'save.jpg')
print('---')
input()
'''

class vae(nn.Module):
    def __init__(self,latent_size,nc,ngf,ndf):
        super(vae,self).__init__()
        self.latent_size = latent_size
        self.nc = nc #3
        self.ngf = ngf #64
        self.ndf = ndf #64
        self.latent_size = latent_size

        #(3,64,64)
        self.e1 = nn.Conv2d(3,32,4,2,1) #(32,32,32)
        self.bn1 = nn.BatchNorm2d(32)

        self.e2 = nn.Conv2d(32,64,4,2,1) #(64,16,16)
        self.bn2 = nn.BatchNorm2d(64)

        self.e3 = nn.Conv2d(64,128,4,2,1) #(128,8,8)
        self.bn3 = nn.BatchNorm2d(128)

        self.e4 = nn.Conv2d(128,256,4,2,1) #(256,4,4)
        self.bn4 = nn.BatchNorm2d(256)

        self.e5 = nn.Conv2d(256,512,4,2,1) #(512,2,2)
        self.bn5 = nn.BatchNorm2d(512)

        self.e6 = nn.Conv2d(512,1024,4,2,1) #(1024,1,1)
        self.bn6 = nn.BatchNorm2d(1024)

        self.mu = nn.Linear(1024,latent_size)
        self.sigma = nn.Linear(1024,latent_size) # 512

        self.d1 = nn.Linear(latent_size, 1024) # (1024,1,1)

        self.up1 = nn.ConvTranspose2d(1024,512,4,2,1)
        self.bn7 = nn.BatchNorm2d(512, 1.e-3)

        self.up2 = nn.ConvTranspose2d(512,256,4,2,1)
        self.bn8 = nn.BatchNorm2d(256, 1.e-3)

        self.up3 = nn.ConvTranspose2d(256,128,4,2,1)
        self.bn9 = nn.BatchNorm2d(128, 1.e-3)

        self.up4 = nn.ConvTranspose2d(128,64,4,2,1)
        self.bn10 = nn.BatchNorm2d(64, 1.e-3)

        self.up5 = nn.ConvTranspose2d(64,64,4,2,1)
        self.bn11 = nn.BatchNorm2d(64, 1.e-3)

        self.up6 = nn.ConvTranspose2d(64,3,4,2,1)

        self.leakyrelu = nn.LeakyReLU(0.2)
        self.relu = nn.ReLU()
        self.sigmoid = nn.Sigmoid()
        self.tanh = nn.Tanh()
    def encode(self,x):
        h1 = self.leakyrelu(self.bn1(self.e1(x)))
        h2 = self.leakyrelu(self.bn2(self.e2(h1)))
        h3 = self.leakyrelu(self.bn3(self.e3(h2)))
        h4 = self.leakyrelu(self.bn4(self.e4(h3)))
        h5 = self.leakyrelu(self.bn5(self.e5(h4)))
        #print(h5.size())
        #input()
        h6 = self.leakyrelu(self.bn6(self.e6(h5)))

        h6 = h6.view(-1,1024)
        return self.mu(h6),self.sigma(h6)

    def reparametrize(self,mu,logvar):
        std = logvar.mul(0.5).exp_()
        eps = torch.cuda.FloatTensor(std.size()).normal_()
        eps = Variable(eps)
        return eps.mul(std).add_(mu)

    def decode(self,z):
        h1 = self.relu(self.d1(z))
        h1 = h1.view(-1, 1024, 1, 1)
        #print('h1:',h1.size())
        h2 = self.leakyrelu(self.bn7(self.up1(h1)))
        #print('h2:',h2.size())
        h3 = self.leakyrelu(self.bn8(self.up2(h2)))
        #print('h3:',h3.size())
        h4 = self.leakyrelu(self.bn9(self.up3(h3)))
        #print('h4:',h4.size())
        h5 = self.leakyrelu(self.bn10(self.up4(h4)))
        #print('h5:',h5.size())
        h6 = self.leakyrelu(self.bn11(self.up5(h5)))
        #print('h6:',h6.size())
        h7 = self.tanh(self.up6(h6))
        #print('h7:',h7.size())
        return h7.permute(0,2,3,1)
        #return h7.view(-1,64,64,3)
        '''
    def get_latent_var(self, x):
        mu, logvar = self.encode(x.view(-1, self.nc, self.ndf, self.ngf))
        z = self.reparametrize(mu, logvar)
        return z
        '''
    def forward(self,x):
        mu, logvar = self.encode(x.permute(0,3,1,2))
        #print('mu:',mu)
        #print('logvar:',logvar)
        z = self.reparametrize(mu, logvar)
        #print('z:',z)
        res = self.decode(z)
        #print('res:',res)
        #input()
        return res, mu, logvar

def mse_loss(inp, target):
    out = (inp - target)**2
    loss = out.sum() # or sum over whatever dimensions
    return loss/(inp.shape[1]*inp.shape[2]*inp.shape[3])
#reconstruction_function = nn.MSELoss()

def loss_function(recon_x, x, mu, logvar):
    #MSE = reconstruction_function(recon_x, x)
    MSE = mse_loss(recon_x, x)
    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
    KLD_element = mu.pow(2).add_(logvar.exp()).mul_(-1).add_(1).add_(logvar)
    KLD = torch.sum(KLD_element).mul_(-0.5)
    #KLD = 0.5 * torch.sum(torch.exp(logvar) + mu**2 - 1. - logvar)
    return MSE + KLD*args.lambda_kl , MSE , KLD

model = vae(nc=3, ngf=64, ndf=64, latent_size = args.latent_size ).cuda()
#print(model)
#input()
optimizer = optim.Adam(model.parameters(), lr=1e-4)

#training_set = get_data('train',num=args.train_num,name='train',batch_size=args.batch_size,shuffle=True)
#testing_set = get_data('test',num=2621,name='test',batch_size=args.batch_size,shuffle=False)
#print('saved data ...')
#input()

'''
arr = np.load('train.npy')[:args.test_num]
#print(arr)
#input()
print('loaded training set')
print(arr.shape)
arr = torch.FloatTensor(arr)
dataset = imagedataset(arr,'train')
training_set = DataLoader(dataset,batch_size=args.batch_size,shuffle=True)
'''

arr = np.load('test.npy')
#print('loaded testing set')
#print(arr.shape)
arr = torch.FloatTensor(arr)
dataset = imagedataset(arr,'test')
testing_set = DataLoader(dataset,batch_size=args.batch_size,shuffle=False)

def timeSince(since, percent):
    now = time.time()
    s = now - since
    es = s / (percent)
    rs = es - s
    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))

def asMinutes(s):
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)

kld_curve = []
mse_curve = []

def train(epoch,iteration):
    model.train()
    train_loss = 0
    start = time.time()
    kld_loss = 0
    mse_loss = 0
    for step , (batch_x,batch_y,_) in enumerate(training_set):
        batch_size = len(batch_x)
        batch_idx = step + 1
        batch_x = Variable(batch_x).cuda()
        batch_y = Variable(batch_y).cuda()
        optimizer.zero_grad()
        recon,mu,logvar = model(batch_x)
        loss , MSE , KLD = loss_function(recon,batch_x,mu,logvar)
        loss.backward()
        train_loss+=loss.data[0]
        optimizer.step()
        kld_loss += KLD / batch_size
        mse_loss += MSE / batch_size
        #print('batch_idx=',batch_idx)
        #print(epoch*len(training_set))
        #input()
        #writer.add_scalar('MSE_train', MSE.data[0] / len(batch_x), (epoch-1)*len(training_set)+batch_idx)
        #writer.add_scalar('KLD_train', KLD.data[0] / len(batch_x), (epoch-1)*len(training_set)+batch_idx)
        print('\rTrain Epoch: {} [{}/{} ({:.0f}%)] | Loss: {:.6f} | MSE: {:.6f} | KLD: {:.6f} | Time: {} '.format(
                   epoch
                   , batch_idx * batch_size
                   , len(training_set.dataset)
                   , 100. * batch_idx * batch_size / len(training_set.dataset)
                   , loss.data[0] / batch_size
                   , MSE.data[0] / batch_size
                   , KLD.data[0] / batch_size
                   , timeSince(start, batch_idx * batch_size / len(training_set.dataset)))
                   , end='')

        if batch_idx % 100 == 0 :
            kld_curve.append(kld_loss/100)
            mse_curve.append(mse_loss/100)
            writer.add_scalar('MSE_loss', mse_loss/100 , iteration )
            writer.add_scalar('KLD_loss', kld_loss/100 , iteration )
            kld_loss = 0
            mse_loss = 0
            iteration += 1

    print('\n ====> Epoch: {} | Time: {} | Average loss: {:.4f}'.format(
        epoch
        , timeSince(start,1)
        , train_loss / len(training_set.dataset)))

    return iteration

testing_mse_loss = []

def test(epoch):
    model.eval()
    test_mse_loss = 0
    for step , (batch_x,_) in enumerate(testing_set):
        batch_idx = step + 1
        batch_x = Variable(batch_x,volatile=True).cuda()
        recon,mu,logvar = model(batch_x)
        loss , MSE , KLD  = loss_function(recon,batch_x,mu,logvar)
        test_mse_loss+=MSE.data[0]
        writer.add_scalar('MSE_test', MSE.data[0] / len(batch_x), (epoch-1)*len(testing_set)+batch_idx)
        writer.add_scalar('KLD_test', KLD.data[0] / len(batch_x), (epoch-1)*len(testing_set)+batch_idx)
        print('\rTest on testing set: | [{}/{} ({:.0f}%)]  Loss: {:.6f} MSE: {:.6f} KLD: {:.6f} '.format(
                   batch_idx * len(batch_x)
                   , len(testing_set.dataset)
                   ,100. * batch_idx * len(batch_x) / len(testing_set.dataset)
                   , loss.data[0] / len(batch_x)
                   , MSE.data[0] / len(batch_x)
                   , KLD.data[0] / len(batch_x))
                   , end='')

    testing_mse_loss.append(test_mse_loss/len(testing_set))

    #print()
    #print('====> Average loss: {:.4f}'.format(
                  #test_loss / len(testing_set.dataset)))
    #print()
    return test_mse_loss / len(testing_set)

def gaussian(ins , mean, stddev):
    noise = Variable(ins.data.new(ins.size()).normal_(mean, stddev))
    return ins + noise

def rand_faces(num,epoch,model):
    model.eval()
    #z = torch.randn(num*num, args.latent_size)
    #z = torch.zeros(num,args.latent_size)
    z = np.random.normal(0, 1, (32, args.latent_size))
    z = Variable(torch.FloatTensor(z), volatile=True)
    #z = gaussian(z,0,1)
    z = z.cuda()
    recon = model.decode(z).permute(0,3,1,2)
    #print('recon:',recon)
    #input()
    recon = recon.data
    final_plot = recon
    #print('recon:',recon)
    #input()
    #img = torchvision.utils.make_grid(recon,nrow=num,normalize=True)
    #writer.add_image(str(epoch)+'_random_sample.jpg', img , epoch)
    final_plot = torchvision.utils.make_grid(final_plot,nrow=num,normalize=True)
    #print('final_plot:',final_plot)
    #input()
    return final_plot.permute(1,2,0).cpu().numpy()

def rec_face(num,epoch,model):
    model.eval()
    seq = np.random.randint(2621, size=10).tolist()
    seq = torch.LongTensor(seq)
    z = torch.index_select(testing_set.dataset.img,0,seq)
    img_orig = torchvision.utils.make_grid(z.permute(0,3,1,2),nrow=num,normalize=True)
    print('img_orig:',img_orig)
    input()
    final_plot = z.permute(0,3,1,2)
    z = Variable(z, volatile=True)
    z = z.cuda()
    recon = model.encode(z.permute(0,3,1,2))[0]
    recon = model.decode(recon).permute(0,3,1,2)
    #print('recon:',recon)
    #input()
    recon = recon.data
    print('final_plot:',final_plot)
    print('recon:',recon)
    input()
    final_plot = torch.cat((final_plot,recon.cpu()),0)
    #print('recon:',recon)
    #input()
    img_recon = torchvision.utils.make_grid(recon,nrow=num,normalize=True)
    final_plot = torchvision.utils.make_grid(final_plot,nrow=num,normalize=True)
    #writer.add_image(str(epoch)+'_original.jpg', img_orig , epoch)
    #writer.add_image(str(epoch)+'_reconstruct.jpg', img_recon , epoch)
    return final_plot.permute(1,2,0).cpu().numpy()

np.random.seed(2)
'''
model = torch.load('model/vae/model_300.pt')
final_plot = rec_face(10,1,model)
#ori = ori.permute(1,2,0)
#print('ori:',ori.shape)
#input()
#ori_grid= Image.fromarray(((ori+1)*127.5).astype(np.uint8))
scipy.misc.imsave('fig1_3.jpg', ((final_plot+1)*127.5))
#ori_grid.save('fig1_3.jpg')
'''
model = torch.load('model/dcgan/model_generator_140.pt')
final_plot = rand_faces(8,1,model)
#ori = ori.permute(1,2,0)
#print('ori:',ori.shape)
#input()
#ori_grid= Image.fromarray(((ori+1)*127.5).astype(np.uint8))
scipy.misc.imsave('fig2_3.jpg', ((final_plot+1)*127.5))


############################################################################################
# NUMBERS ----------------------------------------------------------------------------------
############################################################################################

import cmath  # module for operations with complex numbers
import math  # module for standard math operations

a = 7
b = 2
res1 = a + b
res2 = a - b
res3 = a * b
res4 = a ** b
res5 = a / b  # Division always returns a float
res6 = math.factorial(a)
res7 = math.log(a, math.e)
print("\n1)---------------------------------------------------")
print(f"{a} + {b} = {res1}")
print(f"{a} - {b} = {res2}")
print(f"{a} * {b} = {res3}")
print(f"{a} ** {b} = {res4}")
print(f"{a} / {b} = {res5}")
print(f"{a}! = {res6}")
print(f"ln({a}) = {res7}")

print("\n2)---------------------------------------------------")
rad = 2.5
area = math.pi * (rad ** 2)

print(f"area = π * (r^2) = {area} ")
print(f"cos(π) = {math.cos(math.pi)}")
print(f"e = {math.e}")
print(f"∞ = {math.inf}")
print(f"'Not a Number' = {math.nan}")

print("\n3)---------------------------------------------------")
z1 = complex(1.0, 1.0)
z1_pol = cmath.polar(z1)
print(f"z={z1}; |z| = {z1_pol[0]} , phase(z) = {z1_pol[1]} ")

# For a complete list of math functions and constants read:
# https://docs.python.org/3/library/math.html
# https://docs.python.org/3/library/cmath.html

print("\n4)---------------------------------------------------")
import random  # module for random number generation and random selection

random.seed()  # initialize the random number generator

dice = [1, 2, 3, 4, 5, 6]
count = [0, 0, 0, 0, 0, 0]

for roll in range(1000):  # see 2_4 "for loops"
    face = random.choice(dice)
    count[face - 1] += 1

dice_count = [(d, c) for d, c in zip(dice, count)]  # see 2_5 "list comprehension"

for c in range(len(dice_count)):
    print(f"#{dice_count[c][0]} rolled {dice_count[c][1]} times")

print("\n5)---------------------------------------------------")
# monte-carlo estimation of pi
inside = 0

for step in range(100000):
    x = random.uniform(-1, 1)
    y = random.uniform(-1, 1)
    if (x ** 2 + y ** 2) < 1:
        inside += 1

pi_est = float(4 * inside / 100000)
print(f"π estimation = {pi_est} ; error = {math.pi - pi_est}")

print("\n6)---------------------------------------------------")
# probabilistic distributions generation
import matplotlib.pyplot as plt

samples_1 = []
samples_2 = []
samples_3 = []

for event in range(100000):
    samples_1.append(random.gauss(3, 0.5))
    samples_2.append(random.gammavariate(3, 0.5))
    samples_3.append(random.expovariate(2))

plt.hist(samples_1, color='blue', bins=100, alpha=0.5)
plt.hist(samples_2, color='red', bins=100, alpha=0.5)
plt.hist(samples_3, color='green', bins=100, alpha=0.5)

plt.xlim(0, 5)
plt.show()

# WARNING: The pseudo-random generators of the random module should not be used
# for security purposes. For security or cryptographic uses, see the 'secrets' module.
# Complete reference at https://docs.python.org/3/library/random.html

############################################################################################
# STRINGS ----------------------------------------------------------------------------------
############################################################################################

S = 'Spam'

print(f"STRING = {S}, LENGTH = {len(S)}")
print(f"S[0] = {S[0]}, S[1] = {S[1]}, S[2] = {S[2]}, S[3] = {S[3]}")
print(f"S[-1] = {S[-1]}, S[-2] = {S[-2]}, S[-3] = {S[-3]}, S[-4] = {S[-4]}")

print("\n7)---------------------------------------------------")
#   Strings type objects are immutable. The following assignment would rise an exception:
#   S = 'spam'
#   S[0] = 'E'
#   The correct way to perform this operation is:
#   S = 'E' + S[1:]

name = input("Enter your name (3-20 char):")

while len(name) > 20 or len(name) < 3:
    name = input("Enter a valid name:")

surname = input("Enter your surname (3-20 char):")

while len(surname) > 20 or len(surname) < 3:
    surname = input("Enter a valid surname:")

name_up = name.upper()
name_double = name * 2
name_surname = name + surname
nickname = name[:3] + '_' + surname[:3]
surname_middle = surname[1:-1]

print(f"NAME = {name}\nSURNAME = {surname}")
print(f"NAME * 2 = {name_double}")
print(f"NAME + SURNAME = {name_surname}")
print(f"SURNAME MIDDLE = {surname_middle}")
print(f"NICKNAME = {nickname}")

print("\n8)---------------------------------------------------")
phone_num = '+39 011 13987456'
country_code = phone_num[:3]
prefix = phone_num[4:7]
num = phone_num[8:]
print(f"PHONE NUMBER: {phone_num}")
print(f"COUNTRY CODE: {country_code}")
print(f"PREFIX: {prefix}")
print(f"NUMBER: {num}")

print("\n9)---------------------------------------------------")
fields = ['NAME', 'SURNAME', 'AGE', 'JOB']
profile1 = 'John, Smith, 45, Plumber'
profile2 = 'Alice, Wonder, 32, Engineer'
profile3 = 'Mario, Rossi, 65, Musician'
p1_lst = profile1.split(',')
p2_lst = profile2.split(',')
p3_lst = profile3.split(',')
database = [p1_lst, p2_lst, p3_lst]
print(database)

for record in database:
    print('_________________')
    for f, v in zip(fields, record):
        print(f"{f} = {v.lstrip()}")  # lstrip()/rstrip() methods remove whitespaces on left/right of the string

print("\n10)---------------------------------------------------")
code1 = 'MRT558966544'
code2 = 'AHL-987_1554_44#'
print(code1, code1.isalnum(), code1.replace('MRT', 'HKL'))
print(code2, code2.isalnum(), code2[code2.find('-'):])

print("\n11)---------------------------------------------------")
import re

path = 'usr/home/desktop/notes.txt'
path_lst = re.split('[/.]', path)
print(path, '-->', path_lst)
filename, extension = path_lst[-2:]
print(f"FILENAME: {filename}\tEXTENSION: .{extension}")

############################################################################################
# LISTS ------------------------------------------------------------------------------------
############################################################################################

print("\n12)---------------------------------------------------")
prof = ['Mark', 'Smith', 48, 35500]
print(f"prof = {prof}\t type = {type(prof)}\t len = {len(prof)}")

prof[0] = 'John'  # lists are mutable collections of objects supporting in-place changes
prof + ['Plumber']  # + and * operators do not change the original list, if not specified
print(f"prof = {prof}\t type = {type(prof)}\t len = {len(prof)}")

prof.append('Plumber')  # the 'append' and 'pop' method changes the original list
print(f"prof = {prof}\t type = {type(prof)}\t len = {len(prof)}")
prof.pop()
print(f"prof = {prof}\t type = {type(prof)}\t len = {len(prof)}")

print("\n13)---------------------------------------------------")
cities = ['Rome', 'London', 'Paris', 'New York', 'Amsterdam', 'Buenos Aires']
print(cities)
cities.sort()
print(cities)
cities.reverse()
print(cities)

print("\n14)---------------------------------------------------")

hess = [[12.1, 4.2, 3.3],
        [4.2, 8.3, 41.2],
        [3.3, 41.2, 7.9]]

fxy = hess[0][1]
fxz = hess[0][2]
fyx = hess[1][0]
fyz = hess[1][2]
fzx = hess[2][0]
fzy = hess[2][1]

for line in range(len(hess)):
    print(hess[line])

# for line in hess:
#   print(line)

if fxy == fyx and fxz == fzx and fyz == fzy:
    print(f"The matrix is symmetric")

print("\n15)---------------------------------------------------")

M = [[5, 2, 1],
     [6, 9, 7],
     [4, 0, 3]]

col0 = [row[0] for row in M]  # list comprehension
col1 = [row[1] for row in M]
col2 = [row[2] for row in M]

M_t = [col0,
       col1,
       col2]

print("\nM =")
for line in M:  # iteration on a list
    print(line)

print("\nM_t =")
for line in M_t:
    print(line)

diag = [M[i][i] for i in range(len(M))]
print(f"\ndiagonal elements :\t{diag}")

# extracts even numbers from the first column
even = [row[0] for row in M if row[0] % 2 == 0]
print(f"\neven elements in the first column :\t{even}")

# squared elements of the second column
squared = [row[1] ** 2 for row in M]
print(f"\nsquared elements of the second column:\t{squared}")

print("\n16)---------------------------------------------------")
val_square_cube = [[x, x ** 2, x ** 3] for x in range(5)]
print(val_square_cube)

print("\n17)---------------------------------------------------")
mtx = [[0.5, 3.6, 1.7],
       [6.7, 9.5, 7.8],
       [8.9, 0.9, 5.7],
       [3.6, 0.5, 1.7]]

sum_gen = (sum(row) for row in mtx)  # enclosing a comprehension statements in "()" create a generator
print(sum_gen)
print(next(sum_gen))
print(next(sum_gen))
print(next(sum_gen))
print(next(sum_gen))
# print(next(sum_gen))
# calling next() out of bounds will rise a StopIteration exception

sum_list = list(map(sum, mtx))
print(sum_list)
# the map(func, seq) built-in calls "func" on each item in "seq". Equivalent to:
# sum_list = []
# for item in mtx:
#   sum_l.append(sum(item))
# print(sum_l)

sum_set = {sum(row) for row in mtx}
print(sum_set)
sum_dict = {i: sum(mtx[i]) for i in range(len(mtx))}
print(sum_dict)
# comprehension syntax can also be used yo create sets and dictionaries

############################################################################################
# DICTIONARIES -----------------------------------------------------------------------------
############################################################################################

# There are different ways to create a dictionary
print("\n18)---------------------------------------------------")

cart = []

item1 = {'NAME': 'Ham', 'PRICE': 3.99, 'CODE': '1564775', 'QUANTITY': 5}
cart.append(item1)

item2 = {}
item2['NAME'] = 'Soda'
item2['PRICE'] = 1.99
item2['CODE'] = '7458932'
item2['QUANTITY'] = 2
cart.append(item2)

item3 = dict(NAME='Beer', PRICE=2.59, CODE='1259036', QUANTITY=6)
cart.append(item3)

key_list = ['NAME', 'PRICE', 'CODE', 'QUANTITY']
item4 = dict(zip(key_list, ['Juice', 4.39, '9102004', 3]))
cart.append(item4)

total = 0
for obj in cart:
    print("---------------")
    total += obj['PRICE'] * obj['QUANTITY']
    for key, val in obj.items():
        if key == 'PRICE':
            print(f"{key} :\t€ {val}")
        else:
            print(f"{key} :\t{val}")

print("---------------")
print(f"TOTAL = {total}")

print("\n19)---------------------------------------------------")
record = {'Name': 'Jane', 'Surname': 'Doe', 'Age': 27, 'Sex': 'F', 'Job': ['Engineer', 'Manager']}

for k, v in record.items():
    print(f"{k}:    {v}")

if not 'Married' in record:
    print("Marital status not available")
print("---------------")

record['Married'] = True
record['Job'].append('Director')
record.pop('Name')
record.pop('Surname')
record['Full Name'] = {'First': 'Jane', 'Last': 'Doe'}
record['Full Name']['First'] = 'Mary Jane'

for k, v in record.items():
    print(f"{k}: {v}")

if not 'Married' in record:
    print("Marital status not available")