#!/usr/bin/env python
'''
Classes to parse charset names
'''

#####################
# IMPORT OPERATIONS #
#####################

import GlobalVariables as GlobVars
import MyExceptions as ME

import sys

###############
# AUTHOR INFO #
###############

__author__ = 'Michael Gruenstaeudl <m.gruenstaeudl@fu-berlin.de>'
__copyright__ = 'Copyright (C) 2016-2017 Michael Gruenstaeudl'
__info__ = 'nex2embl'
__version__ = '2017.02.02.1100'

#############
# DEBUGGING #
#############

import pdb
#pdb.set_trace()

###########
# CLASSES #
###########

class GetEntrezInfo:
    ''' This class contains functions to obtain gene information from gene
    symbols. '''

    def __init__(self, email_addr):
        self.email_addr = email_addr

    @staticmethod
    def _id_lookup(gene_sym, retmax=10):
        ''' An internal static function to convert a gene symbol to an Entrez ID
            via ESearch.
        Args:
            gene_sym (str): a gene symbol; example: 'psbI'
        Returns:
            entrez_id_list (list): a list of Entrez IDs; example: ['26835430',
                            '26833718', '26833393', ...]
        Raises:
            none
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> gene_sym = 'psbI'
#                >>> _id_lookup(gene_sym)
#                Out: ['26835430', '26833718', '26833393', ...]

        from Bio import Entrez
        if not gene_sym:
            raise ME.MyException('No gene symbol detected.')
        if '_' in gene_sym:
            raise ME.MyException('Gene symbol `%s` contains an '\
                'underscore, which is not allowed.' % (gene_sym))
        query_term = gene_sym + ' [sym]'
        try:
            esearch_records = Entrez.esearch(db='gene', term=query_term,
                retmax = retmax, retmod='xml')
        except:
            raise ME.MyException('An error occurred while retrieving '\
                'data from %s.' % ('ESearch'))
        parsed_records = Entrez.read(esearch_records)
        entrez_id_list = parsed_records['IdList']
        return entrez_id_list

    @staticmethod
    def _gene_product_lookup(entrez_id_list):
        ''' An internal static function to convert a list of Entrez IDs to a
        list of Entrez gene records via EPost and ESummary.
        Args:
            entrez_id_list (list): a list of Entrez IDs; example: ['26835430',
                                   '26833718', '26833393', ...]
        Returns:
            entrez_rec_list (list): a list of Entrez gene records
        Raises:
            none
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> entrez_id_list = ['26835430', '26833718', '26833393']
#                >>> _record_lookup(entrez_id_list)
#                Out: ???

        from Bio import Entrez
        epost_query = Entrez.epost('gene', id=','.join(entrez_id_list))
        try:
            epost_results = Entrez.read(epost_query)
        except:
            raise ME.MyException('An error occurred while retrieving data from '\
                '%s.' % ('EPost'))
        webenv = epost_results['WebEnv']
        query_key = epost_results['QueryKey']
        try:
            esummary_records = Entrez.esummary(db='gene', webenv=webenv,
                query_key=query_key)
        except:
            raise ME.MyException('An error occurred while retrieving data from '\
                '%s.' % ('ESummary'))
        entrez_rec_list = Entrez.read(esummary_records)
        return entrez_rec_list

    @staticmethod
    def _parse_gene_products(entrez_rec_list):
        ''' An internal static function to parse out relevant information from .
        Args:
            entrez_rec_list (list): a list of Entrez gene records
        Returns:
            gene_info_list (list): a list of dictionaries
        Raises:
            none
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> entrez_rec_list = []
#                >>> _parse_records(entrez_rec_list)

        from collections import Counter
        try:
            documentSummarySet = entrez_rec_list['DocumentSummarySet']
            docs = documentSummarySet['DocumentSummary']
        except:
            raise ME.MyException('An error occurred while parsing the '\
            'data from %s.' % ('ESummary'))
        list_gene_product = [doc['Description'] for doc in docs]
        #list_gene_symbol = [doc['NomenclatureSymbol'] for doc in docs]
        #list_gene_name = [doc['Name'] for doc in docs]

        # Avoiding that spurious first hit biases gene_product
        gene_product = Counter(list_gene_product).most_common()[0][0]
        return gene_product

    @staticmethod
    def _taxname_lookup(taxon_name, retmax=1):
        ''' An internal static function to look up a taxon name at NCBI
            Taxonomy via ESearch.
        Args:
            taxon_name (str): a taxon name; example: 'Pyrus tamamaschjanae'
            retmax (int):     the number of maximally retained hits
        Returns:
            entrez_hitcount (int): an integer
        Raises:
            none
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> taxon_name = 'Pyrus tamamaschjanae'
#                >>> _taxname_lookup(taxon_name)
#                Out: 0

        from Bio import Entrez
        if not taxon_name:
            raise ME.MyException('No taxon name detected.')
        if '_' in taxon_name:
            raise ME.MyException('Taxon name `%s` contains an underscore, '
            'which is not allowed.' % (taxon_name))
        query_term = taxon_name
        try:
            esearch_records = Entrez.esearch(db='taxonomy', term=query_term,
                retmax=retmax, retmod='xml')
        except:
            raise ME.MyException('An error occurred while retrieving data from '\
                '%s.' % ('ESearch'))
        parsed_records = Entrez.read(esearch_records)
        entrez_hitcount = parsed_records['Count']
        return entrez_hitcount

    def obtain_gene_product(self, gene_sym):
        ''' This function performs something.
        '''

#        Examples:
#            Example 1: # Default behaviour
#                >>> gene_sym = 'psbI'
#                >>> GetGeneInfo()._entrezid_lookup(gene_sym)
#                Out: ['26835430', '26833718', '26833393', ...]

        from Bio import Entrez
        Entrez.email = self.email_addr
        try:
            entrez_id_list = GetEntrezInfo._id_lookup(gene_sym)
        except ME.MyException as e:
            raise e
        try:
            entrez_rec_list = GetEntrezInfo._gene_product_lookup(entrez_id_list)
        except ME.MyException as e:
            raise e
        try:
            gene_product = GetEntrezInfo._parse_gene_products(entrez_rec_list)
        except ME.MyException as e:
            raise e
        return gene_product

    def does_taxon_exist(self, taxon_name):
        ''' This function calls _taxname_lookup and thus evaluates if a taxon exists.
        Args:
            taxon_name (str): a taxon name; example: 'Pyrus tamamaschjanae'
            retmax (int):     the number of maximally retained hits
        Returns:
            entrez_id_list (list): a list of Entrez IDs; example: ['26835430',
                            '26833718', '26833393', ...]
        Raises:
            none
        '''
        from Bio import Entrez
        Entrez.email = self.email_addr
        try:
            entrez_hitcount = GetEntrezInfo._taxname_lookup(taxon_name)
        except ME.MyException as e:
            raise e
        if entrez_hitcount == '0':
            return False
        if entrez_hitcount == '1':
            return True

class ConfirmAdjustTaxonName:
    ''' This class contains functions to confirm or adjust a sequence's
    taxon name.
    '''

    def __init__(self):
        pass

    def go(self, seq_record, email_addr):
        ''' This function evaluates a taxon name against NCBI taxonomy;
            if not listed, it adjusts the taxon name and appends it
            as ecotype info.
            Args:
                seq_record (obj):   a seqRecord object
                email_addr (dict):  your email address; example:
                                    "m.gruenstaeudl@fu-berlin.de"
            Returns:
                seq_record (obj):   a seqRecord object
            Raises:
                currently nothing
        '''
        try:
            genus_name, specific_epithet = seq_record.name.split(' ', 1)
        except ME.MyException as e:
            sys.exit('%s annonex2embl ERROR: Could not locate a '\
                'whitespace between genus name and specific epithet '\
                'in taxon name of sequence `%s`.' % ('\n', seq_record.id))
        if not GetEntrezInfo(email_addr).does_taxon_exist(seq_record.name):
            print('%s annonex2embl WARNING: Taxon name of sequence `%s` '\
                'not found in NCBI Taxonomy: `%s`. Please consider sending '\
                'a taxon request to ENA.'
                % ('\n', seq_record.id, seq_record.name))
            if not GetEntrezInfo(email_addr).does_taxon_exist(genus_name):
                sys.exit('%s annonex2embl ERROR: Neither genus name, '\
                    'nor species name of sequence `%s` were found in '\
                    'NCBI Taxonomy.' % ('\n', seq_record.id))
            else:
                species_name_original = seq_record.name
                species_name_new = genus_name + ' sp. ' + specific_epithet
                seq_record.name = species_name_new
                seq_record.features[0].qualifiers['organism'] = species_name_new
                seq_record.description = seq_record.description.\
                    replace(species_name_original, species_name_new)
                print('%s annonex2embl WARNING: Taxon name of sequence '\
                      '`%s` converted to the informal name: `%s`'
                      % ('\n', seq_record.id, species_name_new))
        return seq_record

class ParseCharsetName:
    ''' This class contains functions to parse charset names.
    Args:
        charset_name (str): a string that represents a charset name; example:
                            "psbI_CDS"
        email_addr (dict):  your email address; example:
                            "m.gruenstaeudl@fu-berlin.de"
    Raises:
        currently nothing
    '''

    def __init__(self, charset_name, email_addr):
        self.charset_name = charset_name
        self.email_addr = email_addr

    @staticmethod
    def _extract_charset_type(charset_name):
        ''' An internal static function to extract the charset type from a
        string. '''
        fk_present = [fk for fk in GlobVars.nex2ena_valid_INSDC_featurekeys
            if fk in charset_name]
        if not fk_present:
            raise ME.MyException('%s annonex2embl ERROR: No feature '\
            'key encountered in the name of charset `%s`.' % ('\n',
            charset_name))
        if len(fk_present) > 1:
            raise ME.MyException('%s annonex2embl ERROR: More than '\
            'one feature key encountered in the name of charset '\
            '`%s`.' % ('\n', charset_name))
        charset_type = fk_present[0]
        return charset_type

    @staticmethod
    def _extract_charset_sym(charset_name, charset_type):
        ''' An internal static function to extract the charset symbol from a
        string. '''
        try:
            charset_sym = charset_name.strip(charset_type)
        except:
            raise ME.MyException('%s annonex2embl ERROR: No charset '\
            'symbol encountered in the name of charset `%s`.' % (
            '\n', charset_name))
        charset_sym = charset_sym.strip('_')
        charset_sym = charset_sym.rstrip('_') # Remove trailing underscores
        return charset_sym

    def parse(self):
        ''' This function parses the charset_name.
        Returns:
            tupl.   The return consists of three strings in the order
                    "charset_sym, charset_type, charset_product"
        '''
        try:
            charset_type = ParseCharsetName._extract_charset_type(\
                self.charset_name)
        except ME.MyException as e:
            raise e
        try:
            charset_sym = ParseCharsetName._extract_charset_sym(\
                self.charset_name, charset_type)
        except ME.MyException as e:
            raise e
        entrez_handle = GetEntrezInfo(self.email_addr)
        if charset_type == 'CDS' or charset_type == 'gene':
            try:
                charset_product = entrez_handle.obtain_gene_product(\
                    charset_sym)
            except ME.MyException as e:
                raise e
        else:
            charset_product = None
        return (charset_sym, charset_type, charset_product)

#############
# FUNCTIONS #
#############

########
# MAIN #
########

{'Entrez': 'Bio.Entrez', 'Counter': 'collections.Counter'}._id_lookup

{'Entrez': 'Bio.Entrez', 'Counter': 'collections.Counter'}._gene_product_lookup

{'Entrez': 'Bio.Entrez', 'Counter': 'collections.Counter'}._parse_gene_products

{'Entrez': 'Bio.Entrez', 'Counter': 'collections.Counter'}._taxname_lookup


from tech_news.database import search_news
from datetime import datetime

# Requisito 6
def search_by_title(title):
    titles = []
    title_url = search_news({"title": {"$regex": title, "$options": "i"}})

    for itens in title_url:
        titles.append((itens["title"], itens["url"]))

    return titles

# Requisito 7
def search_by_date(date):
    news_by_date = []
    try:
        date_str = datetime.strptime(date, "%Y-%m-%d").strftime("%d/%m/%Y")
        results = search_news({"timestamp": date_str})
    except ValueError:
        raise ValueError("Data inválida")
    else:
        for itens in results:
            news_by_date.append((itens["title"], itens["url"]))
        return news_by_date

# Requisito 8
def search_by_tag(tag):
    tags = []
    results = search_news({"tags": {"$regex": tag, "$options": "i"}})
    for itens in results:
        tags.append((itens["title"], itens["url"]))

    return tags

# Requisito 9
def search_by_category(category):
    categories = []
    results = search_news({"category": {"$regex": category, "$options": "i"}})
    for itens in results:
        categories.append((itens["title"], itens["url"]))
    return categories


import pandas as pd
import openpyxl

class ErrorAnalysis:

    def __init__(self, path, expected_sheets, expected_columns):

        self.excel_path: str = path
        self.expected_sheets: list = expected_sheets
        self.expected_columns: dict = expected_columns
        self.customers: pd.DataFrame = pd.read_excel(self.excel_path, sheet_name='Customers')
        self.locations: pd.DataFrame = pd.read_excel(self.excel_path, sheet_name='Locations')
        self.suppliers: pd.DataFrame = pd.read_excel(self.excel_path, sheet_name='Suppliers')
        self.customers_names: list = self.customers['Name'].values.tolist()
        self.locations_names: list = self.locations['Name'].values.tolist()
        self.suppliers_names: list = self.suppliers['Name'].values.tolist()

    def sheet_analysis(self, alerts, errors) -> (list, list):

        wb = openpyxl.load_workbook(self.excel_path)
        sheet_names = wb.sheetnames

        alerts += [f'Table {sheet} is not a expected table'
                   for sheet in sheet_names if sheet not in self.expected_sheets]

        errors += [f'Table {sheet} is not in archive tables'
                   for sheet in self.expected_sheets if sheet not in sheet_names]

        return alerts, errors

    def columns_analysis(self, alerts, errors) -> (list, list):

        customer_columns = list(self.customers)
        location_columns = list(self.locations)
        suppliers_columns = list(self.suppliers)

        errors += [f'There is no {column} column in Customers table'
                   for column in self.expected_columns['Customers'] if column not in customer_columns]
        errors += [f'There is no {column} column in suppliers table'
                   for column in self.expected_columns['Suppliers'] if column not in suppliers_columns]
        errors += [f'There is no {column} column in locations table'
                   for column in self.expected_columns['Locations'] if column not in location_columns]

        alerts += [f'Column {column} is not a expected column in customers table'
                   for column in customer_columns if column not in self.expected_columns['Customers']]
        alerts += [f'Column {column} is not a expected column in suppliers table'
                   for column in suppliers_columns if column not in self.expected_columns['Suppliers']]
        alerts += [f'Column {column} is not a expected column in locations table'
                   for column in location_columns if column not in self.expected_columns['Locations']]

        return alerts, errors

    def check_duplicates(self, alerts) -> (list, list):

        duplicates = self.suppliers['Name'].duplicated().tolist()
        if any(duplicates):
            alerts += [f"Name {self.suppliers['Name'][i]} is duplicated in Suppliers"
                      for i in range(len(self.suppliers_names)) if duplicates[i]]
            self.suppliers.drop_duplicates('Name', inplace=True)

        duplicates = self.locations[['Name', 'Status']].duplicated().tolist()
        if any(duplicates):
            alerts += [f"Name {self.locations['Name'][i]} is duplicated with the same status in Locations"
                      for i in range(len(self.locations_names)) if duplicates[i]]
            self.locations.drop_duplicates(['Name', 'Status'], inplace=True)

        duplicates = self.customers['Name'].duplicated().tolist()
        if any(duplicates):
            alerts += [f"Name {self.customers['Name'][i]} is duplicated in Customers"
                      for i in range(len(self.customers_names)) if duplicates[i]]
            self.customers.drop_duplicates('Name', inplace=True)

        duplicates = self.locations['Name'].duplicated().tolist()
        if any(duplicates):
            alerts += [f"Name {self.locations['Name'][i]} is duplicated in Locations"
                      for i in range(len(self.locations_names)) if duplicates[i]]
            self.locations.drop_duplicates('Name', inplace=True)

        return alerts

    def check_names_inconsistency(self, alerts, errors) -> (list, list):

        self.customers.set_index('Name', inplace=True, drop=False)
        self.locations.set_index('Name', inplace=True, drop=False)
        self.suppliers.set_index('Name', inplace=True, drop=False)

        errors += [f'Name {name} in Customers does not exist in Locations table'
                   for name in self.customers_names if name not in self.locations_names]
        errors += [f'Name {name} in Suppliers does not exist in Locations table'
                   for name in self.suppliers_names if name not in self.locations_names]

        if errors:
            return alerts, errors

        errors += [f'Status divergence in the name {name} between tables Customers and Locations'
                   for name in self.customers_names if self.customers['Status'][name] != self.locations['Status'][name]]

        errors += [f'Status divergence in the name {name} between tables Suppliers and Locations'
                   for name in self.suppliers_names if self.suppliers['Status'][name] != self.locations['Status'][name]]

        return alerts, errors

    def check_supply_and_demand_inconsistency(self, alerts, errors) -> (list, list):

        errors += [f' Negative value for Maximum Supply from the supplier {name}'
                  for name in self.suppliers_names if self.suppliers['Maximum Supply'][name] < 0]
        errors += [f' Negative value for Fixed Demand from the customer {name}'
                  for name in self.customers_names if self.customers['Fixed Demand'][name] < 0]

        alerts += [f' Null value for Maximum Supply from the supplier {name}'
                  for name in self.suppliers_names if self.suppliers['Maximum Supply'][name] == 0]
        alerts += [f' Null value for Fixed Demand from the customer {name}'
                  for name in self.customers_names if self.customers['Fixed Demand'][name] == 0]

        return alerts, errors

    def check_status_inconsistency(self, alerts, errors) -> (list, list):

        if (self.suppliers['Status'] == 'Exclude').all():
            alerts += ['All values for status in the table suppliers are Exclude']

        if (self.customers['Status'] == 'Exclude').all():
            alerts += ['All values for status in the table customers are Exclude']

        if (self.locations['Status'] == 'Exclude').all():
            alerts += ['All values for status in the table locations are Exclude']

        errors += [f'Status {status} in Locations is not supported, status must be Exclude or Include'
                   for status in self.locations['Status'].values.tolist() if status not in ['Exclude', 'Include']]

        errors += [f'Status {status} in Suppliers is not supported, status must be Exclude or Include'
                   for status in self.suppliers['Status'].values.tolist() if status not in ['Exclude', 'Include']]

        errors += [f'Status {status} in Customers is not supported, status must be Exclude or Include'
                   for status in self.customers['Status'].values.tolist() if status not in ['Exclude', 'Include']]

        return alerts, errors

    def check_coordenates_inconsistency(self, alerts, errors) -> (list, list):

        alerts += [f'Location {name} latitude is not in the correct format' for name in self.locations_names
                   if abs(self.locations['Latitude'][name]) > 90]

        alerts += [f'Location {name} longitude is not in the correct format' for name in self.locations_names
                   if abs(self.locations['Longitude'][name]) > 180]

        return alerts, errors

    def errors_search(self) -> (list, list):

        alerts = []
        errors = []

        alerts, errors = self.sheet_analysis(alerts, errors)

        if errors:
            return alerts, errors

        alerts, errors = self.columns_analysis(alerts, errors)

        if errors:
            return alerts, errors

        alerts = self.check_duplicates(alerts)

        alerts, errors = self.check_names_inconsistency(alerts, errors)

        if errors:
            return alerts, errors

        alerts, errors = self.check_supply_and_demand_inconsistency(alerts, errors)

        alerts, errors = self.check_coordenates_inconsistency(alerts, errors)

        alerts, errors = self.check_supply_and_demand_inconsistency(alerts, errors)

        return alerts, errors


#!/usr/bin/env python3

import unittest
import time
import psutil
from vpp_papi.vpp_stats import VPPStats

from framework import VppTestCase, VppTestRunner

class StatsClientTestCase(VppTestCase):
    """Test Stats Client"""

    @classmethod
    def setUpClass(cls):
        super(StatsClientTestCase, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        super(StatsClientTestCase, cls).tearDownClass()

    def test_set_errors(self):
        """Test set errors"""
        self.assertEqual(self.statistics.set_errors(), {})
        self.assertEqual(self.statistics.get_counter('/err/ethernet-input/no'),
                         [0])

    def test_client_fd_leak(self):
        """Test file descriptor count - VPP-1486"""

        cls = self.__class__
        p = psutil.Process()
        initial_fds = p.num_fds()

        for _ in range(100):
            stats = VPPStats(socketname=cls.stats_sock)
            stats.disconnect()

        ending_fds = p.num_fds()
        self.assertEqual(initial_fds, ending_fds,
                         "initial client side file descriptor count: %s "
                         "is not equal to "
                         "ending client side file descriptor count: %s" % (
                             initial_fds, ending_fds))

    @unittest.skip("Manual only")
    def test_mem_leak(self):
        def loop():
            print('Running loop')
            for i in range(50):
                rv = self.vapi.papi.tap_create_v2(id=i, use_random_mac=1)
                self.assertEqual(rv.retval, 0)
                rv = self.vapi.papi.tap_delete_v2(sw_if_index=rv.sw_if_index)
                self.assertEqual(rv.retval, 0)

        before = self.statistics.get_counter('/mem/statseg/used')
        loop()
        self.vapi.cli("memory-trace on stats-segment")
        for j in range(100):
            loop()
        print(self.vapi.cli("show memory stats-segment verbose"))
        print('AFTER', before,
              self.statistics.get_counter('/mem/statseg/used'))

if __name__ == '__main__':
    unittest.main(testRunner=VppTestRunner)


#
# @lc app=leetcode id=438 lang=python3
#
# [438] Find All Anagrams in a String
#

# @lc code=start
import collections

class Solution:
    def findAnagrams(self, s: str, p: str):
        if len(s) < len(p) or not s or not p:
            return []
        rg = len(p)
        pd = collections.Counter(p)
        # initialize
        sd = collections.Counter(s[:rg])
        ans = []
        if pd == sd:
            ans = [0]
        for pos in range(1, len(s) - rg + 1):
            if sd[s[pos - 1]] == 1:
                sd.pop(s[pos - 1])
            else:
                sd[s[pos - 1]] -= 1
            if s[pos + rg - 1] not in sd:
                sd[s[pos + rg - 1]] = 1
            else:
                sd[s[pos + rg - 1]] += 1
            if pd == sd:
                ans.append(pos)
            pos += 1
        return ans

if __name__ == '__main__':
    a = Solution()
    b = a.findAnagrams("abab", "ab")
    print(b)

# @lc code=end


import boto3

def create_dax_table(dyn_resource=None):
    """
    Creates a DynamoDB table.

    :param dyn_resource: Either a Boto3 or DAX resource.
    :return: The newly created table.
    """
    if dyn_resource is None:
        dyn_resource = boto3.resource('dynamodb')

    table_name = 'TryDaxTable'
    params = {
        'TableName': table_name,
        'KeySchema': [
            {'AttributeName': 'partition_key', 'KeyType': 'HASH'},
            {'AttributeName': 'sort_key', 'KeyType': 'RANGE'}
        ],
        'AttributeDefinitions': [
            {'AttributeName': 'partition_key', 'AttributeType': 'N'},
            {'AttributeName': 'sort_key', 'AttributeType': 'N'}
        ],
        'ProvisionedThroughput': {
            'ReadCapacityUnits': 10,
            'WriteCapacityUnits': 10
        }
    }
    table = dyn_resource.create_table(**params)
    print(f"Creating {table_name}...")
    table.wait_until_exists()
    return table

if __name__ == '__main__':
    dax_table = create_dax_table()
    print(f"Created table.")


import pygame
import math
import time
import hamilton
import common

pixel_per_meter = 10
width = 800
height = 600
v_inf = 1e4
width_m = width/pixel_per_meter
hegiht_m = height/pixel_per_meter
mass0 = 1

def main():
    dims = (width, height)
    background = (32, 32, 32)
    screen = pygame.display.set_mode(dims)
    pygame.display.set_caption("h")
    screen.fill((0, 0, 0))

    particles = []

    running = True
    currtime = time.time()

    dt = 0.001
    N = len(particles)

    while running:
        if time.time() - currtime >= dt:
            currtime = time.time()
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    running = False
            screen.fill(background)


# -*- coding: utf-8 -*-
"""
Created on Sat Mar 31 20:46:08 2018

@author: Gaurav
"""

#WebCrawler for Dortmund Data Bank

import requests
from bs4 import BeautifulSoup
import scipy as sci
import win32com.client
import os
import sys

#This Function gets the constant value of Temperature or Pressure from above
#the table
def GetConstantValue(useful_header):
    useful_value=useful_header.find_previous("table")
    col=useful_value.find_all("td")
    constvalue=col[0].get_text()
    constvalue=float(constvalue)
    return constvalue

#This function captures the useful headers in the webpage that point to
#Data Tables
def GetUsefulHeaders(soup):
    headers=soup.find_all("h4")
    useful_headers=[]
    for header in headers:
        text=header.get_text()
        if(text=="Data Table"):
            useful_headers.append(header)
    return useful_headers

#This function captures useful tables from the useful headers and stores the
#tables as numpy arrays which are themselves stored in a list

def GetUsefulTables(useful_headers):
    number_of_tables=len(useful_headers)
    list_of_arrays=[]
    constvalues=sci.zeros(number_of_tables)
    stats=sci.zeros(number_of_tables)
    for i in range(number_of_tables):
        useful_header=useful_headers[i]
        useful_table=useful_header.find_next("table")
        rows=useful_table.find_all("tr")
        number_of_cols=len(rows[-1].find_all("td"))
        if(number_of_cols<3):
            continue
        constvalues[i]=GetConstantValue(useful_header)
        stats[i]=GetTableStat(useful_header)
        arr=sci.zeros((len(rows),3))

        row_counter=0
        for row in rows:
            cols=row.find_all("td")
            for cell in range(len(cols)):
                arr[row_counter,cell]=cols[cell].get_text()
            row_counter+=1

        arr=sci.delete(arr,(0),axis=0)
        if(arr[0,1]==0.):
            arr=sci.delete(arr,(0),axis=0)
        if(arr[-1,1]==1.):
            arr=sci.delete(arr,(-1),axis=0)
        list_of_arrays.append(arr)

    final_constvalues=constvalues[constvalues!=0]
    final_stats=stats[constvalues!=0]
    return [list_of_arrays,final_constvalues,final_stats]

#This function is used to print the VLEData to an Excel file (which is created)
#using the win32com library
def PrintToExcel(list_of_tables,compound1,compound2,switch):
    app=win32com.client.Dispatch("Excel.Application")
    app.Visible=False
    wb=app.Workbooks.Add()
    cwdir=os.getcwd()
    filename=f"{compound1}-{compound2}_VLEData.xlsx"
    path=cwdir+"\\"+filename
    wb.SaveAs(path)
    sh=wb.Sheets("Sheet1")

    offset=0
    for table in list_of_tables:
        one_table=table
        [rows,cols]=one_table.shape
        one_table=one_table.tolist()
        for i in range(rows):
            for j in range(cols):
                sh.Cells(i+1+offset,j+1).Value=one_table[i][j]
                if(switch==0):
                    sh.Cells(i+1+offset,1).Interior.ColorIndex=19
                if(switch==1):
                    sh.Cells(i+1+offset,1).Interior.ColorIndex=20
        offset+=rows+1

    wb.Save()
    wb.Close()
    app.Quit()

#This function returns 0 if the table is PXY, else it returns 1 for TXY
def GetTableStat(useful_header):
    useful_value=useful_header.find_previous("table")
    col=useful_value.find_all("td")
    check=col[1].get_text()
    if(check=="K"):
        return 0
    else:
        return 1

#The main function that accepts compound names, calls all the relevant functions
#and returns/prints the final tables
def Crawl(compound1,compound2):
    comp1=f"{compound1}"
    comp2=f"{compound2}"
    page=requests.get(f"http://www.ddbst.com/en/EED/VLE/VLE%20{comp1}%3B{comp2}.php")
    if(page.status_code==200):
        switch=0
    else:
        switch=1
        page=requests.get(f"http://www.ddbst.com/en/EED/VLE/VLE%20{comp2}%3B{comp1}.php")
    soup=BeautifulSoup(page.text,"html.parser")

    useful_headers=GetUsefulHeaders(soup)
    [tables,constantvalues,stats]=GetUsefulTables(useful_headers)

    PrintToExcel(tables,comp1,comp2,switch)
    return [tables,constantvalues,stats,switch]

if __name__=="__main__":
    if(len(sys.argv)==3):
        compound1=f"{sys.argv[1]}"
        compound2=f"{sys.argv[2]}"
    if(len(sys.argv)==4):
        compound1=f"{sys.argv[1]} {sys.argv[2]}"
        compound2=f"{sys.argv[3]}"
    if(len(sys.argv)==5):
        compound1=f"{sys.argv[1]} {sys.argv[2]}"
        compound2=f"{sys.argv[3]} {sys.argv[4]}"

    Crawl(f"{compound1}",f"{compound2}")


from tkinter import *
from tkinter import ttk
from time    import strftime
from PIL     import Image , ImageTk
from io      import BytesIO
import requests

##########################
# Global Variables
##########################
API_key_file   = 'weatherapi.key'
weatherAPI_key = None # Variable to store the API Key
imageTk_icon   = None # Variable for ImageTk icon

##########################
# Common functions
##########################

# Clear root GUI window
def clear_root( root:object ):
    for frame in root.winfo_children():
        frame.destroy()

##########################
# Help frame
##########################

def close_help_form( root:object ):
    clear_root( root )    # Clear the root window
    forecast_form( root ) # Show the forecast form

# Help form will be called by a help button to show query examples
def help_form( root:object ):
    clear_root( root )
    global imageTk_icon
    instructions = '''

For detailed documentation visit:

            www.weatherapi.com

For quick reference, see these query examples:
Latitude and Longitude (Decimal degree) e.g: 48.8567,2.3508
city name e.g.: Paris
US zip e.g.: 10001
UK postcode e.g: SW1
Canada postal code e.g: G2J
metar:<metar code> e.g: metar:EGLL
iata:<3 digit airport code> e.g: iata:DXB
auto:ip IP lookup e.g: auto:ip
IP address (IPv4 and IPv6 supported) e.g: 100.0.0.1

'''
    frm = ttk.Frame( root , padding=5 )
    frm.grid()
    # Row 1 - Logo
    image_icon   = Image.open( 'logo.png' )
    imageTk_icon = ImageTk.PhotoImage( image_icon )
    ttk.Label( frm , image=imageTk_icon ).grid( row=0 , column=0 )
    # Row 2 - Instructions
    ttk.Label( frm , text=instructions ).grid( row=1 , column=0 )
    # Row 3 - Close Button
    ttk.Button( frm, text="Close",
        command = lambda:
            close_help_form( root )
    ).grid( row=2 , column=0 )

########################################
# Weather forecast frame (main frame)
########################################

# Retrive data from API and update form widgets
def fetch_forecast( query:str , form:object ):
    global weatherAPI_key
    global imageTk_icon
    # Send API request like:
    # http://api.weatherapi.com/v1/current.json?key=< API KEY >&q=London&aqi=no
    # For full documentation visit https://www.weatherapi.com site
    url = "http://api.weatherapi.com/v1/forecast.json"
    params = {
        'key': weatherAPI_key,
        'q': query,
        'days':7,
        'aqi': 'no',   # To get air quality data or not
        'alerts':'no'
    }
    res = requests.get( url , params=params )
    # Update widgets
    if res.status_code == 200: # HTTP status code 200 - OK
        # Parse the response to a variable
        data = res.json()
        condition_text = 'The weather condition at {0} is: {1}'.format(
            data['location'][ 'name' ],
            data['current'][ 'condition' ][ 'text' ]
        )
        form.nametowidget( 'condition_text' ).config( text=condition_text )

        res = requests.get( 'http:' + str( data['current'][ 'condition' ][ 'icon' ] ) )
        if res.status_code == 200:
            image_icon   = Image.open( BytesIO( res.content ) )
            imageTk_icon = ImageTk.PhotoImage( image_icon )
            form.nametowidget( 'condition_icon' ).config( image=imageTk_icon )
        else:
            print( res.reason )

        form.nametowidget( 'temp_c' ).config( text='{} °C'.format( str( data['current'][ 'temp_c' ] ) ) )
        form.nametowidget( 'temp_f' ).config( text='{} °F'.format( str( data['current'][ 'temp_f' ] ) ) )
    else:
        form.nametowidget( 'condition_text' ).config( text='Ooops, something went wrong. The API response is:\n{}'.format( res.reason ) )

# Main forecast form
def forecast_form( root:object ):
    global imageTk_icon
    # Initilise Frame for all widgets
    frm = ttk.Frame( root , padding=5)
    frm.grid()

    # Row 1 - title
    ttk.Label( frm , text="Local time" ).grid( row=0 , column=0 )
    # Local Time label
    def my_time( label:object ): # Help function to update the time label every second
        label.config( text= strftime( '%H:%M:%S %p | %x' ) )
        label.after( 1000 , my_time , label ) # recursive call every 1000ms

    time_label = ttk.Label( frm )
    time_label.grid( row=0 , column=1 )
    my_time( time_label ) # start lable update loop

    # Row 2 - current time and date
    query_entry = ttk.Entry( frm )
    query_entry.grid( row=1 , column=0 )
    query_entry.focus()
    ttk.Button( frm , text="Get Weather",
        command = lambda:
            fetch_forecast( query_entry.get() , frm )
    ).grid( row=1 , column=1 )
    ttk.Button( frm , text="Help",
        command = lambda:
            help_form( root )
    ).grid( row=1 , column=2 )

    # Row 3 - Current conditions - icon
    image_icon   = Image.open( 'logo.png' )
    imageTk_icon = ImageTk.PhotoImage( image_icon )
    ttk.Label( frm , name = 'condition_icon' , image=imageTk_icon ).grid( row=2 , column=0 , columnspan=3 )

    # Row 4 - Current conditions - text
    ttk.Label( frm,  name = 'condition_text' ).grid( row=3 , column=0 , columnspan=3 )

    # Row 5 - Current conditions - temperatures
    ttk.Label( frm , name = 'temp_c' ).grid( row=4 , column=0 )
    ttk.Label( frm,  name = 'temp_f' ).grid( row=4 , column=1 , columnspan=2 )

##########################
# API Key frame
##########################

# Save API key into a file
def save_API_key( key:str , root:object ):
    global API_key_file
    global weatherAPI_key
    weatherAPI_key = key
    try:
        f = open( API_key_file , "w" )
        f.write( key )
        f.close()
    except Exception: pass # For now do nothing if fail to write
    clear_root( root )
    forecast_form( root )

# API Key form to be displaied if there is no API Key file
def api_key_form( root:object ):
    global imageTk_icon
    instructions = '''

This application requires an API Key from

           www.weartherapi.com

Please visit the site and read the documentation.

Once you have an API Key, use this form
to save it and use this application.

'''
    frm = ttk.Frame( root , padding=5 )
    frm.grid()
    # Row 1 - Logo
    image_icon   = Image.open( 'logo.png' )
    imageTk_icon = ImageTk.PhotoImage( image_icon )
    ttk.Label( frm , image=imageTk_icon ).grid( row=0 , column=0 , columnspan=2 )
    # Row 2 - Instructions
    ttk.Label( frm , text=instructions ).grid( row=1 , column=0 , columnspan=2 )
    # Row 3 - Enter Key
    query_entry = ttk.Entry( frm )
    query_entry.grid( row=2 , column=0 )
    query_entry.focus()
    ttk.Button( frm, text="Save API Key",
        command = lambda:
            save_API_key( query_entry.get() , root )
    ).grid( row=2 , column=1 )

##################
# MAIN APP
##################

root = Tk(  ) # Creates a window

try:
    # Get WeatherAPI Key from file
    f = open( API_key_file , "r")
    weatherAPI_key = f.read()
    f.close()
    forecast_form( root ) # Display the forecast form
except FileNotFoundError as e:
    # User entery of API Key
    api_key_form( root )

root.mainloop() # Shows the window