import os
from operator import add
from functools import reduce
from itertools import product

BENCHMARKS = {
    'int': [
        "400.perlbench",
        "401.bzip2",
        "403.gcc",
        "429.mcf",
        "445.gobmk",
        "456.hmmer",
        "458.sjeng",
        "462.libquantum",
        "464.h264ref",
        "471.omnetpp",
        "473.astar",
        "483.xalancbmk",
    ],
    'fp': [
        "410.bwaves",
        #"416.gamess", #FIXME: compile error
        "433.milc",
        "434.zeusmp",
        "435.gromacs",
        "436.cactusADM",
        "437.leslie3d",
        "444.namd",
        #"447.dealII", #FIXME: compile error
        #"450.soplex", #FIXME: compile error
        "453.povray",
        "454.calculix",
        "459.GemsFDTD",
        "465.tonto",
        "470.lbm",
        #"481.wrf", #FIXME: underflow flag
        "482.sphinx3",
    ]
}

def _spec_actions(source, target, env, for_signature):
    config = os.path.splitext(source[-1].name)[0]
    prefix = os.path.join(env['SPEC2006_DIR'], 'benchspec', 'CPU2006')
    run_dirs = [
        os.path.join(
            prefix, benchmark, 'run',
            'run_base_%s_%s.0000' % (env['INPUT'], config))
        for benchmark in BENCHMARKS[env['SUITE']]
    ]
    WriteInit = env['WRITE_INIT']
    WriteCommands = env['WRITE_COMMANDS']
    return [
        # Compile SPEC
        Copy(os.path.join(env['SPEC2006_DIR'], source[-1].name), source[-1]),
        ' '.join([
            'cd', env['SPEC2006_DIR'], '&&',
            'source', './shrc', '&&', 'runspec',
            '--action', 'setup',
            '--config', config,
            '--size', env['INPUT'],
            '--parallel_setup', str(GetOption('num_jobs'))
        ] + BENCHMARKS[env['SUITE']])
    ] + reduce(add, [
        [
            # Install overlays
            Delete(binary.dir.abspath),
            Copy(binary.dir.abspath, run_dir),
            Delete(os.path.join(binary.dir.abspath, '..', 'etc', 'init.d')),
            Mkdir(os.path.join(binary.dir.abspath, '..', 'etc', 'init.d')),
            WriteCommands(
                os.path.join(binary.dir.abspath, '..', 'etc', 'init.d', 'rcS'),
                binary.name,
                command.abspath),
            WriteInit(
                os.path.join(binary.dir.abspath, '..', 'etc', 'init.d', 'rcK'),
                '\n'.join(['#!/bin/sh', '# Do nothing']))
        ]
        for run_dir, binary, command in zip(run_dirs, target, source[:-1])
    ], [])

def main():
    Import('env')

    env.Append(BUILDERS={
        'SPEC2006': Builder(generator=_spec_actions)
    })

    spec2006 = dict()
    for suite, input_type in product(['int', 'fp'], ['test', 'ref']):
        commands = [
            os.path.abspath(os.path.join(
                'commands', '%s.%s.cmd' % (benchmark, input_type)))
            for benchmark in BENCHMARKS[suite]
        ]
        binaries = [
            os.path.join(
                env['OVERLAY_DIR'], benchmark + '.' + input_type, 'root',
                'sphinx_livepretend_base.riscv'
                if benchmark == '482.sphinx3'
                else 'Xalan_base.riscv'
                if benchmark == '483.xalancbmk'
                else benchmark[4:] + '_base.riscv')
            for benchmark in BENCHMARKS[suite]
        ]
        build = env.SPEC2006(
            binaries,
            commands + ['riscv.cfg'],
            SUITE=suite,
            INPUT=input_type)
        env.Alias('spec2006%s-%s' % (suite, input_type), build)
        env.SideEffect('#spec2006', build)
        for benchmark, binary in zip(BENCHMARKS[suite], build):
            bblvmlinux = env['BUILD_BBLVMLINUX'](
                env,
                '%s.%s' % (benchmark, input_type),
                os.path.abspath(os.path.join(binary.dir.abspath, '..')),
                [binary.abspath])
            env.Alias(
                'build-spec2006%s-%s' % (suite, input_type), bblvmlinux)
            env.Alias(
                'run-spec2006%s-%s' % (suite, input_type),
                env.Spike('#run-%s.%s' % (benchmark, input_type), bblvmlinux))

    env.Clean(
        'spec2006',
        Glob(os.path.join(env['SPEC2006_DIR'], 'benchspec', 'CPU2006', '*', 'exe')) + \
        Glob(os.path.join(env['SPEC2006_DIR'], 'benchspec', 'CPU2006', '*', 'run')) + \
        Glob(os.path.join(env['SPEC2006_DIR'], 'benchspec', 'CPU2006', '*', 'build')))

    return spec2006

if __name__ == 'SCons.Script':
    main()


import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
import pennylane as qml
import pennylane.qaoa as qaoa

# start with a graph
edges = [(0, 1), (1, 2), (2, 0), (2, 3)]
graph = nx.Graph(edges)

#nx.draw(graph, with_labels=True)
#plt.show()

# cost and mixer is given from pennylane
cost_h, mixer_h = qaoa.min_vertex_cover(graph, constrained=False)
print ('Cost Hamiltonian', cost_h)
print ('Mixer Hamiltonian', mixer_h)

# QAOA with two parameters
def qaoa_layer(gamma, alpha):
    qaoa.cost_layer(gamma, cost_h)
    qaoa.mixer_layer(alpha, mixer_h)

# full circuit
wires = range(4)
depth = 2

# initial state and QAOA
def circuit(params, **kwargs):
    for w in wires:
        qml.Hadamard(wires=w)
    qml.layer(qaoa_layer, depth, params[0], params[1])

# device used as a backend
dev = qml.device("qulacs.simulator", wires=wires)

# cost function - expectation of cost_h on the device
cost_function = qml.ExpvalCost(circuit, cost_h, dev)

# optimization loop
optimizer = qml.GradientDescentOptimizer()
steps = 20
params = [[0.5, 0.5], [0.5, 0.5]]

for i in range(steps):
    params = optimizer.step(cost_function, params)

print("Optimal parameters")
print(params)

@qml.qnode(dev)
def prob_circuit(gamma, alpha):
    circuit([gamma, alpha])
    return qml.probs(wires=wires)

probs = prob_circuit(params[0], params[1])

plt.style.use("seaborn")
plt.bar(range(2**len(wires)), probs)
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 23 15:01:55 2022

@author: Alkios
"""

import os
os.chdir('C:/Users/Alkios/Downloads/signals')

from scipy.spatial import Voronoi
from matplotlib.patches import Polygon

import numpy as np
import cv2 as cv
from matplotlib import pyplot as plt

img = cv.imread('lena.png')
plt.imshow(img)

gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)

edges = cv.Canny(gray,250,250)
plt.imshow(edges,cmap = 'gray')

dst = cv.GaussianBlur(edges,(25,25), 10)
plt.imshow(dst,cmap = 'gray')

ctest = cv.subtract(dst,edges)
plt.imshow(ctest,cmap = 'gray')

modified_test = ctest/np.sum(ctest)
np.sum(modified_test)

# Create a flat copy of the array
flat = modified_test.flatten()

# Then, sample an index from the 1D array with the
# probability distribution from the original array
sample_index = np.random.choice(a=flat.size, p=flat)

img_modif = cv.flip(img, 0)

def voronoi_finite_polygons_2d(vor, radius=None):
    """
    Reconstruct infinite voronoi regions in a 2D diagram to finite
    regions.
    Parameters
    ----------
    vor : Voronoi
        Input diagram
    radius : float, optional
        Distance to 'points at infinity'.
    Returns
    -------
    regions : list of tuples
        Indices of vertices in each revised Voronoi regions.
    vertices : list of tuples
        Coordinates for revised Voronoi vertices. Same as coordinates
        of input vertices, with 'points at infinity' appended to the
        end.
    """

    if vor.points.shape[1] != 2:
        raise ValueError("Requires 2D input")

    new_regions = []
    new_vertices = vor.vertices.tolist()

    center = vor.points.mean(axis=0)
    if radius is None:
        radius = vor.points.ptp().max()*2

    # Construct a map containing all ridges for a given point
    all_ridges = {}
    for (p1, p2), (v1, v2) in zip(vor.ridge_points, vor.ridge_vertices):
        all_ridges.setdefault(p1, []).append((p2, v1, v2))
        all_ridges.setdefault(p2, []).append((p1, v1, v2))

    # Reconstruct infinite regions
    for p1, region in enumerate(vor.point_region):
        vertices = vor.regions[region]

        if all(v >= 0 for v in vertices):
            # finite region
            new_regions.append(vertices)
            continue

        # reconstruct a non-finite region
        ridges = all_ridges[p1]
        new_region = [v for v in vertices if v >= 0]

        for p2, v1, v2 in ridges:
            if v2 < 0:
                v1, v2 = v2, v1
            if v1 >= 0:
                # finite ridge: already in the region
                continue

            # Compute the missing endpoint of an infinite ridge

            t = vor.points[p2] - vor.points[p1] # tangent
            t /= np.linalg.norm(t)
            n = np.array([-t[1], t[0]])  # normal

            midpoint = vor.points[[p1, p2]].mean(axis=0)
            direction = np.sign(np.dot(midpoint - center, n)) * n
            far_point = vor.vertices[v2] + direction * radius

            new_region.append(len(new_vertices))
            new_vertices.append(far_point.tolist())

        # sort region counterclockwise
        vs = np.asarray([new_vertices[v] for v in new_region])
        c = vs.mean(axis=0)
        angles = np.arctan2(vs[:,1] - c[1], vs[:,0] - c[0])
        new_region = np.array(new_region)[np.argsort(angles)]

        # finish
        new_regions.append(new_region.tolist())

    return new_regions, np.asarray(new_vertices)

def voronoi_polygons(n=256):
    random_seeds = points_coord
    vor = Voronoi(random_seeds)
    regions, vertices = voronoi_finite_polygons_2d(vor)

    polygons = []
    for reg in regions:
        polygon = vertices[reg]
        polygons.append(polygon)
    return polygons

def plot_color(t, as_str=True, alpha=0.5):
    rgb = new_l3[t]
    if as_str:
        return "rgba"+str(tuple(rgb+[alpha]))
    else:
        # Normalize & listify
        return list(np.array(rgb)/255) + [alpha]

def plot_polygons2(polygons, ax=None, alpha=0.5, linewidth=0.7, saveas=None, show=True):
    # Configure plot
    if ax is None:
        plt.figure(figsize=(5,5))
        ax = plt.subplot(111)

    # Remove ticks
    ax.set_xticks([])
    ax.set_yticks([])

    ax.axis("equal")

    # Set limits
    ax.set_xlim(0,img.shape[1])
    ax.set_ylim(0,img.shape[0])

    # Add polygons
    m = 0
    for poly in polygons:
        colored_cell = Polygon(poly,
                               linewidth=linewidth,
                               alpha=alpha,
                               facecolor=plot_color(m, as_str=False, alpha=1),
                               edgecolor="black")
        ax.add_patch(colored_cell)
        m += 1

    if not saveas is None:
        plt.savefig(saveas)
    if show:
        plt.show()
    return ax

nbpoints = [500, 1000, 1500, 2000, 2500, 5000, 15000]

for i in nbpoints :

    sample_index = np.random.choice(a=flat.size, size = i, p=flat, replace=False)
    adjusted_index = np.unravel_index(sample_index, modified_test.shape)

    points_coord = np.transpose(np.array([adjusted_index[1],img.shape[0] -1 - adjusted_index[0]]))

    new_l2 = []

    for i in points_coord:
        new_l2.append(list(img_modif[i[1],i[0]]))

    new_l3 = []

    for l in range(len(new_l2)):
        new_l3.append([new_l2[l][2],new_l2[l][1],new_l2[l][0]])

    new_l3 = np.array(new_l3)

    test = plot_polygons2(voronoi_polygons(i),
                              alpha = 1,
                              show = False)


import torch
from torch import tensor
import torch.nn as nn
import os

# from peewee import *
from dbmodel import TrainingDataModel
if(__name__=='__main__'):
    while(True):
        if(os.path.exists('./data/NNModel2')):
            print('loading model')
            model = torch.load('./data/NNModel2')
        else:
            print('creating model')
            # model = nn.Sequential(nn.Linear(7, 200), nn.Linear(200,200), nn.Linear(200,50), nn.Linear(50,3), nn.Sigmoid())
            model = nn.Sequential(nn.Linear(7, 200),nn.ReLU(), nn.Linear(200,50), nn.Linear(50,3), nn.Softmax(dim=0))
            # output, long, short, wait
        X = []
        y = []
        # query = TrainingDataModel.select().where(TrainingDataModel.if_trained==False)
        query = TrainingDataModel.select()
        for rec in query:
            X.append([rec.res_coeffs_0, rec.res_coeffs_1, rec.sup_coeffs_0, rec.sup_coeffs_1, rec.price_coeffs_0, rec.price_coeffs_1, rec.price_coeffs_2])
            # y = [[rec.long, rec.short, rec.wait]]
            y.append([rec.long, rec.short, rec.wait])
        if(not 0 in [len(X), len(y)]):
            X = torch.tensor(X, dtype=torch.float)
            y = torch.tensor(y, dtype=torch.float)
            lossfn = nn.MSELoss(reduction='mean')
            optimizer = torch.optim.RMSprop(model.parameters(), lr=1e-2)
            epochs = 500
            for e in range(epochs):
                optimizer.zero_grad()
                model.train(mode=True)
                predicted = model(X)
                loss = lossfn(predicted, y)
                print('training ', e, end="\r")
                loss.backward()
                optimizer.step()
                # optimizer.zero_grad()
            print('done', ' '*50)
            rec.if_trained = True
            rec.save()
            torch.save(model,'./data/NNModel2')
        res = input('stop training=x? ')
        if(res=='x'):
            print('done')
            break

    # next = True
    # # user interaction mode
    # while next:
    #     study = float(input('Study hour: '))
    #     sleep = float(input('Sleep hour: '))
    #     xPredict = torch.tensor(([study, sleep]), dtype=torch.float)
    #     # xPredict_max, _ = torch.max(xPredict, 0)
    #     xPredictScaled = xPredict / X_max
    #     result = NN.predict(xPredictScaled)
    #     print(f'[{study}, {sleep}] result: {(result * y_max).item()}')
    #     # if(input('next? ')!=''):
    #     #     next = False


#!/usr/bin/env python3

from collections import OrderedDict
from datetime import datetime
from typing import Dict, List, Optional, Sequence, Union

import datetime
import logging
import netCDF4 as nc
import numpy as np
import pandas as pd
import re
import socket
import sys
import time

# from typing import Union, Dict, List, OrderedDict as OrderedDictType, Sequence

from Bandit.bandit_helpers import set_date
# from Bandit.pr_util import print_error

from io import StringIO
from urllib.request import urlopen
from urllib.error import HTTPError, URLError
from html.parser import HTMLParser

class NWISErrorParser(HTMLParser):
    # This is a quick 'n dirty error message parser for NWIS
    inBody = False
    inPara = False
    inBold = False
    lsStartTags = list()
    lsEndTags = list()
    lsStartEndTags = list()
    lsComments = list()
    curr_key = None
    error_info = {}

    # HTML Parser Methods
    def handle_starttag(self, start_tag, attrs):
        if self.inBody and start_tag != 'h1':
            self.lsStartTags.append(start_tag)
            # print('Start tag: ', start_tag)

        if start_tag == 'body':
            self.inBody = True

        if start_tag == 'p':
            self.inPara = True

        if start_tag == 'b':
            self.inBold = True

    def handle_endtag(self, end_tag):
        if end_tag == 'body':
            self.inBody = False

        if self.inBody and end_tag != 'h1':
            self.lsEndTags.append(end_tag)
            # print('End tag: ', end_tag)

        if end_tag == 'p':
            self.inPara = False
        if end_tag == 'b':
            self.inBold = False

    def handle_startendtag(self, startend_tag, attrs):
        self.lsStartEndTags.append(startend_tag)

    def handle_data(self, data):
        if self.inBody:
            # print("Encountered some data  :", data)

            if self.inPara and self.inBold:
                self.curr_key = data
            elif self.inPara:
                if self.curr_key == 'message':
                    self.error_info[self.curr_key] = data.split(',')[0]
                else:
                    self.error_info[self.curr_key] = data

    def handle_comment(self, data):
        self.lsComments.append(data)

# URLs can be generated/tested at: http://waterservices.usgs.gov/rest/Site-Test-Tool.html
BASE_NWIS_URL = 'https://waterservices.usgs.gov/nwis'
RETRIES = 3

nwis_logger = logging.getLogger('bandit.NWIS')

class NWIS:

    """Class for accessing and manipulating streamflow information from the
    National Water Information System (NWIS; https://waterdata.usgs.gov/) provided by the
    United States Geological Survey (USGS; https://www.usgs.gov/).
    """

    # As written this class provides fucntions for downloading daily streamgage observations
    # Additional functionality (e.g. monthyly, annual, other statistics) may be added at a future time.

    def __init__(self, gage_ids: Optional[List] = None,
                 st_date: Optional[datetime.datetime] = None,
                 en_date: Optional[datetime.datetime] = None,
                 verbose: Optional[bool] = False):
        """Create the NWIS object.

        :param list[str] gage_ids: list of streamgages to retrieve
        :param st_date: start date for retrieving streamgage observations
        :type st_date: None or datetime
        :param en_date: end date for retrieving streamgage observations
        :type en_date: None or datetime
        :param bool verbose: output additional debuggin information
        """

        self.logger = logging.getLogger('bandit.NWIS')
        self.logger.info('NWIS instance')

        self.__stdate = None
        self.__endate = None
        self.__gageids = None

        self.start_date = st_date
        self.end_date = en_date
        self.gage_ids = gage_ids
        self.__outdata = None
        self.__date_range = None
        self.__final_outorder = None
        self.__verbose = verbose

        # Regex's for stripping unneeded clutter from the rdb file
        self.__t1 = re.compile('^#.*$\n?', re.MULTILINE)  # remove comment lines
        self.__t2 = re.compile('^5s.*$\n?', re.MULTILINE)  # remove field length lines

    @property
    def data(self):
        return self.__outdata

    @property
    def start_date(self):
        """Get the start date.

        :returns: start date
        :rtype: None or datetime
        """

        return self.__stdate

    @start_date.setter
    def start_date(self, st_date: Union[datetime.datetime, str]):
        """Set the start date.

        :param st_date: start date (either a datetime object or a string of the form YYYY-MM-DD)
        :type st_date: datetime or str
        """

        # Set the starting date for retrieval
        # As written this will clear any streamgage observations that have been downloaded.
        self.__stdate = set_date(st_date)
        self.__outdata = None

    @property
    def end_date(self):
        """Get the end date.

        :returns: end date
        :rtype: None or datetime
        """

        return self.__endate

    @end_date.setter
    def end_date(self, en_date: Union[datetime.datetime, str]):
        """Set the end date.

        :param en_date: end date (either a datetime object or a string of the form YYYY-MM-DD)
        :type en_date: datetime or str
        """
        self.__endate = set_date(en_date)
        self.__outdata = None

    @property
    def gage_ids(self):
        """Get list of streamgage IDs for retrieval.

        :returns: list of streamgage IDs
        :rtype: list[str]
        """

        return self.__gageids

    @gage_ids.setter
    def gage_ids(self, gage_ids):
        """Set the streamgage ID(s) to retrieve from NWIS.

        :param gage_ids: streamgage ID(s)
        :type gage_ids: list or tuple or str
        """

        # Set the gage ids for retrieval this will clear any downloaded observations
        if isinstance(gage_ids, (list, tuple)):
            self.__gageids = gage_ids
        else:
            # Assuming a single value, so convert to a list
            self.__gageids = [gage_ids]
        self.__outdata = None

    def check_for_flag(self, pat, data, col_id):
        """Check for a given pattern in supplied data.

        Checks for a given pattern in the data and log a warning if it occurs.

        :param str pat: pattern to find
        :param data: data for pattern matching
        :param str col_id: column to search in data
        """

        # Check for pattern in data, log error if found and remove from data
        pat_count = data[col_id].str.contains(pat).sum()

        if pat_count > 0:
            pat_first_date = data[data[col_id].str.contains(pat)].index[0].strftime('%Y-%m-%d')

            self.logger.warning(f'{col_id} has {pat_count} records marked {pat}. ' +
                                f'First occurrence at {pat_first_date}. Suffix removed from values')
            data[col_id].replace(pat, '', regex=True, inplace=True)

    def initialize_dataframe(self):
        """Clears downloaded data and initializes the output dataframe.
        """

        if not self.__endate:
            self.__endate = datetime.today()
        if not self.__stdate:
            self.__stdate = datetime(1890, 1, 1)

        # Create an initial dataframe that contains all dates in the date range.
        # Any streamgage missing date(s) will have a NaN value for each missing date.
        # Otherwise it is possible to have dates missing in the output.
        self.__date_range = pd.date_range(start=self.__stdate, end=self.__endate, freq='D')
        self.__outdata = pd.DataFrame(index=self.__date_range)
        self.__final_outorder = ['year', 'month', 'day', 'hour', 'minute', 'second']

    def get_daily_streamgage_observations(self):
        """Retrieves daily observations.

        If gage_ids is set then retrieve observations for those streamgages; otherwise,
        return a single dummy dataset. If st_date and en_date are set then observations
        are restricted to the given date range.
        """

        if not self.__outdata:
            self.initialize_dataframe()

        # Set timeout in seconds - if not set defaults to infinite time for response
        timeout = 30
        socket.setdefaulttimeout(timeout)

        url_pieces = OrderedDict()
        url_pieces['?format'] = 'rdb'
        url_pieces['sites'] = ''
        url_pieces['startDT'] = self.__stdate.strftime('%Y-%m-%d')
        url_pieces['endDT'] = self.__endate.strftime('%Y-%m-%d')
        url_pieces['statCd'] = '00003'  # Mean values
        url_pieces['siteStatus'] = 'all'
        url_pieces['parameterCd'] = '00060'  # Discharge
        url_pieces['siteType'] = 'ST'
        # url_pieces['access'] = '3'  # Allows download of observations for restricted sites/parameters

        if not self.__gageids:
            # If no streamgages are provided then create a single dummy column filled with noData
            self.logger.warning('No streamgages provided - dummy entry created.')
            df = pd.DataFrame(index=self.__date_range, columns=['00000000'])
            df.index.name = 'date'

            self.__outdata = pd.merge(self.__outdata, df, how='left', left_index=True, right_index=True)
            self.__final_outorder.append('00000000')

        # Iterate over new_poi_gage_id and retrieve daily streamflow data from NWIS
        for gidx, gg in enumerate(self.__gageids):
            if self.__verbose:
                sys.stdout.write(f'\rStreamgage: {gg} ({gidx + 1}/{len(self.__gageids)}) ')
                sys.stdout.flush()

            url_pieces['sites'] = gg
            url_final = '&'.join([f'{kk}={vv}' for kk, vv in url_pieces.items()])

            # Read site data from NWIS
            streamgage_obs_page = None
            attempts = 0
            while attempts < RETRIES:
                try:
                    response = urlopen(f'{BASE_NWIS_URL}/dv/{url_final}')
                    encoding = response.info().get_param('charset', failobj='utf8')
                    streamgage_obs_page = response.read().decode(encoding)

                    break
                except HTTPError as err:
                    if err.code == 400:
                        err_parser = NWISErrorParser()
                        err_parser.feed(str(err.read().decode("utf8", 'ignore')))
                        self.logger.warning(f'HTTPError: {err.code}, Site: {gg}, {err_parser.error_info["message"]}')

                        break
                    else:
                        attempts += 1
                        self.logger.warning(f'HTTPError: {err}, Try {attempts} of {RETRIES}')
                        # print('HTTPError: {}, Try {} of {}'.format(err, attempts, RETRIES))
                except URLError as err:
                    attempts += 1
                    self.logger.warning(f'URLError: {err}, reason={err.reason}; try {attempts} of {RETRIES}')
                except ConnectionResetError as err:
                    attempts += 1
                    self.logger.warning(f'ConnectionResetError: {err}, Try {attempts} of {RETRIES}')
                    time.sleep(10)
                except socket.timeout as err:
                    attempts += 1
                    self.logger.warning(f'socket.timeout: {err}; try {attempts} of {RETRIES}')

            if streamgage_obs_page is None:
                # Create a dummy dataframe
                df = pd.DataFrame(index=self.__date_range, columns=[gg])
                df.index.name = 'date'
            elif streamgage_obs_page.splitlines()[0] == '#  No sites found matching all criteria':
                # No observations are available for the streamgage
                # Create a dummy dataset to output
                self.logger.warning(f'{gg} has no data for ' + self.__stdate.strftime('%Y-%m-%d') +
                                    ' to ' + self.__endate.strftime('%Y-%m-%d'))

                df = pd.DataFrame(index=self.__date_range, columns=[gg])
                df.index.name = 'date'
            else:
                streamgage_observations = streamgage_obs_page
                # streamgage_observations = streamgage_obs_page.read()

                # Strip the comment lines and field length lines from the result using regex
                streamgage_observations = self.__t1.sub('', streamgage_observations, count=0)
                streamgage_observations = self.__t2.sub('', streamgage_observations, count=0)

                # Have to enforce site_no as string/text
                col_names = ['site_no']
                col_types = [np.str_]
                cols = dict(zip(col_names, col_types))

                # Read the rdb file into a dataframe
                # TODO: Handle empty datasets from NWIS by creating dummy data and providing a warning
                df = pd.read_csv(StringIO(streamgage_observations), sep='\t', dtype=cols,
                                 parse_dates={'date': ['datetime']}, index_col='date')

                # Conveniently the columns we want to drop contain '_cd' in their names
                drop_cols = [col for col in df.columns if '_cd' in col]
                drop_cols.append('site_no')
                df.drop(drop_cols, axis=1, inplace=True)

                # There should now only be date, site_no, and a Q column named *_00060_00003
                # We will rename the *_00060_00003 to the site_no value
                rename_col = [col for col in df.columns if '_00060_00003' in col]

                if len(rename_col) > 1:
                    self.logger.warning(f'{gg} had more than one Q-col returned; empty dataset used.')
                    df = pd.DataFrame(index=self.__date_range, columns=[gg])
                    df.index.name = 'date'

                    # self.logger.warning('{} had more than one Q-col returned; using {}'.format(gg, rename_col[0]))
                    #
                    # # Keep the first TS column and drop the others
                    # while len(rename_col) > 1:
                    #     curr_col = rename_col.pop()
                    #     df.drop([curr_col], axis=1, inplace=True)
                else:
                    df.rename(columns={rename_col[0]: gg}, inplace=True)

                    try:
                        # If no flags are present the column should already be float
                        df[gg] = pd.to_numeric(df[gg], errors='raise', downcast='float')
                    except ValueError:
                        self.logger.warning(f'{gg} had one or more flagged values; flagged values converted to NaN.')
                        df[gg] = pd.to_numeric(df[gg], errors='coerce', downcast='float')

                    # Check for discontinued gage records
                    # if df[gg].dtype == np.object_:
                    #     # If the datatype of the streamgage values is np.object_ that
                    #     # means some string is appended to one or more of the values.
                    #
                    #     # Common bad data: set(['Eqp', 'Ice', 'Ssn', 'Rat', 'Bkw', '***', 'Dis'])
                    #
                    #     # Check for discontinued flagged records
                    #     self.check_for_flag('_?Dis', df, gg)
                    #
                    #     # Check for ice-flagged records
                    #     self.check_for_flag('_?Ice', df, gg)
                    #
                    #     # Check for eqp-flagged records (Equipment malfunction)
                    #     self.check_for_flag('_?Eqp', df, gg)
                    #
                    #     # Check for _Ssn (parameter monitored seasonally)
                    #     self.check_for_flag('_?Ssn', df, gg)
                    #
                    #     # Check for _Rat (rating being developed)
                    #     self.check_for_flag('_?Rat', df, gg)
                    #
                    #     # Check for _Bkw (Value is affected by backwater at the measurement site)
                    #     self.check_for_flag('_?Bkw', df, gg)
                    #
                    #     # Check for 1 or more astericks
                    #     self.check_for_flag('_?\*+', df, gg)

                    # Resample to daily to fill in the missing days with NaN
                    # df = df.resample('D').mean()

            self.__outdata = pd.merge(self.__outdata, df, how='left', left_index=True, right_index=True)
            self.__final_outorder.append(gg)
            sys.stdout.write('\r                                       \r')

    def write_ascii(self, filename):
        """Writes streamgage observations to a file in PRMS format.

        :param str filename: name of the file to create
        """

        # Create the year, month, day, hour, minute, second columns
        try:
            self.__outdata['year'] = self.__outdata.index.year
            self.__outdata['month'] = self.__outdata.index.month
            self.__outdata['day'] = self.__outdata.index.day
            self.__outdata['hour'] = self.__outdata.index.hour
            self.__outdata['minute'] = self.__outdata.index.minute
            self.__outdata['second'] = self.__outdata.index.second
            self.__outdata.fillna(-999, inplace=True)

            # TODO: 2019-04-03 PAN - Make sure all streamflow data is of type float
        except AttributeError:
            print('AttributeError')
            print(self.__outdata.head())
            print(self.__outdata.info())

        outhdl = open(filename, 'w')
        outhdl.write('Created by Bandit\n')
        outhdl.write('/////////////////////////////////////////////////////////////////////////\n')
        outhdl.write('// Station IDs for runoff:\n')
        outhdl.write('// ID\n')

        if not self.__gageids:
            outhdl.write('// 00000000\n')
        else:
            for gg in self.__gageids:
                outhdl.write(f'// {gg}\n')

        outhdl.write('/////////////////////////////////////////////////////////////////////////\n')
        outhdl.write('// Unit: runoff = cfs\n')
        outhdl.write('/////////////////////////////////////////////////////////////////////////\n')
        outhdl.write(f'runoff {len(self.__gageids)}\n')
        outhdl.write('#########################################################\n')

        self.__outdata.to_csv(outhdl, sep=' ', columns=self.__final_outorder, index=False, header=False)
        outhdl.close()

        if self.__verbose:
            sys.stdout.write('\r                                       ')
            sys.stdout.write(f'\r\tStreamflow data written to: {filename}\n')
            sys.stdout.flush()

    def write_netcdf(self, filename):
        """Write NWIS streamflow to netcdf format file"""

        max_poiid_len = len(max(self.__gageids, key=len))

        # Create a netCDF file for the CBH data
        nco = nc.Dataset(filename, 'w', clobber=True)

        # Create the dimensions
        nco.createDimension('poiid_nchars', max_poiid_len)
        nco.createDimension('poi_id', len(self.__gageids))
        nco.createDimension('time', None)

        reference_time = self.__stdate.strftime('%Y-%m-%d %H:%M:%S')
        cal_type = 'standard'

        # Create the variables
        timeo = nco.createVariable('time', 'f4', 'time')
        timeo.calendar = cal_type
        timeo.units = f'days since {reference_time}'

        poiido = nco.createVariable('poi_id', 'S1', ('poi_id', 'poiid_nchars'), zlib=True)
        poiido.long_name = 'Point-of-Interest ID'
        poiido.cf_role = 'timeseries_id'
        poiido._Encoding = 'ascii'

        varo = nco.createVariable('discharge', 'f4', ('poi_id', 'time'), fill_value=-999.0, zlib=True)
        varo.long_name = 'discharge'
        varo.units = 'ft3 s-1'

        nco.setncattr('Description', 'Streamflow data for PRMS')
        nco.setncattr('FeatureType', 'timeSeries')
        # nco.setncattr('Bandit_version', __version__)
        # nco.setncattr('NHM_version', nhmparamdb_revision)

        # Write the Streamgage IDs
        poiido[:] = nc.stringtochar(np.array(self.__gageids, dtype='S'))

        timeo[:] = nc.date2num(pd.to_datetime(self.__outdata.index).tolist(),
                               units=f'days since {reference_time}',
                               calendar=cal_type)

        # Write the streamgage observations
        varo[:, :] = self.__outdata.to_numpy(dtype=np.float).T

        nco.close()


import torch
import torch.nn as nn
import torch.nn.functional as F

class LSTNet(nn.Module):
    def __init__(self, input_len, input_dim, hidRNN, hidCNN, hidSkip, skip, CNN_kernel, highway_window,
                 dropout, c_out, LIN=True):
        super(LSTNet, self).__init__()

        self.P = input_len
        self.input_dim = input_dim
        self.c_out = c_out
        self.hidR = hidRNN
        self.hidC = hidCNN
        self.hidS = hidSkip
        self.Ck = CNN_kernel
        self.skip = skip
        self.pt = int((self.P - self.Ck) / self.skip)
        self.hw = highway_window
        self.conv1 = nn.Conv2d(1, self.hidC, kernel_size=(self.Ck, input_dim))  #
        self.GRU1 = nn.GRU(self.hidC, self.hidR)
        self.dropout = nn.Dropout(p=dropout)
        if self.skip > 0:
            self.GRUskip = nn.GRU(self.hidC, self.hidS)
            self.linear1 = nn.Linear(self.hidR + self.skip * self.hidS, c_out)
        else:
            self.linear1 = nn.Linear(self.hidR, c_out)
        if self.hw > 0:
            self.highway = nn.Linear(self.hw, 1)
        self.LIN = LIN
        self.norm = nn.LayerNorm(input_len, eps=0, elementwise_affine=False)

    def forward(self, x, drop=0):
        if drop:
            x[:, -2 ** (drop - 1) - 1:-1, :self.c_out] = 0
        if self.LIN:
            x[:, :, :self.c_out] = self.norm(x[:, :, :self.c_out].permute(0, 2, 1)).transpose(1, 2)
        x[torch.isinf(x)] = 0
        x[torch.isnan(x)] = 0

        enc_input = x.clone()
        enc_input[:, -1:, :self.c_out] = 0

        gt = x[:, -1:, :].clone()
        gt = gt[:, :, :self.c_out]

        batch_size = x.size(0)

        # CNN
        c = x.view(-1, 1, self.P, self.input_dim)
        c = F.relu(self.conv1(c))
        c = self.dropout(c)
        c = torch.squeeze(c, 3)

        # RNN
        r = c.permute(2, 0, 1).contiguous()
        _, r = self.GRU1(r)
        r = self.dropout(torch.squeeze(r, 0))

        # skip-rnn
        if self.skip > 0:
            s = c[:, :, int(-self.pt * self.skip):].contiguous()
            s = s.view(batch_size, self.hidC, self.pt, self.skip)
            s = s.permute(2, 0, 3, 1).contiguous()

            s = s.view(self.pt, batch_size * self.skip, self.hidC)
            _, s = self.GRUskip(s)
            s = s.view(batch_size, self.skip * self.hidS)
            s = self.dropout(s)
            r = torch.cat((r, s), 1)
        res = self.linear1(r)

        # highway
        if self.hw > 0:
            z = x[:, -self.hw:, :]
            z = z.permute(0, 2, 1).contiguous().view(-1, self.hw)
            z = self.highway(z)
            z = z.view(-1, self.input_dim)
            res = res + z[:, :self.c_out]

        res = res.contiguous().view(-1, 1, self.c_out)

        return res, gt


from urllib.request import urlopen
from bs4 import BeautifulSoup
from collections import OrderedDict
url = "http://s.cafef.vn/bao-cao-tai-chinh/VNM/IncSta/2017/3/0/0/ket-qua-hoat-dong-kinh-doanh-cong-ty-co-phan-sua-viet-nam.chn"
com = urlopen(url)
rawdata = com.read()
content = rawdata.decode('utf8')
soup = BeautifulSoup(content,'html.parser')
table = soup.find('table',id='tableContent')
tr_list = table.find_all('tr')
cont = ['Mục','Quý 4-2016','Quý 1-2017','Quý 2-2017','Quý 3-2017',]
data = []
vl = {}
for tr in tr_list:
    td_list = tr.find_all('td')
    if len(td_list)==13:
        for i in range(5):
            a = td_list[i].string
            a = str(a).lstrip()
            vl[cont[i]]= a
        data.append(OrderedDict(vl))
import pyexcel
pyexcel.save_as(records=data, dest_file_name="vinamilk.xlsx")