# Import dependencies
import datetime as dt
import numpy as np
import pandas as pd

import sqlalchemy
from sqlalchemy import desc
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func

from flask import Flask, jsonify

# Database Setup
engine = create_engine('sqlite:///Resources/hawaii.sqlite')

#reflect the db into a new model
Base = automap_base()

# reflect the tables
Base.prepare(engine, reflect=True)

# Save a reference to the measurenment table as 'Measurement'
Measurement = Base.classes.measurement
# Save a reference to the station table as 'Station'
Station = Base.classes.station

#create a session
session = Session(engine)

#make an app instance
app = Flask(__name__)

@app.route("/")
def welcome():
    return (
        f"Welcome to Hawaii Weather Analysis Apps API!
"
        f"Available Routes:
"
        f"/api/v1.0/precipitation
"
        f"/api/v1.0/stations
"
        f"/api/v1.0/tobs
"
        f"/api/v1.0/<start>
"
        f"Provide the start date only; calculate `TMIN`, `TAVG`, and `TMAX` for all dates greater than and equal to the start date
"
        f"route example: /api/v1.0/2017-01-01
"
        f"/api/v1.0/<start>/<end>
"
        f"Provide the start and the end date, calculate the `TMIN`, `TAVG`, and `TMAX` for dates between the start and end date inclusive.
"
        f"route example: /api/v1.0/2017-01-01/2017-02-15
"
    )

############
@app.route("/api/v1.0/precipitation")
def query_precipitation():

    date_prcp_query = session.query(Measurement.date,Measurement.prcp).all()
    date_prcp_list =[]
    for i in range(len(date_prcp_query)):
        date_prcp_list.append({'date':date_prcp_query[i][0],'precipitation':date_prcp_query[i][1]})

    return jsonify(date_prcp_list)

############
@app.route("/api/v1.0/stations")
def query_stations():

    station_query = session.query(Station.station,Station.name).all()
    station_list =[]
    for i in range(len(station_query)):
        station_list.append({'station':station_query[i][0],'name':station_query[i][1]})

    return jsonify(station_list)

############
@app.route("/api/v1.0/tobs")
def query_tobs():

    last_date = session.query(Measurement.date).order_by(desc(Measurement.date)).first()
    result = [str(datestr) for datestr in last_date]
    last_date_split = result[0].split('-')
    last_12_month_date = dt.date(int(last_date_split[0]),int(last_date_split[1]),int(last_date_split[2])) - dt.timedelta(365)

    last_12_month_date = dt.date(int(last_date_split[0]),int(last_date_split[1]),int(last_date_split[2])) - dt.timedelta(365)

    tobs_lastyear_query = session.query(Measurement.date,Station.station,Measurement.tobs).\
                        filter(func.datetime(Measurement.date) >= last_12_month_date).\
                        order_by(Measurement.date).\
                        all()
    tobs_lastyear_list =[]
    for i in range(len(tobs_lastyear_query)):
        tobs_lastyear_list.append({'date':tobs_lastyear_query[i][0],'station':tobs_lastyear_query[i][1],'temperature':tobs_lastyear_query[i][2]})

    return jsonify(tobs_lastyear_list)

############
@app.route("/api/v1.0/<start>")
def start_range(start):

    start_date = session.query(Measurement.date,func.avg(Measurement.tobs),func.min(Measurement.tobs),func.max(Measurement.tobs)) \
             .filter(Measurement.date >= start) \
             .group_by(Measurement.date).all()

    return jsonify(start_date)

############
@app.route("/api/v1.0/<start>/<end>")
def start_end(start,end):

    range = session.query(Measurement.date,func.avg(Measurement.tobs),func.min(Measurement.tobs),func.max(Measurement.tobs)) \
             .filter(Measurement.date >= start).filter(Measurement.date <= end) \
             .group_by(Measurement.date).all()

    return jsonify(range)

############
#run the app
if __name__ == '__main__':
    app.run(debug=True)


import os

from flask_app.config.mysqlconnection import connectToMySQL
from flask import session
from flask_app.models.usuarios import Usuario

class Auto:
    def __init__(self, data) -> None:
        self.id = data['id']
        self.modelo = data['modelo']
        self.precio = data['precio']
        self.marca = data['marca']
        self.anio = data['anio']
        self.descripcion = data['descripcion']
        self.usuario_id = data['usuario_id']
        self.usuario = None
        self.comprador_id = data.get('comprador_id')
        self.comprador = None
        self.vendido = data['vendido']

    def __str__(self) -> str:
        return f"{self.modelo}"

    @classmethod
    def validar(cls, formulario):

        errores = []

        if len(formulario['modelo']) == 0:
            errores.append(
                "modelo es obligatorio"
            )
        if len(formulario['marca']) == 0:
            errores.append(
                "marca es obligatorio"
            )
        if len(formulario['descripcion']) == 0:
            errores.append(
                "descripcion es obligatorio"
            )

        if len(formulario['precio']) == 0:
            errores.append(
                "precio es obligatorio"
            )

        if len(formulario['anio']) == 0:
            errores.append(
                "anio es obligatorio"
            )

        if formulario['precio'] != "" and int(formulario['precio']) <= 0:
            errores.append(
                "no puede ser 0 o menor el precio. debe ser mayor"
            )

        if formulario['anio'] != "" and int(formulario['anio']) <= 0:
            errores.append(
                "no puede ser 0 o menor el año. debe ser mayor"
            )

        return errores

    @classmethod
    def get_all(cls, comprador_id = None):
        resultados_instancias = []

        data = {}
        query = "SELECT * FROM autos"

        if comprador_id:
            query += " WHERE comprador_id = %(comprador_id)s"
            data["comprador_id"] = comprador_id

        resultados = connectToMySQL(os.getenv('BASE_DATOS')).query_db(query, data)
        print(resultados)
        for resultado in resultados:
            instancia = cls(resultado)

            instancia.usuario = Usuario.get(instancia.usuario_id)
            if instancia.comprador_id:
                instancia.comprador = Usuario.get(instancia.comprador_id)

            resultados_instancias.append(instancia)

        return resultados_instancias

    @classmethod
    def save(cls, data ):
        query = "INSERT INTO autos (modelo, precio, marca, anio, descripcion, usuario_id) VALUES (%(modelo)s, %(precio)s, %(marca)s, %(anio)s, %(descripcion)s, %(usuario_id)s);"
        data = {
            **data,
            "usuario_id": session['usuario']['id'],
        }

        return connectToMySQL(os.getenv('BASE_DATOS')).query_db( query, data )

    @classmethod
    def get(cls, id ):
        query = "SELECT * FROM autos WHERE id = %(id)s;"
        data = { 'id': id }
        resultados = connectToMySQL(os.getenv('BASE_DATOS')).query_db( query, data )
        if resultados:
            instancia = cls(resultados[0])
            instancia.usuario = Usuario.get(instancia.usuario_id)
            if instancia.comprador_id:
                instancia.comprador = Usuario.get(instancia.comprador_id)
            return instancia

        return None

    @classmethod
    def eliminar(cls, id ):
        query = "DELETE FROM autos WHERE id = %(id)s;"
        data = { 'id': id }
        connectToMySQL(os.getenv('BASE_DATOS')).query_db( query, data )
        return True

    def delete(self):
        query = "DELETE FROM autos WHERE id = %(id)s;"
        data = { 'id': self.id }
        connectToMySQL(os.getenv('BASE_DATOS')).query_db( query, data )
        return True

    def update(self):
        query = "UPDATE autos SET modelo = %(modelo)s, precio = %(precio)s, marca = %(marca)s, anio = %(anio)s, descripcion = %(descripcion)s WHERE id = %(id)s"
        data = {
            'id': self.id,
            'modelo': self.modelo,
            'precio': self.precio,
            'marca': self.marca,
            'anio': self.anio,
            'descripcion': self.descripcion,
        }
        connectToMySQL(os.getenv('BASE_DATOS')).query_db( query, data )
        return True

    def vender(self):
        query = "UPDATE autos SET comprador_id = %(comprador_id)s, vendido = 1 WHERE id = %(id)s"
        data = {
            'id': self.id,
            'comprador_id': session['usuario']['id'],
        }
        connectToMySQL(os.getenv('BASE_DATOS')).query_db( query, data )
        return True


import os
import numpy as np
import xarray as xr
import dask.array as da
from glob import glob
import time
from datetime import datetime, timedelta
from satpy.scene import Scene
import satpy.utils
from satpy.readers.utils import remove_earthsun_distance_correction
from trollsift.parser import globify, Parser
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look

from level1c4pps.calibration_coefs import get_calibration, CalibrationData
from level1c4pps import make_azidiff_angle, get_encoding, compose_filename, update_angle_attributes

try:
    FileNotFoundError
except NameError:
    # Python 2
    FileNotFoundError = IOError

class UnexpectedSatpyVersion(Exception):
    """Exception if unexpected satpy version."""

    pass

BANDNAMES = ['VIS006', 'VIS008', 'IR_016', 'IR_039',
             'IR_087', 'IR_108', 'IR_120',
             'IR_134', 'IR_097', 'WV_062', 'WV_073']
PPS_TAGNAMES = {'VIS006': 'ch_r06',
                'VIS008': 'ch_r09',
                'IR_016': 'ch_r16',
                'IR_039': 'ch_tb37',
                'IR_087': 'ch_tb85',
                'IR_108': 'ch_tb11',
                'IR_120': 'ch_tb12',
                'IR_134': 'ch_tb133',
                'IR_097': 'ch_tb97',
                'WV_062': 'ch_tb67',
                'WV_073': 'ch_tb73'}

HRIT_FILE_PATTERN = ('{rate:1s}-000-{hrit_format:_<6s}-'
                     '{platform_shortname:_<12s}-{channel:_<8s}_-'
                     '{segment:_<9s}-{start_time:%Y%m%d%H%M}-__')
# H-000-MSG3__-MSG3________-IR_120___-000003___-201410051115-__

NATIVE_FILE_PATTERN = ('{platform_shortname:4s}-{instr:4s}-'
                       'MSG{product_level:2d}-{base_algorithm_version:4s}-NA-'
                       '{end_time:%Y%m%d%H%M%S.%f}000Z')
# MSG4-SEVI-MSG15-0100-NA-20190409121243.927000000Z-20190409121300-1329370.nat
# MSG4-SEVI-MSG15-0201-NA-20190409121243.927000000Z.nat
# MSG4-SEVI-MSG15-1234-NA-20190409121243.927000000Z

def load_and_calibrate(filenames, apply_sun_earth_distance_correction, rotate,
                       clip_calib):
    """Load and calibrate data.

    Uses inter-calibration coefficients from Meirink et al.

    Args:
        filenames: List of data files
        apply_sun_earth_distance_correction: If True, apply sun-earth-distance-
            correction to visible channels.
        rotate: Rotate image so that pixel (0, 0) is N-W.
        clip_calib: If True, do not extrapolate calibration coefficients beyond
            the time coverage of the calibration dataset. Instead, clip at the
            boundaries.

    Returns:
        Satpy scene holding calibrated channels
    """
    # Parse filenames
    parser = SEVIRIFilenameParser()
    file_format, info = parser.parse(os.path.basename(filenames[0]))

    calib_coefs = get_calibration(
        platform=info['platform_shortname'],
        time=info['start_time'],
        clip=clip_calib
    )
    scn_ = _create_scene(file_format, filenames, calib_coefs)
    _check_is_seviri_data(scn_)
    _load_bands(scn_, rotate)
    _update_scene_attrs(scn_, {'image_rotated': rotate})

    if not apply_sun_earth_distance_correction:
        remove_sun_earth_distance_correction(scn_)

    return scn_

def _create_scene(file_format, filenames, calib_coefs):
    return Scene(
        reader=file_format,
        filenames=filenames,
        reader_kwargs={'calib_mode': CalibrationData.SATPY_CALIB_MODE.value,
                       'ext_calib_coefs': calib_coefs}
    )

def _check_is_seviri_data(scene):
    if not scene.attrs['sensor'] == {'seviri'}:
        raise ValueError('Not SEVIRI data')

def _load_bands(scene, rotate):
    scene.load(
        BANDNAMES,
        upper_right_corner=_get_upper_right_corner(rotate)
    )

def _get_upper_right_corner(rotation_flag):
    return 'NE' if rotation_flag else 'native'

def _update_scene_attrs(scene, attrs):
    scene.attrs.update(attrs)

def remove_sun_earth_distance_correction(scene):
    """Remove sun-earth-distance correction from visible channels.

    This is required because a downstream CLAAS module (CPP) does not recognize
    the "sun_earth_distance_correction_applied" attribute and applies the
    correction anyway.
    """
    for band in BANDNAMES:
        is_vis = scene[band].attrs['calibration'] == 'reflectance'
        correction_applied = scene[band].attrs.get(
            'sun_earth_distance_correction_applied', False
        )
        if is_vis and correction_applied:
            scene[band] = remove_earthsun_distance_correction(scene[band])

def get_lonlats(dataset):
    """Get lat/lon coordinates."""
    lons, lats = dataset.attrs['area'].get_lonlats()
    lons[np.fabs(lons) > 360] = np.nan
    lats[np.fabs(lats) > 90] = np.nan
    return lons, lats

def get_solar_angles(scene, lons, lats):
    """Compute solar angles.

    Compute angles for each scanline using their acquisition time to account for
    the earth's rotation over the course of one scan.

    Returns:
        Solar azimuth angle, Solar zenith angle in degrees

    """
    suna = np.full(lons.shape, np.nan)
    sunz = np.full(lons.shape, np.nan)
    mean_acq_time = get_mean_acq_time(scene)
    for line, acq_time in enumerate(mean_acq_time.values):
        if np.isnat(acq_time):
            continue
        _, suna_line = get_alt_az(acq_time, lons[line, :], lats[line, :])
        suna_line = np.rad2deg(suna_line)
        suna[line, :] = suna_line
        sunz[line, :] = sun_zenith_angle(acq_time, lons[line, :], lats[line, :])
    return suna, sunz

def get_satellite_angles(dataset, lons, lats):
    """Compute satellite angles.

    Returns:
        Satellite azimuth angle, Satellite zenith angle in degrees

    """
    sat_lon, sat_lat, sat_alt = satpy.utils.get_satpos(dataset)

    # Double check that pyorbital/satpy behave as expected (satpy returning
    # altitude in meters and pyorbital expecting km).
    #
    # if:
    #   1) get_observer_look() gives wrong answer ...
    #   ... for satellite altitude in m. AND
    #   2) get_observer_look() gives correct answer ...
    #   ....  for satellite altitude in km. AND
    #   3) Satellite altitude is m.:
    #    => Satellite alltitude need to be converted to km.
    # else:
    #    => There have been updates to SatPy and this script
    #       need to be modified.
    if not (get_observer_look(0, 0, 36000*1000,
                              datetime.utcnow(), np.array([16]),
                              np.array([58]), np.array([0]))[1] > 30 and
            get_observer_look(0, 0, 36000,
                              datetime.utcnow(), np.array([16]),
                              np.array([58]), np.array([0]))[1] < 23 and
            sat_alt > 38000):
        raise UnexpectedSatpyVersion(
            'Unexpected handling of satellite altitude in pyorbital/'
            'satpy. Conversion to km is probably unneeded and wrong.')

    # Convert altitude from meters to kilometers, as expected by the
    # current version of pyorbital
    sat_alt *= 0.001

    # Compute angles
    sata, satel = get_observer_look(
        sat_lon,
        sat_lat,
        sat_alt,
        dataset.attrs['start_time'],
        lons, lats, 0)
    satz = 90 - satel

    return sata, satz

def set_attrs(scene):
    """Set global and band attributes."""
    # Global
    scene.attrs['platform'] = scene['IR_108'].attrs['platform_name']
    scene.attrs['instrument'] = 'SEVIRI'
    scene.attrs['source'] = "seviri2pps.py"
    scene.attrs['orbit_number'] = 99999
    nowutc = datetime.utcnow()
    scene.attrs['date_created'] = nowutc.strftime("%Y-%m-%dT%H:%M:%SZ")

    # For each band
    for image_num, band in enumerate(BANDNAMES):
        idtag = PPS_TAGNAMES[band]
        scene[band].attrs['id_tag'] = idtag
        scene[band].attrs['description'] = 'SEVIRI ' + str(band)
        scene[band].attrs['wavelength'] = [scene[band].attrs['wavelength'].min,
                                           scene[band].attrs['wavelength'].central,
                                           scene[band].attrs['wavelength'].max]
        if 'sun_earth_distance_correction_factor' not in scene[band].attrs:
            scene[band].attrs['sun_earth_distance_correction_applied'] = False
            scene[band].attrs['sun_earth_distance_correction_factor'] = 1.0
        scene[band].attrs['sun_zenith_angle_correction_applied'] = False
        scene[band].attrs['name'] = "image{:d}".format(image_num)

        # Cosmetics
        for attr in ['orbital_parameters', 'satellite_longitude',
                     'satellite_latitude', 'satellite_altitude',
                     'platform_name', 'sensor', 'georef_offset_corrected']:
            scene[band].attrs.pop(attr, None)

def get_mean_acq_time(scene):
    """Compute mean scanline acquisition time over all bands."""
    dtype = scene['IR_108'].coords['acq_time'].dtype

    # Convert timestamps to float to facilitate averaging. Caveat: NaT is
    # not converted to NaN, but to -9.22E18. So we have to set these elements
    # to NaN manually
    acq_times = []
    for band in BANDNAMES:
        acq_time = scene[band].coords['acq_time'].drop_vars(['acq_time'])
        is_nat = np.isnat(acq_time.values)
        acq_time = acq_time.astype(int).where(np.logical_not(is_nat))
        acq_times.append(acq_time)

    # Compute average over all bands (skip NaNs)
    acq_times = xr.concat(acq_times, 'bands')
    return acq_times.mean(dim='bands', skipna=True).astype(dtype)

def update_coords(scene):
    """Update band coordinates."""
    mean_acq_time = get_mean_acq_time(scene)
    for band in BANDNAMES:
        # Override channel-specific scanline timestamps with mean acquisition
        # time. The differences are not very large and the resulting nc file
        # is much simpler.
        scene[band]['acq_time'] = mean_acq_time

        # Remove area, set lat/lon as coordinates
        scene[band].attrs['coordinates'] = 'lon lat'
        area = scene[band].attrs.pop('area', None)
        if area:
            scene.attrs['area'] = area

        # Add time coordinate to make cfwriter aware that we want 3D data
        scene[band].coords['time'] = scene[band].attrs['start_time']

def add_ancillary_datasets(scene, lons, lats, sunz, satz, azidiff,
                           chunks=(512, 3712)):
    """Add ancillary datasets to the scene.

    Args:
        lons: Longitude coordinates
        lats: Latitude coordinates
        sunz: Solar zenith angle
        satz: Satellite zenith angle
        azidiff: Absolute azimuth difference angle
        chunks: Chunksize

    """
    start_time = scene['IR_108'].attrs['start_time']
    end_time = scene['IR_108'].attrs['end_time']
    angle_coords = scene['IR_108'].coords

    # Latitude
    scene['lat'] = xr.DataArray(
        da.from_array(lats, chunks=chunks),
        dims=['y', 'x'],
        coords={'y': scene['IR_108']['y'], 'x': scene['IR_108']['x']})
    scene['lat'].attrs['long_name'] = 'latitude coordinate'
    scene['lat'].attrs['standard_name'] = 'latitude'
    scene['lat'].attrs['units'] = 'degrees_north'
    scene['lat'].attrs['start_time'] = start_time
    scene['lat'].attrs['end_time'] = end_time

    # Longitude
    scene['lon'] = xr.DataArray(
        da.from_array(lons, chunks=chunks),
        dims=['y', 'x'],
        coords={'y': scene['IR_108']['y'], 'x': scene['IR_108']['x']})
    scene['lon'].attrs['long_name'] = 'longitude coordinate'
    scene['lon'].attrs['standard_name'] = 'longitude'
    scene['lon'].attrs['units'] = 'degrees_east'
    scene['lon'].attrs['start_time'] = start_time
    scene['lon'].attrs['end_time'] = end_time

    # Sunzenith
    scene['sunzenith'] = xr.DataArray(
        da.from_array(sunz[:, :], chunks=chunks),
        dims=['y', 'x'], coords=angle_coords)

    # Satzenith
    scene['satzenith'] = xr.DataArray(
        da.from_array(satz[:, :], chunks=chunks),
        dims=['y', 'x'], coords=angle_coords)

    # Azidiff
    scene['azimuthdiff'] = xr.DataArray(
        da.from_array(azidiff[:, :], chunks=chunks),
        dims=['y', 'x'], coords=angle_coords)

    # Update the attributes
    update_angle_attributes(scene, band=scene['IR_108'])

def add_proj_satpos(scene):
    """Add information on projection and satellite position."""
    orb = scene['IR_108'].attrs['orbital_parameters']

    # Area extent
    try:
        # Traditionally a, b was included in proj_dict
        param_a = scene.attrs['area'].proj_dict['a']
        param_b = scene.attrs['area'].proj_dict['b']
    except KeyError:
        # But sometimes b is missing:
        param_a = scene.attrs['area'].crs.ellipsoid.semi_major_metre
        param_b = scene.attrs['area'].crs.ellipsoid.semi_minor_metre

    scene.attrs.update({
        'projection': 'geos',
        'projection_semi_major_axis': param_a,
        'projection_semi_minor_axis': param_b,
        'projection_longitude': orb['projection_longitude'],
        'projection_latitude': orb['projection_latitude'],
        'projection_altitude': orb['projection_altitude']
    })
    scene['projection_area_extent'] = xr.DataArray(
        np.array([scene.attrs['area'].area_extent]),
        dims=('time', 'corners'),
        attrs={'units': 'meters',
               'long_name': 'Projection area extent',
               'comment': 'Area extent (lower left x, lower left y, '
                          'upper right x, upper right y) in projection '
                          'coordinates'}
    )
    scene['georef_offset_corrected'] = xr.DataArray(
        np.array([scene['IR_108'].attrs['georef_offset_corrected']],
                 dtype='int8'),
        dims=('time', ),
        attrs={'long_name': 'Georeferencing offset correction flag',
               'flag_values': np.array([0, 1], dtype='int8'),
               'flag_meanings': 'georef_offset_present georef_offset_corrected',
               'comment': 'Until December 2017, SEVIRI L1.5 data is shifted '
                          'by 1.5km SSP North and West against the nominal '
                          'GEOS projection. Since December 2017 this offset '
                          'has been corrected by EUMETSAT. This flag indicates '
                          'if the correction has been applied. If not, the area '
                          'extent is adjusted so that the projection always '
                          'matches the data.'}
    )

    # Satellite position
    scene['satellite_longitude'] = xr.DataArray(
        [orb['satellite_actual_longitude']],
        dims=('time',),
        attrs={'units': 'degree_east',
               'long_name': 'Actual satellite longitude',
               'comment': 'Not to be confused with projection longitude'}
    )
    scene['satellite_latitude'] = xr.DataArray(
        [orb['satellite_actual_latitude']],
        dims=('time',),
        attrs={'units': 'degree_north',
               'long_name': 'Actual satellite latitude',
               'comment': 'Not to be confused with projection latitude'}
    )
    scene['satellite_altitude'] = xr.DataArray(
        [orb['satellite_actual_altitude']],
        dims=('time',),
        attrs={'units': 'meters',
               'long_name': 'Actual satellite altitude',
               'comment': 'Relative and perpendicular to the surface of the '
                          'earth ellipsoid. Not to be confused with projection '
                          'altitude.'}
    )

def get_encoding_seviri(scene):
    """Get netcdf encoding for all datasets."""
    # Bands
    chunks = (1, 512, 3712)
    encoding = get_encoding(scene,
                            bandnames=BANDNAMES,
                            pps_tagnames=PPS_TAGNAMES,
                            chunks=chunks)

    # Time
    acq_units = scene.attrs['start_time'].strftime(
        'milliseconds since %Y-%m-%d %H:%M')
    encoding['acq_time'] = {'units': acq_units,
                            'calendar': 'standard',
                            '_FillValue': -9999.0}
    encoding['time'] = {'units': 'days since 2004-01-01 00:00',
                        'calendar': 'standard',
                        '_FillValue': None,
                        'chunksizes': [1]}

    return encoding

def get_header_attrs(scene):
    """Get global netcdf attributes."""
    header_attrs = scene.attrs.copy()
    for key in ['sensor', 'area']:
        header_attrs.pop(key, None)
    return header_attrs

def _drop_seconds_milliseconds(dtime):
    return dtime.replace(second=0, microsecond=0)

def set_nominal_scan_time(dataset):
    """Set time attributes in the dataset to the nominal scan time."""
    dataset = dataset.copy()
    start_time = _drop_seconds_milliseconds(
        dataset.attrs['start_time'])
    end_time = start_time + timedelta(minutes=15)
    dataset.attrs['start_time'] = start_time
    dataset.attrs['end_time'] = end_time
    return dataset

class SEVIRIFilenameParser:
    """SEVIRI filename parser."""

    default_formats = [
        {'name': 'seviri_l1b_hrit',
         'pattern': HRIT_FILE_PATTERN,
         'full_match': True},
        {'name': 'seviri_l1b_native',
         'pattern': NATIVE_FILE_PATTERN,
         'full_match': False},
    ]

    def __init__(self, formats=None):
        self.formats = formats or self.default_formats
        self.format_names = [fmt['name'] for fmt in self.formats]

    def parse(self, filename):
        """Parse the given filename.

        Returns:
            File format, filename info
        """
        for fmt in self.formats:
            parser = Parser(fmt['pattern'])
            try:
                info = parser.parse(filename, full_match=fmt['full_match'])
                break
            except ValueError:
                continue
        else:
            raise ValueError(
                'Filename {} doesn\'t match any of the supported '
                'formats {}.'.format(filename, self.format_names)

            )
        return fmt['name'], self._postproc(fmt['name'], info)

    def _postproc(self, format_name, info):
        """Postprocess filename info."""
        if format_name == 'seviri_l1b_native':
            return self._postproc_native(info)
        elif format_name == 'seviri_l1b_hrit':
            return self._postproc_hrit(info)
        else:
            raise NotImplementedError

    def _postproc_native(self, info):
        """Postprocess Native filename info."""
        # Only end time available, derive start time.
        end_time = info['end_time']
        quarter = end_time.minute // 15
        info['start_time'] = end_time.replace(
            minute=quarter * 15,
            second=0,
            microsecond=0
        )
        return info

    def _postproc_hrit(self, parsed):
        """Postprocess HRIT filename info."""
        return parsed

def process_one_scan(tslot_files, out_path, rotate=True, engine='h5netcdf',
                     use_nominal_time_in_filename=False,
                     apply_sun_earth_distance_correction=True,
                     clip_calib=False):
    """Make level 1c files in PPS-format."""
    for fname in tslot_files:
        if not os.path.isfile(fname):
            raise FileNotFoundError('No such file: {}'.format(fname))

    tic = time.time()
    scn_ = load_and_calibrate(
        tslot_files,
        apply_sun_earth_distance_correction=apply_sun_earth_distance_correction,
        rotate=rotate,
        clip_calib=clip_calib
    )

    # Find lat/lon data
    lons, lats = get_lonlats(scn_['IR_108'])

    # Compute angles
    suna, sunz = get_solar_angles(scn_, lons=lons, lats=lats)
    sata, satz = get_satellite_angles(scn_['IR_108'], lons=lons, lats=lats)
    azidiff = make_azidiff_angle(sata, suna)

    # Update coordinates
    update_coords(scn_)

    # Add ancillary datasets to the scene
    add_ancillary_datasets(scn_, lons=lons, lats=lats, sunz=sunz, satz=satz,
                           azidiff=azidiff)
    add_proj_satpos(scn_)

    # Set attributes. This changes SEVIRI band names to PPS band names.
    set_attrs(scn_)

    # Write datasets to netcdf
    ir108_for_filename = scn_['IR_108']
    if use_nominal_time_in_filename:
        ir108_for_filename = set_nominal_scan_time(ir108_for_filename)
    filename = compose_filename(
        scene=scn_,
        out_path=out_path,
        instrument='seviri',
        band=ir108_for_filename
    )
    scn_.save_datasets(writer='cf',
                       filename=filename,
                       header_attrs=get_header_attrs(scn_),
                       engine=engine,
                       encoding=get_encoding_seviri(scn_),
                       unlimited_dims=['time'],
                       include_lonlats=False,
                       pretty=True,
                       flatten_attrs=True,
                       exclude_attrs=['raw_metadata'])
    print("Saved file {:s} after {:3.1f} seconds".format(
        os.path.basename(filename),
        time.time()-tic))  # About 40 seconds
    return filename

def process_all_scans_in_dname(dname, out_path, ok_dates=None, rotate=False):
    """Make level 1c files for all files in directory dname."""
    parser = Parser(HRIT_FILE_PATTERN)
    fl_ = glob(os.path.join(dname, globify(HRIT_FILE_PATTERN)))
    dates = [parser.parse(os.path.basename(p))['start_time'] for p in fl_]
    unique_dates = np.unique(dates).tolist()
    for uqdate in unique_dates:
        date_formated = uqdate.strftime("%Y%m%d%H%M")
        if ok_dates is not None and date_formated not in ok_dates.keys():
            print("Skipping date {date}".format(date=date_formated))
            continue
        # Every hour only:
        # if uqdate.minute != 0:
        #    continue
        tslot_files = [f for f in fl_ if parser.parse(
            os.path.basename(f))['start_time'] == uqdate]
        try:
            process_one_scan(tslot_files, out_path, rotate=rotate)
        except Exception:
            pass


import unittest
import os
import sys
import logging
from markdowngenerator.markdowngenerator import MarkdownGenerator

from dataclasses import dataclass
from typing import List

# from standardizer.markdown.config.conf import logger

LOGGING_LEVEL = logging.DEBUG
test_logger = logging.getLogger("markdowngenerator.test")
test_logger.setLevel(LOGGING_LEVEL)

# Formatter of log
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(LOGGING_LEVEL)
handler.setFormatter(formatter)
test_logger.addHandler(handler)
test_filename = "unittest.md"

class BaseTestCase(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super(BaseTestCase, self).__init__(*args, **kwargs)
        self.expected_output = EXPECTED_OUTPUT()
        self.test_document = None

    def __call__(self, result=None):
        """
        Does the required configuration for each
        testcase function. No need to repeat when writing actual
        testcase.
        """
        try:
            self._pre_setup()
            super(BaseTestCase, self).__call__(result)
        finally:
            self._post_teardown()

    def _pre_setup(self):
        """
        Generate library instance and file creation.
        Doing this manually, since library is designed to be used
        with 'with' statement to handle all automatically.
        """
        self.test_document = MarkdownGenerator(
            filename=test_filename,
            enable_write=False,
            syntax="gitlab",
        )
        self.test_document.__enter__()

    def _post_teardown(self):
        """
        Manually use exit function to write final file.
        Compare file content to expected values.

        Cleanup afterwards. (delete instance and file)
        """
        self.test_document.__exit__()
        with open(test_filename, "r") as final_file:
            self.assertIsMarkdownEqual(final_file)
        del self.test_document
        try:
            os.remove(test_filename)
        except OSError:
            pass

    def assertIsMarkdownEqual(self, final_file):
        """
        Method for comparing file contents to expected values.
        """
        if self.expected_output.value is None:
            raise AssertionError(
                "Testcase invalid. Expected output value should not be None."
            )
        expected_output_val = self.expected_output.value.split(os.linesep)
        generated_output = final_file.read()
        generated_output = generated_output.split(os.linesep)
        test_logger.debug(
            f"Expected lines: '{self.expected_output.value}', actual lines:'{generated_output}'"
        )
        assert len(expected_output_val) == len(
            generated_output
        ), f"Amount of expected and generated output lines is not equal. Generated: '{len(generated_output)}' Expected: '{len(expected_output_val)}'"

        for index, (expected_line, generated_line) in enumerate(
            zip(expected_output_val, generated_output)
        ):
            test_logger.debug(
                f"\nEXPECTED LINE: {expected_line}\nGENERATED LINE: {generated_line}\n"
            )
            if (
                self.expected_output.if_unequal.enabled
                and index + 1 in self.expected_output.if_unequal.lines
            ):
                assert (
                    expected_line != generated_line
                ), f"Expected line '{expected_line}' is match for final line '{generated_line}'"
            else:
                assert (
                    expected_line == generated_line
                ), f"Expected line '{expected_line}' is not match for final line '{generated_line}' when it shoud not."
        test_logger.info(f"{self.expected_output.func_name} Successful")

    def _insert_test_info(
        self, expected_output, func_name=None, test_if_UNEQUAL=False, unequal_lines=None
    ):
        """
        Method for getting test case function expected values
        and test case function name.

        Additionally using attribute 'test_if_UNEQUAL' for reverse testcase:
        expecting that specific lines are not matching.
        :param func_name: Name of the testcase function
        :param test_if_UNEQUAL: Paramater to define if tested for inequality, defaults to 'False'
        :param unequal_lines: List of lines from the input, which should be unequal.
        """

        test_logger.debug(expected_output)
        self.expected_output.value = "".join(expected_output)
        test_logger.debug(f"Merged list is {self.expected_output.value}")
        self.expected_output.func_name = func_name.upper() if func_name else "Unknown?"
        self.expected_output.if_unequal.enabled = test_if_UNEQUAL
        self.expected_output.if_unequal.lines = unequal_lines

    def _new_line(self):
        """
        Method for generating newline string in
        Markdown syntax to help testing.

        Two spaces before newline mark.
        """
        return f"  {os.linesep}"

@dataclass
class IF_LINES_UNEQUAL:
    lines: List[int] = None
    enabled: bool = False

@dataclass
class EXPECTED_OUTPUT:
    value: str = None
    if_unequal: IF_LINES_UNEQUAL = IF_LINES_UNEQUAL()
    func_name: str = "Unknown"


from pyramid.view import view_config

from substanced.util import Batch

from dace.processinstance.core import DEFAULTMAPPING_ACTIONS_VIEWS
from dace.objectofcollaboration.principal.util import get_current
from pontus.view import BasicView

from novaideo.core import can_access
from novaideo.content.processes.novaideo_view_manager.behaviors import (
    SeeDependencies)
from novaideo.utilities.util import (
    render_small_listing_objs)
from novaideo.core import Node
from novaideo import _

BATCH_DEFAULT_SIZE = 20

WG_MESSAGES = {
    '0': _(u"""No association"""),
    '1': _(u"""One association"""),
    '*': _(u"""${number} associations""")}

@view_config(
    name='seedependencies',
    context=Node,
    renderer='pontus:templates/views_templates/grid.pt',
    )
class SeeDependenciesView(BasicView):
    name = 'seedependencies'
    viewid = 'seedependencies'
    behaviors = [SeeDependencies]
    template = 'novaideo:views/novaideo_view_manager/templates/home.pt'
    wrapper_template = 'pontus:templates/views_templates/simple_view_wrapper.pt'
    view_icon = 'icon ion-android-share'
    title = _('The associations')
    description = _('See the associations')

    def update(self):
        self.execute(None)
        user = get_current()
        objects = [obj for obj
                   in self.context.get_sub_nodes()
                   if can_access(user, obj)]
        objects = sorted(
            objects,
            key=lambda e: getattr(e, 'modified_at'),
            reverse=True)
        url = self.request.resource_url(self.context, self.name)
        batch = Batch(objects, self.request,
                      url=url,
                      default_size=BATCH_DEFAULT_SIZE)
        batch.target = "#results_node_dependencies_" + str(self.context.__oid__)
        len_result = batch.seqlen
        index = str(len_result)
        if len_result > 1:
            index = '*'

        self.title = _(WG_MESSAGES[index], mapping={'number': len_result})
        result = {}
        result_body = render_small_listing_objs(
            self.request, batch, user)

        values = {
            'bodies': result_body,
            'batch': batch,
            'empty_message': _("No association"),
            'empty_icon': 'glyphicon glyphicon-link'
        }
        body = self.content(args=values, template=self.template)['body']
        item = self.adapt_item(body, self.viewid)
        result['coordinates'] = {self.coordinates: [item]}
        return result

DEFAULTMAPPING_ACTIONS_VIEWS.update(
    {SeeDependencies: SeeDependenciesView})


# -*- coding: utf-8 -*-
"""
Forms of customers app
"""

from django import forms
from django.conf import settings

from obp.api import APIError

class CreateCustomerForm(forms.Form):
    bank_id = forms.ChoiceField(
        label='Bank',
        widget=forms.Select(
            attrs={
                'class': 'form-control',
            }
        ),
        choices=[],
    )
    username = forms.CharField(
        label='Username',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'The name of the user',
                'class': 'form-control',
            }
        ),
    )
    customer_number = forms.CharField(
        label='Customer Number',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'E.g. `007`',
                'class': 'form-control',
            }
        ),
    )
    legal_name = forms.CharField(
        label='Legal Name',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'NONE',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    mobile_phone_number = forms.CharField(
        label='Mobile Phone Number',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'E.g. +49 123 456 78 90 12',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    email = forms.CharField(
        label='Email',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'E.g. person@example.com',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    face_image_url = forms.CharField(
        label='Face Image URL',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'https://static.openbankproject.com/images/OBP/favicon.png',  # noqa
                'class': 'form-control',
            }
        ),
        required=False,
    )
    face_image_date = forms.DateTimeField(
        label='Face Image Date',
        input_formats=[settings.API_DATETIMEFORMAT],
        widget=forms.DateTimeInput(
            attrs={
                'placeholder': '2013-01-22T00:08:00Z',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    date_of_birth = forms.DateTimeField(
        label='Date of Birth',
        input_formats=[settings.API_DATETIMEFORMAT],
        widget=forms.DateTimeInput(
            attrs={
                'placeholder': '2013-01-22T00:08:00Z',
                'class': 'form-control',
            }
        ),
        required=True,
    )
    relationship_status = forms.CharField(
        label='Relationship Status',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'Single',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    dependants = forms.IntegerField(
        label='Dependants',
        widget=forms.TextInput(
            attrs={
                'placeholder': '0',
                'class': 'form-control',
            }
        ),
        initial=0,
        required=True,
    )
    dob_of_dependants = forms.CharField(
        label='Date of Birth of Dependants',
        widget=forms.TextInput(
            attrs={
                'placeholder': '2013-01-22T00:08:00Z, 2010-01-22T00:08:00Z',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    credit_rating_rating = forms.CharField(
        label='Credit Rating (Rating)',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'OBP',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    credit_rating_source = forms.CharField(
        label='Credit Rating (Source)',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'OBP',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    credit_limit_currency = forms.CharField(
        label='Credit Limit (Currency)',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'EUR',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    credit_limit_amount = forms.CharField(
        label='Credit Limit (Amount)',
        widget=forms.TextInput(
            attrs={
                'placeholder': '10',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    highest_education_attained = forms.CharField(
        label='Highest Education Attained',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'Bachelor’s Degree',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    employment_status = forms.CharField(
        label='Employment Status',
        widget=forms.TextInput(
            attrs={
                'placeholder': 'Employed',
                'class': 'form-control',
            }
        ),
        required=False,
    )
    kyc_status = forms.BooleanField(
        label='KYC Status',
        widget=forms.CheckboxInput(
            attrs={
                'class': 'form-control',
            }
        ),
        initial=True,
        required=False,
    )
    last_ok_date = forms.DateTimeField(
        label='Last OK Date',
        input_formats=[settings.API_DATETIMEFORMAT],
        widget=forms.DateTimeInput(
            attrs={
                'placeholder': '2013-01-22T00:08:00Z',
                'class': 'form-control',
            }
        ),
        required=True,
    )

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('label_suffix', '')
        super(CreateCustomerForm, self).__init__(*args, **kwargs)

    def clean_face_image_date(self):
        data = self.cleaned_data['face_image_date']
        if data:
            return data.strftime(settings.API_DATETIMEFORMAT)
        else:
            return None

    def clean_date_of_birth(self):
        data = self.cleaned_data['date_of_birth']
        if data:
            return data.strftime(settings.API_DATETIMEFORMAT)
        else:
            return None

    def clean_dob_of_dependants(self):
        data = self.cleaned_data['dob_of_dependants']
        if data:
            return data.split(',')
        else:
            return []

    def clean_username(self):
        username = self.cleaned_data['username']
        if not hasattr(self, 'api'):
            raise forms.ValidationError('No API object available')
        try:
            user = self.api.get('/users/username/{}'.format(username))
        except APIError as err:
            raise forms.ValidationError(err)
        else:
            self.cleaned_data['user_id'] = user['user_id']
        return username


import logging
from collections import OrderedDict, defaultdict
from typing import TYPE_CHECKING, Type

from django.utils.module_loading import import_string
from django.utils.translation import ugettext as _

from alarm_backends.constants import LATEST_POINT_WITH_ALL_KEY
from alarm_backends.core.cache.key import LAST_CHECKPOINTS_CACHE_KEY
from alarm_backends.core.detect_result import ANOMALY_LABEL, CheckResult
from bkmonitor.utils.common_utils import chunks
from bkmonitor.utils.text import camel_to_underscore

logger = logging.getLogger("core.control")

if TYPE_CHECKING:
    from alarm_backends.service.detect.strategy import BasicAlgorithmsCollection  # noqa

def load_detector_cls(_type) -> Type["BasicAlgorithmsCollection"]:
    algorithms_target = camel_to_underscore(_type)
    package_name = "alarm_backends.service.detect"
    cls_target = "{}.strategy.{}.{}".format(package_name, algorithms_target, _type)
    try:
        cls = import_string(cls_target)
    except ImportError:
        logger.error("detector load error: {}".format(cls_target))
        cls = None
    return cls

class DetectMixin(object):
    def detect(self, data_points):
        if not data_points:
            return []

        algorithm_group = defaultdict(list)
        for _config in self.algorithms:
            level = int(_config["level"])
            algorithm_group[level].append(_config)

        levels = sorted(algorithm_group.keys())

        # 初始化检测结果
        detected_result_dict = OrderedDict()

        for level in levels:
            # 同级别算法连接符(and/or)
            algorithm_connector = self.algorithm_connectors[level]

            detector_list = []
            for detect_config in algorithm_group[level]:
                algorithm_type = detect_config["type"]
                algorithm_unit = detect_config.get("unit_prefix", "")
                detector_cls = load_detector_cls(algorithm_type)
                detector = detector_cls(detect_config["config"], algorithm_unit)
                if hasattr(detector, "history_point_fetcher"):
                    # query history point
                    detector.query_history_points(data_points)
                detector_list.append(detector)

            if len(detector_list) == 1:
                anomaly_records = detector_list[0].detect_records(data_points, level)
            else:
                # 组合策略
                anomaly_records = []
                for data_point in data_points:
                    ap = None
                    prefix = suffix = ""
                    for d in detector_list:
                        try:
                            single_ret = d.detect(data_point)

                            # != "or" 兼容connector未配置或配错的情况，默认都使用and
                            if not single_ret:
                                if algorithm_connector != "or":
                                    ap = None
                                    break
                                else:
                                    continue

                            single_ap = d.gen_anomaly_point(data_point, single_ret, level, auto_format=False)
                            if not prefix:
                                prefix, suffix = d.anomaly_message_template_tuple(data_point)
                                # print(prefix, suffix)
                            if ap:
                                ap.anomaly_message += _(" 同时 ") + single_ap.anomaly_message
                            else:
                                ap = single_ap

                            # 如果算法为or，则只需要一个算法匹配即可立即退出
                            if algorithm_connector == "or":
                                break
                        except Exception:
                            if algorithm_connector != "or":
                                ap = None
                                break

                    if ap:
                        ap.anomaly_message = prefix + ap.anomaly_message + suffix
                        logger.info(
                            "[detect] strategy({}) item({}) level[{}] 发现异常点: {}".format(
                                ap.data_point.item.strategy.id, ap.data_point.item.id, level, ap.__dict__
                            )
                        )
                        anomaly_records.append(ap)

            self._update_monitor_d_checkpoint(data_points, anomaly_records, level)

            for ar in anomaly_records:
                collection = detected_result_dict.setdefault(ar.data_point.record_id, {})
                collection.update(self._update_anomaly_info_with_point(ar, level, collection))

        return list(detected_result_dict.values())

    def _update_anomaly_info_with_point(self, anomaly_point, level, info_collection=None):
        info_collection = info_collection or {
            "data": anomaly_point.data_point.as_dict(),
            "anomaly": {},
            "strategy_snapshot_key": self.strategy.snapshot_key,
        }
        anomaly_info = {
            "anomaly_message": anomaly_point.anomaly_message,
            "anomaly_id": anomaly_point.anomaly_id,
            "anomaly_time": anomaly_point.anomaly_time,
        }

        if anomaly_point.context:
            anomaly_info["context"] = anomaly_point.context

        info_collection["anomaly"].update({str(level): anomaly_info})
        return info_collection

    def _update_monitor_d_checkpoint(self, records, anomaly_records, level):
        redis_pipeline = None
        last_checkpoints = {}
        anomaly_record_ids = {i.data_point.record_id for i in anomaly_records}
        latest_point_with_all = 0
        for d in records:
            # data_record 的record_id规则： {dimensions_md5}.{timestamp}
            dimensions_md5, timestamp = d.record_id.split(".")
            timestamp = int(timestamp)
            latest_point_with_all = max([latest_point_with_all, d.timestamp])

            check_result = CheckResult(self.strategy.id, self.id, dimensions_md5, level)
            if redis_pipeline is None:
                redis_pipeline = check_result.CHECK_RESULT

            if d.record_id in anomaly_record_ids:
                name = "{}|{}".format(timestamp, ANOMALY_LABEL)
            else:
                name = "{}|{}".format(timestamp, str(d.value))

            try:
                # 1. 缓存数据(检测结果缓存) type:SortedSet
                kwargs = {name: timestamp}
                check_result.add_check_result_cache(**kwargs)

                # 2. 缓存最后checkpoint type:Hash，先放到内存里，最后再一次性写入redis
                last_point = last_checkpoints.setdefault(dimensions_md5, 0)
                if last_point < timestamp:
                    last_checkpoints[dimensions_md5] = timestamp

            except Exception as e:
                msg = "set check result cache error:%s" % e
                logger.exception(msg)

        if redis_pipeline:
            check_result.expire_key_to_dimension()
            redis_pipeline.execute()

        if latest_point_with_all:
            last_checkpoints[LATEST_POINT_WITH_ALL_KEY] = latest_point_with_all

        # 更新last_checkpoint
        last_checkpoint_cache_key = LAST_CHECKPOINTS_CACHE_KEY.get_key(strategy_id=self.strategy.id, item_id=self.id)

        for chunked_points in chunks(list(last_checkpoints.items()), 5000):
            set_mapping = {}
            for _dimensions_md5, point_timestamp in chunked_points:
                last_checkpoint_cache_field = LAST_CHECKPOINTS_CACHE_KEY.get_field(
                    dimensions_md5=_dimensions_md5,
                    level=level,
                )
                set_mapping[last_checkpoint_cache_field] = point_timestamp
            LAST_CHECKPOINTS_CACHE_KEY.client.hmset(last_checkpoint_cache_key, set_mapping)
        CheckResult.expire_last_checkpoint_cache(strategy_id=self.strategy.id, item_id=self.id)