import logging
import os
import re
import pytest
import sqlalchemy
from datetime import datetime, timedelta, timezone
from sqlalchemy.dialects import oracle

from streamsets.testframework.markers import database, sdc_min_version

from stage.utils.common import cleanup
from stage.utils.utils_migration import LegacyHandler as PipelineHandler
from stage.utils.utils_oracle import (
    DefaultConnectionParameters,
    DefaultStartParameters,
    DefaultTableParameters,
    MIN_ORACLE_VERSION,
    RECORD_FORMATS,
    RELEASE_VERSION,
    database_version,
    oracle_stage_name,
    table_name,
    test_name,
    util_setup,
)

DEFAULT_TIMEOUT_IN_SEC = 120
# These variables need to be loaded only once and will not change from test to test
SERVICE_NAME = ""  # The value will be assigned during setup
SYSTEM_IDENTIFIER = ""  # The value will be assigned during setup
OK_STATUS = "0"

DEFAULT_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
DEFAULT_ZONED_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S %z"
DEFAULT_DATETIME = datetime.strptime("1998-10-31 06:22:33", DEFAULT_DATETIME_FORMAT)
LOCAL_TIMEZONE = datetime.now(timezone.utc).astimezone().tzinfo

logger = logging.getLogger(__name__)
pytestmark = [database("oracle"), sdc_min_version(RELEASE_VERSION)]

def format_timezone_offset(dt):
    """Add a colon separator between timezone hours and minutes.
    Convert YYYY-MM-DDTHH:MM:SS+ZZZZ to YYYY-MM-DDTHH:MM:SS+ZZ:ZZ"""

    dt_str = dt.strftime("%Y-%m-%dT%H:%M:%S%z")

    pattern = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\+(\d{4})"
    match = re.compile(f"^{pattern}$").match(dt_str)
    if not match:
        return dt_str
    main_datetime = match.group(1)
    offset = match.group(2)
    hh, mm = offset[:2], offset[2:]
    postfix = "Z" if hh == mm == "00" else f"+{hh}:{mm}"
    return f"{main_datetime}{postfix}"

@pytest.mark.parametrize("null", [True, False])
@pytest.mark.parametrize(
    "type_name, oracle_type, oracle_value, streamsets_type, streamsets_value",
    [
        ["BINARY_DOUBLE", oracle.BINARY_DOUBLE, 4.2, "DOUBLE", 4.2],
        ["BINARY_FLOAT", oracle.BINARY_FLOAT, 4.2, "FLOAT", 4.2],
        ["CHAR", oracle.CHAR, "A", "STRING", "A"],
        ["DATE", oracle.DATE, DEFAULT_DATETIME, "DATETIME", DEFAULT_DATETIME],
        ["FLOAT", oracle.FLOAT, 4.2, "FLOAT", 4.2],
        ["NCHAR", oracle.NCHAR, "A", "STRING", "A"],
        ["NUMBER", oracle.NUMBER, 4, "DECIMAL", 4],
        ["NVARCHAR2", oracle.NVARCHAR2(length=3), "ABC", "STRING", "ABC"],
        ["TIMESTAMP", oracle.TIMESTAMP, DEFAULT_DATETIME, "DATETIME", DEFAULT_DATETIME],
        [
            "TIMESTAMP WITH LOCAL TIME ZONE",
            oracle.TIMESTAMP(timezone=True),
            DEFAULT_DATETIME.replace(tzinfo=LOCAL_TIMEZONE),
            "ZONED_DATETIME",
            format_timezone_offset(DEFAULT_DATETIME.replace(tzinfo=LOCAL_TIMEZONE)),
        ],
        [
            "TIMESTAMP WITH TIME ZONE",
            oracle.TIMESTAMP(timezone=True),
            DEFAULT_DATETIME.replace(tzinfo=LOCAL_TIMEZONE),
            "ZONED_DATETIME",
            format_timezone_offset(DEFAULT_DATETIME.replace(tzinfo=LOCAL_TIMEZONE)),
        ],
        ["VARCHAR", oracle.VARCHAR(length=3), "ABC", "STRING", "ABC"],
        ["VARCHAR2", oracle.VARCHAR2(length=3), "ABC", "STRING", "ABC"],
        # ["CHAR VARYING", oracle.CHAR VARYING],
        # ["CHARACTER", oracle.CHARACTER],
        # ["CHARACTER VARYING", oracle.CHARACTER VARYING],
        ["DECIMAL", oracle.NUMBER(asdecimal=True), 4, "DECIMAL", 4],
        #! ["DOUBLE PRECISION", oracle.DOUBLE_PRECISION, 4.2, "DOUBLE", 4.2],
        ["DOUBLE PRECISION", oracle.DOUBLE_PRECISION, 4.2, "FLOAT", 4.2],
        #! ["INT", sqlalchemy.INT, 42, "NUMBER", 42],
        ["INT", sqlalchemy.INT, 42, "LONG", 42],
        # ! ["INTEGER", sqlalchemy.INTEGER, 42, "NUMBER", 42],
        ["INTEGER", sqlalchemy.INTEGER, 42, "LONG", 42],
        # ["NATIONAL_CHAR", oracle.NATIONAL CHAR],
        # ["NATIONAL CHAR VARYING", oracle.NATIONAL CHAR VARYING],
        # ["NATIONAL CHARACTER", oracle.NATIONAL CHARACTER],
        # ["NATIONAL CHARACTER VARYING", oracle.NATIONAL CHARACTER VARYING],
        # ["NCHAR VARYING", oracle.NCHAR VARYING],
        # ["NUMERIC", oracle.NUMERIC],
        # ["REAL", oracle.REAL],
        # ["SMALLINT", oracle.SMALLINT],
    ],
)
def test_data_types(
    sdc_builder,
    sdc_executor,
    database,
    database_version,
    oracle_stage_name,
    cleanup,
    table_name,
    test_name,
    null,
    type_name,
    oracle_type,
    oracle_value,
    streamsets_type,
    streamsets_value,
):
    """Test datatypes supported by the Oracle CDC Origin."""

    if database_version < MIN_ORACLE_VERSION:
        pytest.skip(f"Oracle version {database_version} is not officially supported")

    id_column = "ID"
    id_value = 1
    datatype_column = "VALUE"
    table = sqlalchemy.Table(
        table_name,
        sqlalchemy.MetaData(),
        sqlalchemy.Column(id_column, sqlalchemy.Integer, primary_key=True),
        sqlalchemy.Column(datatype_column, oracle_type),
    )
    # Test the null case for each data type
    if null:
        oracle_value = None
        streamsets_value = None

    connection = database.engine.connect()
    cleanup(connection.close)

    table.create(database.engine)
    cleanup(table.drop, database.engine)

    handler = PipelineHandler(sdc_builder, sdc_executor, database, cleanup, test_name, logger)
    pipeline_builder = handler.get_pipeline_builder()

    oracle_cdc_origin = pipeline_builder.add_stage(name=oracle_stage_name)
    oracle_cdc_origin.set_attributes(
        missing_columns="FORWARD",
        **DefaultConnectionParameters(database) | DefaultTableParameters(table_name) | DefaultStartParameters(database),
    )

    wiretap = pipeline_builder.add_wiretap()

    oracle_cdc_origin >> wiretap.destination

    pipeline = pipeline_builder.build(test_name).configure_for_environment(database)
    work = handler.add_pipeline(pipeline)

    cleanup(handler.stop_work, work)
    handler.start_work(work)

    txn = connection.begin()
    connection.execute(table.insert({id_column: id_value, datatype_column: oracle_value}))
    txn.commit()

    handler.wait_for_metric(work, "input_record_count", 1, timeout_sec=DEFAULT_TIMEOUT_IN_SEC)

    assert len(wiretap.output_records) == 1
    record = wiretap.output_records[0]

    # ToDo mikel: assert fail messages for all tests
    assert record.field[id_column].value == id_value
    assert record.field[datatype_column].type == streamsets_type
    assert record.field[datatype_column].value == streamsets_value

@pytest.mark.parametrize(
    "object_name",
    [
        "TABLE",  # reserved words (must be quoted)
        "92TABLE",  # begin with numeric characters (must be quoted)
        "MY#$#TABLE_",  # allowed symbols (even for unquoted names)
        "myTABLE_upperANDlowerCaSEs",  # case sensitiveness (only when quoted)
        "TABLE_7D510B56T_219B_4DAB_AFAB",  # max length for a table name (30 bytes)
        "EVEN THIS & THAT!",  # allowed identifier when quoted
    ],
)
def test_object_named_tables(
    sdc_builder,
    sdc_executor,
    database,
    database_version,
    oracle_stage_name,
    cleanup,
    table_name,
    test_name,
    object_name,
):
    """Test the stage treats table names correctly even if they contain reserved keywords or
    characters that could otherwise break queries. """

    if database_version < MIN_ORACLE_VERSION:
        pytest.skip(f"Oracle version {database_version} is not officially supported")

    id_column = "ID"
    id_value = 1
    other_column = "VALUE"
    other_value = 42
    table = sqlalchemy.Table(
        table_name,
        sqlalchemy.MetaData(),
        sqlalchemy.Column(id_column, sqlalchemy.Integer, primary_key=True),
        sqlalchemy.Column(other_column, sqlalchemy.Integer),
    )

    connection = database.engine.connect()
    cleanup(connection.close)

    table.create(database.engine)
    cleanup(table.drop, database.engine)

    handler = PipelineHandler(sdc_builder, sdc_executor, database, cleanup, test_name, logger)
    pipeline_builder = handler.get_pipeline_builder()

    oracle_cdc_origin = pipeline_builder.add_stage(name=oracle_stage_name)
    oracle_cdc_origin.set_attributes(
        null_columns="FORWARD",
        **DefaultConnectionParameters(database) | DefaultTableParameters(table_name) | DefaultStartParameters(database),
    )

    wiretap = pipeline_builder.add_wiretap()

    oracle_cdc_origin >> wiretap.destination

    pipeline = pipeline_builder.build(test_name).configure_for_environment(database)
    work = handler.add_pipeline(pipeline)

    cleanup(handler.stop_work, work)
    handler.start_work(work)

    txn = connection.begin()
    connection.execute(table.insert({id_column: id_value, other_column: other_value}))
    txn.commit()

    handler.wait_for_metric(pipeline, "input_record_count", 1, timeout_sec=DEFAULT_TIMEOUT_IN_SEC)

    assert len(wiretap.output_records) == 1
    record = wiretap.output_records[0]
    assert record.field[id_column].value == id_value
    assert record.field[other_column].value == other_value

@pytest.mark.parametrize("batches", [1, 10])
@pytest.mark.parametrize("max_batch_size", [1, 10, 1000])
def test_multiple_batches(
    sdc_builder,
    sdc_executor,
    database,
    database_version,
    oracle_stage_name,
    cleanup,
    table_name,
    test_name,
    batches,
    max_batch_size,
):
    """Test the stage produces the expected number of batches with the expected size."""

    if database_version < MIN_ORACLE_VERSION:
        pytest.skip(f"Oracle version {database_version} is not officially supported")

    id_column = "ID"
    other_column = "VALUE"
    expected_values = [i for i in range(batches * max_batch_size)]
    table = sqlalchemy.Table(
        table_name,
        sqlalchemy.MetaData(),
        sqlalchemy.Column(id_column, sqlalchemy.Integer, primary_key=True),
        sqlalchemy.Column(other_column, sqlalchemy.Integer),
    )

    connection = database.engine.connect()
    cleanup(connection.close)

    table.create(database.engine)
    cleanup(table.drop, database.engine)

    handler = PipelineHandler(sdc_builder, sdc_executor, database, cleanup, test_name, logger)
    pipeline_builder = handler.get_pipeline_builder()

    oracle_cdc_origin = pipeline_builder.add_stage(name=oracle_stage_name)
    oracle_cdc_origin.set_attributes(
        max_batch_size_in_records=max_batch_size,
        **DefaultConnectionParameters(database) | DefaultTableParameters(table_name) | DefaultStartParameters(database),
    )

    wiretap = pipeline_builder.add_wiretap()

    oracle_cdc_origin >> wiretap.destination

    pipeline = pipeline_builder.build(test_name).configure_for_environment(database)
    work = handler.add_pipeline(pipeline)

    cleanup(handler.stop_work, work)
    handler.start_work(work)

    txn = connection.begin()
    for value in expected_values:
        connection.execute(table.insert({id_column: value, other_column: value}))
    txn.commit()

    handler.wait_for_metric(pipeline, "input_record_count", len(expected_values), timeout_sec=DEFAULT_TIMEOUT_IN_SEC)

    assert len(wiretap.output_records) == len(expected_values)
    record_values = set(
        (record.field[id_column].value, record.field[other_column]) for record in wiretap.output_records
    )
    assert all([(value, value) in record_values] for value in expected_values)

def test_data_format(sdc_builder, sdc_executor, database):
    pytest.skip("Oracle CDC Origin doesn't deal with data formats.")

@pytest.mark.parametrize("iterations", [5])
def test_resume_offset(
    sdc_builder, sdc_executor, database, database_version, oracle_stage_name, cleanup, table_name, test_name, iterations
):
    """Test the stage resumes from the expected offset so it does"""

    if database_version < MIN_ORACLE_VERSION:
        pytest.skip(f"Oracle version {database_version} is not officially supported")

    records_per_iteration = 5
    id_column = "ID"
    other_column = "VALUE"
    expected_values = [
        [i for i in range(j, j + records_per_iteration)]
        for j in range(0, iterations * records_per_iteration, records_per_iteration)
    ]
    table = sqlalchemy.Table(
        table_name,
        sqlalchemy.MetaData(),
        sqlalchemy.Column(id_column, sqlalchemy.Integer, primary_key=True),
        sqlalchemy.Column(other_column, sqlalchemy.Integer),
    )

    connection = database.engine.connect()
    cleanup(connection.close)

    table.create(database.engine)
    cleanup(table.drop, database.engine)

    handler = PipelineHandler(sdc_builder, sdc_executor, database, cleanup, test_name, logger)
    pipeline_builder = handler.get_pipeline_builder()

    oracle_cdc_origin = pipeline_builder.add_stage(name=oracle_stage_name)
    oracle_cdc_origin.set_attributes(
        **DefaultConnectionParameters(database) | DefaultTableParameters(table_name) | DefaultStartParameters(database)
    )

    wiretap = pipeline_builder.add_wiretap()

    oracle_cdc_origin >> wiretap.destination

    pipeline = pipeline_builder.build(test_name).configure_for_environment(database)
    work = handler.add_pipeline(pipeline)

    cleanup(handler.stop_work, work)

    for iteration in range(iterations):
        handler.start_work(work)

        txn = connection.begin()
        for value in expected_values[iteration]:
            connection.execute(table.insert({id_column: value, other_column: value}))
        txn.commit()

        handler.wait_for_metric(
            pipeline, "input_record_count", records_per_iteration, timeout_sec=DEFAULT_TIMEOUT_IN_SEC
        )
        handler.stop_work(work)

    assert len(wiretap.output_records) == iterations * records_per_iteration
    record_values = set(
        (record.field[id_column].value, record.field[other_column]) for record in wiretap.output_records
    )
    assert all(
        [[(value, value) in record_values for value in iteration_values] for iteration_values in expected_values]
    )

@pytest.mark.parametrize("threads", [1, 4, 8])
def test_multithreading(
    sdc_builder, sdc_executor, database, database_version, oracle_stage_name, cleanup, test_name, table_name, threads
):
    """Ensure the behaviour of the stage does not change with the number of parser threads."""

    if database_version < MIN_ORACLE_VERSION:
        pytest.skip(f"Oracle version {database_version} is not officially supported")

    primary_column = "ID_COLUMN"
    secondary_column = "OTHER_COLUMN"
    base_time = datetime.strptime("2000-01-01 00:00:00", "%Y-%m-%d %H:%M:%S")
    record_count = 2000

    table = sqlalchemy.Table(
        table_name,
        sqlalchemy.MetaData(),
        sqlalchemy.Column(primary_column, sqlalchemy.Integer, primary_key=True),
        sqlalchemy.Column(secondary_column, sqlalchemy.DATE),
    )
    connection = database.engine.connect()
    cleanup(connection.close)

    table.create(database.engine)
    cleanup(table.drop, database.engine)

    records = [{primary_column: i, secondary_column: base_time + timedelta(days=1) * i} for i in range(record_count)]

    handler = PipelineHandler(sdc_builder, sdc_executor, database, cleanup, test_name, logger)
    pipeline_builder = handler.get_pipeline_builder()

    oracle_cdc_origin = pipeline_builder.add_stage(name=oracle_stage_name)
    oracle_cdc_origin.set_attributes(
        **DefaultConnectionParameters(database) | DefaultTableParameters(table_name) | DefaultStartParameters(database)
    )
    oracle_cdc_origin.set_attributes(sql_parser_threads=threads)

    wiretap = pipeline_builder.add_wiretap()

    oracle_cdc_origin >> wiretap.destination

    pipeline = pipeline_builder.build(test_name).configure_for_environment(database)
    work = handler.add_pipeline(pipeline)

    txn = connection.begin()
    connection.execute(table.insert(), records)
    txn.commit()

    cleanup(handler.stop_work, work)
    handler.start_work(work)

    handler.wait_for_metric(work, "input_record_count", len(records), timeout_sec=DEFAULT_TIMEOUT_IN_SEC)

    output_records = [{k: v for k, v in record.field.items()} for record in wiretap.output_records]
    assert output_records == records


# Copyright (C) 2020 Claudio Marques - All Rights Reserved
from enum import Enum

class Lists(Enum):
    VOWEL = 'aeiou'
    CONSOANT = "bcdfghjklmnpqrstvwxyz"
    NUMERIC = "0123456789"
    SPECIALCHAR = "!\"#|\\$%&/()=?«»´`*+ºª^~;,-_@£€{[]}'"

class DatesEnum(Enum):
    SemDados = 0
    UmMes = 1
    SeisMeses = 2
    UmAno = 3
    MaisDeUmAno = 4

class HttpResponseEnum(Enum):
    SemDados = 0
    Codigos100 = 1
    Codigos200 = 2
    Codigos300 = 3
    Codigos400 = 4
    Codigos500 = 5


"""
Get Options Action for App ID
"""
from urllib.parse import urlencode

from api.api_samples.python_client.ext import requests
from common.methods import set_progress
from itsm.servicenow.models import ServiceNowITSM
import json

def get_options_list(field, **kwargs):
    options = [('', '--- Select an App ID ---')]
    snowitsm = ServiceNowITSM.objects.first()
    wrapper = snowitsm.get_api_wrapper()
    base_url = wrapper.service_now_instance_url.replace("/login.do", "")
    req_for_data = lookup_ci(table_name='cmdb_ci_appl',
                             base_url=base_url,
                             conn=snowitsm,
                             sysparm_fields=[
                                 'name',
                                 'u_application_id',
                                 'u_systemid',
                                 'install_status',
                                 'owned_by',
                                 'support_group',
                                 'u_priority',
                                 'version',
                                 'operational_status',
                                 'sys_created_on',
                             ]
                             )
    response_json = json.loads(req_for_data)
    results = response_json["result"]
    for result in results:
        if result["support_group"]:
            support_group = result["support_group"]["display_value"]
            result["support_group"] = support_group
        if result["owned_by"]:
            support_group = result["owned_by"]["display_value"]
            result["owned_by"] = support_group
        options.append((json.dumps(result), result["name"]))
    return options

def lookup_ci(table_name=None, base_url=None, conn=None,
              sysparm_fields: list = None, url=None):
    if not url:
        query = ''
        if sysparm_fields:
            query = urlencode({'sysparm_display_value': 'true',
                               'sysparm_fields': ",".join(sysparm_fields)})

        url = base_url + f"/api/now/table/{table_name}?{query}"

    response = requests.get(
        url=url,
        auth=(conn.service_account, conn.password),
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json"
        },
        timeout=5.0
    )

    return response.content


from django.shortcuts import render

# Create your views here.
from django.http import HttpResponse
from wmh_server.models import LocationData as lc
from django.views.decorators.csrf import csrf_exempt
import json
import datetime as dt
from django.conf import settings

def index(request):
    res = "Base Path:{}. This is a site of 'WalkMeHome' Project. The site is under construction!".format(settings.BASE_DIR)
    return HttpResponse(res)

@csrf_exempt
def send_loc(request):
    try:
        if (request.method == 'POST') \
                and ('email' in request.POST) \
                and ('loc_date' in request.POST) \
                and ('LAT' in request.POST) \
                and ('LON' in request.POST):
            loc_date_str = request.POST['loc_date']
            loc_date = dt.datetime.strptime(loc_date_str,"%d.%m.%Y %H:%M:%S")
            loc = lc(email=request.POST['email'], loc_date=loc_date, LON=float(request.POST['LON']), LAT=float(request.POST['LAT']))
            loc.save()
            res = "0"
        else:
            res = "Ошибка"
    except NameError:
        res = "Ошибка"

    return HttpResponse(res)

@csrf_exempt
def send_track(request):
    try:
        if (request.method == 'POST'):
            json_data = request.read()
            # json_data contains the data uploaded in request
            data = json.loads(json_data)
            for rec in data:
                loc_date_str = rec['loc_date']
                loc_date = dt.datetime.strptime(loc_date_str,"%d.%m.%Y %H:%M:%S")
                loc = lc(email=rec['email'], loc_date=loc_date, LON=float(rec['LON']), LAT=float(rec['LAT']))
                loc.save()
            res = "0"
        else:
            res = "Ошибка"
    except NameError:
        res = "Ошибка"

    return HttpResponse(res)

@csrf_exempt
def get_last_location(request):
    if (request.method == 'GET') and ('email' in request.GET):
        locations = lc.objects.filter(email=request.GET['email'])
        if locations.count()>0:
            sortlocations = locations.order_by("-loc_date")
            last_record = sortlocations[0]
            loc_date_str = last_record.loc_date.strftime("%d.%m.%Y %H:%M:%S")
            last_location = dict(email=last_record.email, loc_date=loc_date_str,
                                 LAT=last_record.LAT,
                                 LON=last_record.LON)

            res = json.dumps(last_location)
        else:
            res = 'Нет данных'
    else:
        res = 'Ошибка'
    return HttpResponse(res)


# Sytulacja rzutu dwoma kośćmi.

from die import Die
from plotly.graph_objs import Bar, Layout
from plotly import offline

die_1 = Die()
die_2 = Die(10)

results = [die_1.roll()+die_2.roll() for roll_num in range(50_000)]

max_result = die_1.num_sides + die_2.num_sides
frequencies = [results.count(value) for value in range(2, max_result+1)]

x_values = list(range(2, max_result+1))
data = [Bar(x=x_values, y=frequencies)]

x_axis_config = {'title': 'Wynik', 'dtick': 1} # dtick : 1 -> nakazanie dodawania podpisu do każdego słupka
y_axis_config = {'title': 'Częstotliwośc występowania wartości'}
my_layout = Layout(title='Wynik rzucania kośćmi D6 i D10 pięćdziesiąt tysięcy razy',
                   xaxis=x_axis_config, yaxis=y_axis_config)
offline.plot({'data':data, 'layout': my_layout}, filename='d6_d10.html')


#!/usr/bin/env python3
"""
cron: 0 40 22 * * *
new Env('明日天气');
"""
import sys
import requests
import json
import time
from bs4 import BeautifulSoup
import os, re

# 获取WxPusher appToken  WxPusher_appToken
if "WxPusher_appToken" in os.environ:
    if len(os.environ["WxPusher_appToken"]) > 1:
        WxPusher_appToken = os.environ["WxPusher_appToken"]
else:
    print('未配置环境变量 WxPusher_appToken')
    sys.exit()

# 获取WxPusher uid  WxPusher_uids ','分隔
if "WxPusher_uids" in os.environ:
    if len(os.environ["WxPusher_uids"]) > 1:
        WxPusher_uids = os.environ["WxPusher_uids"].split(',')
else:
    print('未配置环境变量 WxPusher_uids')
    sys.exit()

def get_weather_data(city_id):
    """
    获取当前到次日凌晨的每小时气温和未来5天的每天最低气温
    """
    head = {
        "referer": "http://m.weather.com.cn/",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44 "
    }
    timestamp = str(int(time.time())) + '761'
    url = r'https://d1.weather.com.cn/wap_40d/{}.html?_={}'.format(city_id, timestamp)
    print('正在请求天气信息...')
    r = requests.get(url, headers=head)
    if r.status_code != 200:
        print('status code:', r.status_code)
        return False

    r.encoding = r.apparent_encoding
    html = r.text.split(';')  # html[0]为40天预报  html[2]为48小时预报

    weather40 = html[0][html[0].find('['):]
    tomorrow = {}
    i = 0
    for day in json.loads(weather40):
        if int(day['009']) < int(time.strftime('%Y%m%d')):
            continue  # 009是日期，如果日期小于今天则舍去
        if int(day['009']) == int(time.strftime('%Y%m%d')):
            i = i + 1
            continue
        if i == 1:
            tomorrow = day
            i = i + 1
        else:
            break
    return tomorrow

def analyse_weather(tomorrow):
    """
    分析气温变化，判断要不要发送消息提醒，如果不需要提醒，返回空字符串
    :param tomorrow: 明天天气信息
    :return: 返回发送消息提醒的文本内容
    """

    message = ''

    moji_url = 'https://tianqi.moji.com/weather/china/henan/anyang'

    headers = {
        'Host': 'tianqi.moji.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
    }

    r = requests.get(moji_url, headers=headers)
    r.encoding = "utf-8"
    soup = BeautifulSoup(r.text, 'lxml')
    today_temp = soup.select('body > div:nth-child(5) > div.left > div.forecast.clearfix > ul:nth-child(2) > '
                             'li:nth-child(3)')[0].text.replace(" ", "").replace("°", "")
    today_min_temp, today_max_temp = today_temp.split('/')
    tomorrow_temp = soup.select('body > div:nth-child(5) > div.left > div.forecast.clearfix > ul:nth-child(3) > '
                                'li:nth-child(3)')[0].text.replace(" ", "")
    tomorrow_min_temp, tomorrow_max_temp = tomorrow_temp.replace("°", "").split('/')
    tomorrow_weather = soup.select('body > div:nth-child(5) > div.left > div.forecast.clearfix > ul:nth-child(3) > '
                                   'li:nth-child(2)')[0].text.strip()

    # 明日天气
    today_avg_temp = (int(today_min_temp) + int(today_max_temp)) / 2
    tomorrow_avg_temp = (int(tomorrow_min_temp) + int(tomorrow_max_temp)) / 2
    diff = today_avg_temp - tomorrow_avg_temp
    print(diff)
    if diff > 5:
        message += '明日降温,平均温度下降{}℃\n'.format(diff)
    elif diff < -5:
        message += '明日升温,平均温度上升{}℃\n'.format(diff)
    year, month, days = time.strftime('%Y-%m-%d', time.strptime(tomorrow['009'], '%Y%m%d')).split('-')
    message += '明 日: {}年{}月{}日 {}\n'.format(year, month, days, tomorrow['016'])
    message += '天 气: {}\n'.format(tomorrow_weather)
    message += '温 度: {}\n'.format(tomorrow_temp)
    message += '农 历: {}\n'.format(tomorrow['010'])
    if tomorrow['017'] != '':
        message += '节 日: {}\n'.format(tomorrow['017'])
    if tomorrow['018'] != '':
        message += '节 气: {}\n'.format(tomorrow['018'])
    return message

def send_reminder(message):

    url = 'http://wxpusher.zjiecode.com/api/send/message'

    headers = {
        'Content-Type': 'application/json'
    }

    data = {
        'appToken': WxPusher_appToken,
        'content': message,
        # 'summary': '消息摘要', # 消息摘要，显示在微信聊天页面或者模版消息卡片上，限制长度100，可以不传，不传默认截取content前面的内容。
        'contentType': 1,  # 内容类型 1表示文字 2表示html(只发送body标签内部的数据即可，不包括body标签) 3表示markdown
        # 'topicIds': [], # 发送目标的topicId，是一个数组！！！，也就是群发，使用uids单发的时候， 可以不传。
        # 'uids': ['', '']  # 发送目标的UID，是一个数组。注意uids和topicIds可以同时填写，也可以只填写一个。
        'uids':WxPusher_uids
        # 'url': "" # 原文链接，可选参数
    }

    if not message:
        print('气温变化适宜，不用发送消息提醒')
        return
    r = requests.post(headers=headers, url=url, data=json.dumps(data))
    if r.status_code == 200:
        print('发送提醒成功,message:')
        print(message)
    else:
        print('发送提醒失败，status_code:', r.status_code)

if __name__ == '__main__':
    cityID = '101180201'  # 安阳
    assert cityID

    time5 = time24 = time5_2 = time24_2 = 0
    print('现在是{}'.format(time.strftime('%Y-%m-%d %H:%M')))
    tomorrow = get_weather_data(cityID)
    assert tomorrow
    message = analyse_weather(tomorrow)
    assert message
    send_reminder(message)


from bs4 import BeautifulSoup
import requests
from rus import send_mail
from soc import messages_to_string

def parse_hearpwn_page(page):
    page = requests.get(page)
    soup = BeautifulSoup(page.text, 'html.parser')
    messages = soup.find_all('div', itemprop='text')
    return messages

def parse_salty_thread_page(pageid):
    page = 'http://www.hearthpwn.com/forums/hearthstone-general/general-discussion/28947-group-' \
           'therapy-need-to-blow-off-steam-mega-salty?page=' + str(pageid)
    return parse_hearpwn_page(page)

def pretty_print(messages):
    for el in messages:
        print(el.getText(separator = ' '))

def main():
    print(messages_to_string(parse_salty_thread_page(2180)))
    send_mail(messages_to_string(parse_salty_thread_page(2180)))

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Course: CS 4365/5354 [Computer Vision]
Author: Jose Perez [ID: 80473954]
Assignment: Lab 1
Instructor: Olac Fuentes
Last Modification: September 2, 2016 by Jose Perez
"""
from timeit import default_timer as timer
from PIL import Image
from numpy import *

# Page 42-43, exercise 5
# Gradient direction and magnitude
# Detect lines in an image
# Estimate extent of the lines and parameters
# Plot lines overlaid on the image
def problem1_exercise5(filename, threshold=50):
    im = array(Image.open(filename).convert('L'))
    background = Image.open(filename).convert('RGBA')

    # Begin calculation
    start_time = timer()

    # Sobel
    imx_sobel = zeros(im.shape)
    filters.prewitt(im, 1, imx_sobel)

    imy_sobel = zeros(im.shape)
    filters.prewitt(im, 0, imy_sobel)

    magnitude = sqrt(imx_sobel**2 + imy_sobel**2)

    # Filter (AKA set a threshold)
    low_values = magnitude < threshold
    high_values = magnitude >= threshold
    magnitude[low_values] = 255 # White for values below threshold
    magnitude[high_values] = 0 # Black for values above threshold

    magnitude_color = Image.fromarray(magnitude).convert('RGBA')
    magnitude_color_arr = array(magnitude_color)
    red, green, blue, alpha = magnitude_color_arr.T

    black_areas = (red == 0) & (blue == 0) & (green == 0)
    white_areas = (red == 255) & (blue == 255) & (green == 255)
    magnitude_color_arr[:,:,:][black_areas.T] = (255, 0, 0, 255) # Convert black to red for outline
    magnitude_color_arr[:,:,:][white_areas.T] = (0, 0, 255, 0) # Make white transparent

    # Overlay over the original image in color
    overlay = Image.fromarray(magnitude_color_arr)
    background.paste(overlay, None, overlay)

    # End calculation
    end_time = timer()
    print('[Problem 1 - Exercise 5] Duration: ' + str(end_time - start_time))

    imshow(background)


# -*- coding: utf-8 -*-
"""
This model takes the winemag data & filters on the north_america and europe continents.
After cleaning and prepping the date,
we run two bayesian hierarchical models (one for each continent) on the data using variational inference & pymc3
"""
import pandas as pd
import numpy as np
import pymc3 as pm
import seaborn as sns
from sklearn.model_selection import train_test_split
import theano
from sklearn.metrics import classification_report

# Use Feature Engineering file to clean data beforehand
from FE import FE

wine = pd.read_csv('winemag-data-130k-v2.csv')
wine_clean = FE(wine)

# For reproducibility
SEED = 20200514

# %% Data cleaning/prep

# Clone initial data sets for cleaning
NA = wine_clean.copy().query('continent == "north_america"')
EU = wine_clean.copy().query('continent == "europe"')

# Convert province to codes to be used in hierarchical priors
province_dict = wine_clean.set_index('province').to_dict()['continent']

NA_dict = np.asarray([i for i,j in province_dict.items() if j == "north_america"])
NA_lookup = dict(zip(NA_dict, range(len(NA_dict))))
NA['province'] = NA.province.replace(NA_lookup).values

EU_dict = np.asarray([i for i,j in province_dict.items() if j == "europe"])
EU_lookup = dict(zip(EU_dict, range(len(EU_dict))))
EU['province'] = EU.province.replace(EU_lookup).values

# Set Other & Nan Values to 0
NA_province = NA['province'] = NA['province'].replace(to_replace={np.nan :0}).astype(int).values
EU_province = NA['province'] = NA['province'].replace(to_replace={np.nan :0}).astype(int).values

NA_unique = NA['province'].unique()
EU_unique = EU['province'].unique()

cont_vars = ['price']
cat_vars = ['year','color']
hierarchical = 'province'
pred = 'points'

NA_data = NA.copy()
EU_data = EU.copy()

# List of variables to be used in regression
colList = cont_vars + cat_vars + [hierarchical] + [pred]

# Drop where we don't have region data
regression_NA = NA_data.query('country.notnull() & price.notnull()' , engine='python')[colList]
regression_EU = EU_data.query('country.notnull() & price.notnull()' , engine='python')[colList]

# Break out categorical & one-hot encode
NA_regression_categorical = pd.get_dummies(regression_NA[cat_vars])
EU_regression_categorical = pd.get_dummies(regression_EU[cat_vars])

# Scale Price Logarithmically & scale
NA_X_mean = np.log(regression_NA[cont_vars]).mean()
NA_X_std = np.log(regression_NA[cont_vars]).std()
NA_regression_continuous = ((np.log(regression_NA[cont_vars]) - NA_X_mean)/NA_X_std)

EU_X_mean = np.log(regression_EU[cont_vars]).mean()
EU_X_std = np.log(regression_EU[cont_vars]).std()
EU_regression_continuous = ((np.log(regression_EU[cont_vars]) - EU_X_mean)/EU_X_std)

NA_X = pd.concat([NA_regression_categorical, NA_regression_continuous, regression_NA[hierarchical]], axis=1)
EU_X = pd.concat([EU_regression_categorical, EU_regression_continuous, regression_EU[hierarchical]], axis=1)

# Scale Y
NA_Y_mean = regression_NA[pred].mean()
NA_Y_std = regression_NA[pred].std()
NA_Y = (regression_NA[pred] - NA_Y_mean)/NA_Y_std

# Scale Y
EU_Y_mean = regression_EU[pred].mean()
EU_Y_std = regression_EU[pred].std()
EU_Y = (regression_EU[pred] - EU_Y_mean)/EU_Y_std

# Split out a train/test set
NA_X_train, NA_X_test, NA_y_train, NA_y_test = train_test_split(NA_X, NA_Y, random_state=SEED, test_size = 0.20)
EU_X_train, EU_X_test, EU_y_train, EU_y_test = train_test_split(EU_X, EU_Y, random_state=SEED, test_size = 0.20)

NA_train_cat = NA_X_train.drop(cont_vars + [hierarchical], axis=1)
EU_train_cat = EU_X_train.drop(cont_vars + [hierarchical], axis=1)

# Set up shape of continents priors
NA_len = len(NA_unique)+1
EU_len = len(EU_unique)+1

# Set up tensors for train/test data
NA_y_tensor = theano.shared(NA_y_train.values.astype('float64'))
NA_x_cont_tensor = theano.shared(NA_X_train[cont_vars].values.astype('float64'))
NA_x_cat_tensor = theano.shared(NA_X_train.drop(cont_vars + [hierarchical], axis=1).values.astype('float64'))
NA_x_province_tensor = theano.shared(NA_X_train['province'].values.astype('int64'))

EU_y_tensor = theano.shared(EU_y_train.values.astype('float64'))
EU_x_cont_tensor = theano.shared(EU_X_train[cont_vars].values.astype('float64'))
EU_x_cat_tensor = theano.shared(EU_X_train.drop(cont_vars + [hierarchical], axis=1).values.astype('float64'))
EU_x_province_tensor = theano.shared(EU_X_train['province'].values.astype('int64'))

# %% Bayesian Linear Regression models
with pm.Model() as NA_model:
    # Hyperpriors
    mu_a = pm.Normal('mu_a',mu=0,sd=1e5)
    sigma_a = pm.HalfCauchy('sigma_a',5)

    # Hierarcical intercept
    a = pm.Normal('a', mu=mu_a, sd=sigma_a, shape=NA_len)

    # Regular intercept
    zbetaj_cont = pm.Normal('zbetaj_cont', mu=0, sd=2)
    zbetaj_cat = pm.Beta('zbetaj_cat', alpha = 1, beta = 1, shape=NA_train_cat.shape[1])

    zmu =  a[NA_x_province_tensor] + pm.math.dot(zbetaj_cont, NA_x_cont_tensor.T) + pm.math.dot(zbetaj_cat, NA_x_cat_tensor.T)

    sigma = pm.HalfCauchy('sigma', beta=10, testval=1.)

    likelihood = pm.Normal('likelihood', mu=zmu, sigma=sigma, observed=NA_y_tensor)

with pm.Model() as EU_model:
    # Hyperpriors
    mu_a = pm.Normal('mu_a',mu=0,sd=1e5)
    sigma_a = pm.HalfCauchy('sigma_a',5)

    # Hierarcical intercept
    a = pm.Normal('a', mu=mu_a, sd=sigma_a, shape=EU_len)

    # Regular intercept
    zbetaj_cont = pm.Normal('zbetaj_cont', mu=0, sd=2)
    zbetaj_cat = pm.Beta('zbetaj_cat', alpha = 1, beta = 1, shape=EU_train_cat.shape[1])

    zmu =  a[EU_x_province_tensor] + pm.math.dot(zbetaj_cont, EU_x_cont_tensor.T) + pm.math.dot(zbetaj_cat, EU_x_cat_tensor.T)

    sigma = pm.HalfCauchy('sigma', beta=10, testval=1.)

    likelihood = pm.Normal('likelihood', mu=zmu, sigma=sigma, observed=EU_y_tensor)

with NA_model:
     NA_advi_fit = pm.fit(n=15000, method = pm.ADVI())

with EU_model:
     EU_advi_fit = pm.fit(n=15000, method = pm.ADVI())
# %% Plot Elbo Results

# plot the ELBO over time

NA_advi_elbo = pd.DataFrame(
    {'ELBO': -NA_advi_fit.hist,
     'n': np.arange(NA_advi_fit.hist.shape[0])})

_ = sns.lineplot(y='ELBO', x='n', data=NA_advi_elbo)

EU_advi_elbo = pd.DataFrame(
    {'ELBO': -EU_advi_fit.hist,
     'n': np.arange(EU_advi_fit.hist.shape[0])})

_ = sns.lineplot(y='ELBO', x='n', data=EU_advi_elbo)

# %% Run data on test split

# Sample to look at regression fit
NA_advi_trace = NA_advi_fit.sample(10000)
EU_advi_trace = EU_advi_fit.sample(10000)

# Switch tensor to test data
NA_y_tensor.set_value(NA_y_test)
NA_x_cont_tensor.set_value(NA_X_test[cont_vars].values)
NA_x_cat_tensor.set_value(NA_X_test.drop(cont_vars + [hierarchical], axis=1).values)
NA_x_province_tensor.set_value(NA_X_test['province'].values)

EU_y_tensor.set_value(EU_y_test)
EU_x_cont_tensor.set_value(EU_X_test[cont_vars].values)
EU_x_cat_tensor.set_value(EU_X_test.drop(cont_vars + [hierarchical], axis=1).values)
EU_x_province_tensor.set_value(EU_X_test['province'].values)

# Generate predictions for test set
NA_advi_posterior_pred = pm.sample_posterior_predictive(NA_advi_trace, 1000, NA_model)
NA_advi_predictions = np.mean(NA_advi_posterior_pred['likelihood'], axis=0)

EU_advi_posterior_pred = pm.sample_posterior_predictive(EU_advi_trace, 1000, EU_model)
EU_advi_predictions = np.mean(EU_advi_posterior_pred['likelihood'], axis=0)

# %% Convert back to original scale & plot results
NA_advi_predictions_orig = NA_advi_predictions*NA_Y_std + NA_Y_mean
NA_y_test_orig = NA_y_test*NA_Y_std + NA_Y_mean

EU_advi_predictions_orig = EU_advi_predictions*EU_Y_std + EU_Y_mean
EU_y_test_orig = EU_y_test*EU_Y_std + EU_Y_mean

NA_prediction_data = pd.DataFrame(
    {'ADVI': NA_advi_predictions_orig[0],
     'actual': NA_y_test_orig,
     'error_ADVI': NA_advi_predictions_orig[0] - NA_y_test_orig})

EU_prediction_data = pd.DataFrame(
    {'ADVI': EU_advi_predictions_orig[0],
     'actual': EU_y_test_orig,
     'error_ADVI': EU_advi_predictions_orig[0] - EU_y_test_orig})

final_prediction_data = pd.concat([NA_prediction_data, EU_prediction_data], axis=0)

# Calculate RMSE
RMSE = np.sqrt(np.mean(final_prediction_data.error_ADVI ** 2))
print(f'RMSE for ADVI predictions = {RMSE:.3f}')

_ = sns.lmplot(y='ADVI', x='actual', data=final_prediction_data,
               line_kws={'color': 'red', 'alpha': 0.5})

class_grid = final_prediction_data.copy()
class_grid['cat_actual'] = np.where(class_grid['actual'] < 86, "BottomTier",
                                      np.where(class_grid['actual'] < 93, "MiddleTier","TopTier"))
class_grid['cat_predicted'] = np.where(class_grid['ADVI'] < 86, "BottomTier",
                                      np.where(class_grid['ADVI'] < 93, "MiddleTier","TopTier"))

print(classification_report(class_grid['cat_actual'],class_grid['cat_predicted'],target_names=['TopTier','MiddleTier','BottomTier']))

pm.model_to_graphviz(NA_model)

# %%


import pytest
from thefuck.types import Command

from thefuck_contrib_scoop.rules.scoop_unknown_command import get_new_command, match
from thefuck_contrib_scoop.scoop import get_aliases, get_commands

@pytest.mark.parametrize(
    "script, output",
    [
        (
            "scoop bucke add versions",
            "scoop: 'bucke' isn't a scoop command. See 'scoop help'.",
        ),
    ],
)
def test_match(script, output):
    assert match(Command(script, output))

@pytest.mark.parametrize(
    "script",
    [(f"scoop {command}") for command in get_commands() + get_aliases()],
)
def test_not_match(script):
    assert not match(Command(script, ""))

@pytest.mark.parametrize(
    "script, output, new_command",
    [
        (
            "scoop inst thefuck",
            "scoop: 'inst' isn't a scoop command. See 'scoop help'.",
            [
                "scoop install thefuck",
                "scoop uninstall thefuck",
                "scoop list thefuck",
            ],
        ),
        (
            "scoop instaa thefuck",
            "scoop: 'instaa' isn't a scoop command. See 'scoop help'.",
            [
                "scoop install thefuck",
                "scoop uninstall thefuck",
                "scoop list thefuck",
            ],
        ),
    ],
)
def test_get_new_command(script, output, new_command):
    command = Command(script, output)
    assert sorted(get_new_command(command)) == sorted(new_command)


#!/usr/bin/env python3
from Crypto.Util import number
from binascii import hexlify, unhexlify
from gmpy2 import next_prime, powmod, gcdext, gcd
from itertools import count
from random import randint

class MPRSA(object):
    def __init__(self):
        self.public_key = None
        self.secret_key = None

    def key_gen(self, bits, prime_numbers=4):
        delta = randint(5, 15)
        bit_prime = int(bits // prime_numbers)

        P = [next_prime(number.getPrime(bit_prime) + 1)]
        for i in range(1, prime_numbers):
            P.append(next_prime(P[i - 1] * delta))

        n = self.__compute_module(P)
        phi = self.__compute_phi(P)

        for d_next in count(int(pow(P[0] // 2, 0.5)), -1):
            g, e, __ = gcdext(d_next, phi)
            if (1 < e < n) and (g == 1) and (gcd(phi, e) == 1):
                d = d_next
                break

        self.public_key = (e, n)
        self.secret_key = (d, n)

    def import_keys(self, public_key, secret_key):
        self.public_key = public_key
        self.secret_key = secret_key

    def export_keys(self):
        return self.public_key, self.secret_key

    @staticmethod
    def __compute_module(primes):
        n = 1
        for prime in primes:
            n *= prime
        return n

    @staticmethod
    def __compute_phi(primes):
        phi = 1
        for prime in primes:
            phi *= (prime - 1)
        return phi

    @staticmethod
    def __encode_message(data):
        return int(hexlify(data), 16)

    @staticmethod
    def __decode_message(data):
        return unhexlify(format(data, "x"))

    def encryption(self, ptext):
        data = self.__encode_message(ptext)
        return powmod(data, self.public_key[0], self.public_key[1])

    def decryption(self, ctext):
        data = powmod(ctext, self.secret_key[0], self.secret_key[1])
        return MPRSA.__decode_message(data)


import pygame

world_size = (1600, 768 + 64)
import random
from data.src.utils.utils import wait

class ScreenShake:

    def __init__(self, display):
        self.display = display
        self.timer = 0
        self.screen_position = [0, 0]
        self.shake = False

    def update(self):
        if self.shake:
            return self.screen_position

    def shake_screen(self):
        if self.shake:
            self.screen_shake()

    def screen_shake(self, amount, intensity):
        """Intensity should be relatively low, 3 is good enough"""
        if not wait(self.display.game, self.timer, amount):
            self.screen_position = [random.randint(-intensity, intensity), random.randint(-intensity, intensity)]

class Display:

    def __init__(self, game):
        self.game = game
        self.display = pygame.display.set_mode(world_size, pygame.RESIZABLE)
        self.screen = pygame.Surface(world_size).convert()
        self.screen_position = [0, 0]
        self.particle_screen = pygame.Surface((world_size[0] / 4, world_size[1] / 4),
                                              pygame.SRCALPHA).convert_alpha()
        self.dest_surf = pygame.Surface((world_size[0], world_size[1]), pygame.SRCALPHA).convert_alpha()
        self.surface = None
        self.timer = 0

    def blit_default_particle_screen(self):
        # if self.game.particle_manager.particles:
        self.screen.blit(
            pygame.transform.scale(self.particle_screen, (world_size[0], world_size[1]), self.dest_surf),
            (0, 0))

    def blit_resized_particle_screen(self):
        x = (0 - self.game.camera.zoom_target_x) * self.game.camera.zoom
        y = (0 - self.game.camera.zoom_target_y) * self.game.camera.zoom
        position = (x + self.game.camera.zoom_target_x, y + self.game.camera.zoom_target_y)

        self.surface = pygame.transform.scale(self.particle_screen.copy(),
                                              (world_size[0] * self.game.camera.zoom,
                                               world_size[1] * self.game.camera.zoom))
        self.screen.blit(self.surface, position)

    def blit_particle_screen(self):
        if self.game.camera.zoom == 1:
            self.blit_default_particle_screen()
        else:
            self.blit_resized_particle_screen()

    def update_screen(self):
        self.screen_shake()

    # self.screen.fill((0, 0, 0))

    def blit_display(self):
        position = ((self.display.get_width() - world_size[0]) / 2, (self.display.get_height() - world_size[1]) / 2)
        self.display.blit(self.screen, self.screen_position)

    def screen_shake(self):
        if not wait(self.game, self.timer, 150):
            self.screen_position = [random.randint(-3, 3), random.randint(-3, 3)]


# Code source: Jaques Grobler
# License: BSD 3 clause
# Code from https://scikit-learn.org/stable/auto_examples/linear_model/plot_ols.html

import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from joblib import dump, load
from numpy import savetxt

# Load the diabetes dataset
diabetes_X, diabetes_y = datasets.load_diabetes(return_X_y=True)

# Use only one feature
#diabetes_X = diabetes_X[:, np.newaxis, 2]

diabetes_X = np.expand_dims(diabetes_X[:, 2], axis=1)

# Split the data into training/testing sets
diabetes_X_train = diabetes_X[:-20]
diabetes_X_test = diabetes_X[-20:]

# Split the targets into training/testing sets
diabetes_y_train = diabetes_y[:-20]
diabetes_y_test = diabetes_y[-20:]

# Create linear regression object
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(diabetes_X_train, diabetes_y_train)

data_test = np.concatenate((diabetes_X_test, np.expand_dims(diabetes_y_test, axis=1)),axis=1)

savetxt('data_test.csv', data_test, delimiter=',')

#print(type(diabetes_X_test))
#np.save("X_test.npy",diabetes_X_test)

#
dump(regr, 'Model/model.joblib')
#regr_loaded = load('C:/Users/dioge/Desktop/Docker_Machine_Learning/Model/model.joblib')

# Make predictions using the testing set
#diabetes_y_pred = regr_loaded.predict(diabetes_X_test)

diabetes_y_pred = regr.predict(diabetes_X_test)

# The coefficients
print("Coefficients: \n", regr.coef_)
# The mean squared error
print("Mean squared error: %.2f" % mean_squared_error(diabetes_y_test, diabetes_y_pred))
# The coefficient of determination: 1 is perfect prediction
print("Coefficient of determination: %.2f" % r2_score(diabetes_y_test, diabetes_y_pred))

"""Coefficients: [938.23786125]
Mean squared error: 2548.07
Coefficient of determination: 0.47"""


import json
import os
import socket
import sys
import threading
import time

class Server:
    __instance = None

    @staticmethod
    def getInstance(callback=None):
        if Server.__instance == None:
            Server(callback)
        return Server.__instance

    def __init__(self, callback):

        if Server.__instance != None:
            raise Exception("This class is a singleton!")
        else:
            Server.__instance = self

        self.callback = callback
        self.clients = {}
        self.size = 2048
        self.port = os.getenv("PORT_SOCKET",2693)
        self.request_time = 10
        self.timeout = 5
        self.addres = os.getenv("HOST","0.0.0.0")
        self.command = {}
        self.data = {}

        self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            self.server.bind((self.addres, int(self.port)))
        except Exception as e:
            print(f"[SERVER] Error: {e}")
            exit()
        print("[SERVER] Starting...")
        thread = threading.Thread(name="server", target=self.run, daemon=True)
        thread.start()

    def handle_client(self, conn, addr):
        print(f"[SERVER] {addr} connected.")
        conn.settimeout(self.timeout)
        left = 3
        while left > 0:
            msg = conn.recv(self.size).decode()
            if msg:
                try:
                    code, name = msg.split(":", 1)
                except:
                    code, name = None, None
                if code == "CON":
                    if self.clients.get(name):
                        if self.ping(name):
                            conn.send(
                                str.encode(
                                    "ERR:Name is taken! Pleace selcet other name."
                                )
                            )
                            left -= 1
                            continue
                    print(f"[SERVER] {addr} registered with name: {name}.")
                    conn.send(str.encode("OK:Connected sucessfuly"))
                    self.clients[name] = conn
                    self.command[name] = None
                    self.data[name] = None
                    self.listen(name)
            left -= 1
        print(f"[SERVER] {addr} timeout.")
        conn.shutdown(socket.SHUT_RDWR)
        sys.exit()

    def listen(self, name):
        while True:
            try:
                msg = self.clients[name].recv(self.size).decode()
            except Exception as e:
                if type(e) == socket.timeout:
                    continue
                print(f"[SERVER] Error: {e}, {self.clients}")
                try:
                    self.clients[name].shutdown(socket.SHUT_RDWR)
                    self.clients.pop(name)
                    self.command.pop(name)
                    self.data.pop(name)
                except:
                    pass
                sys.exit()

            if not msg:
                continue

            code, info = msg.split(":", 1)
            print(f"[SERVER] {code} : {info}")
            if self.command[name]:
                if self.command[name] == "GET":
                    if code != "DATA":
                        self.data[name] = {"error": True, "info": "Wrong response"}
                    else:
                        self.data[name] = {"error": False, "info": json.loads(info)}
                elif self.command[name] == "SET":
                    if code == "ERR":
                        self.data[name] = {"error": True, "info": info}
                    elif code != "OK":
                        self.data[name] = {"error": True, "info": "Wrong response"}
                    else:
                        self.data[name] = {"error": False, "info": "OK"}
                self.command[name] = None
                continue
            if code == "PING":
                self.send("OK:OK", name)
            elif code == "DATA":
                data = json.loads(info)
                self.callback("get", {"name": name, "payload": {"data": data}})
                self.send("OK:OK", name)
            elif code == "OK":
                continue

    def ping(self, name):
        other = self.clients.get(name)
        try:
            other.send(str.encode(f"PING:{name}"))
            msg = other.recv(self.size).decode()
            if not "OK" in msg:
                raise Exception("Not responding")
        except:
            print(f"[SERVER] {name} diconnected (not responding).")
            try:
                other.shutdown(socket.SHUT_RDWR)
            except:
                pass
            self.clients.pop(name)
            self.command.pop(name)
            self.data.pop(name)
            return False
        print(f"[SERVER] {name} is still alive.")
        return True

    def async_ping(self, name):
        thread = threading.Thread(name=f"ping {name}", target=self.ping, args=(name,))
        thread.start()

    def send_set(self, name, data):
        self.data[name] = None
        self.command[name] = "SET"
        self.send(f"SET:{json.dumps(data)}", name)
        start_time = time.perf_counter()
        while self.data[name] is None:
            if time.perf_counter() - start_time > self.request_time:
                self.async_ping(name)
                return {"error": True, "info": "Timeout"}
            pass
        return self.data[name]

    def send_get(self, name):
        self.data[name] = None
        self.command[name] = "GET"
        self.send(f"GET:GET", name)
        start_time = time.perf_counter()
        while self.data[name] is None:
            if time.perf_counter() - start_time > self.request_time:
                self.async_ping(name)
                return {"error": True, "info": "Timeout"}
            pass
        return self.data[name]

    def send_slient_get(self, name):
        self.send(f"GET:GET", name)

    def isName(self, name):
        c = self.clients.get(name)
        if c is None:
            return False
        return True

    def send(self, msg, name):
        c = self.clients.get(name)
        message = msg.encode()
        c.send(message)

    def run(self):
        self.server.listen()
        print(f"[SERVER] Server is listening on {self.port}")
        while True:
            conn, addr = self.server.accept()
            thread = threading.Thread(target=self.handle_client, args=(conn, addr))
            thread.start()

if __name__ == "__main__":
    Server()
    while True:
        pass


#!/usr/bin/python3
# coding: utf-8
import os
import glob
import re
import sys
import shutil
import logging
import subprocess

def logging_init():
    '''
    紀錄檔格式初始化
    格式：
         [ 2015 Nov 19 21:13:06 ] ryHalign.py:090 [INFO] : Generate 4 scp files: genScpFiles() Success!
         [ 2015 Nov 19 21:13:07 ] ryHalign.py:030 [DEBUG] : mkdir hmms_p
    '''
    #Log文件輸出格式定義
    logging.basicConfig(level=logging.DEBUG, format='[ %(asctime)s ] %(filename)s:%(lineno)03d [%(levelname)s] : %(message)s', datefmt='%Y %b %d %H:%M:%S', filename='ryHalign.log', filemode='w')

    #Log訊息輸出到螢幕
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('[%(filename)s:%(lineno)03d](%(levelname)s): %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)

def genScpFiles(dirName):
	'''
	產生 4 個 .scp 檔案：.log
		spWav.scp, spLab.scp, spMfc.scp, spWav2Mfc.scp
	'''
	if os.path.exists(dirName):
		fnName = '*.wav'
		pathName = os.path.join(dirName, fnName)
		wavL = glob.glob(pathName)
		wavScp = '\n'.join(wavL)

		wavScp= wavScp.replace('\\','/')  # 寫在.scp內的path 分隔用正斜/
		labScp= re.sub('.wav','.lab', wavScp)
		mfcScp= re.sub('.wav','.mfc', wavScp)

		wmScp= ''
		if wavScp!='' and mfcScp!='':
			for w,m in zip(wavScp.split('\n'), mfcScp.split('\n')):
				wmScp += w+'\t'+m+'\n'

		lost_file = []

		if wavScp!='':
			try:
				with open('spWav.scp', 'w') as  f:
					f.write(wavScp)
			except IOError as e:
				logging.error(e)
				lost_file.append("spWav.scp")

		if labScp!='':
			try:
				with open('spLab.scp', 'w') as f:
					f.write(labScp)
			except IOError as e:
					logging.error(e)
					lost_file.append("spLab.scp")

		if wmScp!='':
			try:
				with open('spWav2Mfc.scp', 'w') as f:
					f.write(wmScp)
			except IOError as e:
					logging.error(e)
					lost_file.append("spWav2Mfc.scp")

		if mfcScp!='':
			try:
				with open('spMfc.scp', 'w') as f:
					f.write(mfcScp)
			except IOError as e:
					logging.error(e)
					lost_file.append("spMfc.scp")

		#check size of 4 files all greater than 0
		if len(lost_file) > 0:
			logging.error('Need file: '+', '.join(lost_file))
			logging.error("Missing %d files: genScpFiles() Failed ...", len(lost_file))
			return False
		else:
			logging.info("Generate 4 files: genScpFiles() Success ...")
			return True
	else:
		logging.error('No such directory as ' + dirName + '.')

def 建立Hmm原型(N=1, M=1, D=39, fileName=''):
    '''
    產生myHmmPro檔案
    '''
    hmm原型檔名='myHmmPro'
    if fileName=='':
        fileName=hmm原型檔名

    dotPosition=fileName.find('.')
    if dotPosition>=0:
        fileName=fileName[0:dotPosition]

    logging.info('CreateHProto... ' + fileName + ', N=' + str(N) + ', M=' + str(M))
    hProto= '~h ' + '"' + fileName + '"' + '\n' + '~o <VecSize> ' + str(D) + '\n<MFCC_0_D_A> <StreamInfo> 1 ' + str(D) + '\n\n<BeginHMM>\n'

    hProto += '<NumStates> ' + str(N+2) + '\n'

    for state in range(2,N+2): # 2...(N+1)
        hProto += '\n'
        hProto += ' <State> ' + str(state) + '  <NumMixes> ' + str(M) + ' <Stream> 1 \n'

        for mixture in range(1, M+1): # 1...M
            hProto += '  <Mixture> '+str(mixture) + ' ' +str(1.0/M) +'\n'
            hProto += '   <Mean> '+str(D) +'\n'
            hProto += '    '
            for i in range(D):
                hProto += str(0)+' '
            hProto += '\n'
            hProto += '   <Variance> ' + str(D) + '\n'
            hProto += '    '
            for i in range(D):
                hProto += str(1) + ' '
            hProto += '\n'

    hProto += '\n<TransP> ' + str(N+2) + '\n'

    for i in range(0,1):
        hProto += ' '
        for j in range(0,N+2):
            if j==1:
                aij=1.0
            else:
                aij=0.0
            hProto += str(aij) + ' '
        hProto += '\n'
    for i in range(1,N+1):
        hProto += ' '
        for j in range(0,N+2):
            if j==i:
                aij=0.6
            elif j==(i+1):
                aij=0.4
            else:
                aij=0.0
            hProto += str(aij)+' '
        hProto += '\n'
    for i in range(N+1,N+2):
        hProto += ' '
        for j in range(0,N+2):
            aij=0.0
            hProto += str(aij) + ' '
        hProto += '\n'

    hProto += '\n<EndHMM>\n'

    try:
        with open(fileName, 'w') as f:
            f.write(hProto)
    except IOError as e:
        logging.error(e)
        logging.error('建立Hmm原型() Failed ...')
        return False
    else:
        logging.info('Build ‘myHmmPro‘ file Success ...')
        logging.info('建立Hmm原型() Success ...')
        return True

def htk00_製造各個HtkTool所需的參數檔():
    hLed_led='''
    #### hLed.led
    EX    # expand word into phone
    '''

    hLed00_led='''
    #### hLed00.led
    #EX    # expand word into phone
    '''

    hCopy_conf='''
    SOURCEKIND     = WAVEFORM
    TARGETKIND     = MFCC_0     #12d(MFCC)+1d(E) = 13d
    SOURCEFORMAT   = WAV #NIST    #TIMIT
    SOURCERATE     = 625        # T=625e-7 sec ==> Fs = 16KHz
    TARGETRATE     = 100000      # 10 ms

    ENORMALISE=F   # for realtime test
    '''

    hInit_conf='''
    TARGETKIND = MFCC_0_D_A
    SAVEGLOBOPTS = TRUE
    KEEPDISTINCT=F
    '''

    hRest_conf='''
    TARGETKIND = MFCC_0_D_A
    SAVEGLOBOPTS = TRUE
    KEEPDISTINCT=F
    '''

    hErest_conf='''
    TARGETKIND = MFCC_0_D_A
    SAVEGLOBOPTS = TRUE
    KEEPDISTINCT=T
    BINARYACCFORMAT=T
    '''

    hVite_conf='''
    TARGETKIND = MFCC_0_D_A
    SAVEGLOBOPTS = TRUE
    KEEPDISTINCT=F
    BINARYACCFORMAT=F
    '''

    lost_file = []
    try:
        with open('hLed.led','w') as f:
            f.write(hLed_led)
    except IOError as e:
        logging.error(e)
        lost_file.append('hLed.led')

    try:
        with open('hLed00.led','w') as f:
            f.write(hLed00_led)
    except IOError as e:
        logging.error(e)
        lost_file.append('hLed00.led')

    try:
        with open('hCopy.conf','w') as f:
            f.write(hCopy_conf)
    except IOError as e:
        logging.error(e)
        lost_file.append('hCopy.conf')

    try:
        with open('hInit.conf','w') as f:
            f.write(hInit_conf)
    except IOError as e:
        logging.error(e)
        lost_file.append('hInit.conf')

    try:
        with open('hRest.conf','w') as f:
            f.write(hRest_conf)
    except IOError as e:
        logging.error(e)
        lost_file.append('hRest.conf')

    try:
        with open('hErest.conf','w') as f:
            f.write(hErest_conf)
    except IOError as e:
        logging.error(e)
        lost_file.append('hErest.conf')

    try:
        with open('hVite.conf','w') as f:
            f.write(hVite_conf)
    except IOError as e:
        logging.error(e)
        lost_file.append('hVite.conf')

    if len(lost_file) > 0:
        logging.error('Need file: ' + ', '.join(lost_file))
        logging.error("Missing %d files: htk00() Failed ...", len(lost_file))
        return False
    else:
        if 建立Hmm原型(N=3, M=6):
            logging.info("Generate 7 files: htk00() Success ...")
            return True
        else:
            return False

def htk01_處裡語音標籤及詞典():
	if os.path.exists('spLab.scp') and os.path.exists('hLed00.led') and os.path.exists('hLed.led'):
		try:
			cmd = 'HLEd -A -i spLab00.mlf -n spLab00.lst -S spLab.scp  hLed00.led >> HTK.log'
			subprocess.check_call( [cmd], shell=True )
			logging.debug(cmd)

			cmd = 'HLEd -A -i spLab.mlf -n spLab.lst -S spLab.scp  hLed.led >> HTK.log'
			subprocess.check_call( [cmd], shell=True )
			logging.debug(cmd)
		except Exception as e:
			logging.error(e)
		else:
			if lst2dic():
				try:
					cmd = 'HLEd -A -i spLab_p.mlf -n spLab_p.lst -S spLab.scp -I spLab.mlf -d spLab_p.dic hLed.led >> HTK.log'
					subprocess.check_call( [cmd], shell=True )
					logging.debug(cmd)
				except Exception as e:
					logging.error(e)
				else:
					logging.info('htk01() Success ...')
					return True
	else:
		lost_file = []
		for f in ['spLab.scp', 'hLed00.led', 'hLed.led']:
			if not os.path.exists(f):
				lost_file.append(f)
		if len(lost_file) > 0:
			logging.error('Need file: ' + ', '.join(lost_file))
	logging.error('htk01() Failed ...')
	return False

def lst2dic():
	lost_file = []
	try:
		with open('spLab.lst') as f:
			lines = f.readlines()
	except IOError as e:
		logging.error(e)
		logging.error('spLab.lst sort Failed ...')
	else:
		if lines!='':
			lines.sort()  # htk.dic 需要 sort
		try:
			with open('spLab.lst','w') as f:
				for l in lines:
					f.write(l)
		except IOError as e:
			logging.error(e)
			logging.error('spLab.lst sort Failed ...')

	D = ''
	Dp = ''
	for l in lines:
		d=l.rstrip('\n')
		D += d + ' ' + d + '\n'

		if d=='sil' or len(d)<2:
			ps = d
		else:
			ps = ''
			for n in range(len(d)-1):
				ps += d[n] + d[n+1] + ' '

		Dp += d + ' ' + ps + '\n'

	try:
		with open('spLab.dic','w') as f:
			f.write(D)
	except IOError as e:
		logging.error(e)
		lost_file.append('spLab.dic')

	try:
		with open('spLab_p.dic','w') as f:
			f.write(Dp)
	except IOError as e:
		logging.error(e)
		lost_file.append('spLab_p.dic')

	if len(lost_file) > 0:
		logging.error('Need file: ' + ', '.join(lost_file))
		logging.error("Missing %d files: lst2dic() Failed ...", len(lost_file))
		logging.error('lst2dic() Failed ...')
		return False
	else:
		logging.info('Generate 3 files: lst2dic() Success ...')
		return True

def htk02_擷取語音特徵及訓練語音模型():
	if os.path.exists('spWav2Mfc.scp'):
		try:
			cmd = 'HCopy -A -C hCopy.conf -S spWav2Mfc.scp >> HTK.log'
			subprocess.check_call( [cmd], shell=True )
			logging.debug(cmd)
		except Exception as e:
			logging.error(e)

		try:
			with open('spLab_p.lst') as f:
				lines = f.readlines()
		except IOError as e:
			logging.error(e)
		else:
			#if hmms_p folder exists, remove and create again.
			if os.path.exists('hmms_p'):
				try:
					shutil.rmtree('hmms_p')
				except Exception as e:
					logging.error(e)

			try:
				os.mkdir('hmms_p')
			except Exception as e:
				logging.error(e)
			else:
				mList = []
				for l in lines:
					m = l.strip('\n')
					mList.append(m)

				try:
					cmd = 'HCompV -A -C hInit.conf -S spMfc.scp -m -I spLab_p.mlf -M hmms_p -o myHCompV myHmmPro >> HTK.log'
					subprocess.check_call( [cmd], shell=True )
					logging.debug(cmd)
				except Exception as e:
					logging.error(e)
				else:
					try:
						with open(os.path.join('hmms_p', 'myHCompV')) as f:
							myHCompV = f.read()
					except IOError as e:
						logging.error(e)
					else:
						for m in mList:
							myModel = myHCompV.replace('myHCompV',m)
							try:
								with open( os.path.join('hmms_p', m),'w') as f:
									f.write(myModel)
							except IOError as e:
								logging.error(e)

						repeat_time = 0
						for i in range(5):
							logging.debug('[%d]HERest '%i)
							try:
								cmd = 'HERest -A -C hErest.conf -S spMfc.scp -p 1 -t 2000.0 -w 3 -v 0.05 -I spLab_p.mlf -M hmms_p -d hmms_p spLab_p.lst >> HTK.log'
								subprocess.check_call( [cmd], shell=True )
								logging.debug(cmd)

								cmd = 'HERest -A -C hErest.conf -p 0 -t 2000.0 -w 3 -v 0.05 -I spLab_p.mlf -M hmms_p -d hmms_p spLab_p.lst ' + os.path.join('hmms_p', 'HER1.acc') + ' >> HTK.log'
								subprocess.check_call( [cmd], shell=True )
								logging.debug(cmd)
							except Exception as e:
								logging.error(e)
							else:
								repeat_time += 1
						if repeat_time == 5:
							logging.info('htk02() Success ...')
							return True
	logging.error('htk02() Failed ...')
	return False

def htk03_語音文字對齊():
	try:
		cmd = 'HVite -A -C hVite.conf  -S spMfc.scp  -a -d hmms_p/ -i spLab_aligned.mlf -I spLab.mlf spLab_p.dic spLab_p.lst >> HTK.log'
		subprocess.check_call( [cmd], shell=True )
		logging.debug(cmd)
		logging.debug('generate ‘spLab_aligned.mlf‘')
	except Exception as e:
		logging.error(e)
		logging.error('htk03() Failed ...')
		return False
	else:
		logging.info('htk03() Success ...')
		return True

def 主程式():
	try:
		f_index = sys.argv.index('-d')
	except ValueError as e:
		dirName = 'wavDir'
	else:
		if len(sys.argv) >= f_index+2 and sys.argv[f_index+1][0]!='-':
			dirName = sys.argv[f_index+1]
		else:
			print('ryHalign.py: error: missing filename after ‘-d‘')
			print('Try \'ryHalign.py -d wavDir\'.')
			exit(1)

	logging_init()
	r = genScpFiles(dirName)
	if r:
		r = htk00_製造各個HtkTool所需的參數檔()
	if r:
		r = htk01_處裡語音標籤及詞典()
	if r:
		r = htk02_擷取語音特徵及訓練語音模型()
	if r:
		#----> 'spLab_aligned.mlf'
		r = htk03_語音文字對齊()
	if r:
		保存結果(dirName)

	清理垃圾(dirName)

def 保存結果(dirName):
    save_file = 0
    # moving alignedFile
    try:
        shutil.copy( "spLab_aligned.mlf", dirName+".align" )
    except IOError as e:
        logging.error(e)
    else:
        save_file += 1

    #moving  _all.mp3 file
    try:
        shutil.copy( os.path.join(dirName,"_all.mp3"), dirName+".mp3" )
    except IOError as e:
        logging.error(e)
    else:
        save_file += 1

    #moving _all.txt file
    try:
        shutil.copy( os.path.join(dirName,"_all.txt"), dirName+".txt" )
    except IOError as e:
        logging.error(e)
    else:
        save_file += 1

    if save_file == 3:
        logging.info('保存結果() Success ...')
        return True
    else:
        logging.error('保存結果() Failed ...')
        return False

def 清理垃圾(dirName):
    #remove "myHmmPro" file
    try:
        os.remove("myHmmPro")
    except IOError as e:
        logging.debug(e)

    #remove file with extension in rm_extension_list
    rm_extension_list = ['.mlf', '.conf', '.lst', '.scp', '.led', '.dic']
    for file in os.listdir(os.getcwd()):
        if os.path.splitext(file)[1] in rm_extension_list:
            try:
                os.remove(file)
            except IOError as e:
                logging.debug(e)

    #remove "hmms_p" folder and it's child
    try:
        shutil.rmtree('hmms_p')
    except IOError as e:
        logging.debug(e)

    #remove "dirName" folder and it's child
    '''
    try:
        shutil.rmtree(dirName)
    except IOError as e:
        logging.debug(e)
    '''

if __name__=='__main__':
    主程式()


# Import the dependencies.
import numpy as np
import pandas as pd
import datetime as dt
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func
from flask import Flask, jsonify

#################################################
# Database Setup
#################################################
# reflects hawaii.sqlite database into a new model
engine = create_engine("sqlite:///Resources/hawaii.sqlite",
                       connect_args={'check_same_thread': False})
base = automap_base()

# reflects the tables
base.prepare(autoload_with=engine)

# Save references to each table, measurement and station
measurement = base.classes.measurement
station = base.classes.station

# Create our session (link) from Python to the DB
session = Session(engine)

#################################################
# Flask Setup
#################################################
app = Flask(__name__)

#################################################
# Flask Routes
#################################################

# Home page - gives list of possible routes
@app.route("/")
def index():
    return (
            f"Available Routes:"
            f"
"
            f"
Precipitation :  /api/v1.0/precipitation
"
            f"
Stations:  /api/v1.0/stations
"
            f"
Temperature:  /api/v1.0/tobs
"
            f"
Insert Start Date (YYYY-MM-DD): /api/v1.0/<start>
"
            f"
Insert Date Range (YYYY-MM-DD): /api/v1.0/<start>/<end>
"
            f"
"
            )

# Precipitation - gives latest year precipitation data in JSON format
@app.route("/api/v1.0/precipitation")
def precipitation():
    # Finds the most recent date in the data
    latest_date = session.query(measurement.date).\
                        order_by(measurement.date.desc()).\
                        first()

    # Converts latest date to datetime format
    latest_date_dt = dt.datetime.strptime(latest_date[0], '%Y-%m-%d').date()
    # Finds the date exactly one year from the latest date
    query_date = latest_date_dt - dt.timedelta(weeks = 52.2)

    # Queries the date and precipitation from the latest year in descending order
    latest_year_data =  session.query(measurement.date, measurement.prcp).\
    filter(measurement.date >= query_date).\
    order_by(measurement.date.desc()).all()

    # Formats data into a dictionary
    latest_year_precip = []
    for date, prcp in latest_year_data:
        dates_precip = {}
        dates_precip["date"] = date
        dates_precip["prcp"] = prcp
        latest_year_precip.append(dates_precip)

    # Returns in json format
    return jsonify(latest_year_precip)

# Stations - gives all distinct stations in json format
@app.route("/api/v1.0/stations")
def stations():
    # Queries all station information
    all_stations_query = session.query(station.id,
                                       station.station,
                                       station.name,
                                       station.latitude,
                                       station.longitude,
                                       station.elevation).\
                        all()

    # Formats data into a dictionary
    all_stations = []
    for id, stn, name, lat, lng, elev in all_stations_query:
        stations_dict = {}
        stations_dict["id"] = id
        stations_dict["station"] = stn
        stations_dict["name"] = name
        stations_dict["lat"] = lat
        stations_dict["long"] = lng
        stations_dict["elevation"] = elev
        all_stations.append(stations_dict)

    # Returns in json format
    return jsonify(all_stations)

@app.route("/api/v1.0/tobs")
def tobs():
    # Finds the most recent date in the data
    latest_date_temp = session.query(measurement.date).\
                        order_by(measurement.date.desc()).\
                        first()

    # Converts latest date to datetime format
    latest_date_temp_dt = dt.datetime.strptime(latest_date_temp[0], '%Y-%m-%d').date()
    # Finds the date exactly one year from the latest date
    query_date_temp = latest_date_temp_dt - dt.timedelta(weeks = 52.2)

    # Queries most active stations and the number of measurements per station
    station_query = session.query(measurement.station, func.count(measurement.station)).\
                            group_by(measurement.station).\
                            order_by(func.count(measurement.station).desc()).\
                            all()

    # Queries temperature from the latest year and from the most active station
    latest_year_temp_query =  session.query(measurement.date, measurement.tobs).\
                                filter(measurement.station == station_query[0][0]).\
                                filter((measurement.date >= query_date_temp)).\
                                order_by(measurement.date).\
                                all()

    # Formats data into a dictionary
    latest_year_temp = []
    for date, temp in latest_year_temp_query:
        dates_temp = {}
        dates_temp["date"] = date
        dates_temp["temp"] = temp
        latest_year_temp.append(dates_temp)

    # Returns in json format
    return jsonify(latest_year_temp)

@app.route("/api/v1.0/<start>")
def start(start):
    # Converts user date input into datetime format
    start_date = dt.datetime.strptime(start, '%Y-%m-%d').date()

    # Queries all temperature measurement from start date to latest date
    start_temp_query = session.query(measurement.date, measurement.tobs).\
                               filter(measurement.date >= start_date).\
                               order_by(measurement.date).\
                               all()

    # Formats data into a dictionary
    start_date_temp = []
    for date, temp in start_temp_query:
        dates_temp = {}
        dates_temp["date"] = date
        dates_temp["temp"] = temp
        start_date_temp.append(dates_temp)

    # Returns in json format
    return jsonify(start_date_temp)

@app.route("/api/v1.0/<start>/<end>")
def end(start, end):
    # Finds the most recent date in the data
    latest_date = session.query(measurement.date).\
                          order_by(measurement.date.desc()).\
                          first()

    # Converts latest date to datetime format
    latest_date_dt = dt.datetime.strptime(latest_date[0], '%Y-%m-%d').date()

    # Converts user date input into datetime format
    start_date = dt.datetime.strptime(start, '%Y-%m-%d').date()
    end_date = dt.datetime.strptime(end, '%Y-%m-%d').date()

    # Queries all temperatures between start and end dates
    start_end_temp_query = session.query(measurement.date, measurement.tobs).\
                               filter(measurement.date >= start_date).\
                               filter(measurement.date <= end_date).\
                               order_by(measurement.date).\
                               all()

    # Formats data into a dictionary
    start_end_date_temp = []
    for date, temp in start_end_temp_query:
        dates_temp = {}
        dates_temp["date"] = date
        dates_temp["temp"] = temp
        start_end_date_temp.append(dates_temp)

    # Returns in json format
    return jsonify(start_end_date_temp)

if __name__ == "__main__":
    app.run(debug=True)


"""
It is a file that crops the mp3 and srt file by the durations
which is getting from the srt file.
"""
import os
from ..helper.helper import run_bash, parse_time

class CropMp3Srt:
    """
    Gets the mp3 and srt files to crop
    """

    def __init__(self, filepath):
        self._path = filepath

    def crop(self):
        """
        It is a public function which is used from the other packages or files
        to perform main purpose
        """
        dirs = os.listdir(self._path)
        for file in dirs:
            if not file.startswith("crop_"):
                if file.endswith(".srt"):
                    self._crop_mp3_srt(file)

    def _crop_mp3_srt(self, filename):
        # extract video_id from the filename
        _video_id = filename[:-4]
        # read from the given file.
        with open(self._path + "/" + filename, 'r') as _filehandle:
            _filecontent_list = _filehandle.readlines()

            # it starts at index one and increases at adding 4
            for line in range(1, len(_filecontent_list), 4):
                _current_sub_num = _filecontent_list[line - 1][:-1]

                # to split the times (start and end times)
                _split__current_times = _filecontent_list[line].split(' ')

                # crop mp3 by the current times
                # PARSE TIMES
                # before given time, the duration time will get
                # as _split__current_times[:-1],
                # because it has "\n"
                _current_start = parse_time(_split__current_times[0])
                _current_end = parse_time(_split__current_times[2])
                # CALL THE CROPPING MP3 func
                self._crop_mp3(_video_id, _current_sub_num,
                               _current_start, _current_end)

                # CALL the function which WILL SAVE THE CURRENT
                # SUBTITLE to the TXT FILE
                self._save_sub_txt(_video_id, _current_sub_num,
                                   _filecontent_list[line + 1])

    def onetime_save_cropped_mp3_srt(self, video_id, sub_id,
                                     current_sub, start_time, end_time):
        """
        Saves each cropped file and performing cropping of the mp3 file
        :param video_id: is the current video id
        which is using to save cropped files
        :param sub_id: is using for naming to save cropped files
        :param current_sub: it saves to the new cropped text file
        :param start_time: is used to crop the mp3 file
        :param end_time: is used to crop the mp3 file
        """
        _current_start = parse_time(start_time)
        _current_end = parse_time(end_time)
        self._crop_mp3(video_id, str(sub_id), _current_start, _current_end)
        self._save_sub_txt(video_id, str(sub_id), current_sub)

    def _save_sub_txt(self, video_id, sub_id, current_subtitle):
        _new_srt_name = self._path + "/" + video_id\
                        + "_crop_" + sub_id + ".txt"
        with open(_new_srt_name, 'w+') as new_cropped_file:
            new_cropped_file.write(current_subtitle)
        new_cropped_file.close()

    def _crop_mp3(self, video_id, sub_id, start_time, end_time):
        """
        video_id is the video id from the youtube
        sub_id is the index from the subtitle list
        start time the starting point to crop
        """
        input_filename = self._path + "/" + video_id + ".mp3"
        new_file_name = self._path + "/" + video_id+"_crop_" + sub_id + ".mp3"

        # -n means never override
        command = "ffmpeg -n -i " + input_filename + " -ss  " + \
                  start_time + " -to " + end_time + " -c copy " \
                  + new_file_name + " 2> /dev/null"

        run_bash(command)


from app import app
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import json

class IneffableStorage(object):

    def __init__(self):
        """ Initialize the class """
        self.connection = None
        self.bucket = None

    def setup_connection(self):
        """ Setup the connection to S3 """
        if self.connection is None:
            self.connection = S3Connection(
                aws_access_key_id=app.config['AWS_ACCESS_KEY_ID'],
                aws_secret_access_key=app.config['AWS_SECRET_ACCESS_KEY']
            )

    def setup_bucket(self):
        """ Setup the connection to the S3 bucket """
        self.setup_connection()
        if self.bucket is None:
            self.bucket = self.connection.get_bucket(app.config['AWS_S3_BUCKET'])

    def get_bucket(self):
        """ Get an S3 bucket """
        self.setup_bucket()
        return self.bucket

    def get_key(self, key_name):
        """ Get a key for a specific bucket """
        self.setup_bucket()

        key = Key(self.bucket)
        key.key = key_name

        return key

ineffable_storage = IneffableStorage()


from __future__ import unicode_literals
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from hazm import *
import fasttext
import string
import emoji
import hazm
import json
import os
import re
#
normalizer = Normalizer()

def remove_extra_chars(text):
    eng_alphabets = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',\
                     'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
    eng_pattern = r'(?:{})'.format('|'.join([r"{}".format(ch) for ch in eng_alphabets]))

    with open("words_with_2_duplicate_letters.txt", encoding="utf-8") as text_file:
        certain_words = [line.strip() for line in text_file.readlines()]

    text = text.lower()
    # if url exists in the text
    url_pattern = r"(?:(?:https?):\/\/)?[\w/\-?=%.]+\.[\w/\-?=%.]+"
    if re.findall(url_pattern, text, re.IGNORECASE):
        text = re.sub(url_pattern,"",text)

    if re.findall(r"@(\w+[.]*\b)", text):
        text = re.sub(r"@(\w+[.]*\b)","",text)

    # if number length is more than 3 numbers
    text = re.sub(r'\d{3,}', '', text)

    # remove english characters
    text = re.sub(eng_pattern, '', text)

    # if text contains @
    # text = re.sub(r'\s@\s', '', text)

    # if text contains extra character
    # remove all punctuation in fasttext or glove in ai_model
    regex = re.compile('[%s]' % re.escape(string.punctuation))
    text = regex.sub(' ', text)

    text = unifing_alphabets(text)

    duplicate_letter_pattern = re.compile(r"(.)\1{2,}")
    text_with_pattern2 = duplicate_letter_pattern.sub(r"\1\1", text)
    differences2 = [word for word in text_with_pattern2.split(" ") if word not in text.split(" ")]
    if differences2:
        curse_pattern = r'\b(?:{})\b'.format('|'.join(certain_words))
        if re.findall(curse_pattern, " ".join(differences2)):
            text = text_with_pattern2
        else:
            text = duplicate_letter_pattern.sub(r"\1", text)

    text = (" ").join([word.strip() for word in text.split()])
    return text

def emoji_space(text):
    regex = re.compile(r'\d+(.*?)(?:\u263a|\U0001f645)')
    all_emoji = regex.findall(text)

    edited_text = []
    for c in text:
        if c in emoji.UNICODE_EMOJI:
            edited_text.append(" "+c+" ")
        else:
            edited_text.append(c)

    edited_text = ("").join(edited_text)
    return(edited_text)

def unifing_alphabets(text):
    text = re.sub(r'اَ|اِ|اُ|اٌ|اٍ|اً|أ|إ', 'ا', text)
    text = re.sub(r'بَ|بِ|بُ|بّ', 'ب', text)
    text = re.sub(r'پَ|پِ|پُ', 'پ', text)
    text = re.sub(r'تَ|تِ|تُ|تّ', 'ت', text)
    text = re.sub(r'ثَ|ثِ|ثُ', 'ث', text)
    text = re.sub(r'جَ|جِ|جُ|جّ', 'ج', text)
    text = re.sub(r'چَ|چِ|چُ|چّ', 'چ', text)
    text = re.sub(r'حَ|حِ|حُ', 'ح', text)
    text = re.sub(r'خَ|خِ|خُ', 'خ', text)
    text = re.sub(r'دَ|دِ|دُ|دّ', 'د', text)
    text = re.sub(r'ذَ|ذِ|ذُ', 'ذ', text)
    text = re.sub(r'رَ|رِ|رُ|رّ', 'ر', text)
    text = re.sub(r'زَ|زِ|زُ|زّ', 'ز', text)
    text = re.sub(r'ژَ|ژِ|ژُ', 'ژ', text)
    text = re.sub(r'سَ|سِ|سُ|سّ', 'س', text)
    text = re.sub(r'شَ|شِ|شُ', 'ش', text)
    text = re.sub(r'صَ|صِ|صُ', 'ص', text)
    text = re.sub(r'ضَ|ضِ|ضُ', 'ض', text)
    text = re.sub(r'طَ|طِ|طُ', 'ط', text)
    text = re.sub(r'ظَ|ظِ|ظُ', 'ظ', text)
    text = re.sub(r'عَ|عِ|عُ', 'ع', text)
    text = re.sub(r'غَ|غِ|غُ', 'غ', text)
    text = re.sub(r'فَ|فِ|فُ|فّ', 'ف', text)
    text = re.sub(r'قَ|قِ|قُ|قّ', 'ق', text)
    text = re.sub(r'کَ|کِ|کُ|ك|كَ|كِ|كُ|کّ|كّ', 'ک', text)
    text = re.sub(r'گَ|گِ|گُ', 'گ', text)
    text = re.sub(r'لَ|لِ|لُ|لّ', 'ل', text)
    text = re.sub(r'مَ|مِ|مُ', 'م', text)
    text = re.sub(r'نَ|نِ|نُ|نّ', 'ن', text)
    text = re.sub(r'وَ|وِ|وُ|ؤ|ؤَ|ؤُ|ؤِ|وّ|ؤّ', 'و', text)
    text = re.sub(r'هَ|هِ|هُ|ة', 'ه', text)
    text = re.sub(r'یَ|یِ|یُ|ي|يَ|يِ|يُ|يّ|یّ', 'ی', text)
    text = re.sub(r'ئَ|ئِ|ئُ|ئّ', 'ئ', text)
    return text

train_dataset_path ="fastai_7lbl_train2.txt"
test_dataset_path= "fastai_7lbl_test2.txt"

model = fasttext.train_supervised(train_dataset_path,
                                 dim=100,
                                 lr=0.9, #[0.1 - 1.0]
                                 loss="hs",
                                 wordNgrams=7, #[1 - 5]
                                 ws=3,
                                 epoch=100, #[5 - 50]
                                 minn=3,#3
                                 maxn=6,#6
                                 lrUpdateRate=10,
                                 neg=10,
                                 t=0.0001
                                 )

model.quantize(input=train_dataset_path,
               qnorm=True,
               retrain=True,
               epoch=1,
               cutoff=100000
               )

quantized_model_path = "fastai_7lbl.ftz"
model.save_model(quantized_model_path)

print("test: ", model.test(test_dataset_path))

#number label
labels = set()
with open("fastai_7lbl_test2.txt", 'r') as f:
    for line in f:
        # print(line)
        labels.add(line.split(' ')[0])

print(labels,"test")

labels = set()
with open("fastai_7lbl_train2.txt", 'r') as f:
    for line in f:
        # print(line)
        labels.add(line.split(' ')[0])

print(labels,"train")

fasttext_model = fasttext.load_model("fastai_7lbl.ftz")
count=0
y_pred=[]
y_true=[]
with open ('fastai_7lbl_test2.txt', 'r') as f :
    # print(f)
    for line in f:
        # print(y_true[line])
        # print(line)
        label = line.split(' ', maxsplit=1)[0].strip()

        y_true.append(label)

        sentence = line.split(' ', maxsplit=1)[1].strip()

        outputs = fasttext_model.predict(sentence)

        y_pred.append(outputs[0])

def plot_confusion_matrix(y_true, y_pred, classes,
                          normalize=False,
                          title=None,
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if not title:
        if normalize:
            title = 'Normalized confusion matrix'
        else:
            title = 'Confusion matrix, without normalization'

    # Compute confusion matrix
    cm = confusion_matrix(y_true, y_pred)
    # Only use the labels that appear in the data
    #     classes = classes[unique_labels(y_true, y_pred)]
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    fig, ax = plt.subplots()
    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)
    # We want to show all ticks...
    ax.set(xticks=np.arange(cm.shape[1]),
           yticks=np.arange(cm.shape[0]),
           # ... and label them with the respective list entries
           xticklabels=classes, yticklabels=classes,
           title=title,
           ylabel='True label',
           xlabel='Predicted label')

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")

    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            ax.text(j, i, format(cm[i, j], fmt),
                    ha="center", va="center",
                    color="white" if cm[i, j] > thresh else "black")
    fig.tight_layout()
    return ax

class_names = ['__label__4', '__label__7', '__label__5', '__label__1', '__label__6', '__label__2', '__label__3']

# Plot non-normalized confusion matrix
plot_confusion_matrix(y_true, y_pred, classes=class_names,
                      title='Confusion matrix, without normalization')

# Plot normalized confusion matrix
plot_confusion_matrix(y_true, y_pred, classes=class_names, normalize=True,
                      title='Normalized confusion matrix')

plt.show()


# Capture multiple Faces from multiple users to be stored on a DataBase (dataset directory)
# ==> Faces will be stored on a directory: dataset/ (if does not exist, pls create one)
# ==> Each face will have a unique numeric integer ID as 1, 2, 3, etc

import cv2
from scripts.database_connection import Connection
from tkinter import *

class Photographer:
    def __init__(self, f, i, o="", face_id=None):
        self._face_id = face_id
        _to_database = [self._face_id, f, i, o]

        conn, cursor = Connection.connect("WorkAttendance")
        print (conn, cursor)
        try:
            cursor.execute("""INSERT INTO
                            WorkAttendance.dbo.FIO_Tabels(Табельный_номер, Фамилия, Имя, Отчество)
                            VALUES(?,?,?,?)""", _to_database)
            cursor.commit()
            cursor.close()
            conn.close()
        except Exception as e:
            print("[Ошибка!] " + str(e))
            window = Tk()
            window.wm_title("Ошибка!")
            Label(window, text=str(e),
                  font="arial 14").grid(row=0, column=0)
            window.mainloop()
            cv2.destroyAllWindows()

    def make_photo_set(self):
        cam = cv2.VideoCapture(0)
        cam.set(5, 1900)  # set video width
        cam.set(5, 1000)  # set video height

        face_detector = cv2.CascadeClassifier('Cascades/haarcascade_frontalface_default.xml')
        eye_detector = cv2.CascadeClassifier('Cascades/haarcascade_eye.xml')

        print("[INFO] Инициализация захвата лица. " +
              "\nСмотрите в камеру и равномерно, медленно поворачивайте лицо по сторонам.")

        # Initialize individual sampling face count
        count = 0

        while True:
            ret, img = cam.read()
            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
            faces = face_detector.detectMultiScale(
                gray,
                scaleFactor=1.1,
                minNeighbors=5,
                minSize=(20, 20)
            )

            for (x, y, w, h) in faces:
                face_saved = False
                count += 1
                roi_gray = gray[y:y+h, x:x+w]

                eyes = eye_detector.detectMultiScale(
                    roi_gray,
                    scaleFactor=1.1,
                    minNeighbors=5,
                    minSize=(5, 5)
                )

                for (ex, ey, ew, eh) in eyes:
                    cv2.rectangle(img, (x, y), (x+w, y+h), (255, 0, 0), 2)
                    cv2.imshow('video', img)
                    if not face_saved:
                        # Save the captured image into the dataset folder
                        cv2.imwrite("dataset/User." + str(self._face_id) + '.' + str(count) + ".jpg",
                                    gray[y:y+h, x:x+w])
                        face_saved = True

            k = cv2.waitKey(100) & 0xff  # Press 'ESC' for exiting video
            if k == 27:
                break
            elif count >= 150:  # Take 150 face sample and stop video
                break

        # Do a bit of clean up
        print("[INFO] Exiting Program and cleaning up stuff")
        cam.release()
        cv2.destroyAllWindows()

if __name__ == "__main__":
    face_id = input("Введите табельный номер работика и нажмите <Return> ==> ")
    f, i, o = input("Введите ФИО ==> ").split(" ")
    Photographer(f, i, o, face_id).make_photo_set()
else:
    "Вы исользуете frontal_face_dataset_with_database_with_control как библиотеку."