import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import PIL
import csv
import glob

def storageImg(files):
    images = [PIL.Image.open(file) for file in files]
    return images

def csvWriter(fil_name, nparray):
    example = nparray.tolist()
    with open(fil_name+'.csv', 'w', newline='') as csvfile:
        writer = csv.writer(csvfile, delimiter=',')
        writer.writerows(example)

def generateCSV(images, files):
    for image, name in zip(images, files):
        image = np.array(image)
        CSVlist = [csvWriter(name, image)]

def run(files):
    images = storageImg(files)
    generateCSV(images, files)
    return images

def intervals(locRound, size):
    intervaledLoc = [[[int(loc[0] - size), int(loc[0] + size)], [int(loc[1] - size), int(loc[1] + size)]] for loc in
                     locRound]
    return np.abs(intervaledLoc)

def finalMean(concentrationsSliced):
  means = np.array([np.mean(matrices) for location in concentrationsSliced for matrices in location ])
  return means

def csvWriterPandas(fil_name, example):
  with open(fil_name+'.csv', 'w', newline='') as csvfile:
    writer = csv.writer(csvfile, delimiter=',')
    writer.writerows(example)

from sklearn.impute import SimpleImputer

# Imputation
def imputation(dataframe):
  my_imputer = SimpleImputer()
  imputed_dataframe = pd.DataFrame(my_imputer.fit_transform(dataframe))
  imputed_dataframe.describe()
  return imputed_dataframe

def sliceMap(coodinates, mapwidth, mapheight):
    max = np.max(coordinates, axis=0)
    min = np.min(coordinates, axis=0)
    rounded = [np.round(max), np.round(min)]
    factors = rounded[0] - rounded[1]
    translated = (coordinates + [90, 180]) * 4  # [mapwidth, mapheight]/factors
    coordOffset = np.around(translated).astype(int)
    return np.around(translated).astype(int)

def unitmean(imputed_dataframe, range, loctranslated):
  inflim_lat  = np.around(loctranslated[:,0]-range).astype(int)
  suplim_lat  = np.around(loctranslated[:,0]+range).astype(int)
  inflim_long = np.around(loctranslated[:,1]-range).astype(int)
  suplim_long = np.around(loctranslated[:,1]+range).astype(int)
  means = []
  for i,_ in enumerate(inflim_lat):
    lat = np.arange(inflim_lat[i],suplim_lat[i])
    long = np.arange(inflim_long[i],suplim_long[i])
    holder = np.asarray(imputed_dataframe)[lat, long]
    meanHolder = np.mean(holder)
    means.append(meanHolder)
  return means

def concentrationMean(arrayImages, intervaled):
  region = [arrayImages[interval[0][0]:interval[0][1], #latitudes
                    interval[1][0]:interval[1][1], #longitedes
                    1:] for interval in intervaled] #RGB channels
  return np.array(region)

years = ['2018']
months = ['04', '05', '06', '07', '08', '09', '10', '11', '12']

for year in years:
 for month in months:
     folder = '../images/'+year+'/'+month+'/*.bmp'
     files = glob.glob(folder)
     print(files)

     images = run(files)

     coordinates = np.array([[33.009729, -86.767986],
                             [65.662974, -152.680225],
                             [34.861482, -92.445789],
                             [34.490558, -111.889376],
                             [36.991554, -119.763117],
                             [38.607915, 98.530059],
                             [35.637753, -79.171863],
                             [33.888385, -80.772328],
                             [39.085679, -105.753043],
                             [41.555102, -72.674349],
                             [47.301324, -100.406133],
                             [44.403356, -100.362945],
                             [38.938108, -75.472131],
                             [28.180683, -81.566901],
                             [32.759085, -83.419519],
                             [19.625318, -155.524609],
                             [44.383125, -114.627169],
                             [41.657752, -71.51467],
                             [40.056942, -89.320978],
                             [39.824493, -86.145119],
                             [41.895684, -93.383126],
                             [37.6265311, -88.0921685],
                             [30.532277, -91.859954],
                             [45.112176, -69.21317],
                             [38.969392, -76.511818],
                             [42.348583, -71.886931],
                             [44.431373, -84.723837],
                             [45.999328, -94.490988],
                             [32.62873, -89.902164],
                             [38.481367, -92.538641],
                             [47.025732, -109.784619],
                             [41.537191, -100.20987],
                             [39.328261, -117.089878],
                             [43.645746, -71.591645],
                             [39.995839, -74.584874],
                             [43.015146, -75.696376],
                             [34.396468, -105.949359],
                             [35.493897, -97.311148],
                             [40.189677, -82.848862],
                             [44.052884, -120.763447],
                             [40.750438, -77.670567],
                             [35.857756, -86.331751],
                             [31.325608, -99.375351],
                             [39.274899, -111.745956],
                             [43.944735, -72.735493],
                             [37.482976, -78.703476],
                             [38.601185, -80.727662],
                             [47.242266, -120.507092],
                             [44.609367, -89.748056],
                             [42.896405, -107.59752]])

     print(np.max(coordinates, axis=0), np.min(coordinates, axis=0))

     max = np.max(coordinates, axis=0)
     min = np.min(coordinates, axis=0)
     rounded = [np.round(max), np.round(min)]
     print(rounded)

     arrayImages = [np.array(img) for img in images]
     arrayImages[0].shape
     factors = rounded[0] - rounded[1]
     print(factors)

     LAT = factors[0]
     LON = factors[1]
     MIN_LAT = rounded[1][0]
     MIN_LON = rounded[1][1]

     coordOffset = coordinates - rounded[1]
     translated = coordOffset * 512 / factors
     transRound = np.round(translated)
     print(factors / 512)

     kms = factors * 111 / 512

     intervaled = intervals(transRound, 2)

     concentrationExample = concentrationMean(arrayImages[0], intervaled)

     concentrationsSliced = np.array([concentrationMean(arrayImg, intervaled) for arrayImg in arrayImages])

     concentrationsSliced.shape

     means = finalMean(concentrationsSliced)

     finalData = np.reshape(means, concentrationsSliced.shape)

     dataframe = pd.DataFrame(finalData.T, columns=['CO', 'NO2', 'SO2', 'O3'], )

     print(dataframe.head())

     dataframe = dataframe.fillna(0)

     CO_MAX = 0.1
     NO2_MAX = 0.0003
     SO2_MAX = 0.01
     O3_MAX = 0.36

     dataframe['CO'] = dataframe['CO'] * CO_MAX / 255
     dataframe['NO2'] = dataframe['NO2'] * NO2_MAX / 255
     dataframe['SO2'] = dataframe['SO2'] * SO2_MAX / 255
     dataframe['O3'] = dataframe['O3'] * O3_MAX / 255

     print(dataframe.head())

     # 0: "RR", 1: "AP", 2: "AM", 3: "AC", 4: "RO", 5: "PA", 6: "MA",
     # 7: "PI", 8: "CE", 9: "RN", 10: "PB", 11: "PE", 12: "AL",
     # 13: "SE", 14: "BA", 15: "TO", 16: "MT", 17: "GO", 18: "MS",
     # 19: "MG", 20: "ES", 21: "RJ", 22: "SP", 23: "PR", 24: "SC",
     # 25: "RS", 26: "DF"
     dataframe.rename(index={0: "Alabama",
                             1: "Alaska",
                             2: "Arkansas",
                             3: "Arizona",
                             4: "California",
                             5: "Kansas",
                             6: "NorthCarolina",
                             7: "SouthCarolina",
                             8: "Colorado",
                             9: "Connecticut",
                             10: "NorthDakota",
                             11: "SouthDakota",
                             12: "Delaware",
                             13: "Florida",
                             14: "Georgia",
                             15: "Hawaii",
                             16: "Idaho",
                             17: "RhodeIsland",
                             18: "Illinois",
                             19: "Indiana",
                             20: "Iowa",
                             21: "Kentucky",
                             22: "Louisiana",
                             23: "Maine",
                             24: "Maryland",
                             25: "Massachusetts",
                             26: "Michigan",
                             27: "Minnesota",
                             28: "Mississippi",
                             29: "Missouri",
                             30: "Montana",
                             31: "Nebraska",
                             32: "Nevada",
                             33: "NewHampshire",
                             34: "NewJersey",
                             35: "NewYork",
                             36: "NewMexico",
                             37: "Oklahoma",
                             38: "Ohio",
                             39: "Oregon",
                             40: "Pennsylvania",
                             41: "Tennessee",
                             42: "Texas",
                             43: "Utah",
                             44: "Vermont",
                             45: "Virginia",
                             46: "WestVirginia",
                             47: "Washington",
                             48: "Wisconsin",
                             49: "Wyoming"})

     print(dataframe.head())

     csvWriterPandas('../datas_satelites/process/concentrations', dataframe)

     temperatura = pd.read_csv('../datas_satelites/MOD_LSTD_M_2020-10-01_gs_1440x720.CSV', header=None)
     umidade = pd.read_csv('../datas_satelites/MYDAL2_M_SKY_WV_2018-12-01_rgb_1440x720.CSV', header=None)

     temp = temperatura.replace(99999.0, None)
     imputed_temp = imputation(temp)
     temptranslated = sliceMap(coordinates, 720, 1440)
     tempMeans = unitmean(imputed_temp, 2, temptranslated)
     statesTempdataframe = pd.DataFrame(tempMeans, columns=['Temperatura'])
     statesTempdataframe.rename(index={0: "Alabama",
                                       1: "Alaska",
                                       2: "Arkansas",
                                       3: "Arizona",
                                       4: "California",
                                       5: "Kansas",
                                       6: "NorthCarolina",
                                       7: "SouthCarolina",
                                       8: "Colorado",
                                       9: "Connecticut",
                                       10: "NorthDakota",
                                       11: "SouthDakota",
                                       12: "Delaware",
                                       13: "Florida",
                                       14: "Georgia",
                                       15: "Hawaii",
                                       16: "Idaho",
                                       17: "RhodeIsland",
                                       18: "Illinois",
                                       19: "Indiana",
                                       20: "Iowa",
                                       21: "Kentucky",
                                       22: "Louisiana",
                                       23: "Maine",
                                       24: "Maryland",
                                       25: "Massachusetts",
                                       26: "Michigan",
                                       27: "Minnesota",
                                       28: "Mississippi",
                                       29: "Missouri",
                                       30: "Montana",
                                       31: "Nebraska",
                                       32: "Nevada",
                                       33: "NewHampshire",
                                       34: "NewJersey",
                                       35: "NewYork",
                                       36: "NewMexico",
                                       37: "Oklahoma",
                                       38: "Ohio",
                                       39: "Oregon",
                                       40: "Pennsylvania",
                                       41: "Tennessee",
                                       42: "Texas",
                                       43: "Utah",
                                       44: "Vermont",
                                       45: "Virginia",
                                       46: "WestVirginia",
                                       47: "Washington",
                                       48: "Wisconsin",
                                       49: "Wyoming", })
     csvWriterPandas('../datas_satelites/process/temperatura_estados', statesTempdataframe)

     umid = umidade.replace(99999.0, None)
     imputed_umid = imputation(umid)
     umidtranslated = sliceMap(coordinates, 720, 1440)
     umidMeans = unitmean(imputed_umid, 2, umidtranslated)
     statesUmidataframe = pd.DataFrame(umidMeans, columns=['Umidade'])
     statesUmidataframe.rename(index={0: "Alabama",
                                      1: "Alaska",
                                      2: "Arkansas",
                                      3: "Arizona",
                                      4: "California",
                                      5: "Kansas",
                                      6: "NorthCarolina",
                                      7: "SouthCarolina",
                                      8: "Colorado",
                                      9: "Connecticut",
                                      10: "NorthDakota",
                                      11: "SouthDakota",
                                      12: "Delaware",
                                      13: "Florida",
                                      14: "Georgia",
                                      15: "Hawaii",
                                      16: "Idaho",
                                      17: "RhodeIsland",
                                      18: "Illinois",
                                      19: "Indiana",
                                      20: "Iowa",
                                      21: "Kentucky",
                                      22: "Louisiana",
                                      23: "Maine",
                                      24: "Maryland",
                                      25: "Massachusetts",
                                      26: "Michigan",
                                      27: "Minnesota",
                                      28: "Mississippi",
                                      29: "Missouri",
                                      30: "Montana",
                                      31: "Nebraska",
                                      32: "Nevada",
                                      33: "NewHampshire",
                                      34: "NewJersey",
                                      35: "NewYork",
                                      36: "NewMexico",
                                      37: "Oklahoma",
                                      38: "Ohio",
                                      39: "Oregon",
                                      40: "Pennsylvania",
                                      41: "Tennessee",
                                      42: "Texas",
                                      43: "Utah",
                                      44: "Vermont",
                                      45: "Virginia",
                                      46: "WestVirginia",
                                      47: "Washington",
                                      48: "Wisconsin",
                                      49: "Wyoming", })
     csvWriterPandas('../datas_satelites/process/temperatura_estados', statesUmidataframe)

#coordinates = np.array([
#[1.982412, -61.426880],
#[1.593689, -52.016992],
#[-3.873574, -64.283420],
#[-8.963201, -70.363310],
#[-11.079112, -62.785181],
#[-4.871411, -52.721684],
#[-5.209189, -45.468431],
#[-7.474627, -42.355572],
#[-5.345183, -39.556922],
#[-5.909861, -36.599925],
#[-7.306302, -36.699159],
#[-8.323194, -37.968741],
#[-9.756657, -36.643270],
#[-10.659885, -37.374312],
#[-12.435302, -41.798611],
#[-9.977960, -48.263765],
#[-13.183693, -55.655239],
#[-16.202376, -49.638259],
#[-20.262102, -54.639455],
#[-18.979756, -44.796579],
#[-19.694772, -40.563513],
#[-22.308735, -42.415923],
#[-22.510024, -48.554745],
#[-24.733255, -51.312321],
#[-27.089118, -50.756816],
#[-29.697457, -53.240563],
#[-15.791799, -47.765179]
#])


try:
    from dna.data_base import DataBase
except ImportError:
    print("Need to fix the installation")
    raise
'''
BatchRun Command.
command: run <@batch_name>
'''

class RunCommand:

    def __init__(self, args):
        data_base = DataBase()
        name = args[0][1:]
        self.__commands = data_base.get_commands(name)

    def execute(self):
        self.__commands.notify()


import json
import os
import requests
import logging

from datetime import datetime
from airflow.utils.email import send_email

log = logging.getLogger(__name__)

OUTPUT_DIR = os.environ.get("AV_OUTPUT_DIR", default='/opt/output')
VIDEO_LIST = os.path.join(OUTPUT_DIR, 'videofiles.json')
AMS_API = os.environ.get("AMS_API", default='http://ams.osaarchivum.org/api/')
AMS_API_TOKEN = os.environ.get("AMS_API_TOKEN", default='<api_token>')
AV_STAFF_EMAIL_LIST = os.environ.get("AV_STAFF_EMAIL_LIST", default='')

def push_to_ams():
    if not os.path.exists(VIDEO_LIST):
        log.error("Video list file '%s' doesn't exists" % VIDEO_LIST)
        raise Exception

    with open(VIDEO_LIST, 'r') as video_list_file:
        video_list = json.load(video_list_file)

    for barcode, path in video_list.items():
        barcode_dir = os.path.join(OUTPUT_DIR, barcode)
        metadata_dir = os.path.join(barcode_dir, 'Metadata', 'Preservation')
        metadata_file = os.path.join(metadata_dir, "%s_md_tech.json" % barcode)

        with open(metadata_file, 'r') as metadata:
            technical_metadata = metadata.read()

        headers = {"Authorization": "Bearer %s" % AMS_API_TOKEN}
        data = {
            'barcode': barcode,
            'digital_version_exists': True,
            'digital_version_technical_metadata': technical_metadata,
            'digital_version_creation_date': datetime.now().strftime('%Y-%m-%d')
        }
        url = "%s/%s/%s/" % (AMS_API, 'containers', barcode)
        log.info("Pushing metadata through %s" % url)
        r = requests.put(url=url, data=data, headers=headers, allow_redirects=True)
        if r.status_code == 200:
            log.info("OK - Data ingested to AMS...")
        elif r.status_code == 404:
            log.info("Sending notification email...")
            email_title = "AV Digitization Workflow - Missing Barcode: %s" % barcode
            email_body = """
            Dear AV team,

            It seems that the barcode %s is not registered in the AMS. Please give it a look!

            Your sincerely,

            AV workflow

            """ % barcode
            emails = AV_STAFF_EMAIL_LIST.split(',')
            send_email(emails, email_title, email_body)
        else:
            log.error("Bad request to: %s" % r.url)
            log.error("Response: %s - %s" % (r.status_code, r.reason))
            log.error("Error message: %s" % r.content)
            raise Exception

if __name__ == '__main__':
    push_to_ams()


import io
import os
from argparse import ArgumentParser

from django.core.management import BaseCommand
from django.core.management.base import OutputWrapper
from django.core.management.color import no_style
from django.template.loader import render_to_string
from django.utils.translation import gettext as _

from df_config import __version__ as version
from df_config.config.base import merger
from df_config.utils import guess_version, remove_arguments_from_help

__author__ = "Matthieu Gallet"

from df_config.config.values_providers import (
    EnvironmentConfigProvider,
    IniConfigProvider,
)

class Command(BaseCommand):
    help = (
        "show the current configuration."
        'Can display as python file ("config python") or as .ini file ("config ini"). Use -v 2 to display more info.'
    )
    requires_system_checks = []
    options = {
        "python": "display the current config as Python module",
        "ini": "display the current config as .ini file",
        "env": "display the current config as environment variables",
    }

    def add_arguments(self, parser: ArgumentParser):
        parser.add_argument(
            "action",
            default="show",
            choices=self.options,
            help=",\n".join(['"%s": %s' % x for x in self.options.items()]),
        )
        parser.add_argument(
            "--filename", default=None, help="write output to this file"
        )
        remove_arguments_from_help(
            parser, {"--settings", "--traceback", "--pythonpath"}
        )

    def handle(self, *args, **options):
        try:
            self.handle_head(**options)
        except BrokenPipeError:
            # this exception is raised when `manage.py config python | head` is used
            pass

    def handle_head(self, **options):
        action = options["action"]
        verbosity = options["verbosity"]
        filename = options["filename"]
        fd = None
        if filename:
            fd = io.StringIO()
            self.stdout = OutputWrapper(fd)
            self.style = no_style()
        if action == "python":
            self.show_python_config(verbosity)
        elif action == "ini":
            self.show_ini_config(verbosity)
        elif action == "env":
            self.show_env_config(verbosity)

        if filename and action in {"python", "env"}:
            content = fd.getvalue()
            if action == "python":
                # noinspection PyBroadException
                try:
                    # noinspection PyPackageRequirements,PyUnresolvedReferences
                    import black

                    mode = black.FileMode()
                    # noinspection PyArgumentList
                    content = black.format_file_contents(content, fast=False, mode=mode)
                except Exception:  # nosec  # nosec
                    pass
            os.makedirs(os.path.dirname(filename), exist_ok=True)
            with open(filename, "w") as dst_fd:
                dst_fd.write(content)

    def show_external_config(self, config):
        content = render_to_string(config, merger.settings)
        self.stdout.write(content)

    def show_ini_config(self, verbosity):
        if verbosity >= 2:
            p = merger.fields_provider
            self.stdout.write(
                self.style.SUCCESS(f"# configuration fields read from {p}")
            )
            self.stdout.write(self.style.SUCCESS("# read configuration files:"))
        for provider in merger.providers:
            if not isinstance(provider, IniConfigProvider):
                continue
            elif provider.is_valid():
                self.stdout.write(
                    self.style.SUCCESS('    #  - %s "%s"' % (provider.name, provider))
                )
            elif verbosity >= 2:
                self.stdout.write(
                    self.style.ERROR(
                        '    #  - %s "%s" (not found)' % (provider.name, provider)
                    )
                )
        provider = IniConfigProvider()
        merger.write_provider(provider, include_doc=verbosity >= 2)
        self.stdout.write(provider.to_str())

    def show_env_config(self, verbosity):
        prefix = None
        for provider in merger.providers:
            if not isinstance(provider, EnvironmentConfigProvider):
                continue
            prefix = provider.prefix
        if not prefix:
            self.stderr.write("Environment variables are not used•")
            return
        if verbosity >= 2:
            p = merger.fields_provider
            self.stdout.write(
                self.style.SUCCESS(f"# configuration fields read from {p}")
            )
            self.stdout.write(self.style.SUCCESS("# read environment variables:"))
        provider = EnvironmentConfigProvider(prefix)
        merger.write_provider(provider, include_doc=verbosity >= 2)
        self.stdout.write(provider.to_str())

    def show_python_config(self, verbosity):
        self.stdout.write(self.style.SUCCESS("# " + "-" * 80))
        self.stdout.write(
            self.style.SUCCESS(
                _("# df_config version %(version)s") % {"version": version}
            )
        )
        self.stdout.write(
            self.style.SUCCESS(
                _("# %(project)s version %(version)s")
                % {
                    "version": guess_version(merger.settings),
                    "project": merger.settings["DF_PROJECT_NAME"],
                }
            )
        )
        self.stdout.write(self.style.SUCCESS("# Configuration providers:"))
        for provider in merger.providers:
            if provider.is_valid():
                self.stdout.write(
                    self.style.SUCCESS('#  - %s "%s"' % (provider.name, provider))
                )
            elif verbosity > 1:
                self.stdout.write(
                    self.style.ERROR(
                        '#  - %s "%s" (not found)' % (provider.name, provider)
                    )
                )
        self.stdout.write(self.style.SUCCESS("# " + "-" * 80))
        setting_names = list(merger.raw_settings)
        setting_names.sort()

        # first, compute all imports to do
        imports = {}

        def add_import(val):
            if not isinstance(val, type):
                val = val.__class__
            if val.__module__ != "builtins":
                imports.setdefault(val.__module__, set()).add(val.__name__)

        for setting_name in setting_names:
            if setting_name not in merger.settings:
                continue
            value = merger.settings[setting_name]
            add_import(value)
        if imports:
            self.stdout.write("\n")
            for module_name in sorted(imports):
                objects = ", ".join(sorted(imports[module_name]))
                self.stdout.write(
                    self.style.WARNING("from %s import %s" % (module_name, objects))
                )
            self.stdout.write("\n")

        for setting_name in setting_names:
            if setting_name not in merger.settings:
                continue
            value = merger.settings[setting_name]
            self.stdout.write(self.style.SUCCESS("%s = %r" % (setting_name, value)))
            if verbosity <= 1:
                continue
            for provider_name, raw_value in merger.raw_settings[setting_name].items():
                self.stdout.write(
                    self.style.WARNING(
                        "    #   %s -> %r" % (provider_name or "built-in", raw_value)
                    )
                )


# -*- coding:UTF-8 -*-
"""
-------------------------------------------------
   File Name：     SoldHouseCrawler.py
   Description :   to crawler sold house from http://house.ksou.cn/p.php
   Author :       elvisfang
   date：          2017/8/2
-------------------------------------------------
   Change Activity:
                   2017/8/2:
-------------------------------------------------
"""
__author__ = 'elvisfang'

import basecrawler
import time
from Database import MongoDBClient
from bs4 import BeautifulSoup
import re
from Log import logger

class SoldhouseCrawler():

    __crawl_url = 'http://house.ksou.cn/p.php'
    __porxy_enabled = False

    def __init__(self, proxy_enable=False):
        self.__proxy_enabled = proxy_enable
        self.__crawler = basecrawler.BaseCrawler(self.__crawl_url,self.__proxy_enabled)

    #generate the query string of the search data for http://house.ksou.cn/p.php
    def __generate_query_string(self,query,page,region,state):
        Query_String = {}
        Query_String['q'] = query
        Query_String['p'] = page
        Query_String['s'] = 1
        Query_String['st'] = ''
        Query_String['type'] = ''
        Query_String['count'] = '300'
        Query_String['region'] = region
        Query_String['lat'] = '0'
        Query_String['lng'] = '0'
        Query_String['sta'] = state
        Query_String['htype'] = ''
        Query_String['agent'] = '0'
        Query_String['minprice'] = '0'
        Query_String['maxprice'] = '0'
        Query_String['minned'] = '0'
        Query_String['maxbed'] = '0'
        Query_String['minland'] = '0'
        Query_String['maxland'] = '0'
        return Query_String

    def __absctract_house_info(self,bs):
        if not isinstance(bs,BeautifulSoup):
            raise TypeError('bs must be bs4.BeautifulSoup, now: ' + type(bs))

        _house_info = {}
        addr_tag = bs.find('span', class_='addr')
        detail_tag = addr_tag.parent.parent.next_sibling
        # find detail info in the currnet page

        # Get Address sample "37 Clendon Road, Toorak"
        _house_info['Addr'] = addr_tag.text.split(',')[0]

        # Get Sold Price sample Sold $8,790,000 unit AU dollar
        if detail_tag.find(text=re.compile('Sold')):
            _house_info['SoldPrice'] = detail_tag.find(text=re.compile('Sold'))[6:]
        else:
            _house_info['SoldPrice'] = 'null'
        # Get Sold Date sample in Jun 2017(Auction)
        if detail_tag.find(text=re.compile('Sold')):
            _house_info['SoldDate'] = detail_tag.find(text=re.compile('Sold')).parent.next_sibling[3:].replace('(Auction)','').strip()
        else:
            _house_info['SoldDate'] = 'null'
        # Get Last Sold Price sample " $1,505,000 in Oct 1997"
        if detail_tag.find(text=re.compile('Last Sold')):
            _sl = detail_tag.find(text=re.compile('Last Sold')).parent.next_sibling
            _house_info['LastSoldPrice'] = _sl.split('in')[0].strip()[1:]
            _house_info['LastSoldDate'] = _sl.split('in')[1].replace('(Auction)','').strip()
        else:
            _house_info['LastSoldPrice'] = 'null'
            _house_info['LastSoldDate'] = 'null'
        # Get Rent Price
        if detail_tag.find(text=re.compile('Rent')):
            if detail_tag.find(text=re.compile('Rent')).parent.parent.find('a'):
                _rp = detail_tag.find(text=re.compile('Rent')).parent.parent.a.string  # with Rent link
            else:
                _rp = detail_tag.find(
                    text=re.compile('Rent')).parent.next_sibling  # Rent $1,150pw in Jun 2014
            _house_info['RentPrice'] = _rp.split('in')[0].strip()[1:-2]
            _house_info['RentDate'] = _rp.split('in')[1].strip()
        else:
            _house_info['RentPrice'] = 'null'
            _house_info['RentDate'] = 'null'
        # get property type
        if detail_tag.find(text=re.compile('Apartment|Unit|House')):
            _house_info['Type'] = detail_tag.find(text=re.compile('Apartment|Unit|House'))[:-1]
        else:
            _house_info['Type'] = 'null'
        # get Bed Room Number
        if detail_tag.find(title='Bed rooms'):
            _house_info['BedRoom'] = detail_tag.find(title='Bed rooms').previous_sibling
        else:
            _house_info['BedRoom'] = '0'
        # get Bath Room Number
        if detail_tag.find(title='Bath rooms'):
            _house_info['BathRoom'] = detail_tag.find(title='Bath rooms').previous_sibling
        else:
            _house_info['BathRoom'] = '0'
        # Get car space
        if detail_tag.find(title='Car spaces'):
            _house_info['CarSpace'] = detail_tag.find(title='Car spaces').previous_sibling
        else:
            _house_info['CarSpace'] = '0'
        # Get Land Size
        if detail_tag.find(text=re.compile('Land size:')):
            _house_info['LandSize'] = detail_tag.find(text=re.compile('Land size:')).parent.next_sibling.split('sqm')[0].strip()
            if _house_info['LandSize'] == '':
                _house_info['LandSize'] = 'null'
        else:
            _house_info['LandSize'] = 'null'
        # get building size
        if detail_tag.find(text=re.compile('Building size:')):
            _house_info['BuildingSize'] = detail_tag.find(text=re.compile('Building size:')).parent.next_sibling.split('sqm')[0].strip()
            if _house_info['BuildingSize'] == '':
                _house_info['BuildingSize'] = 'null'
        else:
            _house_info['BuildingSize'] = 'null'
        # get Build Year
        if detail_tag.find(text=re.compile('Build year:')):
            _house_info['BuildYear'] = detail_tag.find(text=re.compile('Build year:')).parent.next_sibling
        else:
            _house_info['BuildYear'] = 'null'
        # Get Agent Name
        if detail_tag.find(text=re.compile('Agent')):
            # Agent: Rodney Morley Pty Ltd -
            if detail_tag.find(text=re.compile('Agent')).parent.parent.find('a'):
                _house_info['Agent'] = detail_tag.find(text=re.compile('Agent')).parent.parent.a.string.strip()
            else:  # Agent: hockingstuart  Armadale
                _house_info['Agent'] = detail_tag.find(text=re.compile('Agent')).parent.next_sibling.strip()
        else:
            _house_info['Agent'] = 'null'
        # get distance to cbd sample 5.3 km to CBD; 825 metres to Heyington Station, unit meters
        if detail_tag.find(text=re.compile('Distance')):
            _dist = detail_tag.find(text=re.compile('Distance')).parent.next_sibling
            _house_info['Distance2CBD'] = _dist.split(';')[0].split('to')[0].strip()
            _house_info['Distance2Stat'] = _dist.split(';')[1].split('to')[0].strip()
        else:
            _house_info['Distance2CBD'] = 'null'
            _house_info['Distance2Stat'] = 'null'

        #return one house info
        return _house_info

    def crawl_sold_house(self,state,mode='CONTINUE',excluderegion=''):
        _region_list = self.__crawler.load_region_list(state)
        _excludelist = excluderegion
        #create a new crawler to fro detail data
        _crawler = basecrawler.BaseCrawler()
        #-------------------
        # set both crawler proxy enabled
        _crawler.proxy_enabled = True
        self.__crawler.proxy_enabled = True
        #------------------
        #new collction for detail property
        _mongoclient = MongoDBClient.MongodbClient('PropertyDetails')
        for _region in _region_list:
            if _region in _excludelist:
                continue
            for i in range(30):
                _query = self.__generate_query_string(_region,i,_region,state)
                #_bs_searchresult = self.__crawler.crawl_data(**_query)
                _bs_searchresult = self.__crawler.requests_crawl_data(**_query)
                for addr in _bs_searchresult.find_all('span', class_='addr'):
                    _crawler.crawl_url = self.__crawl_url + addr.a.get('href')[5:]
                    try:
                        if mode == 'FULL' or \
                                (mode == 'CONTINUE' and (not _mongoclient.get_one_value_by_key({'URL':_crawler.crawl_url}))):
                            #_bs_detailinfo = _crawler.crawl_data()
                            _bs_detailinfo = _crawler.requests_crawl_data()
                            _house_data = self.__absctract_house_info(_bs_detailinfo)
                            _house_data['State'] = state
                            _house_data['Region'] = _region
                            _house_data['URL'] = self.__crawl_url + addr.a.get('href')[5:]
                            #save data to mongodb
                            _mongoclient.update_one_record({'Addr':_house_data['Addr']},_house_data,'PropertyDetails')
                            # sleep 2 second
                            time.sleep(2)
                            print('save data:' + _house_data['URL'])
                        else:
                            print(_crawler.crawl_url + 'existed')
                    except BaseException as e:
                        logger.log_to_file('SoldHouseCrawler.log', 'error when processing '+ _crawler.crawl_url +'err: ' + str(e))
                #time.sleep(10)
        _mongoclient.close_bd()

if __name__ =='__main__':
    _crawler1 = SoldhouseCrawler()
    _crawler1.crawl_sold_house('vic',mode='FULL')
    #TODO exclude regions if 300 entries are existed in CONTINUE Mode, and how to process crawler in different mode
    #TODO using proxy and change IP address when error occures


from __future__ import print_function
import tifffile
import argparse
import numpy as np
from copy import deepcopy
import math

import neuroglancer

ap = argparse.ArgumentParser()
ap.add_argument(
    '-a',
    '--bind-address',
    help='Bind address for Python web server.  Use 127.0.0.1 (the default) to restrict access '
         'to browers running on the local machine, use 0.0.0.0 to permit access from remote browsers.')
ap.add_argument(
    '--static-content-url', help='Obtain the Neuroglancer client code from the specified URL.')
args = ap.parse_args()
if args.bind_address:
    neuroglancer.set_server_bind_address(args.bind_address)
if args.static_content_url:
    neuroglancer.set_static_content_source(url=args.static_content_url)

im = tifffile.imread("testFile3d.tif")
a = np.array(im)
a = np.moveaxis(a, 0, -1)

# creating the numpy "sphere" array
r0 = 30
r1 = 40
r2 = 20
r3 = 25
r4 = 50
r5 = 35
x0, y0, z0 = 642, 370, 70
x1, y1, z1 = 200, 500, 25
x2, y2, z2 = 650, 550, 64
x3, y3, z3 = 660, 280, 64
x4, y4, z4 = 642, 370, 77

dimensions = (a.shape[0], a.shape[1], a.shape[2])
zeros = np.zeros(dimensions)

sphere0 = deepcopy(zeros)
sphere1 = deepcopy(zeros)
sphere2 = deepcopy(zeros)
sphere3 = deepcopy(zeros)
sphere4 = deepcopy(zeros)

for x in range(x0 - r0, x0 + r0):
    for y in range(y0 - r0, y0 + r0):
        for z in range(z0 - r0, z0 + r0):
            distance0 = math.sqrt(math.pow(x - x0, 2) + math.pow(y - y0, 2) + math.pow(z - z0, 2))
            distance1 = math.sqrt(math.pow(x - x1, 2) + math.pow(y - y1, 2) + math.pow(z - z1, 2))
            distance2 = math.sqrt(math.pow(x - x2, 2) + math.pow(y - y2, 2) + math.pow(z - z2, 2))
            distance3 = math.sqrt(math.pow(x - x3, 2) + math.pow(y - y3, 2) + math.pow(z - z3, 2))
            distance4 = math.sqrt(math.pow(x - x4, 2) + math.pow(y - y4, 2) + math.pow(z - z4, 2))
            if distance0 <= r0 :    sphere0[x,y,z] = 1
            if distance1 <= r1 :    sphere1[x,y,z] = 2
            if distance2 <= r2 :    sphere2[x,y,z] = 3
            if distance3 <= r3 :    sphere3[x,y,z] = 4
            if distance4 <= r4 :    sphere4[x,y,z] = 5

viewer = neuroglancer.Viewer()
dimensions = neuroglancer.CoordinateSpace(
    names=['x', 'y', 'z'],
    units='nm',
    scales=[10, 10, 10]
)
with viewer.txn() as s:
    s.dimensions = dimensions
    s.layers.append(
        name='a',
        layer=neuroglancer.LocalVolume(
            data=a,
            dimensions=neuroglancer.CoordinateSpace(
                names=['x', 'y', 'z'],
                units=['nm', 'nm', 'nm'],
                scales=[10, 10, 10]),
            voxel_offset=(20, 30, 15),
        ),
        shader="""
void main() {
  emitRGB(vec3(toNormalized(getDataValue(0)),
               toNormalized(getDataValue(1)),
               toNormalized(getDataValue(2))));
}
""")
    s.layers.append(
        name = 'First sphere (Q1) overlay',
        layer = neuroglancer.LocalVolume(
            data = sphere0,
            dimensions = dimensions,
        ),
        shader="""
        void main() {
          emitRGB(vec3(toNormalized(getDataValue(0)),
                       toNormalized(getDataValue(1)),
                       toNormalized(getDataValue(2))));
        }
        """)

    s.layers.append(
        name = "Second sphere (200, 500, 25) Overlay",
        layer = neuroglancer.LocalVolume(
            data=sphere1,
            dimensions = dimensions,
        ),
        shader="""
        void main() {
          emitRGB(vec3(toNormalized(getDataValue(0)),
                       toNormalized(getDataValue(1)),
                       toNormalized(getDataValue(2))));
        }
        """)
    s.layers.append(
        name="Third sphere (200, 500, 25) Overlay",
        layer=neuroglancer.LocalVolume(
            data=sphere2,
            dimensions=dimensions,
        ),
        shader="""
            void main() {
              emitRGB(vec3(toNormalized(getDataValue(0)),
                           toNormalized(getDataValue(1)),
                           toNormalized(getDataValue(2))));
            }
            """)

    s.layers.append(
        name="Fourth sphere (200, 500, 25) Overlay",
        layer=neuroglancer.LocalVolume(
            data=sphere3,
            dimensions=dimensions,
        ),
        shader="""
            void main() {
              emitRGB(vec3(toNormalized(getDataValue(0)),
                           toNormalized(getDataValue(1)),
                           toNormalized(getDataValue(2))));
            }
            """)

    s.layers.append(
        name="Fifth sphere (200, 500, 25) Overlay",
        layer=neuroglancer.LocalVolume(
            data=sphere4,
            dimensions=dimensions,
        ),
        shader="""
            void main() {
              emitRGB(vec3(toNormalized(getDataValue(0)),
                           toNormalized(getDataValue(1)),
                           toNormalized(getDataValue(2))));
            }
            """)

def acquire_mouse_action(s):
    print('Getting some mouse action')
    print('  Mouse position: %s' % (s.mouse_voxel_coordinates,))
    coordinates = s.mouse_voxel_coordinates
    with viewer.config_state.txn() as s:
        s.status_messages['Welcome to this example'] = 'You pressed key t'
        s.status_messages['You pressed key t'] = 'The mouse position is'
        s.status_messages['The mouse position is'] = coordinates

viewer.actions.add('mouse-action', acquire_mouse_action)
with viewer.config_state.txn() as s:
    s.input_event_bindings.viewer['keyt'] = 'mouse-action'
    s.status_messages['hello'] = 'Welcome to this example'

def acquire_layer_action(s):
    print('Getting selected layer action')
    print(' Layer selected values: %s' % (s.selected_values,))
    coordinates = s.selected_values
    with viewer.config_state.txn() as s:
        s.status_messages['Welcome to this example'] = 'You pressed key m'
        s.status_messages['You pressed key m'] = 'The selected layer info is'
        s.status_messages['The selected layer info is'] = coordinates

viewer.actions.add('layer-action', acquire_layer_action)
with viewer.config_state.txn() as s:
    s.input_event_bindings.viewer['keym'] = 'layer-action'
    s.status_messages['hello'] = 'Welcome to this example'

print(viewer)


from django.db.models import Count, Sum
from django.utils import timezone
from rest_framework.generics import GenericAPIView
from rest_framework.response import Response

from accounts.models import Group
from competitions.api.serializers import CompetitionSerizlizers
from competitions.models import (AnswerRecords, Competitions,
                                 get_active_competition)

class Stats(GenericAPIView):
    """大屏统计"""

    # base: 基本信息
    # gs:   队伍排行
    # qs:   答题动态
    # rate：队伍答题进度
    # trend: 答题趋势(态势)
    TYPE = ['base', 'gs', 'qs', 'rate', 'trend']

    def get(self, request, *args, **kwargs):
        t = kwargs['types']
        if t in self.TYPE:
            method = getattr(self, t, None)
            if not method:
                raise ValueError('服务器错误')
            competi = get_active_competition()
            if not competi:
                raise ValueError('无活动赛事')
            return method(request, competi)
        raise ValueError('未找到')

    def base(self, request, competi: Competitions):
        serializer = CompetitionSerizlizers(instance=competi)
        gs = competi.groups.all()
        hc = sum([g.users.count() for g in gs])
        data = dict(serializer.data)
        data.update(people_number=hc)
        return Response(data=data)

    def gs(self, request, competi: Competitions):
        records = AnswerRecords.objects.filter(competition=competi)
        gid_sum = records.values('gid').annotate(Sum('score'))[:10]
        # records.values('gid').annotate(Sum('score')).order_by('-score__sum')[:10]
        data = []
        for gs in gid_sum:
            g = Group.objects.get(pk=gs['gid'])
            data.append({'group': g.name, 'score_sum': gs['score__sum']})
        return Response(data=data)

    def qs(self, request, competi: Competitions):
        qs = AnswerRecords.objects.filter(competition=competi)[:10]
        data = []
        for q in qs:
            data.append(f'{q.respondent.name} 提交了 {q.question.topic}')
        return Response(data=data)

    def rate(self, request, competi: Competitions):
        recoreds = AnswerRecords.objects.filter(competition=competi)
        gid_count = recoreds.values('gid').annotate(Count('question'))
        data = []
        for gc in gid_count:
            g = Group.objects.get(pk=gc['gid'])
            data.append({'group': g.name, 'rate': (gc['question__count']/competi.questions.count())*100})
        return Response(data=data)

    def trend(self, request, competi: Competitions, duration=24):
        befor = timezone.localtime(timezone.now()) - timezone.timedelta(hours=duration)
        befor_records = AnswerRecords.objects.filter(competition=competi, create_at__lt=befor)
        records = AnswerRecords.objects.filter(competition=competi, create_at__gte=befor)
        gid_sum = AnswerRecords.objects.filter(competition=competi, score__gt=0).values('gid').annotate(Sum('score'))[:10]
        data = {}
        for gs in gid_sum:
            name = Group.objects.get(pk=gs['gid']).name
            packet_data = []
            p = befor_records.filter(gid=gs['gid']).aggregate(Sum('score'))['score__sum'] or 0
            for h in range(duration):
                this_time = records.filter(gid=gs['gid'], create_at__range=(
                    befor+timezone.timedelta(hours=h), befor + timezone.timedelta(hours=h+1)
                )
                )
                if not this_time:
                    packet_data.append(p)
                    continue
                this_sum = this_time.aggregate(Sum('score'))['score__sum']
                p += this_sum
                packet_data.append(p)
            data.update({name: packet_data})
        return Response(data=data)


############################################################
#
#dataAnalyser.py
#
############################################################
#
# Author: romano <sromanos@cern.ch>
#
# May 2013
#
# Goal:
# - Produce the evolution plots for: LV, temp, reference voltage, HV, HVset and DeltaHV
#
# Input parameters are:
#
# -> Period for data recovery
#
#
#
#############################################################

from src.ReadGenericCalibration import *
from src.oscalls import *
from src.run import *
import datetime
import time
import sys

#from ROOT import *
import ROOT

# insert HV tools
from src.HV.hvtools import *

class dataAnalyser(GenericWorker):
	"Evolution plots for all the variables stored in DCS"

	def __init__(self, ini_Date, fin_Date, mode, do_extraPlots):

		self.fin_Date = fin_Date
		self.ini_Date = ini_Date
		self.HVTool = hvtools()
		self.mode = mode
		self.do_extraPlots = do_extraPlots

	def ProcessStart(self):

		# definition of loops limits
		print('The analysis start at', datetime.datetime.now())

		max_partitions, max_modules, max_channels = 4, 64, 48

		#for the plots

		ini_volt  = 450
		end_volt = 1000
		binning_volt = (end_volt-ini_volt)*2

		ini_low_volt = 0
		end_low_volt = 20
		binning_low_volt = (end_low_volt-ini_low_volt)*2

		ini_Dvolt     = -400.5
		end_Dvolt     = 200.5
		binning_Dvolt = abs(end_Dvolt-ini_Dvolt)
		binning_Dvolt = 5*int(binning_Dvolt)

		ini_ref_volt   = 1180
		end_ref_volt   = 1280
		binning_ref_volt = (end_ref_volt-ini_ref_volt)*4

		ini_temp  = 15
		end_temp  = 40
		binning_temp = (end_temp-ini_temp)*4

		period = 1 #number of points per day

		if self.mode=='RegularJob':
			inDir = os.path.join(getResultDirectory(),'HV/files/mergefile/RegularJob')
			outDir = os.path.join(getResultDirectory(),'HV/files/results/RegularJob')
			inFile_name = 'HV_RegularJob_Merged.root'

		else:
			inDir = os.path.join(getResultDirectory(),'HV/files/mergefile/' + str(ini_Date))
			outDir = os.path.join(getResultDirectory(),'HV/files/results/'+ str(ini_Date))
			inFile_name = 'HV_' + str(ini_Date) + '_Merged.root'

		mergedFile = TFile(inDir + '/' + inFile_name,'READ')
		tree = mergedFile.Get('ZeTree')

		tree.Print()
		#print tree.GetMinimun("Date")

		n_entries = tree.GetEntries()

		# output file tree: branches definitions
		Measure     = array('f',[ 0 ])
		Reference   = array('f',[ 0 ])
		Date        = array('i',[ 0 ])

		MeasureType = array('i',[ 0 ])
		Channel     = array('i',[ 0 ])
		Module      = array('i',[ 0 ])
		Partition   = array('i',[ 0 ])

		tree.SetBranchAddress('Measure', Measure)
		tree.SetBranchAddress('Reference', Reference)
		tree.SetBranchAddress('Date', Date)

		tree.SetBranchAddress('MeasureType', MeasureType)
		tree.SetBranchAddress('Channel', Channel)
		tree.SetBranchAddress('Module', Module)
		tree.SetBranchAddress('Partition', Partition)

		# For the evolution plots

		ini_day = tree.GetMinimum("Date")
		end_day = tree.GetMaximum("Date")
		binning_day = (end_day-ini_day) / (86400 * period)
		binning_day = int(round(binning_day))

		self.HVTool.mkdir(outDir)
		outFile = TFile(outDir + '/A_histograms.root', 'RECREATE')

		outFile.mkdir('DHV')
		outFile.mkdir('HV')
		outFile.mkdir('HVset')
		outFile.mkdir('T')
		outFile.mkdir('Ref')
		outFile.mkdir('LV')
		outFile.mkdir('Time')
		outFile.cd()

		# Lists of histograms and TProfiles:

		h_means, h_sigmas, h_DHV_part, h_final_DHV_part, max_diff_part, h_HV_vs_t_part, h_DHV_vs_t_part, h_T_vs_t_part, h2_map = [], [], [], [], [], [], [], [], []
		h_HV_vs_t_mod, h_DHV_vs_t_mod, h_T_vs_t_mod = [], [], []
		h_HV, h_HV_vs_t, h_HVset, h_HVset_vs_t, h_DHV, h_DHV_vs_t, h_T, h_T_vs_t, h_LV, h_LV_vs_t, h_ref, h_ref_vs_t = [], [], [], [], [], [], [], [], [], [], [], []

		difference_date, h_intertime, h_intertime_long = [], [], [] # To study the time

		# Histrograms declaration
		for ipart in range(4): # loop in all the partitions

			part_name = self.HVTool.get_partition_name(ipart)

			# histos of means
			h_means.append(TH1F("h_means_"+part_name,"All means in partition "+part_name+";#mu_{i}; #events",150,-15,15))
			h_means[ipart].SetDirectory(0)

			# histos of sigmas
			h_sigmas.append(TH1F("h_sigmas_"+part_name,"All sigmas in partition "+part_name+";#sigma_{i}; #events",500,0,5))
			h_sigmas[ipart].SetDirectory(0)

			# histos of DHV
			h_DHV_part.append(TH1F("h_DHV_"+part_name,"#DeltaHV ("+part_name+");#DeltaHV (V); #events",binning_Dvolt,ini_Dvolt,end_Dvolt))
			h_DHV_part[ipart].SetDirectory(0)

			# histos of DHV for good channels
			h_final_DHV_part.append(TH1F("h_final_DHV_"+part_name,"#DeltaHV ("+part_name+") for Good Channels;#DeltaHV (V); #events",binning_Dvolt,ini_Dvolt,end_Dvolt))
			h_final_DHV_part[ipart].SetDirectory(0)

			# histos of Max diff
			max_diff_part.append(TH1F("h_max_diff_"+part_name,"Max diff ("+part_name+");Max diff; #events",1000,0,100))
			max_diff_part[ipart].SetDirectory(0)

			# TProfiles of HV
			h_HV_vs_t_part.append(TProfile("HV_vs_t_"+part_name,"High Voltage ("+part_name+") vs Time; Day; HV (V)",binning_day ,ini_day,end_day,ini_volt,end_volt))
			h_HV_vs_t_part[ipart].GetXaxis().SetTimeDisplay(1)
			h_HV_vs_t_part[ipart].SetDirectory(0)

			# TProfiles of DHV
			h_DHV_vs_t_part.append(TProfile("DHV_vs_t_"+part_name,"#DeltaHV ("+part_name+") vs Time; Day; #DeltaHV (V)",binning_day,ini_day,end_day,ini_Dvolt,end_Dvolt))
			h_DHV_vs_t_part[ipart].GetXaxis().SetTimeDisplay(1)
			h_DHV_vs_t_part[ipart].SetDirectory(0)

			# TProfiles of Temperatures
			h_T_vs_t_part.append(TProfile("T_vs_t_"+part_name,"T ("+part_name+") vs Time; Day; T (*C)",binning_day,ini_day,end_day,ini_temp,end_temp))
			h_T_vs_t_part[ipart].GetXaxis().SetTimeDisplay(1)
			h_T_vs_t_part[ipart].SetDirectory(0)

			# maps for good and bad channels
			h2_map.append(TH2F("Map_"+part_name,"Mapping of channels for partition "+part_name+";Module;Channel",max_modules+2,0,max_modules+2,max_channels+2,0,max_channels+2))
			h2_map[ipart].SetDirectory(0)

			h_HV_vs_t_mod.append([])
			h_DHV_vs_t_mod.append([])
			h_T_vs_t_mod.append([])

			h_HV.append([])
			h_HVset.append([])
			h_DHV.append([])
			h_T.append([])
			h_LV.append([])
			h_ref.append([])
			h_intertime.append([])
			h_intertime_long.append([])

			h_HV_vs_t.append([])
			h_HVset_vs_t.append([])
			h_DHV_vs_t.append([])
			h_T_vs_t.append([])
			h_LV_vs_t.append([])
			h_ref_vs_t.append([])

			difference_date.append([])
			for imod in range(64): # loop in all the modules

				mod_name = part_name + str(imod+1).zfill(2)

				# TProfiles of HV
				h_HV_vs_t_mod[ipart].append(TProfile("HV_vs_t_"+mod_name,"High Voltage ("+mod_name+") vs Time; Day; HV (V)",binning_day ,ini_day,end_day,ini_volt,end_volt))
				h_HV_vs_t_mod[ipart][imod].GetXaxis().SetTimeDisplay(1)
				h_HV_vs_t_mod[ipart][imod].SetDirectory(0)

				# TProfiles of DHV
				h_DHV_vs_t_mod[ipart].append(TProfile("DHV_vs_t_"+mod_name,"#DeltaHV ("+mod_name+") vs Time; Day; #DeltaHV (V)",binning_day,ini_day,end_day,ini_Dvolt,end_Dvolt))
				h_DHV_vs_t_mod[ipart][imod].GetXaxis().SetTimeDisplay(1)
				h_DHV_vs_t_mod[ipart][imod].SetDirectory(0)

				# TProfiles of Temperatures
				h_T_vs_t_mod[ipart].append(TProfile("T_vs_t_"+mod_name,"T ("+mod_name+") vs Time; Day; T (*C)",binning_day,ini_day,end_day,ini_temp,end_temp))
				h_T_vs_t_mod[ipart][imod].GetXaxis().SetTimeDisplay(1)
				h_T_vs_t_mod[ipart][imod].SetDirectory(0)

				h_HV[ipart].append([])
				h_HVset[ipart].append([])
				h_DHV[ipart].append([])
				h_T[ipart].append([])
				h_LV[ipart].append([])
				h_ref[ipart].append([])
				h_intertime[ipart].append([])
				h_intertime_long[ipart].append([])

				h_HV_vs_t[ipart].append([])
				h_HVset_vs_t[ipart].append([])
				h_DHV_vs_t[ipart].append([])
				h_T_vs_t[ipart].append([])
				h_LV_vs_t[ipart].append([])
				h_ref_vs_t[ipart].append([])

				difference_date[ipart].append([])
				for ichan in range(48): # loop in all the channels

					chan_name = mod_name + '_' + str(ichan+1).zfill(2)

					if self.HVTool.IsThisChannelInstrumented(ipart,imod+1,ichan+1):

						# HV
						h_HV[ipart][imod].append(TH1F("h_HV_"+chan_name,"High Voltage ("+chan_name+"); HV (V); #events", binning_volt, ini_volt, end_volt))
						h_HV[ipart][imod][ichan].SetDirectory(0)

						h_HV_vs_t[ipart][imod].append(TProfile("HV_vs_t_"+chan_name,"High Voltage ("+chan_name+") vs Time; Day; HV (V)", binning_day, ini_day,end_day, ini_volt,end_volt))
						h_HV_vs_t[ipart][imod][ichan].GetXaxis().SetTimeDisplay(1)
						h_HV_vs_t[ipart][imod][ichan].SetDirectory(0)

						# DHV
						h_DHV[ipart][imod].append(TH1F("h_DHV_"+chan_name,"#DeltaHV ("+chan_name+"); #DeltaHV (V); #events", binning_Dvolt, ini_Dvolt, end_Dvolt))
						h_DHV[ipart][imod][ichan].SetDirectory(0)

						h_DHV_vs_t[ipart][imod].append(TProfile("DHV_vs_t_"+chan_name,"#DeltaHV ("+chan_name+") vs Time; Day; #DeltaHV (V)",binning_day,ini_day,end_day,ini_Dvolt,end_Dvolt))
						h_DHV_vs_t[ipart][imod][ichan].GetXaxis().SetTimeDisplay(1)
						h_DHV_vs_t[ipart][imod][ichan].SetDirectory(0)

						# Setting
						h_HVset[ipart][imod].append(TH1F("h_HVset_"+chan_name,"#Requested HV ("+chan_name+"); HVset (V); #events", binning_volt, ini_volt, end_volt))
						h_HVset[ipart][imod][ichan].SetDirectory(0)

						h_HVset_vs_t[ipart][imod].append(TProfile("HVset_vs_t_"+chan_name,"Requested HV ("+chan_name+") vs Time; Day; HVset (V)",binning_day,ini_day,end_day,ini_volt,end_volt))
						h_HVset_vs_t[ipart][imod][ichan].GetXaxis().SetTimeDisplay(1)
						h_HVset_vs_t[ipart][imod][ichan].SetDirectory(0)

					else:
						# fill with strings the channels that are not intrumented
						h_HV[ipart][imod].append('no instrumented channel')
						h_HV_vs_t[ipart][imod].append('no instrumented channel')
						h_DHV[ipart][imod].append('no instrumented channel')
						h_DHV_vs_t[ipart][imod].append('no instrumented channel')
						h_HVset[ipart][imod].append('no instrumented channel')
						h_HVset_vs_t[ipart][imod].append('no instrumented channel')

					if ichan<7:
						# There are 7 probes for the measurement of the temp
						h_T[ipart][imod].append(TH1F("h_T_"+chan_name,"T ("+chan_name+"); T (*C); #events", binning_temp, ini_temp, end_temp))
						h_T[ipart][imod][ichan].SetDirectory(0)

						h_T_vs_t[ipart][imod].append(TProfile("T_vs_t_"+chan_name,"T ("+chan_name+") vs Time; Day; T (*C)",binning_day,ini_day,end_day,ini_temp,end_temp))
						h_T_vs_t[ipart][imod][ichan].GetXaxis().SetTimeDisplay(1)
						h_T_vs_t[ipart][imod][ichan].SetDirectory(0)

						# There are 7 values of low voltage
						h_LV[ipart][imod].append(TH1F("h_LV_"+chan_name,"Low Voltage ("+chan_name+"); Low Voltage (mV); #events", binning_low_volt, ini_low_volt, end_low_volt))
						h_LV[ipart][imod][ichan].SetDirectory(0)

						h_LV_vs_t[ipart][imod].append(TProfile("LV_vs_t_"+chan_name,"Low Voltage ("+chan_name+") vs Time; Day; Low Voltage (mV)",binning_day,ini_day,end_day,ini_low_volt,end_low_volt))
						h_LV_vs_t[ipart][imod][ichan].GetXaxis().SetTimeDisplay(1)
						h_LV_vs_t[ipart][imod][ichan].SetDirectory(0)

					if ichan<2:
						# There are 2 values for the reference voltage
						h_ref[ipart][imod].append(TH1F("h_ref_"+chan_name," Reference Voltage ("+chan_name+"); Ref Voltage(mV); #events", binning_ref_volt, ini_ref_volt, end_ref_volt))
						h_ref[ipart][imod][ichan].SetDirectory(0)

						h_ref_vs_t[ipart][imod].append(TProfile("Ref_vs_t_"+chan_name,"Reference Voltage ("+chan_name+") vs Time; Day; Ref Voltage (mV)",binning_day,ini_day,end_day,ini_ref_volt,end_ref_volt))
						h_ref_vs_t[ipart][imod][ichan].GetXaxis().SetTimeDisplay(1)
						h_ref_vs_t[ipart][imod][ichan].SetDirectory(0)

					difference_date[ipart][imod].append([])
					difference_date[ipart][imod][ichan] = 0

					h_intertime[ipart][imod].append(TH1F(chan_name+"_inter_time", "Time difference between two consecutive measurements ("+chan_name+"); #Deltat (s); #events", 600,0,600))
					h_intertime[ipart][imod][ichan].SetDirectory(0)
					h_intertime_long[ipart][imod].append(TH1F(chan_name+"_inter_time_long" , "Time difference between two consecutive measurements ("+chan_name+"); #Deltat (s); Events", 588,0,84640))
					h_intertime_long[ipart][imod][ichan].SetDirectory(0)

		#Fill Histograms
		print(n_entries)

		point = n_entries/100
		increment = n_entries/20

		counter = int(0)
		while counter < n_entries:
		#while counter < 30000000:

			# progress bar

			if (counter % (5*point) == 0):
				sys.stdout.write("\r[" + "=" * (counter / increment) +  " " * ((n_entries - counter)/ increment) + "]" +  str(counter / point) + "%")
				sys.stdout.flush()
				print('\n')

			tree.GetEntry(counter)
			#print "Event", counter, "Partition = ",Partition[0], "Module = ", Module[0], "Channel = ", Channel[0], 'measureType =', MeasureType[0], 'measure = ', Measure[0], 'refe =', Reference[0]
			counter += 1

			measureType = MeasureType[0]
			partition = Partition[0]
			module = Module[0]-1
			channel = Channel[0]-1
			measure = Measure[0]
			reference = Reference[0]
			date = Date[0]

			deltaHV = Measure[0] - Reference[0]

			# Histograms concerning Temperatures
			if measureType==1:
				if channel<7:
					h_T[partition][module][channel].Fill(measure)
					h_T_vs_t[partition][module][channel].Fill(date,measure)

					if self.do_extraPlots:
						h_T_vs_t_mod[partition][module].Fill(date,measure)
						h_T_vs_t_part[partition].Fill(date,measure)

			elif measureType==2:
				if self.HVTool.IsThisChannelInstrumented(partition,module+1,channel+1):

					h_HV[partition][module][channel].Fill(measure)
					h_HV_vs_t[partition][module][channel].Fill(date,measure)

					h_HVset[partition][module][channel].Fill(reference)
					h_HVset_vs_t[partition][module][channel].Fill(date,reference)

					h_DHV[partition][module][channel].Fill(deltaHV)
					h_DHV_vs_t[partition][module][channel].Fill(date,deltaHV)

					if self.do_extraPlots:
						h_DHV_part[partition].Fill(deltaHV)
						h_DHV_vs_t_part[partition].Fill(date,deltaHV)
						h_DHV_vs_t_mod[partition][module].Fill(date,deltaHV)

						h_HV_vs_t_part[partition].Fill(date,measure)
						h_HV_vs_t_mod[partition][module].Fill(date,measure)

						if difference_date[partition][module][channel]==0:
							difference_date[partition][module][channel] = date
						else:
							h_intertime[partition][module][channel].Fill(date-difference_date[partition][module][channel])
							h_intertime_long[partition][module][channel].Fill(date-difference_date[partition][module][channel])

			elif measureType==3:
				if channel<2:
					h_ref[partition][module][channel].Fill(measure)
					h_ref_vs_t[partition][module][channel].Fill(date,measure)

			elif measureType==4:
				if channel<7:
					h_LV[partition][module][channel].Fill(measure)
					h_LV_vs_t[partition][module][channel].Fill(date,measure)

		# Analysis

		# Computing other plots
		for ipart in range(4): # loop in all the partitions

			threshold_mv_mean   = 1
			threshold_RMS_mean  =  1

			threshold_mv_sigmas = 1
			threshold_RMS_sigmas= 1

			badChan_threshold_mean  = 5
			badChan_threshold_sigma = 1

			for imod in range(64): # loop in all the modules

				for ichan in range(48): # loop in all the modules

					if self.HVTool.IsThisChannelInstrumented(ipart,imod+1,ichan+1):

						if h_DHV[ipart][imod][ichan].Integral() == 0:
							continue

						if not self.HVTool.HasThisChannelHVData(ipart, imod, ichan, h_HV_vs_t[ipart][imod][ichan]):
							continue

						isGood, imu, _, _= self.HVTool.IsThisChannelGood(h_DHV[ipart][imod][ichan],ini_Dvolt, end_Dvolt, binning_Dvolt, threshold_mv_mean, threshold_RMS_mean, badChan_threshold_mean, badChan_threshold_sigma)
						if isGood:
							h_means[ipart].Fill(imu)

		# Calculation of the thresholds

		h_means_mv = []
		h_means_rms = []

		h_sigmas_mv = []
		h_sigmas_rms = []

		for ipart in range(4): # loop in all the partitions
			imean, irms = self.HVTool.MeanAndSigmaThresholds(h_means[ipart])
			#print imean, irms
			h_means_mv.append(imean)
			h_means_rms.append(irms)

		for ipart in range(4): # loop in all the partitions
			for imod in range(64): # loop in all the modules
				for ichan in range(48): # loop in all the modules

					if self.HVTool.IsThisChannelInstrumented(ipart,imod+1,ichan+1):

						if h_DHV[ipart][imod][ichan].Integral() == 0:
							continue

						isigma = self.HVTool.SigmasThresholds(h_DHV[ipart][imod][ichan])
						#print isigma
						h_sigmas[ipart].Fill(isigma)

			fit_sigmas = TF1("f2","gaus",0,10)
			h_sigmas[ipart].Fit("f2","RSQ")

			imean = fit_sigmas.GetParameter(1)
			irms = fit_sigmas.GetParameter(2)
			h_sigmas_mv.append(imean)
			h_sigmas_rms.append(irms)

		# Separating good and bad channels
		outputFile = TFile(outDir + '/HV_HVset_plots.root', 'RECREATE')
		self.HVTool.mkdir(outDir + '/Summary')

		self.HVTool.FilesHeaderWriter(outDir + '/good_channels.txt',outDir + '/bad_channels.txt',outDir + '/for_latex.txt' , ini_Date, fin_Date)
		for ipart in range(4): # loop in all the partitions
			part_name = self.HVTool.get_partition_name(ipart)
			print('Partition ', part_name, ' proceeding ...')

			threshold_mv_mean    = h_means_mv[ipart]
			threshold_RMS_mean   = h_means_rms[ipart]

			threshold_mv_sigmas  = h_sigmas_mv[ipart]
			threshold_RMS_sigmas = h_sigmas_rms[ipart]

			self.HVTool.PartitionHeader(part_name, threshold_mv_mean, threshold_RMS_mean, threshold_mv_sigmas, threshold_RMS_sigmas, outDir + '/good_channels.txt', outDir + '/bad_channels.txt')

			for imod in range(64): # loop in all the modules
				for ichan in range(48): # loop in all the modules

					if self.HVTool.IsThisChannelInstrumented(ipart,imod+1,ichan+1):

						status = 0

						if h_DHV[ipart][imod][ichan].Integral()==0:
							status = 4
						else:
							if not self.HVTool.HasThisChannelHVData(ipart, imod, ichan, h_HV_vs_t[ipart][imod][ichan]):
								status = 4
							else:
								isGood, imu, isigma, causes = self.HVTool.IsThisChannelGood(h_DHV[ipart][imod][ichan],ini_Dvolt, end_Dvolt, binning_Dvolt, threshold_mv_mean, threshold_RMS_mean, badChan_threshold_mean, badChan_threshold_sigma)
								if isGood:

									isUnstable, causes_un, count_error = self.HVTool.IsThisChannelUnStable(h_DHV_vs_t[ipart][imod][ichan], imu)
									if isUnstable:
										# Unstable channels
										status = 3
										self.HVTool.WriteBadChannel(part_name, imod+1, ichan+1, imu, isigma, outDir + '/bad_channels.txt', causes_un)
										self.HVTool.PerformDiagnosticForBadChannel(outDir + '/bad_channels.txt', h_HV[ipart][imod][ichan], h_HVset[ipart][imod][ichan], h_T[ipart][imod][5], h_ref[ipart][imod][1])
										self.HVTool.threesholdPlots_DHV(ipart, imod+1, ichan+1, imu, outputFile, h_DHV_vs_t[ipart][imod][ichan] , h_HV_vs_t[ipart][imod][ichan], h_HVset_vs_t[ipart][imod][ichan])

										#print part_name, imod+1, ichan+1, count_error
									else:
										# Stable and without offset channel
										status = 1
										self.HVTool.WriteGoodChannel(part_name, imod+1, ichan+1, imu, isigma, outDir + '/good_channels.txt')
										self.HVTool.WriteSummary(part_name, imod+1, ichan+1, outDir + '/short_good_channels.txt')
										self.HVTool.threesholdPlots_DHV(ipart, imod+1, ichan+1, imu, outputFile, h_DHV_vs_t[ipart][imod][ichan] , h_HV_vs_t[ipart][imod][ichan], h_HVset_vs_t[ipart][imod][ichan])

										#print part_name, imod+1, ichan+1, count_error
								else:
									#  Channel with offset
									status = 2

									isUnstable, causes_un, count_error = self.HVTool.IsThisChannelUnStable(h_DHV_vs_t[ipart][imod][ichan], imu)
									if isUnstable:
										status = 3
										causes += causes_un

									self.HVTool.WriteBadChannel(part_name, imod+1, ichan+1, imu, isigma, outDir + '/bad_channels.txt', causes)
									self.HVTool.PerformDiagnosticForBadChannel(outDir + '/bad_channels.txt', h_HV[ipart][imod][ichan], h_HVset[ipart][imod][ichan], h_T[ipart][imod][5], h_ref[ipart][imod][1])
									self.HVTool.offset_forLatex(part_name, imod+1, ichan+1, outDir + '/for_latex.txt' , imu, h_HV[ipart][imod][ichan], h_HVset[ipart][imod][ichan])
									self.HVTool.threesholdPlots_DHV(ipart, imod+1, ichan+1, imu, outputFile, h_DHV_vs_t[ipart][imod][ichan] , h_HV_vs_t[ipart][imod][ichan], h_HVset_vs_t[ipart][imod][ichan])

									#print part_name, imod+1, ichan+1, count_error

						# Performing mapping for this channel
						h2_map[ipart].Fill(imod+1, ichan+1, status)

						self.HVTool.mkdir(outDir + '/LISTS')
						date_format = time.strftime("%y%m%d", fin_Date.timetuple())
						# writes summary text files
						if status==2:
							self.HVTool.WriteSummary(part_name, imod+1, ichan+1, outDir + '/LISTS/'+date_format+'_bad_list.txt')
						if status==3:
							self.HVTool.WriteSummary(part_name, imod+1, ichan+1, outDir + '/LISTS/'+date_format+'_unstable_list.txt')
						if status==4:
							self.HVTool.WriteSummary(part_name, imod+1, ichan+1, outDir + '/LISTS/'+date_format+'_noHV_list.txt')

		outputFile.Close()

		# Write
		for ipart in range(4): # loop in all the partitions
			outFile.cd()

			h_means[ipart].Write()
			h_sigmas[ipart].Write()
			h2_map[ipart].Write()

			if self.do_extraPlots:
				h_DHV_part[ipart].Write()
				h_DHV_vs_t_part[ipart].Write()
				h_HV_vs_t_part[ipart].Write()
				h_T_vs_t_part[ipart].Write()

			for imod in range(64): # loop in all the modules

				if self.do_extraPlots:
					outFile.cd('HV/')
					h_HV_vs_t_mod[ipart][imod].Write()
					outFile.cd('DHV/')
					h_DHV_vs_t_mod[ipart][imod].Write()
					outFile.cd('T/')
					h_T_vs_t_mod[ipart][imod].Write()

				for ichan in range(48): # loop in all the channels

					if self.HVTool.IsThisChannelInstrumented(ipart,imod+1,ichan+1):
						outFile.cd("HV/")
						h_HV[ipart][imod][ichan].Write()
						h_HV_vs_t[ipart][imod][ichan].Write()

						outFile.cd('DHV/')
						h_DHV[ipart][imod][ichan].Write()
						h_DHV_vs_t[ipart][imod][ichan].Write()

						outFile.cd('HVset/')
						h_HVset[ipart][imod][ichan].Write()
						h_HVset_vs_t[ipart][imod][ichan].Write()

					if ichan<7:
						outFile.cd("T/")
						h_T[ipart][imod][ichan].Write()
						h_T_vs_t[ipart][imod][ichan].Write()

					if ichan<2:
						outFile.cd('Ref/')
						h_ref[ipart][imod][ichan].Write()
						h_ref_vs_t[ipart][imod][ichan].Write()

					if ichan<7:
						outFile.cd('LV/')
						h_LV[ipart][imod][ichan].Write()
						h_LV_vs_t[ipart][imod][ichan].Write()

					outFile.cd('Time/')
					if self.do_extraPlots:
						h_intertime[ipart][imod][ichan].Write()
						h_intertime_long[ipart][imod][ichan].Write()

		outFile.Write()
		outFile.Close()

	def ProcessRegion(self, region):
		if region.GetHash() == 'TILECAL':
			print("You produced the evolutions plots for HV, HVset, DHV, T, LV and refVol. The next step is perfom the HV analysis")
			print('The analysis finished at', datetime.datetime.now())


from Body.listen import Listen
from Body.speak import speak
from Features.clap import Tester

Tester()

# speak("Welcome Sohal i am MARK 2.0 ")

import speech_recognition as sr
import os
from mark import MainExe
def Listen():

    r = sr.Recognizer()

    with sr.Microphone() as source :
        print("Listning...")
        r.pause_threshold=1
        audio= r.listen(source,0,8)
    try:
        print("Recognizing...")
        query=r.recognize_google(audio,language="en")
    except:
        return ""

    query=str(query).lower()
    print(query)

    return query

def WakeupDetected():
    queery =Listen().lower()

    if "Wake up" is queery:
        print("DOne ")

        MainExe()
    else:
        pass

while True:
    WakeupDetected()


import os
import tweepy
import json
from pprint import pprint

# fetch the secrets from our virtual environment variables
CONSUMER_KEY = os.environ['CONSUMER_KEY']
CONSUMER_SECRET = os.environ['CONSUMER_SECRET']
ACCESS_TOKEN = os.environ['ACCESS_TOKEN']
ACCESS_SECRET = os.environ['ACCESS_SECRET']

# authenticate to the service we're accessing
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)

# create the connection
api = tweepy.API(auth)

# setup query
search_words = "#datascience"
date_since = "2000-09-01"
tweets = tweepy.Cursor(api.search, q=search_words, lang="en", since=date_since).items(100)
count = 1
print(type(tweets))
for tweet in tweets:
    print(tweet.coordinates)


from datetime import timedelta
from ebedke.utils.date import on_workdays, days_lower_ascii
from ebedke.utils.http import get_dom
from ebedke.utils.text import remove_accents, pattern_slice
from ebedke.pluginmanager import EbedkePlugin

URL = "http://www.magyarqtr.com/index.php?option=com_k2&view=item&layout=item&id=13&Itemid=231&lang=hu"

@on_workdays
def get_menu(today):
    dom = get_dom(URL)
    menu_week_number = dom.xpath("/html/body//div[@class='itemBody']//h2[@class='itemTitle']//text()")
    menu_week_number = ''.join(char for char  in ''.join(menu_week_number) if char.isnumeric())
    _, current_week_number, _ = today.date().isocalendar()

    if menu_week_number == str(current_week_number):
        week_menu = dom.xpath("/html/body//div[@class='itemBody']//div[@class='heti-menu']//text()")
        lower = lambda line: remove_accents(line).lower()
        menu = pattern_slice(week_menu, [days_lower_ascii[today.weekday()]], days_lower_ascii, modifier=lower)
    else:
        menu = []

    return menu

plugin = EbedkePlugin(
    enabled=True,
    groups=["ferenciek"],
    name="MagyarQTR",
    id="mqtr",
    url=URL,
    downloader=get_menu,
    ttl=timedelta(hours=24),
    cards=[],
    coord=(47.489049, 19.054093)
)


import os
import sys
import argparse
from pathlib import Path
from pdf2image import convert_from_path
from shutil import copyfile

def convert(input_pdf_path, output_images_dir_path):
    print(input_pdf_path, output_images_dir_path)

    # poppler/binを環境変数PATHに追加する
    # poppler_dir = Path(__file__).parent.absolute() / "poppler/bin"
    # os.environ["PATH"] += os.pathsep + str(poppler_dir)

    # PDFファイルのパス
    pdf_path = Path("./pdf_file/pdf3009p.pdf")

    # PDF -> Image に変換（150dpi）
    pages = convert_from_path(input_pdf_path, 150)

    # 画像ファイルを１ページずつ保存
    image_dir = Path(output_images_dir_path)
    for i, page in enumerate(pages):
        file_name = pdf_path.stem + "_{:02d}".format(i + 1) + ".jpeg"
        image_path = image_dir / file_name
        # JPEGで保存
        page.save(str(image_path), "JPEG")

def main():
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument('input', help='Relative or absolute path of the input PDF file')
    parser.add_argument('-o', '--out', help='Relative or absolute path of the output directory of images')
    args = parser.parse_args()

    # In case no output directory is specified, store in ./temp
    if not args.out:
        args.out = './temp'
        copyfile(args.input, args.input.replace(".pdf", "_BACKUP.pdf"))

    # Run
    convert(args.input, args.out)

if __name__ == '__main__':
    main()


from datetime import datetime
import os
import configparser
import sys
import shutil
from requests_oauthlib import OAuth1Session
from distutils.dir_util import copy_tree

# 現ディレクトリ
os.chdir(os.path.dirname(os.path.abspath(__file__)))

# config.ini読み込み
config = configparser.ConfigParser()
config.read("config.ini")

# ファイルパス・Twitterトークン定義
blogpath = config["path"]["blog_folder"]
blogurl = config["path"]["blog_url"]
ckey = config["twitter"]["consumer_key"]
csec = config["twitter"]["consumer_secret"]
atkn = config["twitter"]["access_token"]
asec = config["twitter"]["access_secret"]

# ディレクトリ生成
os.chdir(blogpath)
os.makedirs("articles/", exist_ok=True)
os.makedirs("pictures/", exist_ok=True)
os.chdir(os.path.dirname(os.path.abspath(__file__))) # 戻ってくる

# ファイルパスの合成
print("記事のタイトルを入力してください")
title_atc = input(">> ") # タイトル取得
current_time = datetime.now().strftime("%Y%m%d%H%M%S") # 日付・時刻取得
crtimestr = str(current_time) # 時間の文字列化
title_code = "
{}
\n".format(title_atc) # タイトル生成

file_path = f"{blogpath}articles/{crtimestr}.html" # 記事ファイルパス合成
file_code = f"./articles/{crtimestr}.html" # コード上に挿入するファイルパス文字列

# トップページの合成
day_str = str(datetime.now().strftime("%Y-%m-%d")) # yyyy-mm-dd形式で日付テキストを生成

index_path = f"{blogpath}index.html" # ファイル生成(初回)/上書き用パス
insert_code = f'\n

{day_str}

{title_atc}

' # 挿入するコード

# top1.html
with open("assets/top1.html", mode="r", encoding="utf-8") as f:
    top1 = f.read()

# insert_code
if os.path.isfile("assets/insert_code_file.html"):
    with open("assets/insert_code_file.html", mode="r", encoding="utf-8") as f:
        conv = f.read()
else:
    conv = ""

with open("assets/insert_code_file.html", mode="w", encoding="utf-8") as f:
    icf = insert_code + "\n" + conv
    f.write(icf)

# top2.html
with open("assets/top2.html", mode="r", encoding="utf-8") as f:
    top2 = f.read()

# 合成したものをindex_pathに保存
with open(index_path, mode="w", encoding="utf-8") as f:
    f.write(top1 + icf + top2)

# 記事の合成

# 1.html
with open("assets/1.html", mode="r", encoding="utf-8") as f:
    t1 = f.read()

# 2.html
with open("assets/2.html", mode="r", encoding="utf-8") as f:
    t2 = f.read()

# article_base.html
with open("assets/article_base.html", mode="r", encoding="utf-8") as f:
    atc = f.read()

# 3.html
with open("assets/3.html", mode="r", encoding="utf-8") as f:
    t3 = f.read()

# 合成したものをfile_pathに保存
with open(file_path, mode="w", encoding="utf-8")as f:
    f.write(t1 + title_atc + t2 + title_code + atc + t3)

# 画像を移動
copy_tree("pictures", f"{blogpath}pictures")

# CSSを移動
shutil.copy("assets/style.css", f"{blogpath}style.css")

input("ファイルの準備ができました。サーバーへのアップロードが完了したら、キーを押して続行してください。Twitterに更新通知が投稿されます")

# Twitterに更新通知をツイートする
endpoint = "https://api.twitter.com/1.1/statuses/update.json"
tweet = "ブログを更新しました: " + title_atc + f" {blogurl}articles/" + crtimestr + ".html"
params = {"status": tweet}
twAuth = OAuth1Session(ckey, csec, atkn, asec)
postReq = twAuth.post(endpoint, params = params)
if postReq.status_code != 200: # ここも自動化できたらうれしい
    print("HTTP Status Code:" + str(postReq.status_code))
    print("エラーが発生した可能性があります。")
    print("articleフォルダ内の最新記事とindex.htmlにある最新記事へのリンクを削除し、再試行してください。")
    print("もしくは、更新通知のみを手動で行ってください。")
    input("Enterを押すとプログラムが終了します。")
    sys.exit()
input("更新通知が正常に投稿されました。キーを押して終了します")


"""
直接在 ipython 里复制输出结果
参考：
https://blog.michaelyin.info/send-content-to-clipboard-in-ipython/
https://gist.github.com/bwagner/270da7c7d31af7ffaca32674557fc172

Add copy to clipboard from IPython!
To install, just copy it to your profile/startup directory, typically:
    ~/.ipython/profile_default/startup/
Example usage:
    %clip hello world
    # will store "hello world"
    a = [1, 2, 3]
    %clip a
    # will store "[1, 2, 3]"
You can also use it with cell magic
    In [1]: %%clip
       ...: Even multi
       ...: lines
       ...: work!
       ...:
If you don't have a variable named 'clip' you can rely on automagic:
    clip hey man
    a = [1, 2, 3]
    clip a
# This version removes the dependency of AppKit, but maintains compatibility with linux and osx.
# using ideas from: https://gist.github.com/vpontis/46e5d3154cda92ce3e0f
# It also provides infrastructure to easily add further platforms (function _get_implementation).
"""

import sys
from subprocess import Popen, PIPE

from IPython.core.magic import register_line_cell_magic

def _get_implementation():

    try:
        _get_implementation.impls
    except AttributeError:
        _get_implementation.impls = {}

        if sys.platform.startswith('linux'):
            def _clip(arg):
                p = Popen(['xsel', '-pi'], stdin=PIPE)
                p.communicate(arg)
            _get_implementation.impls['linux'] = _clip

        elif sys.platform == 'darwin':
            def _clip(arg):
                p = Popen('pbcopy', env={'LANG': 'en_US.UTF-8'}, stdin=PIPE)
                p.communicate(arg.encode('utf-8'))
            _get_implementation.impls[sys.platform] = _clip

        # add further platforms below here:

        else:
            raise ImportError(
                "Clip magic doesn't work on your platform: '{}'".format(sys.platform))

    return _get_implementation['linux'] \
        if sys.platform.startswith('linux') \
        else _get_implementation.impls[sys.platform]

def _copy_to_clipboard(arg):
    arg = str(globals().get(arg) or arg)

    _get_implementation()(arg)

    print('Copied to clipboard!')

@register_line_cell_magic
def clip(line, cell=None):
    if line and cell:
        cell = '\n'.join((line, cell))

    _copy_to_clipboard(cell or line)

# We delete it to avoid name conflicts for automagic to work
del clip


import pandas as pd
import cv2
import numpy as np
import time
import mpi4py.MPI
rank = mpi4py.MPI.COMM_WORLD.Get_rank()
size = mpi4py.MPI.COMM_WORLD.Get_size()

q=pd.read_csv('eeg_raw_values.csv').to_numpy()
#print(q)

task_list = range(len(q))
z=time.time()
def fft_v(task,i):
	x=q[i]
	x = np.asarray(x, dtype=float)
	N = x.shape[0]
	if np.log2(N) % 1 > 0:
		raise ValueError("must be a power of 2")

	N_min = min(N, 2)

	n = np.arange(N_min)
	k = n[:, None]
	M = np.exp(-2j * np.pi * n * k / N_min)
	X = np.dot(M, x.reshape((N_min, -1)))
	while X.shape[0] < N:
		X_even = X[:, :int(X.shape[1] / 2)]
		X_odd = X[:, int(X.shape[1] / 2):]
		terms = np.exp(-1j * np.pi * np.arange(X.shape[0])
			        / X.shape[0])[:, None]
		X = np.vstack([X_even + terms * X_odd,
			       X_even - terms * X_odd])
	return X.ravel()
def fft():
	x=q[i]
	x = np.asarray(x, dtype=float)
	N = x.shape[0]
	if np.log2(N) % 1 > 0:
		raise ValueError("must be a power of 2")

	N_min = min(N, 2)

	n = np.arange(N_min)
	k = n[:, None]
	M = np.exp(-2j * np.pi * n * k / N_min)
	X = np.dot(M, x.reshape((N_min, -1)))
	while X.shape[0] < N:
		X_even = X[:, :int(X.shape[1] / 2)]
		X_odd = X[:, int(X.shape[1] / 2):]
		terms = np.exp(-1j * np.pi * np.arange(X.shape[0])
			        / X.shape[0])[:, None]
		X = np.vstack([X_even + terms * X_odd,
			       X_even - terms * X_odd])
	return X.ravel()

for i,task in enumerate(task_list):
  #print(i,task)
  #This is how we split up the jobs.
  #The % sign is a modulus, and the "continue" means
  #"skip the rest of this bit and go to the next time
  #through the loop"
  # If we had e.g. 4 processors, this would mean
  # that proc zero did tasks 0, 4, 8, 12, 16, ...
  # and proc one did tasks 1, 5, 9, 13, 17, ...
  # and do on.
  if i%size!=rank: continue
  #print ("Task number %d (%d) being done by processor %d of %d" % (i, task, rank, size))
  fft_v(task,i)
b=time.time()
print("PARALLEL",b-z)


#!/usr/bin/env python
# coding: utf-8

import datetime
import time
import numpy as np
import pandas as pd
import warnings
import os
import xlwings as xw
import function
warnings.filterwarnings("ignore")  # 取消警告

data_path = os.getcwd() + "/Data"
PCMapping = pd.read_excel(data_path + "/Profit Center Hierarchy Flattened (MDG).xlsx")
popdata = data_path + "/POPData"
popyearlydata1 = data_path + "/Data1"
popyearlydata2 = data_path + "/Data2"

function.confirmversion()
pop_period = function.pop_period
pop_version = function.pop_version
period_list = function.period_list

#计算POP
file_names = []
file_paths = []
for roots, dirs, files in os.walk(popdata):
    for file in files:
        if file[-5:] == '.xlsx':
            file_names.append(file[:file.rfind(' Data')])
            file_paths.append(popdata + "//" + file)
for file_path, file_name in zip(file_paths, file_names):
    function.cal_pop(file_path, file_name)

#计算POP Yearly
file_names = []
file_data1paths = []
file_data2paths = []
file_dataRYpaths =[]
for roots, dirs, files in os.walk(popyearlydata1):
    for file in files:
        if file[-5:] == '.xlsx':
            file_names.append(file[:file.rfind(' Data1')])
            file_data1paths.append(popyearlydata1 + "//" + file)
for o in file_data1paths:
    file_data2paths.append(o.replace("Data1", "Data2"))
for o in file_data1paths:
    file_dataRYpaths.append(o.replace("Data1", "DataRY"))
for file_path1, file_path2, file_pathRY,file_name in zip(file_data1paths, file_data2paths,file_dataRYpaths,
                                             file_names):
    function.cal_yearlypop(file_path1, file_path2, file_pathRY,file_name)

lastfilepath = os.getcwd() + r"/LastFiles"
pathdata = os.getcwd() + r'/POP/%s.csv'
pathyearlydata = os.getcwd() + r'/POPYearly/%s-yearly.csv'
file_names = []
file_poppaths = []
file_poppath1s = []
file_poppath2s = []
for roots, dirs, files in os.walk(lastfilepath):
    for file in files:
        if file[:1] != '~' and file[-5:] == '.xlsx':
            file_poppaths.append(lastfilepath + "//" + file)
for i in file_poppaths:
    file_name = i.split("//")[-1][:3]
    file_name = ("JGP" if (i.split("//")[-1][:7] == 'JABIL G') else "JRI" if
                 (i.split("//")[-1][:7] == 'JABIL R') else "JPS" if
                 (i.split("//")[-1][:7] == 'JABIL P') else file_name)
    file_names.append(file_name)
    file_poppath1s.append(pathdata % file_name)
    file_poppath2s.append(pathyearlydata % file_name)

#把结果复制到对应的POP文件

popfilesavepath = os.getcwd() + r"/POPFiles"
popperiodsort = []
for i in range(0, 4):
    if int(pop_period[1:2]) + i > 4:
        a = int(pop_period[1:2]) + i - 4
    else:
        a = int(pop_period[1:2]) + i
    popperiodsort.append("Q%s" % str(a))
current_version = "8 Quarter (%s %s)" % (pop_period, pop_version)
try:
    for path, path1, path2, file_name in zip(file_poppaths, file_poppath1s,
                                             file_poppath2s, file_names):

        app = xw.App(visible=True, add_book=False)  # 程序可见，只打开不新建工作薄
        app.display_alerts = False  # 警告关闭
        app.screen_updating = True  # 屏幕更新关闭
        wb = app.books.open(path)  # 打开该excel
        Sheets = wb.sheets
        wb1 = app.books.open(path1)  # 打开该excel
        wb2 = app.books.open(path2)

        sheet_POPdata = wb1.sheets[file_name]  # 指定sheet
        sheet_POPyearlydata = wb2.sheets['%s-yearly' % file_name]

        sheet_POP = wb.sheets['Data']  # 指定sheet
        if pop_version == 'Bid1':
            popdata = sheet_POPdata.range('A1').expand('table').value
            sheet_POP.clear()
            sheet_POP.range('A1').options(expand='table').value = popdata
        else:
            popdata = sheet_POPdata.range('A1').expand('table').value
            row = sheet_POP.range('A1').expand('table').last_cell.row + 1
            sheet_POP.range('A' +
                            str(row)).options(expand='table').value = popdata
            sheet_POP.range('A' + str(row)).api.EntireRow.Delete()
        sheet_YearlyPOP = wb.sheets["YearlyData"]  # 指定sheet
        popyearlydata = sheet_POPyearlydata.range('A1').expand('table').value
        sheet_YearlyPOP.clear()
        sheet_YearlyPOP.range('A1').options(
            expand='table').value = popyearlydata

        #sheet_POP.api.visible = False  #hide sheet
        #sheet_YearlyPOP.api.visible = False
        wb1.close()
        wb2.close()

        #time.sleep(15)
        wb1 = app.books.open(os.getcwd()+r"/refreshall.xlsm")
        refreshall = wb1.macro('refreshall')
        for Sheet in Sheets:
            wb.sheets[Sheet].activate()
            if (Sheet.name == 'PoP'):
                refreshall()
                time.sleep(15)
            #wb.api.ActiveSheet.PivotTables('PivotTable1').PivotCache().refreshall

            if (Sheet.name == 'PoP') or (Sheet.name == 'Capex'):
                for p, i in zip(popperiodsort, range(1, 5)):
                    wb.sheets[Sheet].activate()
                    wb.api.ActiveSheet.PivotTables("PivotTable1").PivotFields(
                        "Version").PivotItems("%s Locked Forecast" %
                                              p).Position = i

            if Sheet.name not in [
                    'Data', 'YearlyData', 'PoP', 'Yearly PoP', 'Capex',
                    'CHECK', 'Yearly Capex'
            ]:
                wb.sheets[Sheet].activate()
                wb.api.ActiveSheet.PivotTables('PivotTable1').PivotFields(
                    "Version").CurrentPage = current_version

            if Sheet.name not in [
                    'Data', 'YearlyData', 'Yearly PoP', 'CHECK', 'Yearly Capex'
            ]:
                wb.sheets[Sheet].activate()
                if (wb.sheets[Sheet].range('C11') == period_list[0]) | (
                        wb.sheets[Sheet].range('D11') == period_list[0]):
                    wb.api.ActiveSheet.PivotTables("PivotTable1").PivotFields(
                        "Period").PivotItems("%s" %
                                             period_list[0]).Visible = False

            if Sheet.name in ['Yearly PoP', 'Yearly Capex']:
                wb.sheets[Sheet].activate()
                wb.api.ActiveSheet.PivotTables("PivotTable1").PivotFields(
                    "Period").PivotItems("FY 2017").Visible = False
                wb.api.ActiveSheet.PivotTables("PivotTable1").PivotFields(
                    "Period").PivotItems("FY 2022").Visible = True
        file_name = function.changename(file_name)
        wb.sheets['PoP'].activate()
        #Del Account:  Other Profit/(Loss)
        #wb.api.ActiveSheet.PivotTables("PivotTable1").PivotFields(
        #                "Account").PivotItems(" Other Profit/(Loss)").Visible = False
        wb.save(popfilesavepath + '/%s POP File_%s.xlsx' %
                (file_name, current_version))
        wb.close()
        #app.quit()
        print("%s POP File Done!" % file_name)
except Exception as e:
    print(e)
finally:
    app.quit()


#!/bin/env python
# -*- coding: utf-8 -*-
## chart_window ##

import gtk
from cairo_canva import cairo_canva
from background_plotter import background_plotter
from legend_plotter import legend_plotter
from mesh_plotter import mesh_plotter
from charts_plotter import charts_plotter
from complex_plotter import complex_plotter
from drawing_rectangle import drawing_rectangle

class chart_window(object):
    """\brief chart window controller and creator
    """
    def __init__(self, parent):
        self._parent = parent
        self.window = gtk.Dialog(title = "График", buttons = (gtk.STOCK_CLOSE, gtk.RESPONSE_CANCEL))
        if self._parent != None:
            self.window.set_transient_for(self._parent.window.builder.get_object('main_window'))
            self.window.set_position(gtk.WIN_POS_CENTER_ON_PARENT)
        else:
            self.window.set_position(gtk.WIN_POS_CENTER)
        self.chart_area = cairo_canva()
        self.background = background_plotter()
        self.legend = legend_plotter()
        self.mesh = mesh_plotter()
        self.charts = charts_plotter()
        self.rectangle = drawing_rectangle()
        self.plotter = complex_plotter(self.rectangle,
                                       legend = self.legend,
                                       mesh = self.mesh,
                                       charts = self.charts)
        self.chart_area.add_drawer(self.background)
        self.chart_area.add_drawer(self.plotter)
        self.window.get_content_area().pack_start(self.chart_area, True, True)
        self.window.set_default_size(640, 480)

    def show(self, ):
        """\brief show chart window
        """
        self.window.show_all()

    def plot(self, data):
        """\brief plot data in the chart area
        \param data - \ref data_chart.data_chart instance
        """
        self.plotter.plot(data)

    def autoscale(self, top_indent = 0, bottom_indent = 0):
        """\brief scale rectangle when data already ploted
        \param top_indent float, top chart indent in percents
        \param bottom_indent float, bottom chart indent in percents
        """
        self.plotter.autoscale()
        self.rectangle.indent(top_indent, bottom_indent)

    def redraw(self, ):
        self.plotter.redraw()

    def set_background_color(self, color_string):
        """\brief set color of background
        \param color_string str, background color
        """
        self.background.set_color(color_string)

    def set_mesh_line_width(self, line_width):
        """\brief
        \param line_width - float
        """
        self.mesh.set_line_width(line_width)

    def set_mesh_color(self, color_string):
        """\brief set color of the mesh
        \param color_string str, mesh color string
        """
        self.mesh.set_color(color_string)

    def set_legend_font(self, font):
        """\brief set font for legend
        \param font
        """
        self.legend.set_font(font)

    def set_legend_color(self, color):
        """\brief
        \param color - str, representing color
        """
        self.legend.set_color(color)

    def set_mesh_font(self, font):
        """\brief set font for mesh text prints
        \param font
        """
        self.mesh.set_font(font)

    def flush(self, ):
        """\brief delete all data before new plot
        """
        self.plotter.flush()

    def set_legend_on_top(self, legend_on_top):
        """\brief if true legend rawing on top
        \param legend_on_top
        """
        self.plotter.set_legend_on_top(legend_on_top)

    def get_legend_on_top(self, ):
        """\brief return true if legend drawing on top
        """
        return self.plotter.get_legend_on_top()

if __name__ == '__main__':
    from datetime import datetime
    from random import random
    from data_chart import data_chart
    win = chart_window(None)
    x = [datetime(2010, 10, 10), datetime(2010,10, 11), datetime(2010, 10, 12)]
    ys = []
    colors = []
    for i in xrange(7):
        pk = []
        for j in xrange(3):
            pk.append(random())
        ys.append(pk)
        colors.append((random(), random(), random()))
    legends = ['every', 'hunter', 'whant', 'to know', 'where', 'pheasant', 'is sitting']
    data_charts = []
    for i in xrange(7):
        data_charts.append(data_chart(zip(x, ys[i]), color = colors[i], legend = legends[i], line_width = 1.5))

    win.set_legend_font('Terminus 20')
    win.set_mesh_font('Terminus 20')
    win.set_background_color((0, 0, 0))
    win.set_mesh_color((1, 1, 1))
    for dd in data_charts:
        win.plot(dd)
    win.autoscale()
    win.mesh.set_line_width(2)
    win.show()
    win.window.run()