import numpy as np
from sklearn.decomposition import PCA

def get_fg_bg_epochs(rec):
    """
    Get fg / bg epoch names
    """

    fgs = np.unique([e.split('STIM_null_')[1] for e in rec.epochs.name.unique() if e.startswith('STIM_null_')]).tolist()
    bgs = np.unique([e.split('_null')[0].split('STIM_')[1] for e in rec.epochs.name.unique() if e.endswith('_null')]).tolist()

    return fgs, bgs

def get_fg_bg_transitions(rec):
    """
    get epochs that transitioned from only fg to fg + bg
    """
    trans_epochs = [e for e in rec.epochs.name.unique() if ('null' not in e) & ('STIM_' in e)]
    fgs, bgs = get_fg_bg_epochs(rec)
    lfgs = [f for f in fgs if '-0-1' in f]
    sbgs = [b for b in bgs if '-0.5-1' in b]
    fg_bg = [e for e in trans_epochs if any(ele in e for ele in lfgs) & any(ele in e for ele in sbgs)]
    return fg_bg

def get_bg_fg_transitions(rec):
    """
    get epochs that transitioned from only bg to fg + bg
    """
    trans_epochs = [e for e in rec.epochs.name.unique() if ('null' not in e) & ('STIM_' in e)]
    fgs, bgs = get_fg_bg_epochs(rec)
    sfgs = [f for f in fgs if '-0.5-1' in f]
    lbgs = [b for b in bgs if '-0-1' in b]
    bg_fg = [e for e in trans_epochs if any(ele in e for ele in sfgs) & any(ele in e for ele in lbgs)]
    return bg_fg

def get_fg_bg_full_overlap(rec):
    """
    get epochs that overlapped for the full period
    """
    trans_epochs = [e for e in rec.epochs.name.unique() if ('null' not in e) & ('STIM_' in e)]
    fgs, bgs = get_fg_bg_epochs(rec)
    lfgs = [f for f in fgs if '-0-1' in f]
    lbgs = [b for b in bgs if '-0-1' in b]
    bg_fg = [e for e in trans_epochs if any(ele in e for ele in lfgs) & any(ele in e for ele in lbgs)]
    return bg_fg

def get_fg_pcs(rec, collapse=False):
    """
    Get top pc axes of FG evoked responses
    """
    prestim = rec['resp'].extract_epoch('PreStimSilence').shape[-1]
    poststim = rec['resp'].extract_epoch('PostStimSilence').shape[-1]
    nCells = rec['resp'].shape[0]

    fgs, bgs = get_fg_bg_epochs(rec)
    fgs1 = [f for f in fgs if f.endswith('0-1')]
    if collapse:
        fg_resp = np.zeros((len(fgs1), nCells))
    else:
        fg_resp = []
    for i, fg in enumerate(fgs1):
        if collapse:
            resp = rec['resp'].extract_epoch('STIM_null_'+fg)[:, :, prestim:-poststim].sum(axis=-1).mean(axis=0)
            prespont = rec['resp'].extract_epoch('STIM_null_'+fg)[:, :, 0:prestim].sum(axis=-1).mean(axis=0)
            fg_resp[i, :] = resp - prespont
        else:
            resp = rec['resp'].extract_epoch('STIM_null_'+fg)[:, :, prestim:-poststim].mean(axis=0)
            prespont = rec['resp'].extract_epoch('STIM_null_'+fg)[:, :, 0:prestim].mean(axis=(0, -1))
            fg_resp.append((resp.T - prespont))

    if type(fg_resp) == list:
        fg_resp = np.concatenate(fg_resp, axis=0)
    fg_pca = PCA()
    fg_pca.fit(fg_resp)
    return fg_pca.components_

def get_bg_pcs(rec, collapse=False):
    """
    Get top pc axes of BG evoked responses
    """
    prestim = rec['resp'].extract_epoch('PreStimSilence').shape[-1]
    poststim = rec['resp'].extract_epoch('PostStimSilence').shape[-1]
    nCells = rec['resp'].shape[0]

    fgs, bgs = get_fg_bg_epochs(rec)
    bgs1 = [b for b in bgs if b.endswith('0-1')]
    if collapse:
        bg_resp = np.zeros((len(bgs1), nCells))
    else:
        bg_resp = []
    for i, bg in enumerate(bgs1):
        if collapse:
            resp = rec['resp'].extract_epoch('STIM_' + bg + '_null')[:, :, prestim:-poststim].sum(axis=-1).mean(axis=0)
            prespont = rec['resp'].extract_epoch('STIM_' + bg + '_null')[:, :, 0:prestim].sum(axis=-1).mean(axis=0)
            bg_resp[i, :] = resp - prespont
        else:
            resp = rec['resp'].extract_epoch('STIM_' + bg + '_null')[:, :, prestim:-poststim].mean(axis=0)
            prespont = rec['resp'].extract_epoch('STIM_' + bg + '_null')[:, :, 0:prestim].mean(axis=(0, -1))
            bg_resp.append((resp.T - prespont))

    if type(bg_resp) == list:
        bg_resp = np.concatenate(bg_resp, axis=0)
    bg_pca = PCA()
    bg_pca.fit(bg_resp)
    return bg_pca.components_

def get_decoder(rec, fg_epoch, bg_epoch, collapse=False):
    """
    Get decoding axis for each time bin for the given epoch pair
    """

    prestim = rec['resp'].extract_epoch('PreStimSilence').shape[-1]
    poststim = rec['resp'].extract_epoch('PostStimSilence').shape[-1]
    tbins = rec['resp'].extract_epoch('REFERENCE').shape[-1]
    nCells = rec['resp'].shape[0]

    fg_resp = rec['resp'].extract_epoch('STIM_null_'+fg_epoch)
    bg_resp = rec['resp'].extract_epoch('STIM_'+bg_epoch+'_null')

    decoding_axes = np.zeros((tbins, nCells))
    if collapse:
        # single decoding axis, repped out for all tbins
        fr = fg_resp[:, :, prestim:-poststim].transpose([0, 2, 1]).reshape(-1, nCells)
        br = bg_resp[:, :, prestim:-poststim].transpose([0, 2, 1]).reshape(-1, nCells)
        axis = fr.mean(axis=0) - br.mean(axis=0)
        axis /= np.linalg.norm(axis)
        decoding_axes = np.tile(axis, [tbins, 1])
    else:
        # different decoding axis for each time point
        for t in range(fg_resp.shape[-1]):
            fr = fg_resp[:, :, t]
            br = bg_resp[:, :, t]
            axis = fr.mean(axis=0) - br.mean(axis=0)
            axis /= np.linalg.norm(axis)
            decoding_axes[t, :] = axis

    return decoding_axes


import os
import datetime as dt

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from scrapy.utils.project import get_project_settings

from orflaedi.models import Model, Retailer, VehicleClassEnum

def get_or_create_retailer(db, slug):
    retailer = db.query(Retailer).filter(Retailer.slug == slug).first()
    if retailer is None:
        retailer = Retailer(slug=slug, name=slug)
        db.add(retailer)
        db.commit()
    return retailer

def get_connection_string():
    database_url = get_project_settings().get("DATABASE_URL")
    if database_url is not None:
        return database_url
    return os.environ.get("DATABASE_URL", "postgresql://orflaedi:@localhost/orflaedi")

class DatabasePipeline(object):
    def __init__(self):
        self.scraped_skus = set()

    def open_spider(self, spider):
        connection_string = spider.settings.get("DATABASE_URL")
        if connection_string is None:
            connection_string = get_connection_string()
        engine = create_engine(connection_string, connect_args={})
        SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
        self.db = SessionLocal()
        self.retailer = get_or_create_retailer(self.db, slug=spider.name)

    def close_spider(self, spider):
        # we can deactivate these since they are no longer available
        if not getattr(spider, "skip", False):
            self.db.query(Model).filter(
                ~Model.sku.in_(self.scraped_skus), Model.retailer == self.retailer
            ).update({"active": False}, synchronize_session=False)
        self.db.commit()
        self.db.close()

    def process_item(self, item, spider):
        model = (
            self.db.query(Model)
            .filter(Model.sku == item["sku"], Model.retailer == self.retailer)
            .first()
        )
        if model is None:
            model = Model(retailer=self.retailer, sku=item["sku"])
        model.name = item["name"]
        model.make = item.get("make") or model.make
        model.last_scraped = dt.datetime.utcnow()
        model.scrape_url = item["scrape_url"]
        model.price = item["price"]
        model.image_url = len(item["file_urls"]) and item["file_urls"][0] or None
        model.active = True
        model.name = model.name or item.get("name")
        if not model.motor_model:
            model.motor_model = item.get("motor_model") or None
        if model.name and model.make:
            if model.name.startswith(model.make):
                model.name = model.name[len(model.make) :]
        model.name = model.name.strip()
        model.classification = item.get(
            "classification", model.classification or VehicleClassEnum.bike_b
        )
        self.db.add(model)
        self.db.commit()
        self.scraped_skus.add(item["sku"])
        return item


from django.shortcuts import render, redirect
from dashboard.form import Formkomik, Formdaftarkomik
from dashboard.models import komik, daftarkomik
from django.contrib import messages

# Create your views here.

def Komik_View(request):
    komiks=komik.objects.all()

    konteks={
        'komiks':komiks,
    }
    return render(request,'Tampil_komik.html',konteks)

def Daftarkomik_View(request):
    daftarkomiks=daftarkomik.objects.all()

    konteks={
        'daftarkomiks':daftarkomiks,
    }
    return render(request,'Tampil_daftarkomik.html',konteks)

def tambah_komik(request):
    if request.POST:
        form=Formkomik(request.POST)
        if form.is_valid():
            form.save()
            messages.success(request,"Data Berhasil Ditambahkan")
            form = Formkomik()
            konteks={
                'form':form,
            }
            return render(request,'tambah_komik.html',konteks)
    else:
        form=Formkomik()
        konteks = {
            'form':form,
        }
        return render(request,'tambah_komik.html',konteks)

def ubah_Komik(request,id_komik):
    komiks=komik.objects.get(id=id_komik)
    if request.POST:
        form=Formkomik(request.POST,instance=komiks)
        if form.is_valid():
            form.save()
            messages.success(request,"Data Berhasil Diubah")
            return redirect('ubah_komik',id_komik=id_komik)
    else:
        form=Formkomik(instance=komiks)
        konteks = {
            'form':form,
            'komiks':komiks
        }
        return render(request,'ubah_komik.html',konteks)

def hapuskomik(request,id_komik):
    komiks=komik.objects.filter(id=id_komik)
    komiks.delete()
    messages.success(request,'Data Terhapus')
    return redirect('/tabelkomik')

def tambah_daftarkomik(request):
    if request.POST:
        form=Formdaftarkomik(request.POST)
        if form.is_valid():
            form.save()
            messages.success(request,"Data Berhasil Ditambahkan")
            form = Formdaftarkomik()
            konteks={
                'form':form,
            }
            return render(request,'tambah_daftarkomik.html',konteks)
    else:
        form=Formdaftarkomik()
        konteks = {
            'form':form,
        }
        return render(request,'tambah_daftarkomik.html',konteks)

def ubahdaftarkomik(request,id_daftarkomik):
    daftarkomiks=daftarkomik.objects.get(id=id_daftarkomik)
    if request.POST:
        form=Formdaftarkomik(request.POST,instance=daftarkomiks)
        if form.is_valid():
            form.save()
            messages.success(request,"Data Berhasil Diubah")
            return redirect('ubahdaftarkomik',id_daftarkomik=id_daftarkomik)
    else:
        form=Formdaftarkomik(instance=daftarkomiks)
        konteks = {
            'form':form,
            'daftarkomiks':daftarkomiks
        }
        return render(request,'ubah_daftarkomik.html',konteks)

def hapusdaftarbarang(request,id_daftarkomik):
    daftarkomiks=daftarkomik.objects.filter(id=id_daftarkomik)
    daftarkomiks.delete()
    messages.success(request,'Data Terhapus')
    return redirect('/daftarkomik')


from django.conf.urls import url, include
from wardrobe.views import *

urlpatterns = [
    # url(r'^test/', Test, name='test'),
    url(r'^reload-flex', ReloadFlex, name='reload_flex'),
    url(r'^product-consultation', ProductConsultation, name='wardrobe_consultation'),
    # url(r'^wardrobe-response', WardrobeResponse, name='wardrobe-response'),
    url(r'^(?P<w_type_slug>[-\w]+)/(?P<theme_slug>[-\w]+)/(?P<wardrobe_slug>[-\w]+)/$', ServeProduct),
    url(r'^(?P<w_type_slug>[-\w]+)/(?P<theme_slug>[-\w]+)/$', ServeWardrobe),
    url(r'^(?P<w_type_slug>[-\w]+)/$', ServeTheme),
    url(r'^$', ServeType, name='wardrobe')
]


from pathlib import Path
from htmlStruct import HtmlStruct

def main():

    tests = []
    tests.append(("test1.1.html", "test1.2.html"))
    tests.append(("test2.1.html", "test2.2.html"))
    tests.append(("test3.1.html", "test3.2.html"))
    tests.append(("test4.1.html", "test4.2.html"))
    tests.append(("test5.1.html", "test5.2.html"))
    tests.append(("test6.1.html", "test6.2.html"))
    tests.append(("test7.1.html", "test7.1.html"))

    for old, new in tests:
        oldfn = Path("./tests/" + old)
        newfn = Path("./tests/" + new)

        with open(oldfn, "r", encoding="utf-8") as oldf:
            with open(newfn, "r", encoding="utf-8") as newf:
                oldStr = oldf.read()
                newStr = newf.read()
                #diagnose(newStr)
                oldStruc = HtmlStruct()
                oldStruc.parseHtml(oldStr, keepAttributes=True)

                newStruc = HtmlStruct()
                newStruc.parseHtml(newStr, keepAttributes=True)

                diff = oldStruc.structDiff(newStruc)

                print ("===============================================\n")
                print("    ---------- old document ----------")
                print(HtmlStruct.presentNodes(oldStruc.getNodes()))
                print("\n    ---------- new document ----------")
                print(HtmlStruct.presentNodes(newStruc.getNodes()))
                if diff != None:
                    print("\n    ---------- number of injected structures: {} ----------\n".format(len(diff)))
                else:
                    print("    ---------- document changed beyond simple injections ----------\n")
                i = 1
                if diff:
                    for insertedStruct in diff:
                        print("    ---------- injected structure {} ----------\n".format(i))
                        i += 1
                        print (HtmlStruct.presentNodes(insertedStruct))
                        print("\n")

if __name__ == '__main__':
    main()


from hypothesis import given, settings
from hypothesis import strategies as st

from gattaca.fast_non_dominated_sort import dominates, fast_non_dominated_sort

@st.composite
def list_of_tuple_floats(draw, tuple_len=st.integers(min_value=1, max_value=5)):
    tuple_len = draw(tuple_len)
    int_list = [st.floats() for _ in range(tuple_len)]
    return draw(st.lists(elements=st.tuples(*int_list), max_size=100, min_size=1))

@settings(max_examples=200)
@given(tuple_list=list_of_tuple_floats())
def test_sort_levels_dominate_correctly(tuple_list):
    pareto_fronts = fast_non_dominated_sort(tuple_list)
    pareto_len = len(pareto_fronts)
    assert pareto_len <= len(tuple_list)
    if pareto_len > 1:
        for i in range(pareto_len - 1):
            front_1 = pareto_fronts[i]
            front_2 = pareto_fronts[i + 1]
            for i_2 in front_2:
                assert any(
                    [dominates(tuple_list[i_1], tuple_list[i_2]) for i_1 in front_1]
                )


import sys
sys.path.append("../..")
sys.path.append("..")

from limits.limits import *
from hypothesis.strategies import *
from hypothesis import given
from timeout import run_with_timeout
from typing import *

@composite
def create(draw):
  n = draw(integers(min_value=1, max_value=MAX_SEQUENCE_LEN))
  arr = draw(lists(integers(), min_size=n, max_size=n))
  i = draw(integers(min_value=0, max_value=n-1))
  return arr, i

arr = shared(create(), key='eval').map(lambda x: x[0])
i = shared(create(), key='eval').map(lambda x: x[1])

strategy = arr, i
if not isinstance(strategy, tuple):
    strategy = (strategy,)

def check_min_heap_helper(arr, i):
    if 2 * i + 2 > len(arr):
        return True
    left_child = (arr[i] <= arr[2 * i + 1]) and check_min_heap_helper(arr, 2 * i + 1)
    right_child = (2 * i + 2 == len(arr)) or (arr[i] <= arr[2 * i + 2]
                                      and check_min_heap_helper(arr, 2 * i + 2))
    return left_child and right_child

def check_min_heap(arr):
  return check_min_heap_helper(arr, 0)

@given(tuples(*strategy))
def test_fuzz(args):
    run_with_timeout(0.3, check_min_heap_helper, *args)


import warnings

warnings.simplefilter('ignore')

import os
import logging
import torch
from torchaudio.backend import soundfile_backend

from os import remove
from os.path import dirname, abspath

import telebot
from utils import init_jit_model, apply_tts, replace_accents, pluses_to_accents

TOKEN = os.environ['TOKEN']

if not TOKEN:
    print('You must set the TOKEN environment variable')
    exit(1)

SYMBOLS = '_~абвгдеёжзийклмнопрстуфхцчшщъыьэюя +.,!?…:;–'
SAMPLE_RATE = 16000

START_MSG = '''Привет!

Этот бот создан для тестирования Silero TTS: https://github.com/snakers4/silero-models
'''

FIRST_STEP = '''Использовать бот просто:

1) Добавьте ударения в свой текст с помощью https://morpher.ru/accentizer/

2) Отправьте текст и ждите аудиосообщение
'''

device = torch.device('cpu')

jit_model = dirname(__file__) + '/model/v1_kseniya_16000.jit'
model = init_jit_model(jit_model, device=device)

bot = telebot.TeleBot(TOKEN, parse_mode=None)

@bot.message_handler(commands=['start', 'help'])
def send_welcome(message):
    bot.reply_to(message, START_MSG)
    bot.reply_to(message, FIRST_STEP)

@bot.message_handler(func=lambda message: True)
def process_voice_message(message):
    if not message.text:
        bot.reply_to(message, 'А где текст??!')
        return

    # normalize the input

    # convert pluses into accents
    text_with_accents = pluses_to_accents(message.text)

    # replace accents into pluses
    text_normalized = replace_accents(text_with_accents)

    if not text_normalized:
        bot.reply_to(message, 'Ошибка: а где текст?')
        return

    # add a dot if it is not set
    last_char = text_normalized[-1]
    if last_char not in ['.', '!', '?']:
        text_normalized = text_normalized + '.'

    text_len = len(text_normalized)
    if text_len > 150:
        bot.reply_to(message, f'Ошибка: ваш текст больше 150 символов, а именно {text_len}')
        return

    bot.reply_to(message, f'Текст c ударениями:\n\n{text_with_accents}\n\nНормализованный текст:\n\n{text_normalized}')

    # do the synthesizing
    audios = apply_tts(texts=[text_normalized],
                       model=model,
                       sample_rate=SAMPLE_RATE,
                       symbols=SYMBOLS,
                       device=device)

    s = 0
    for n, audio_tensor in enumerate(audios):
        # form the filename
        filename = dirname(abspath('__file__')) + f'/files/file_{n}.wav'

        # save to the disk
        soundfile_backend.save(filename, audio_tensor, SAMPLE_RATE)

        # send back to the user
        audio = open(filename, 'rb')
        bot.send_voice(message.chat.id, audio)

        # remove WAV file
        remove(filename)

        # increment the counter
        s = s + 1

    if s == 0:
        bot.reply_to(message, "Ничего не смог синтезировать :(")

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    bot.polling()


from django import forms

def ModelFormFunc(model, data=None, instance=None, files=None):
	global model_
	model_ = model

	class ModelAddForm(forms.ModelForm):
		# forms.ModelChoiceField(queryset= model_.objects.all(), initial=0)
		def __init__(self, *args, **kwargs):
			# user = kwargs.pop('user','')
			super(ModelAddForm, self).__init__(*args, **kwargs)

		class Meta:
			model = model_
			fields = ('__all__')

	form = ModelAddForm()
	if data or instance or files:
		form = ModelAddForm(data=data, instance=instance, files=files)

	return form


import csv
import logging

import requests

HEADERS_CSV = (
    'Телефон',
    'Цена',
    'Название товара',
    'Фото',
    'Метро',
    'Ссылка',
)

headers_csv = HEADERS_CSV

def get_html(url: str, product_name=None, max_price=None, min_price=None, page=None):
    """
    Функция создает подключение к серверу по полученному url и параметрам переданными в params,
    и возвращает responce.text.

    : param url: Ссылка для подключения к указанному ресурсу.
    : param product_name: Название товара(марка) которое будет парситися.
    : param max_price: Максимальная цена, которая будет отображенна в поиске.
    : param min_price: Минимальная цена, если указана max_price и min_price, то поиск производится в этом диапазоне цен.
    : param page: Номер страницы, на которую будет произведет переход.

    timeout=5 сек, при следующих подключениях, если идет парсинг более 1-ой страницы.
    В случает ошибки сервера возвращает False и пишет в логи.
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36'
                      ' (KHTML, like Gecko) Chrome/40.0.2214.38 Safari/537.36',
        'Accept-Language': 'ru',
    }
    params = {
        'pmax': max_price,
        'pmin': min_price,
        'q': product_name,
        'p': page,
    }

    try:
        with requests.Session() as session:
            responce = session.get(url, params=params, headers=headers, timeout=5)
        if responce.url == 'https://www.avito.ru/blocked':
            logging.info('IP temporarily blocked')

        responce.raise_for_status()
        logging.info(f'response from server : {responce} {responce.url}')
        return responce.text
    except (requests.RequestException, requests.exceptions.HTTPError, ValueError) as errors:
        logging.error(f'Network error: ERROR - {errors}')
        return False

def writing_in_csv(parser_page_data):
    """
    Функция принимает следующие данные:
    [[{'Название товара': 'ZTE ', 'Фото': 'https://', 'Ссылка': 'https://', ...},...],
    ...[{'Название товара': 'ZTE ', 'Фото': 'https://', 'Ссылка': 'https://', ...}]].
    Полученный данные записываются в файл csv.
    """
    with open('parser_avito.csv', 'w', newline='', encoding='utf-8') as data_parser:
        writer = csv.DictWriter(data_parser, headers_csv, delimiter=',')
        writer.writeheader()
        [writer.writerow(data) for datas_page in parser_page_data for data in datas_page]

if __name__ == "__main__":
    headers_csv = HEADERS_CSV


from settings import CSV_CITY, CSV_SCHOOL, INSEE_CODE
from database import db
from apps.cities.models import City
from apps.schools.models import School
from apps.doctors.models import Doctor
import pandas as pd
import matplotlib.pyplot as plt
import apps.cities.city as ct
import apps.schools.school as sch
import apps.doctors.doctor as dc
import apps.data as dt
import argparse

if __name__ == '__main__':

    db.connect(reuse_if_open=True)

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "action",
        help="Choose an action to execute",
        nargs="?",
        choices=[
            "compute_city",
            "compute_school",
            "compute_doctor",
            "show_city_chart",
            "show_school_chart",
            "show_doctor_chart"
        ],
    )
    args = parser.parse_args()

    if args.action == "compute_city":
        # compute and store result in sql
        cities = ct.extract_and_prepare_cities_from_csv()
        cities_rename = ct.rename_city_column(cities)
        dt.save_data_into_database(cities_rename, City)

    if args.action == "compute_school":
        # compute and store result in sql
        cities = ct.extract_and_prepare_cities_from_csv()
        schools = sch.extract_and_prepare_schools_from_csv(cities)
        schools_rename = sch.rename_school_column(schools)
        dt.save_data_into_database(schools_rename, School)

    if args.action == "compute_doctor":
        # compute and store result in sql
        cities = ct.extract_and_prepare_cities_from_csv()
        doctors = dc.extract_and_prepare_doctors_from_csv(cities)
        doctors_rename = dc.rename_doctor_column(doctors)
        dt.save_data_into_database(doctors_rename, Doctor)

    if args.action == "show_city_chart":
        # show result in a chart
        cities = pd.DataFrame(City.select().order_by('population').limit(10).dicts())
        print(cities)
        city_rank = dt.show_data_in_graph(cities, 10, 'name', 'population', 'barh')
        plt.show()

    if args.action == "show_school_chart":
        # show result in a chart
        schools = pd.DataFrame(School.select(City, School).join(City).order_by(School.success_rate_spread.desc()).dicts())
        print(schools)
        school_rank = dt.show_data_in_graph(schools, 50, 'name', 'success_rate_spread', 'bar')
        plt.show()

    if args.action == "show_doctor_chart":
        # show result in a chart
        doctors = pd.DataFrame(Doctor.select(City, Doctor).join(City).order_by(Doctor.citizens_per_doctor.desc()).dicts())
        doctor_rank = dt.show_data_in_graph(doctors, 20, 'name', 'citizens_per_doctor', 'barh')
        plt.show()

    db.close()


#!/usr/bin/env python

# Using the ROS api to control different joints in the hsr with the
# aim of extending the arm towrads an object to try to pick it up
# Then afterwards, it will move back towards its default position

import sys
import time
import tf
import controller_manager_msgs.srv
import rospy
import trajectory_msgs.msg
from myvis.msg import Object
from myvis.msg import Objects

DEG_90 = 1.5708
MAX_LIFT = 0.69
MIN_LIFT = 0
GRIPPER_OPEN_ANGLE = 1.239
GRIPPER_CLOSE_ANGLE = -0.105
OPEN_GRIPPER_OFFSET = 0.07
ARM_LOWERING_SPEED = float(sys.argv[1])
HAND_CAMERA_TOPIC = '/hsrb/hand_camera/image_raw'
HEAD_RGB_TOPIC = '/hsrb/head_rgbd_sensor/rgb/image_rect_color'
HEAD_DEPT_TOPIC = '/hsrb/head_rgbd_sensor/depth_registered/image_rect_raw'
OBJECT_POS_TOPIC = 'ObjectsMap'

class Arm_Controller:
    def __init__(self):
        rospy.init_node('arm_test')
        self.object_to_pick = Object()
        self.objects_list = Objects()
        # initialize ROS publisher
        self.arm_pub = rospy.Publisher('/hsrb/arm_trajectory_controller/command',
                              trajectory_msgs.msg.JointTrajectory, queue_size=10)
        self.gripper_pub = rospy.Publisher('/hsrb/gripper_controller/command',
                                trajectory_msgs.msg.JointTrajectory, queue_size=10)
        self.omni_pub = rospy.Publisher('/hsrb/omni_base_controller/command',
                                trajectory_msgs.msg.JointTrajectory, queue_size=10)
        self.object_sub = rospy.Subscriber(OBJECT_POS_TOPIC, Objects, self.object_pos_callback)

        self.head_pub = rospy.Publisher('/hsrb/head_trajectory_controller/command',
                                trajectory_msgs.msg.JointTrajectory, queue_size=10)

        # wait to establish connection between the controller
        while self.arm_pub.get_num_connections() == 0:
            rospy.sleep(0.1)

        # make sure the controller is running
        rospy.wait_for_service('/hsrb/controller_manager/list_controllers')
        list_controllers = (
            rospy.ServiceProxy('/hsrb/controller_manager/list_controllers',
                               controller_manager_msgs.srv.ListControllers))

        # Check that both the arm trajectory and gripper controllers are running
        running = False
        # Assume gripper always works, some reason it doesn't detect the controller but when
        # publishing, the gripper receives the message fine
        running_check = [False, True, False,False]
        while not running:
            rospy.sleep(0.1)
            for c in list_controllers().controller:
                if c.name == 'arm_trajectory_controller' and c.state == 'running':
                    running_check[0] = True
                if c.name == 'gripper_controller' and c.state == 'running':
                    running_check[1] = True
                if c.name == 'omni_base_controller' and c.state == 'running':
                    running_check[2] = True
                if c.name == 'head_trajectory_controller' and c.state == 'running':
                    running_check[3] = True
                # Check all controllers are running
                running = True
                for flag in running_check:
                    if flag == False:
                        running = False

        # Arm joints
        self.al_joint = 0
        self.af_joint = 0
        self.ar_joint = DEG_90
        self.wf_joint = DEG_90
        self.wr_joint = 0
        #self.reset_default_pos()
        # Gripper joints
        self.hm_joint = 0
        # Initial position
        trans = self.lookup_transform("/map", "/base_link")
        self.base_x = trans[0]
        self.base_y = trans[1]
        self.base_z = trans[2]
        print("Initial position: x: %d, y: %d" %(self.base_x, self.base_y))
        print("Arm controller initialised")

    # Reset the arm to the default position
    def reset_default_pos(self):
        self.move_arm(0, 0, -DEG_90, -DEG_90, 0)
        print("Resetting arm to default position")

    # Turns the arm roll and wrist flex joint to face the hand north of the robot
    def face_hand_forward(self):
        self.move_arm(self.al_joint, self.af_joint, 0, -DEG_90, self.wr_joint)
        print("Arm facing forward published")

    # Lower the arm and adjust the wrist flex joint so that the hand is always 90 degrees
    def lower_arm_sync_wrist(self):
        while self.af_joint > -DEG_90:
            temp_af_joint = max(self.af_joint-ARM_LOWERING_SPEED, -DEG_90)
            temp_wf_joint = -(DEG_90-abs(temp_af_joint))
            self.move_arm(self.al_joint, temp_af_joint, self.ar_joint, temp_wf_joint, self.wr_joint)
            rospy.sleep(0.5)

    # Move the arm by pusblishing a joint trajectory message with the relevant arm joints
    def move_arm(self, al_joint, af_joint, ar_joint, wf_joint, wr_joint):
        # fill ROS message
        traj = trajectory_msgs.msg.JointTrajectory()
        traj.joint_names = ["arm_lift_joint", "arm_flex_joint",
                            "arm_roll_joint", "wrist_flex_joint", "wrist_roll_joint"]
        p = trajectory_msgs.msg.JointTrajectoryPoint()
        p.positions = [al_joint, af_joint, ar_joint, wf_joint, wr_joint]
        p.velocities = [0, 0, 0, 0, 0]
        p.time_from_start = rospy.Time(3)
        traj.points = [p]
        # Update joint angles
        self.al_joint = al_joint
        self.af_joint = af_joint
        self.ar_joint = ar_joint
        self.wf_joint = wf_joint
        self.wr_joint = wr_joint
        # publish ROS message
        self.arm_pub.publish(traj)

    def move_arm_lift(self, target_z):
        currTrans = self.lookup_transform('/base_link', '/hand_motor_dummy_link')
        curr_z = currTrans[2]
        if curr_z < target_z:
            temp_al_joint = target_z - curr_z
        else:
            temp_al_joint = curr_z - target_z
        self.move_arm(temp_al_joint, self.af_joint, self.ar_joint, self.wf_joint, self.wr_joint)

    # Move the gripper by publishing a joint trajectory message with the hand motor joint
    def move_gripper(self, hm_joint):
        traj = trajectory_msgs.msg.JointTrajectory()
        traj.joint_names = ["hand_motor_joint"]
        p = trajectory_msgs.msg.JointTrajectoryPoint()
        p.positions = [hm_joint]
        p.velocities = [0]
        p.effort = [0.1]
        p.time_from_start = rospy.Time(3)
        # Update joint
        self.hm_joint = hm_joint
        traj.points = [p]
        # publish ROS message
        self.gripper_pub.publish(traj)

    def open_gripper(self):
        self.move_gripper(GRIPPER_OPEN_ANGLE)

    def close_gripper(self):
        self.move_gripper(GRIPPER_CLOSE_ANGLE)

    def move_base(self, x_pos, y_pos):
        traj = trajectory_msgs.msg.JointTrajectory()
        traj.joint_names = ["odom_x", "odom_y", "odom_t"]
        p = trajectory_msgs.msg.JointTrajectoryPoint()
        p.positions = [x_pos, y_pos, 0]
        p.velocities = [0, 0, 0]
        p.time_from_start = rospy.Time(6)
        traj.points = [p]
        # Update the base pos
        self.base_x = x_pos
        self.base_y = y_pos
        # publish ROS message
        # For some reason publishing the message once doesn't get the robot
        for i in range(10):
            self.omni_pub.publish(traj)

    # Gets transform transform of the hand to offset the position where the robot should be
    # so the gripper is right in front of the object
    def align_hand_with_object(self, target_x, target_y):
        trans = self.lookup_transform('/base_link', '/hand_motor_dummy_link')
        new_target_x = target_x-trans[0]-OPEN_GRIPPER_OFFSET
        new_target_y = target_y-trans[1]
        print("Original target x: %f y: %f Target x: %f y: %f" %(target_x, target_y, new_target_x, new_target_y))
        return new_target_x, new_target_y

    def lookup_transform(self, source_link, target_link):
        listener = tf.TransformListener()
        seconds_passed = 0
        timeout = 5
        delay_rate = 0.5
        print("Looking up transform")
        while not rospy.is_shutdown():
            try:
                (trans,rot) = listener.lookupTransform(source_link, target_link, rospy.Time(0))
                print("Transform received")
                return trans
            except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException):
                continue
            if seconds_passed >= timeout:
                print("Transform lookup timed out")
                return (0, 0, 0)
            rospy.sleep(delay_rate)
            seconds_passed += delay_rate

    def objects_print(self):
        print("Received Object")
        print("Objects array has length: ",  self.objects_list.length)

    def get_object(self,name):
        for xa in range(0, self.objects_list.length):
            if self.objects_list.Objects[xa].name == name:
                self.object_to_pick = self.objects_list.Objects[xa]

    def object_print(self):
        print("Objects name: ", self.object_to_pick.name)

    def object_x(self):
        return self.object_to_pick.x

    def object_y(self):
        return self.object_to_pick.y

    def object_z(self):
        return self.object_to_pick.z

    def object_name(self):
        return self.object_to_pick.name

    def move_head(self):
        traj = trajectory_msgs.msg.JointTrajectory()
        traj.joint_names = ["head_pan_joint", "head_tilt_joint"]
        p = trajectory_msgs.msg.JointTrajectoryPoint()
        p.positions = [0.5, 0.5]
        p.velocities = [0, 0]
        p.time_from_start = rospy.Time(3)
        traj.points = [p]
        self.head_pub.publish(traj)

    # Increasing head pan turns head left, decreasing turns head right
    # Increasing head tilt tilts head up, decreasing tilts head down
    def move_headxy(self ,x,y):
        traj = trajectory_msgs.msg.JointTrajectory()
        traj.joint_names = ["head_pan_joint", "head_tilt_joint"]
        p = trajectory_msgs.msg.JointTrajectoryPoint()
        p.positions = [x, y]
        p.velocities = [0, 0]
        p.time_from_start = rospy.Time(3)
        traj.points = [p]
        self.head_pub.publish(traj)

    def grab_obj_mode1(self, target_x, target_y, target_z):
        self.open_gripper()
        self.face_hand_forward()
        self.lower_arm_sync_wrist()
        rospy.sleep(10)
        self.move_arm_lift(target_z)
        # Move to the target location and close the gripper when it is reached
        target_x, target_y = self.align_hand_with_object(target_x, target_y)
        error_bound = 0.05
        # Move Y axis
        print("Moving Y axis")
        trans = self.lookup_transform('/map', '/base_link')
        curr_x = trans[0]
        curr_y = trans[1]
        self.move_base(curr_x, target_y)
        while curr_y < target_y - error_bound or curr_y > target_y + error_bound:
            trans = self.lookup_transform('/map', '/base_link')
            curr_y = trans[0]
            print(trans)
        # Move X axis
        rospy.sleep(5)
        print("Moving X axis")
        trans = self.lookup_transform('/map', '/base_link')
        curr_x = trans[0]
        curr_y = trans[1]
        self.move_base(target_x, curr_y)
        while curr_x < target_x - error_bound or curr_x > target_x + error_bound:
            trans = self.lookup_transform('/map', '/base_link')
            curr_x = trans[0]
            print(trans)
        # Moving in both axis at once can cause hand to collide with the object
        # However if an offset is given in the axis the robot is facing the object, then this can be done
        '''
        while ((curr_x < target_x - error_bound or curr_x > curr_x > target_x + error_bound) or
            (curr_y < target_y - error_bound or curr_y > curr_y > target_y + error_bound)):
            trans = self.lookup_transform('/map', '/base_link')
            curr_x = trans[0]
            curr_y = trans[1]
            print(trans)
        '''
        print("Reached target point on map")
        self.close_gripper()
        rospy.sleep(10)
        # STRANGE BUG: It won't go back to its original position but somewhere halfway but if the self.move_base is called again
        # then it moves back to its original posiiton
        print("Going back to original position")
        self.reset_default_pos()
        self.move_base(ac.base_x, ac.base_y)

    def object_pos_callback(self, objects):
        self.objects_list = objects
        objectTarget = "bottle"
        self.get_object(objectTarget)
        if(self.object_name() == objectTarget):
            print("Objects x: ", self.object_x())
            print("Objects y: ", self.object_y())
            print("Objects z: ", self.object_z())
            self.grab_obj_mode1(self.object_x(), self.object_y(), self.object_z())

def main(args):
    ac = Arm_Controller()
    ac.reset_default_pos()
    ac.open_gripper()
    target_x = float(sys.argv[2])
    target_y = float(sys.argv[3])
    target_z = float(sys.argv[4])

    try:
        rospy.spin()
    except KeyboardInterrupt:
        print("Shutting down")

if __name__ == '__main__':
    main(sys.argv)


import copy
import csv
import os
import pickle
import librosa
import numpy as np
from scipy import signal
import torch
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms
import pdb, cv2
import random

import models.vggish_src.mel_features as mel_features
import models.vggish_src.vggish_params as vggish_params
import numpy as np
import resampy
import soundfile as sf

from sklearn.model_selection import train_test_split
from glob import glob
import pandas as pd

import albumentations as A
from albumentations.pytorch import ToTensorV2
import audiomentations as Au

def process_annotation(Annotation):
    with open(Annotation) as f:
        lines = f.readlines()
    print("totally {} lines".format(len(lines)))

    anno_dict = {}

    for i in lines[1:]:
        i = i.split('&')
        anno_dict[i[1]] = i[0]

    label_list = set(i for i in anno_dict.values())
    labels_double_ref = {}
    for i, label in enumerate(label_list):
        labels_double_ref[i] = label
        labels_double_ref[label] = i

    return anno_dict, labels_double_ref

class x360_Basic_Dataset(Dataset):

    def get_img_info(self, img_path, img, byshortmax=224, label=None):

        H, W, C = img.shape

        if W < H:
            ratio = byshortmax / W    #

        elif H <= W:
            ratio = byshortmax / W

        Wcrop = int(W * ratio)  # size
        Hcrop = int(H * ratio)

        if label == 'crop':
            Hcrop = byshortmax
            Wcrop = int(2 * byshortmax)

        rnd_h = random.randint(0, max(0, H - Hcrop))  # loc
        rnd_w = random.randint(0, max(0, W - Wcrop))

        info = {"H":H, "W":W,
                "Hcrop":Hcrop, "Wcrop":Wcrop,
                "rnd_h":rnd_h, "rnd_w":rnd_w}

        return info

    def crop_by_ratio(self, img, center=False, info=None):

        if center:
            crop = A.Compose([
                A.CenterCrop(height=info['Hcrop'], width=info['Wcrop'], always_apply=True),
            ])
            cropped = crop(image=img)['image']

        else:
            cropped = img[info['rnd_h']:info['rnd_h'] + info['Hcrop'],
                          info['rnd_w']:info['rnd_w'] + info['Wcrop']]

        return cropped

    def waveform_to_examples(self, data, sample_rate, return_tensor=True):
        """Converts audio waveform into an array of examples for VGGish.

    Args:
        data: np.array of either one dimension (mono) or two dimensions
        (multi-channel, with the outer dimension representing channels).
        Each sample is generally expected to lie in the range [-1.0, +1.0],
        although this is not required.
        sample_rate: Sample rate of data.
        return_tensor: Return data as a Pytorch tensor ready for VGGish

    Returns:
        3-D np.array of shape [num_examples, num_frames, num_bands] which represents
        a sequence of examples, each of which contains a patch of log mel
        spectrogram, covering num_frames frames of audio and num_bands mel frequency
        bands, where the frame length is vggish_params.STFT_HOP_LENGTH_SECONDS.

    """
        # Convert to mono.
        if len(data.shape) > 1:
            data = np.mean(data, axis=1)
        # Resample to the rate assumed by VGGish.
        if sample_rate != vggish_params.SAMPLE_RATE:
            data = resampy.resample(data, sample_rate, vggish_params.SAMPLE_RATE)

        # Compute log mel spectrogram features.
        log_mel = mel_features.log_mel_spectrogram(
            data,
            audio_sample_rate=vggish_params.SAMPLE_RATE,
            log_offset=vggish_params.LOG_OFFSET,
            window_length_secs=vggish_params.STFT_WINDOW_LENGTH_SECONDS,
            hop_length_secs=vggish_params.STFT_HOP_LENGTH_SECONDS,
            num_mel_bins=vggish_params.NUM_MEL_BINS,
            lower_edge_hertz=vggish_params.MEL_MIN_HZ,
            upper_edge_hertz=vggish_params.MEL_MAX_HZ)

        # Frame features into examples.
        features_sample_rate = 1.0 / vggish_params.STFT_HOP_LENGTH_SECONDS
        example_window_length = int(round(
            vggish_params.EXAMPLE_WINDOW_SECONDS * features_sample_rate))
        example_hop_length = int(round(
            vggish_params.EXAMPLE_HOP_SECONDS * features_sample_rate))
        log_mel_examples = mel_features.frame(
            log_mel,
            window_length=example_window_length,
            hop_length=example_hop_length)

        return log_mel_examples

    def handle_audio_sample(self, sample, rate):

        start_point = random.randint(a=0, b=np.max([0, len(sample) - rate]))

        sample = self.au_transform(samples=sample, sample_rate=rate)
        sample[sample > 1.] = 1.
        sample[sample < -1.] = -1.

        tuple_spec = []

        l = sample.shape[0] // self.tuple_len

        sample = np.concatenate([sample, sample])

        for i in range(self.tuple_len):
            new_sample = sample[start_point: start_point + l]

            start_point = start_point + l

            spectrogram = self.waveform_to_examples(new_sample, sample_rate=rate)[:5]

            if spectrogram.shape[0] < 5:
                s = spectrogram.shape[0]
                try:
                    ratio = (5 // s + 1) // 2 + 1
                except:
                    print("failed to get ratio:",
                          spectrogram.shape, new_sample.shape, sample.shape)

                for i in range(ratio):
                    spectrogram = np.concatenate([spectrogram, spectrogram], axis=0)

            spectrogram = self.spec_transform(spectrogram)

            tuple_spec.append(torch.from_numpy(spectrogram[:5]))

        return tuple_spec

    def read_audio(self, audio_path, sr_in):

        base_sr = 16000 // sr_in
        samples, sr = librosa.load(audio_path, offset=0.0, mono=False, sr=base_sr)  # sr=16000,   - > 220618,)

        left_spectrogram = self.handle_audio_sample(samples[0], sr)
        right_spectrogram = self.handle_audio_sample(samples[:, 1], sr)

        return [left_spectrogram, right_spectrogram, audio_path]

    def read_audio_tracking(self, at):
        # read in directional audio time delay
        return np.load(at) # [250]

    def read_frames(self, frame_folder, clip_sr=1, label=None):

        tuple_clip = []

        tuple_start = random.randint(0, 36)  # 64
        clip_start = tuple_start

        frame_list = glob(os.path.join(frame_folder, '*.jpg'))
        frame_list.sort()

        num_frames = len(frame_list)

        try:
            ratio = int(np.ceil(self.clip_len / len(frame_list))) + 1
        except:
            print("frame_folder:", frame_folder)

        cache_frame_list = frame_list.copy()
        for i in range(ratio):

            frame_list.extend(cache_frame_list)

        self.transforms = torch.nn.Sequential(
                transforms.RandomVerticalFlip(p= (1.0 if np.random.random() > 0.5 else 0) ),
                transforms.RandomHorizontalFlip(p=(1.0 if np.random.random() > 0.5 else 0))
            )

        idx = 0
        tuble_id_list = []

        for _ in range(self.tuple_len):
            clip_id_list = []

            for i in range(self.clip_len):
                # clip_sr is the pace of video pace, usually randomly sampled from 1~4

                if (clip_start + (idx + 1) * clip_sr) >= num_frames:
                    clip_start = (clip_start + (idx + 1)* clip_sr) - num_frames
                    idx = 0

                try:
                    img = cv2.imread(frame_list[clip_start + idx * clip_sr])
                    clip_id_list.append(clip_start + idx * clip_sr)

                except:
                    print(f"frame {clip_start + idx * clip_sr} of {num_frames} and len {len(frame_list)}")

                img = self.resize(image=img)["image"]

                if i == 0:
                    info = self.get_img_info(frame_list[i], img, byshortmax=self.final_img_size, label=label)
                    images = torch.zeros((self.clip_len, 3, info['Hcrop'], info['Wcrop']))

                img = self.crop_by_ratio(img, info=info,
                                         center=(self.mode == 'test'))

                img = self.transform(image=img)["image"]

                images[i] = img

                idx += 1

            images = images.permute(1, 0, 2, 3)

            images = self.transforms(images)

            tuple_clip.append(images)
            tuble_id_list.append(clip_id_list)

            clip_start = clip_start + self.clip_len

        return tuple_clip

    # Frame Order Shuffle
    def order_shuffle(self, tuple_clip, tuple_audio):

        tuple_order = list(range(0, self.tuple_len))

        clip_and_order = list(zip(tuple_clip, tuple_audio, tuple_order))
        random.shuffle(clip_and_order)
        tuple_clip, tuple_audio, tuple_order = zip(*clip_and_order)

        return torch.stack(tuple_clip), torch.cat(tuple_audio), \
               torch.tensor(tuple_order)              # np.concatenate(tuple_audio), \

    def idx2data(self, idx, frame_list, audio_list):

        data = self.csvfile.iloc[idx]

        path = data['path']
        classnum = data['classnum']

        # has n cut videos
        frame_list = glob(os.path.join(path, frame_list))
        frame_list.sort()

        audio_list = glob(os.path.join(path, audio_list))
        audio_list.sort()

        return classnum, frame_list, audio_list

    def get_at_data(self, idx):
        data = self.csvfile.iloc[idx]
        path = data['path']

        at_list = glob(os.path.join(path, self._front_view_audio_tracking_list))
        at_list.sort()

        return at_list

    def read_data(self, frame, audio, select_cut_id, clip_sr=1, label=None):

        # clip_sr is the pace of video pace, usually randomly sampled from 1~4

        return self.read_frames(frame[select_cut_id],  label=label, clip_sr=clip_sr), \
               self.read_audio(audio[select_cut_id], clip_sr)

    def __len__(self):
        return len(self.csvfile)

class x360Dataset(x360_Basic_Dataset):
    """UCF101 dataset for video clip order prediction. Generate clips and permutes them on-the-fly.
        Need the corresponding configuration file exists.

        Args:
            root_dir (string): Directory with videos and splits.
            train (bool): train split or test split.
            clip_len (int): number of frames in clip, 16/32/64.
            interval (int): number of frames between clips, 16/32.
            tuple_len (int): number of clips in each tuple, 3/4/5.
            transforms_ (object): composed transforms which takes in PIL image and output tensors.
        """

    def __init__(self, args, csvfile, mode='train',
                 use_360=False, use_front=True, use_clip=False):

        self.args = args
        self.clip_len = args.cl   # length of video clip
        self.max_sr = args.max_sr          # max sampling rate

        self.interval = args.it   # args.cl, args.it, args.tl
        self.tuple_len = args.tl  # number of clips in each tuple, 3/4/5.

        self.mode = mode
        self.csvfile = pd.read_csv(csvfile)
        self.data_list = []
        self.cut_number = 6

        self.final_img_size = 128  #args.crop_sz
        self.resize_img_size = 512

        # $Root/ (Inside_Ouside)/ Location(Label)/ Video_ID
        root = '/bask/projects/j/jiaoj-3d-vision/360XProject/Data'

        self._360cut_frames_list = "360/360_panoramic_cut/all_frames/*"
        self._360cut_audios_list = "360/360_panoramic_cut/audios/*.wav"

        self._frontcut_frames_list = "360/front_view_cut/all_frames/*"
        self._frontcut_audios_list = "360/front_view_cut/audios/*.wav"

        self._Clip_frames_list = "Snapchat/clip1/binocular_cut/all_frames/*"
        self._Clip_audios_list = "Snapchat/clip1/binocular_cut/audios/*.wav"

        self._front_view_audio_tracking_list = "360/front_view_cut/at/*/front_audio_tracking.npy"

        self.use_360 = use_360
        self.use_front = use_front
        self.use_clip = use_clip

        self.resize = A.Compose([
                A.SmallestMaxSize(self.resize_img_size, interpolation=3)
                ])

        if self.mode == 'train':

            self.transform = A.Compose([
                # A.HorizontalFlip(p=0.5),
                # A.VerticalFlip(p=0.5),
                # A.ShiftScaleRotate(scale_limit=0.3, rotate_limit=15, p=0.5),

                A.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1),
                                           contrast_limit=(-0.1, 0.1), p=0.5),
                A.ImageCompression(p=0.3),  #

                # A.Affine(p=0.3),
                ToTensorV2(),
            ])

            # , keypoint_params=A.KeypointParams(format='xy', remove_invisible=False)

            self.au_transform = Au.Compose([
                    Au.AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.5),
                    Au.TimeStretch(min_rate=0.8, max_rate=1.25, p=0.5),
                    Au.PitchShift(min_semitones=-4, max_semitones=4, p=0.5),
                    Au.Shift(min_fraction=-0.5, max_fraction=0.5, p=0.5),
                ])

            self.spec_transform = Au.SpecCompose([
                        # Au.SpecChannelShuffle(p=0.5),
                        Au.SpecFrequencyMask(p=0.5),
                        # ToTensorV2()
                ])

        else:
            self.transform = A.Compose([
                ToTensorV2(),
            ])

            self.au_transform = Au.Compose([])
            self.spec_transform = Au.SpecCompose([])

    def __getitem__(self, idx):

        length_of_list = 100
        clip_sr = random.randint(1, self.max_sr)

        # Load Data list
        if self.use_360:
            cls, _360frames_list, _360audio_list = \
                self.idx2data(idx, self._360cut_frames_list, self._360cut_audios_list)
            length_of_list = np.min([length_of_list, len(_360frames_list), len(_360audio_list)])

        if self.use_front:
            cls, _frontframes_list, _frontaudio_list = \
                self.idx2data(idx, self._frontcut_frames_list, self._frontcut_audios_list)
            length_of_list = np.min([length_of_list, len(_frontframes_list), len(_frontaudio_list)])

        if self.use_clip:
            cls, _Clipframes_list, _Clipaudio_list = \
                self.idx2data(idx, self._Clip_frames_list, self._Clip_audios_list)
            length_of_list = np.min([length_of_list, len(_Clipframes_list), len(_Clipaudio_list)])

        Output_dict = {"cls": cls, "sr": clip_sr}

        if self.args.use_directional_audio:
            at_list = self.get_at_data(idx)
            length_of_list = np.min([length_of_list, len(at_list)])

        select_cut_id = np.random.randint(0, length_of_list)   # number of clips

        if self.args.use_directional_audio:
            at = self.read_audio_tracking(at_list[select_cut_id])[:250]  # clip at 250
            if len(at) < 255:
                at = np.pad(at, (0,255-len(at)), 'constant', constant_values=(0, 0))

            Output_dict['at'] = np.expand_dims(at, 0)

        # select Data
        if self.use_360:
            # tuple
            f, a = self.read_data(_360frames_list, _360audio_list, select_cut_id,
                                  clip_sr=clip_sr)
            f, a, o = self.order_shuffle(f, a[0])

            Output_dict["360"] = {"f": f, "order": o, "a": a}

        if self.use_front:
            f, a = self.read_data(_frontframes_list, _frontaudio_list, select_cut_id,
                                   clip_sr=clip_sr)

            f, a, o = self.order_shuffle(f, a[0])

            Output_dict["front"] = {"f": f, "order": o, "a": a}

        if self.use_clip:
            f, a = self.read_data(_Clipframes_list, _Clipaudio_list, select_cut_id,
                                  label='crop', clip_sr=clip_sr)
            f, a, o = self.order_shuffle(f, a[0])

            Output_dict["clip"] = {"f": f, "order": o, "a": a}

        return Output_dict

if __name__ == "__main__":
    Annotation = '../360data/AVE_Dataset/Annotations.txt'
    with open(Annotation) as f:
        lines = f.readlines()
    print("totally {} lines".format(len(lines)))

    label_dict = {}

    for i in lines[1:]:
        i = i.split('&')
        label_dict[i[1]] = i[0]

    label_list = set(i for i in label_dict.values())
    print(label_list)


import pandas as pd
import numpy as np
import os
from io import StringIO
import matplotlib.pyplot as plt
from scipy.stats import norm
import pymc3

import ForceSpectroscopyHelperMCM.Utils.ConsoleColor as color
import ForceSpectroscopyHelperMCM as mcm
import ForceSpectroscopyHelper as fsh
import scipy.stats as stats

"""
    Bayesで出力されたcsvファイルの解析
"""

class BayesData:
    def __init__(self, file_path):
        if not mcm.utility.IsBayesianSaveResultFile(file_path):
            print(color.color.Error + "Error File Type Error!" + color.color.Error, file_path)
            return

        self._file = open(file_path)
        headers = str(self._file.read().split("HeaderEnd")[0]).replace(" ", "").split('\n')
        try:
            self.name = headers[1].replace("csvdata,", "")
            self.source_file_name = headers[1].split(",")[1]
            self.norm_min = float(headers[2].split(",")[1])
            self.norm_delta = float(headers[3].split(",")[1])
            self.index = headers[4].replace("index,", "")
            self.z0 = int(headers[5].split(",")[1])
            self.sample = str(headers[6].split(",")[1])
            self.chainIndex = int(headers[7].split(",")[1])
            self.model_name = mcm.BayesianModel.ModelEnum.value_of(str(headers[8].split(",")[1]))
            self.f_equation = mcm.BayesianModel.ModelEnum.get_model(self.model_name)

            # self.traceDir = headers[8].split(",")[1]
        except:
            import traceback
            traceback.print_exc()

        self._data = open(file_path).read()

    def get_result_param(self, percent):

        param_mat = np.array(self.trace_data)[:, 1:6]
        param_mat = sorted(param_mat, key=lambda x: self.f_equation(0, x[0], x[1], x[2], x[3]))
        # print(param_mat)

        count = int(len(param_mat))
        if count * percent > count - 1:
            param = param_mat[count-1]
        else:
            param = param_mat[int(count * percent)]

        # sigma_mat = np.array(self.trace_data)[:, 5:]
        # sigma = np.percentile(sigma_mat, int(percent * 100))
        return param[:-1], param[4]

    def get_lr_curve(self, x, percent):
        param, sigma = self.get_result_param(percent)
        return self.f_equation(x, *param), sigma

    @property
    def summary_data(self) -> pd.DataFrame:
        data = self._data.split("HeaderEnd")[1].split("SummaryEnd")[0]
        f_io = StringIO(data)
        df = pd.read_csv(f_io, sep=",")
        return df

    @property
    def trace_data(self) -> pd.DataFrame:
        data = self._data.split("SummaryEnd")[1].split("TraceEnd")[0]
        f_io = StringIO(data)
        df = pd.read_csv(f_io, sep=",")
        return df

    @property
    def y_range_data(self) ->pd.DataFrame:
        data = self._data.split("TraceEnd")[1]
        f_io = StringIO(data)
        df = pd.read_csv(f_io, sep=",")
        return df

    @property
    def a(self):
        return self.get_result_param(0.5)[0][0]

    @property
    def b(self):
        return self.get_result_param(0.5)[0][1]

    @property
    def c(self):
        return self.get_result_param(0.5)[0][2]

    @property
    def d(self):
        return self.get_result_param(0.5)[0][3]

    @property
    def a_range(self):
        return np.asarray(self.summary_data)[:, 4][0], np.asarray(self.summary_data)[:, 5][0]

    @property
    def b_range(self):
        return np.asarray(self.summary_data)[:, 4][1], np.asarray(self.summary_data)[:, 5][1]

    @property
    def c_range(self):
        return np.asarray(self.summary_data)[:, 4][2], np.asarray(self.summary_data)[:, 5][2]

    @property
    def d_range(self):
        return np.asarray(self.summary_data)[:, 4][3], np.asarray(self.summary_data)[:, 5][3]

    @property
    def real_a(self):
        return (self.a + 1) * self.norm_delta + self.norm_min

    @property
    def real_b(self):
        return self.b * self.norm_delta

    @property
    def y_range(self):
        return np.array(self.y_range_data)[0, 1:], np.array(self.y_range_data)[1, 1:]

    def __str__(self):
        return "a: " + str(self.a) + " b: " + str(self.b) + " c: " + str(self.c) + " d: " + str(self.d)

    def __repr__(self):
        return "a: " + str(self.a) + " b: " + str(self.b) + " c: " + str(self.c) + " d: " + str(self.d)

    def plot_curve(self, x, y):
        plt.plot(x, y, color="black")
        plt.axhline(0, color='green', linestyle='dashdot')
        plt.axvline(x[self.z0], color='green', linestyle='dashdot')

        plt.plot(x, self.f_equation(x, self.real_a, self.real_b, self.c, self.d), color="red")
        plt.plot(x, y - self.f_equation(x, self.real_a, self.real_b, self.c, self.d))
        plt.tight_layout()
        plt.show()

    @staticmethod
    def to_bayesData(trace, bayesianFitParam, z0, index,
                     norm_data=None, model_name=None, path=None):
        if norm_data is None:
            norm_data = [0, -1, 1]
        summary = pymc3.summary(trace, var_names=["a", "b", "c", "d",
                                                 "a0", "sa", "b0", "sb", "c0", "sc", "sigma"])

        print(summary)
        result_mat = np.asarray(summary)
        # summary save
        df = pd.DataFrame(result_mat,
                         columns=list(summary.columns.values),
                         index=["a", "b", "c", "d", "a0", "sa", "b0", "sb", "c0", "sc", "sigma"])
        df.to_csv(bayesianFitParam.Csv_saveResultName+"_temp1")
        # trace save
        df = pymc3.trace_to_dataframe(trace, varnames=["a", "b", "c", "d", "sigma"])
        df.to_csv(bayesianFitParam.Csv_saveResultName+"_temp2")
        # y-range save
        if trace.nchains == 1:
            # mu = np.array([pymc3.quantiles(trace)["mu-all"][2.5], pymc3.quantiles(trace)["mu-all"][97.5]])
            # eps = np.array(pymc3.quantiles(trace)["sigma"][50], pymc3.quantiles(trace)["sigma"][50])
            # y1, y2 = map(list, zip(*norm.ppf(q=[0.05, 0.95], loc=mu.T, scale=eps)))
            # y1 = pymc3.quantiles(trace)["mu-all"][2.5]
            # y2 = pymc3.quantiles(trace)["mu-all"][97.5]
            y1 = np.array(pymc3.summary(trace, var_names=["mu-all"])["hpd_3%"])
            y2 = np.array(pymc3.summary(trace, var_names=["mu-all"])["hpd_97%"])
            # y3 = norm.ppf(q=0.5, loc=mu.T, scale=eps)
            mu_mat = pd.DataFrame(np.asarray([y1, y2]))
            mu_mat.to_csv(bayesianFitParam.Csv_saveResultName + "_temp3")

        if path is None:
            dir = os.path.join(bayesianFitParam.work_abs_directory, bayesianFitParam.Csv_saveResultName)
        else:
            dir = os.path.join(path, bayesianFitParam.Csv_saveResultName)

        with open(dir, "w") as f:
            f.write("BayesianSaveResultFile" + "\n")
            f.write("csvdata," + bayesianFitParam.name + "\n")
            f.write("min," + str(float(norm_data[1])) + "\n")
            f.write("delta," + str(float(norm_data[2])) + "\n")
            f.write("index," + str(index) + "\n")
            f.write("z0," + str(int(z0)) + "\n")
            f.write("sampler," + str(bayesianFitParam.sampler) + "\n")
            f.write("chainIndex," + str(bayesianFitParam.chainIndex) + "\n")
            if model_name is not None:
                f.write("model," + str(model_name) + "\n")
            # f.write("traceDir," + trace_dir + "\n")
            f.write("HeaderEnd" + "\n")

            f.write(open(bayesianFitParam.Csv_saveResultName+"_temp1").read())
            os.remove(bayesianFitParam.Csv_saveResultName+"_temp1")
            f.write("SummaryEnd" + "\n")
            f.write(open(bayesianFitParam.Csv_saveResultName+"_temp2").read())
            os.remove(bayesianFitParam.Csv_saveResultName+"_temp2")
            f.write("TraceEnd" + "\n")
            if os.path.isfile(bayesianFitParam.Csv_saveResultName+"_temp3"):
                f.write(open(bayesianFitParam.Csv_saveResultName + "_temp3").read())
                os.remove(bayesianFitParam.Csv_saveResultName + "_temp3")

        return BayesData(os.path.join(bayesianFitParam.work_abs_directory, bayesianFitParam.Csv_saveResultName))

class Extension:

    plot_hyper_range = 450

    @staticmethod
    def get_best_data(fileName, data_path, index, z0=None) -> BayesData:
        data_dict = mcm.BayesSelector.select(data_path, fileName=fileName, index=index, z0=z0)
        data = None
        for v in data_dict.values():
            data, _ = mcm.BayesSelector.select_best_result_from_verification(v)
        if data is None:
            raise ValueError("can not find bayes data in path: " + data_path)
        return data

    @staticmethod
    def convert_real_param(bayes_data: BayesData, percentage=0.5):
        param, sigma = bayes_data.get_result_param(percentage)
        param[0] = (param[0] + 1) * bayes_data.norm_delta + bayes_data.norm_min
        param[1] = param[1] * bayes_data.norm_delta
        return param, sigma

    @staticmethod
    def plot_percentage_result(percentage, bayes_data: BayesData, x, y):
        param, eps = Extension.convert_real_param(bayes_data, percentage)

        plt.scatter(x, y, s=3)
        plt.axhline(0, color='green', linestyle='dashdot')
        plt.axvline(x[bayes_data.z0], color='green', linestyle='dashdot')
        f_lr = mcm.BayesianModel.ModelEnum.get_model(bayes_data.model_name)(x, param[0], param[1], param[2], param[3])
        plt.plot(x, f_lr, alpha=0.9, color="black", linewidth=1.0, label="long-range_median")
        plt.plot(x, y - f_lr, alpha=0.9, color="red", linewidth=1.0, label="short-range")

        if Extension.plot_hyper_range < len(x):
            plt.xlim(0, x[Extension.plot_hyper_range])
        plt.legend(fontsize=18)
        plt.tick_params(labelsize=18)

    @staticmethod
    def plot_range_result(bayes_data: BayesData, x, y, show_legend=True, show_observed_range=True, show_lr_range=False,
                          show_short_range=True):
        eps = bayes_data.get_result_param(0.5)[1]

        plt.scatter(x, y, s=3, label="measured data")
        # plt.axhline(0, color='black', linestyle='dashdot')
        plt.axvline(x[bayes_data.z0], color='black', linestyle='dashdot')
        f_lr = bayes_data.f_equation(x, bayes_data.real_a, bayes_data.real_b, bayes_data.c, bayes_data.d)

        mu = np.array([bayes_data.y_range[0], bayes_data.y_range[1]])
        nan_limit = -1
        for index in range(len(mu[0])):
            if np.isnan(mu[0][index]):
                nan_limit = index
            else:
                break
        for index in range(len(mu[1])):
            if np.isnan(mu[1][index]):
                nan_limit = index
            else:
                break

        param1, sigma1 = bayes_data.get_result_param(0.0)
        f1 = bayes_data.f_equation(x, param1[0], param1[1], param1[2], param1[3])
        param2, sigma2 = bayes_data.get_result_param(1.0)
        f2 = bayes_data.f_equation(x, param2[0], param2[1], param2[2], param2[3])

        if show_observed_range:
            y1, y2 = map(list, zip(*norm.ppf(q=[0.03, 0.97], loc=mu.T, scale=eps)))
            plt.fill_between(x, y1=y1, y2=y2, alpha=0.4, color="orange", label="$F_{fit}$ range(bayes inference)")

        # plt.plot(x, y - f_lr, alpha=0.9, color="red", linewidth=1.0, label="short-range")
        if nan_limit == -1:
            if show_lr_range:
                y1, y2 = bayes_data.f_equation(x, *param1), bayes_data.f_equation(x, *param2)
                plt.fill_between(x, y1=y1, y2=y2, alpha=0.4, color="blue", label="long-range")
            if show_short_range:
                plt.fill_between(x, y1=y-f1, y2=y-f2, alpha=0.6, color="red", label="short-range")

            # plt.plot(x, f_lr, alpha=0.9, color="black", linewidth=1.0, label="long-range_median")

        else:
            if show_lr_range:
                y1, y2 = bayes_data.f_equation(x[nan_limit:], *param1), bayes_data.f_equation(x[nan_limit:], *param2)
                plt.fill_between(x[nan_limit:], y1=y1, y2=y2, alpha=0.4, color="blue", label="long-range")
            if show_short_range:
                plt.fill_between(x[nan_limit:], y1=(y-f1)[nan_limit:], y2=(y-f2)[nan_limit:],
                             alpha=0.6, color="red", label="short-range")

            plt.plot(x[nan_limit:], f_lr[nan_limit:], alpha=0.9, color="black", linewidth=1.0, label="long-range_median")

        if Extension.plot_hyper_range < len(x):
            plt.xlim(x[0]-0.5, x[Extension.plot_hyper_range])
        if show_legend:
            plt.legend(fontsize=16)
        plt.tick_params(labelsize=18)

    @staticmethod
    def plot_histogram(datas, label="", color="blue", bin=100, alpha=0.5, figsize=(5, 10)):
        # point = literal_eval(data.index)
        # height_offset = analyzer.topo(fileName)[point[0], point[1]] * 10 ** 10
        axes = mcm.BayesianUtility.build_ax(5, figsize=figsize)
        if not isinstance(datas, list):
            datas = [datas]
            label = [label]
            color = [color]

        a_bins = np.histogram(np.hstack([data.trace_data["a"] for data in datas]), bins=bin)[1]
        b_bins = np.histogram(np.hstack([data.trace_data["b"] for data in datas]), bins=bin)[1]
        c_bins = np.histogram(np.hstack([data.trace_data["c"] for data in datas]), bins=bin)[1]
        d_bins = np.histogram(np.hstack([data.trace_data["d"] for data in datas]), bins=bin)[1]
        sigma_bins = np.histogram(np.hstack([data.trace_data["sigma"] for data in datas]), bins=bin)[1]

        for j, data in enumerate(datas):
            a = np.array(data.trace_data["a"])
            b = np.array(data.trace_data["b"])
            c = np.array(data.trace_data["c"])
            d = np.array(data.trace_data["d"])
            sigma = np.array(data.trace_data["sigma"])

            axes[0].hist(a, bins=a_bins, alpha=alpha, label=label[j], color=color[j])
            axes[0].set_xlabel("$A^\prime$")
            axes[1].hist(b, bins=b_bins, alpha=alpha, label=label[j], color=color[j])
            axes[1].set_xlabel("$B^\prime$")
            axes[2].hist(c, bins=c_bins, alpha=alpha, label=label[j], color=color[j])
            axes[2].set_xlabel("$C^\prime$")
            axes[3].hist(d, bins=d_bins, alpha=alpha, label=label[j], color=color[j])
            axes[3].set_xlabel("$D^\prime$")
            axes[4].hist(sigma, bins=sigma_bins, alpha=alpha, label=label[j], color=color[j])
            axes[4].set_xlabel("sigma")
        return axes

    @staticmethod
    def plot_smooth_histogram(datas, label="", color=None, bin=100, alpha=0.5, figsize=(5, 12)):
        # point = literal_eval(data.index)
        # height_offset = analyzer.topo(fileName)[point[0], point[1]] * 10 ** 10
        axes = mcm.BayesianUtility.build_ax(5, figsize=figsize)
        if not isinstance(datas, list):
            datas = [datas]
            label = [label]
            color = [color]

        a_bins = np.histogram(np.hstack([data.trace_data["a"] for data in datas]), bins=bin)[1]
        b_bins = np.histogram(np.hstack([data.trace_data["b"] for data in datas]), bins=bin)[1]
        c_bins = np.histogram(np.hstack([data.trace_data["c"] for data in datas]), bins=bin)[1]
        d_bins = np.histogram(np.hstack([data.trace_data["d"] for data in datas]), bins=bin)[1]
        sigma_bins = np.histogram(np.hstack([data.trace_data["sigma"] for data in datas]), bins=bin)[1]

        for j, data in enumerate(datas):
            a = np.array(data.trace_data["a"])
            b = np.array(data.trace_data["b"])
            c = np.array(data.trace_data["c"])
            d = np.array(data.trace_data["d"])
            sigma = np.array(data.trace_data["sigma"])

            den_a = stats.gaussian_kde(a)
            den_b = stats.gaussian_kde(b)
            den_c = stats.gaussian_kde(c)
            den_d = stats.gaussian_kde(d)
            n, x, _ = plt.hist(a, bins=a_bins, histtype=u'step', density=True)
            axes[0].plot(x, den_a(x), alpha=alpha, label=label[j], color=color[j])
            axes[0].set_xlabel("$A$[Hz]")
            n, x, _ = plt.hist(b, bins=b_bins, histtype=u'step', density=True)
            axes[1].plot(x, den_b(x), alpha=alpha, label=label[j], color=color[j])
            axes[1].set_xlabel("$B$[Hz/$\AA^D$]")
            n, x, _ = plt.hist(c, bins=c_bins, histtype=u'step', density=True)
            axes[2].plot(x, den_c(x), alpha=alpha, label=label[j], color=color[j])
            axes[2].set_xlabel("$C$[$\AA$]")
            n, x, _ = plt.hist(d, bins=d_bins, histtype=u'step', density=True)
            axes[3].plot(x, den_d(x), alpha=alpha, label=label[j], color=color[j])
            axes[3].set_xlabel("$D$")
            axes[4].hist(sigma, bins=sigma_bins, alpha=alpha, label=label[j], color=color[j])
            axes[4].set_xlabel("sigma")
        return axes

    # @staticmethod
    # def custom_plot_range_result(bayes_data: BayesData, x, y):
    #     plt.style.use('grayscale')
    #     plt.scatter(x[1:], y[1:], label="measured force", c="black", s=20, marker=".", alpha=1)
    #     plt.axvline(x[bayes_data.z0], linestyle='dashdot', c="black", alpha=0.8)
    #     f_lr = bayes_data.f_equation(x, bayes_data.real_a, bayes_data.real_b, bayes_data.c, bayes_data.d)
    #
    #     param1, sigma1 = bayes_data.get_result_param(0.001)
    #     param2, sigma2 = bayes_data.get_result_param(0.999)
    #
    #     y1, y2 = bayes_data.f_equation(x, *param1), bayes_data.f_equation(x, *param2)
    #     plt.fill_between(x, y1=y1, y2=y2, alpha=0.5, label="$\Delta f_{LR}$ inference range")
    #     # plt.plot(x, f_lr, linewidth=1.0, label="$F_{LR}$ median value")
    #     plt.xlim(-0.2, 5)
    #
    #     plt.legend(fontsize=13)
    #     plt.tick_params(labelsize=14)

if __name__ == "__main__":
    fileName = "Si_H_0041.f.ch1"
    data = BayesData(fileName+",51,42,Num0.bayes")
    print(data)
    Extension.plot_hyper_range = 10000
    print(fsh.default_project_path)

    cvt = fsh.PakConvertor(fileName)
    x, y = cvt.get_x_data(point=51), cvt.get_y_data(point=51)
    # x, y = cvt.get_x_data(point=(24,22), offset=None) - 30, cvt.get_y_data(point=(24,22))

    # plt.style.use('grayscale')
    Extension.plot_range_result(data, x, y)
    # Extension.plot_histogram(data)

    # Extension.plot_percentage_result(0.001, data, np.flipud(cvt.GetXData()), np.flipud(cvt.GetYData(50)))
    # Extension.plot_percentage_result(0.999, data, np.flipud(cvt.GetXData()), np.flipud(cvt.GetYData(50)))
    plt.show()
    # plt.savefig("fig_bayes1.png", transparent=True, dpi=300)


import struct
import numpy as np      # 支持大量的维度数组与矩阵运算
import matplotlib.pyplot as plt     # Python的绘图库

train_images_path = "E:\\zxy\\svm_learn\\minist\\train-images-idx3-ubyte\\train-images.idx3-ubyte"
train_labels_path = "E:\\zxy\\svm_learn\\minist\\train-labels-idx1-ubyte\\train-labels.idx1-ubyte"
test_images_path = "E:\\zxy\\svm_learn\\minist\\t10k-images-idx3-ubyte\\t10k-images.idx3-ubyte"
test_labels_path = "E:\\zxy\\svm_learn\\minist\\t10k-labels-idx1-ubyte\\t10k-labels.idx1-ubyte"

def read_images(inputFile):
    binaryData = open(inputFile, 'rb').read()
    offest = 0
    head = '>iiii'
    _, imageNum, imageRow, imageCol = struct.unpack_from(head, binaryData, offest)
    print("读取文件：{}， 图片数：{}，规格：{}*{}".format(inputFile, imageNum, imageRow, imageCol))

    iamgeSize = imageCol * imageRow  # 28*28 = 784
    offest += struct.calcsize(head)
    imageFmt = '>' + str(iamgeSize) + 'B'  # >784B
    # imageFmt = ">784B"
    images = np.empty((imageNum, imageRow * imageCol))
    for i in range(imageNum):
        image = struct.unpack_from(imageFmt, binaryData, offest)
        tmp = np.array(image)  # 1*784
        #plt.imshow(tmp.reshape((28,28)), cmap='binary')
        #plt.show()
        #tmp = tmp.reshape((imageRow, imageCol)) # 28*28
        images[i] = tmp
        offest += struct.calcsize(imageFmt)
    print('Load image done')
    return images

def read_labels(inputFile):
    binaryData = open(inputFile, 'rb').read()
    head = '>ii'
    offset = 0
    _, labelNum = struct.unpack_from(head, binaryData, offset)
    print('读取文件：{}， 标签数：{}'.format(inputFile, labelNum))
    labelHead = '>B'
    offset += struct.calcsize(head)
    labels = np.empty(labelNum)
    for i in range(labelNum):
        labels[i] = struct.unpack_from(labelHead, binaryData, offset)[0]
        offset += struct.calcsize(labelHead)
    return labels

def ReadTrainImages():
    return read_images(train_images_path)

def ReadTrainLabels():
    return read_labels(train_labels_path)

def ReadTestImages():
    return read_images(test_images_path)

def ReadTestLabels():
    return read_labels(test_labels_path)

if __name__ == "__main__":
    train_images = ReadTrainImages()
    train_labels = ReadTrainLabels()
    for i in range(60000):
        print("这个图片里的数字是{}".format(train_labels[i]))
        plt.imshow(np.array(train_images[i]).reshape((28,28)), cmap="binary")
        plt.show()


import os
import nltk
from nltk import sent_tokenize
from nltk import word_tokenize
from nltk.probability import FreqDist
from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np

# nltk.download('punkt')

# 1 - Importando o texto
with open(os.path.join('automacoes/PLN_IA/data', 'texto.txt', ), 'r', encoding='utf-8') as file:
    texto = file.read()
    # print(texto)

# 2 - Tokenizando o texto (separando por palavras, ou setença (ponto final))
sent_tokens = sent_tokenize(texto)
print(sent_tokens)
print(len(sent_tokens))

word_tokens = word_tokenize(texto)
print(word_tokens)
print(len(word_tokens))

# 3 - Frequencia de distribuição
fdist = FreqDist(word_tokens)
print(fdist)
print(fdist.most_common(10))
fdist.plot(10)

# 4 - WordCloud / WordCloud Customizado
def plot_cloud(wordcloud):
    plt.figure(figsize=(40,30))
    plt.imshow(wordcloud)
    plt.axis('off')
    plt.show()

wordcloud = WordCloud(
    width = 3000,
    height = 2000,
    random_state = 1,
    background_color= 'salmon',
    colormap='Pastel1',
    collocations=False,
    stopwords = STOPWORDS
).generate(texto)

plot_cloud(wordcloud)

mask = np.array(
    Image.open('automacoes/PLN_IA/data/upvote.png')
)

print(mask)
wordcloud = WordCloud(
    width = 3000,
    height = 2000,
    random_state = 1,
    background_color= 'salmon',
    colormap='Pastel1',
    collocations=False,
    stopwords = STOPWORDS,
    mask = mask
).generate(texto)

# plot_cloud(wordcloud)


"""
Applies a rule-based approach for labeling unambiguous animated entities for the training data.
It uses an actor noun list and an inanimate adjective list as patterns.
NER is used to identify organizations and people, and a name database is used for labeling names.
"""

import os
import spacy
from spacy.matcher import Matcher
from names_dataset import NameDataset

nlp = spacy.load("de_core_news_lg")
matcher = Matcher(nlp.vocab)
name_dataset = NameDataset()

with open("gold_actor_10_0.7.txt", "r", encoding="utf-8") as f:
    for line in f:
        words = line.strip().lower().split()
        for word in words:
            if word:
                pattern = [{"LOWER": word}]
                matcher.add(word, [pattern])

inanimated_adjectives = set()
with open("inanimated_adjectives_5_0.9.txt", "r", encoding="utf-8") as f:
    for line in f:
        adjective = line.strip().lower()
        inanimated_adjectives.add(adjective)

def should_animate_chunk(chunk_tokens):
    doc = nlp(" ".join(chunk_tokens))
    matches = matcher(doc)
    contains_inanimated_adjective = any(token.lower_ in inanimated_adjectives for token in doc if token.pos_ == "ADJ")
    contains_person_entity = any(ent.label_ == "PERSON" for ent in doc.ents)
    if matches and not contains_inanimated_adjective:
        for match_id, start, end in matches:
            span = doc[start:end]
            if span.root.pos_ == "NOUN":
                return True
    if contains_person_entity:
        return True
    if any(token.lower() in ['wir', 'ich', 'man', 'jemand'] for token in chunk_tokens):
        return True
    if any(ent.label_ == "ORG" for ent in doc.ents):
        return True
    first_name_count = 0
    last_name_count = 0
    for token in doc:
        token_lower = token.text.lower()
        name_info = name_dataset.search(token_lower)
        if 'first_name' in name_info and token.pos_ == "PROPN":
            first_name_count += 1
        if 'last_name' in name_info and token.pos_ == "PROPN":
            last_name_count += 1
    if first_name_count >= 2 and last_name_count >= 2:
        return True
    return False

def process_chunk(chunk):
    chunk_tokens = [token_tuple[2] for token_tuple in chunk]
    is_animated = should_animate_chunk(chunk_tokens)
    if is_animated:
        for i in range(len(chunk)):
            try:
                chunk[i][3] = 'Animated' if chunk[i][4] in ['B', 'I'] else '_'
            except IndexError:
                continue
    else:
        for i in range(len(chunk)):
            try:
                chunk[i][3] = '_'
            except IndexError:
                continue
    return ['\t'.join(token_tuple) for token_tuple in chunk]

input_folder = 'C:/Users/fredh/Documents/BA/Labeling/Chunks'
output_folder = 'C:/Users/fredh/Documents/BA/Labeling/Animated'

file_list = os.listdir(input_folder)

for filename in file_list:
    if os.path.isfile(os.path.join(input_folder, filename)):
        input_file = os.path.join(input_folder, filename)
        output_file = os.path.join(output_folder, filename.rsplit('.', 1)[0] + '_animated.csv')

        output_lines = []

        with open(input_file, 'r', encoding="utf-8") as file:
            chunk = []
            inside_text_segment = False
            line_count = 0
            for line in file:
                line = line.strip()
                line_count += 1
                if line_count <= 5:
                    output_lines.append(line)
                    continue
                if line.startswith("#Text="):
                    if inside_text_segment and chunk:
                        output_lines.extend(process_chunk(chunk))
                        chunk = []
                    inside_text_segment = True
                    output_lines.append(line)
                    continue
                if not line:
                    inside_text_segment = False
                    if chunk:
                        output_lines.extend(process_chunk(chunk))
                        chunk = []
                    output_lines.append(line)
                    continue
                if inside_text_segment:
                    parts = line.split('\t')
                    chunk_label = parts[4] if len(parts) > 4 else "_"
                    if chunk_label == 'B' and chunk:
                        output_lines.extend(process_chunk(chunk))
                        chunk = []
                    chunk.append(parts)
            if chunk:
                output_lines.extend(process_chunk(chunk))
        with open(output_file, 'w', encoding="utf-8") as out_file:
            for line in output_lines:
                out_file.write(line + '\n')


"""main URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from webapp.views import IndexView, ProductView, ProductUpdateView, ProductCreateView, ProductDeleteView, CategoryView, BasketOrderView, BasketCountView, BasketDeleteView

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', IndexView.as_view(), name='index'),
    path('product/<int:pk>/', ProductView.as_view(), name='product_view'),
    path('product/<int:pk>/update/', ProductUpdateView.as_view(), name='product_update'),
    path('product/create/', ProductCreateView.as_view(), name='product_create'),
    path('product/<int:pk>/delete/', ProductDeleteView.as_view(), name='product_delete'),
    path('product/category/<int:pk>/', CategoryView.as_view(), name='product_category'),

    path('basket/product/<int:pk>/', BasketCountView.as_view(), name='basket_count'),
    path('basket/', BasketOrderView.as_view(), name='basket_view'),
    path('basket/<int:pk>/', BasketDeleteView.as_view(), name='basket_delete')
]


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging
logging.basicConfig(level=logging.INFO)
logging.getLogger("boto3").setLevel(logging.WARNING)

import boto3
import os
import feedparser
from boto3 import Session
from boto3 import resource
from botocore.exceptions import BotoCoreError, ClientError
from contextlib import closing
from HTMLParser import HTMLParser
from feedgen.feed import FeedGenerator
from bs4 import BeautifulSoup
import datetime
import dateutil.parser
import hashlib

MAX_TPS = 10
MAX_CONCURENT_CONNECTIONS = 20
TASKS = 100
REQUEST_LIMIT = 1200

def split_content_by_dot(soup, max_len):
    """
    split HTML soup into parts not bigger than max_len may break prosody where
    dot is not at the end of the sentence (like "St. Louis") in some cases may
    be synthesized as two separate sentences
    """
    text = soup.get_text(" ", strip=True)
    start = 0
    while start < len(text):
        if len(text) - start <= max_len:
            yield text[start:]
            return
        max = start + max_len
        index = text.rfind(".", start, max)
        if index == start:
            start += 1
        elif index < 0:
            yield text[start:max]
            start = max
        else:
            yield text[start:index]
            start = index

def get_entries(feed):
    NEW_POST = u"""New post, author {author}, title {title} {content}"""
    for entry in feed.entries:
        if "http" in entry.id:
            nid = hashlib.md5(str(entry.id))
            entry.id = nid.hexdigest()
        entry_content = entry.content[0].value
        soup = BeautifulSoup(entry_content, 'html.parser')
        chunks = split_content_by_dot(soup, REQUEST_LIMIT-len(NEW_POST))
        chunks = list(chunks)
        published = dateutil.parser.parse(entry.published)
        for i, chunk in enumerate(chunks):
            if i == 0:
                chunk = NEW_POST.format(
                        author=entry.author,
                        title=entry.title,
                        content=chunk)
            yield dict(
                content=chunk,
                id="%s_%d" % (entry.id, i),
                title=entry.title,
                published=published - datetime.timedelta(0, i),
            )
            remaining = chunk

def lambda_handler(event, context):
    rss = event['rss']
    bucket_name = event['bucket']
    logging.info("Processing url: %s" % rss)
    logging.info("Using bucket: %s" % bucket_name)

    session = Session(region_name="us-west-2")
    polly = session.client("polly")
    s3 = resource('s3')
    bucket = s3.Bucket(bucket_name)

    logging.info("getting list of existing objects in the given bucket")
    files = set(o.key for o in bucket.objects.all())

    feed = feedparser.parse(rss)

    title = feed['feed']['title']
    fg = FeedGenerator()
    fg.load_extension('podcast')
    fg.title('Audio podcast based on: %s' % title)
    fg.link(href=feed.feed.link, rel='alternate')
    fg.subtitle(feed.feed.description)

    ENTRY_URL = "http://{bucket}.s3-website.{region}.amazonaws.com/{filename}"

    for entry in get_entries(feed):
        filename = "%s.mp3" % entry['id']
        fe = fg.add_entry()
        fe.id(entry['id'])
        fe.title(entry['title'])
        fe.published(entry['published'])
        entry_url = ENTRY_URL.format(bucket=bucket_name, filename=filename, region=os.environ["AWS_REGION"])
        fe.enclosure(entry_url, 0, 'audio/mpeg')
        if filename in files:
            logging.info('Article "%s" with id %s already exist, skipping.'
                         % (entry['title'], entry['id']))
            continue
        try:
            logging.info("Next entry, size: %d" % len(entry['content']))
            logging.debug("Content: %s" % entry['content'])
            response = polly.synthesize_speech(
                    Text=entry['content'],
                    OutputFormat="mp3",
                    VoiceId="Joanna")
            with closing(response["AudioStream"]) as stream:
                bucket.put_object(Key=filename, Body=stream.read())
        except BotoCoreError as error:
            logging.error(error)
    bucket.put_object(Key='podcast.xml', Body=fg.rss_str(pretty=True))