from __future__ import unicode_literals
import youtube_dl
import soundfile as sf
import os
import pandas as pd
import librosa
import pickle
from collections import Counter
from pathlib import Path
import json
import numpy as np
from praatio import tgio

from config import config as c
from preprocess import combine_intervals

folder_audioset = os.path.join(c['folder_raw'],'audioset')
folder_wavs = os.path.join(folder_audioset, 'wavs')
bal = 'unbalanced'
ydl_opts = {
    'quiet': True,
    'format': 'bestaudio/best',
    'postprocessors': [{
        'key': 'FFmpegExtractAudio',
        'preferredcodec': 'wav',
        'preferredquality': '192'
    }],
    'postprocessor_args': [
        '-ar', str(c['sr']),
        '-ac', '1']
}

json_path = os.path.join(folder_audioset, 'ontology.json')
df_ont = pd.read_json(json_path).set_index('name')

def n2id(name):
    return(df_ont.loc[name, 'id'])
def id2name(id):
    return(df_ont[df_ont['id']==id].iloc[0].name)
# Gets the ids for the classes that have been chosen
def get_all_ids(chosen_classes):
    chosen_ids = []

    # check the chosen classes. if they have child ids, add those to the list
    # if not, add that id to the list of chosen ids
    for cc in chosen_classes:
        row = df_ont.loc[cc]
        childs = row['child_ids']
        chosen_ids.append(row.id)
        # if not(childs):
        #     chosen_ids.append(row.id)
        # else:
        if childs:
            #add these classes to list so we can get their ids
            chosen_classes += [id2name(f) for f in childs]
    return(chosen_ids)

# Gets a dataframe with the youtube IDs in a good format
def get_df_YTID():

    path_df_ytid = os.path.join(folder_audioset,'df_ytid.csv')
    if os.path.exists(path_df_ytid):
        df_ytid = pd.read_csv(path_df_ytid)
        print("Loaded df_ytid")
    else:
        youtube_id_path = os.path.join(folder_audioset, '{}_train_segments.csv'.format(bal))
        columns = ['YTID', 'start_seconds', 'end_seconds', 'positive_labels']
        df_ytid = pd.read_csv(youtube_id_path, skiprows=3, sep=' ', header=None, names=columns)
        df_ytid_strip = df_ytid[['YTID', 'start_seconds', 'end_seconds']]

        df_ytid[['YTID', 'start_seconds', 'end_seconds']] = df_ytid_strip.apply(lambda x: x.str.strip(','))
        df_ytid[['start_seconds', 'end_seconds']] = df_ytid[['start_seconds', 'end_seconds']].apply(pd.to_numeric)
        df_ytid = df_ytid.set_index('YTID')
        df_ytid.to_csv(os.path.join(folder_audioset, 'df_ytid.csv'))

        print("Created df_ytid and saved to csv")

    return(df_ytid)

def get_YTIDs_from_chosen_ids(df_ytid, chosen_ids, typ):

    suff = '_music' if typ == 'music' else ''
    pickle_path = os.path.join(folder_audioset, '{}_ytids_and_ytids_dict{}.pkl'.format(bal, suff))
    print("PICKLE PATH:", pickle_path)

    if os.path.exists(pickle_path):
        with open(pickle_path, 'rb') as handle:
            ytids, ytids_by_id_dict = pickle.load(handle)
        print("Loaded ytids from file")
    else:
        ytids = []
        ytids_by_id_dict = {}
        for c_i in chosen_ids:
            ytids_by_id_dict[c_i] = []
        for i in range(len(df_ytid.index)):
            row = df_ytid.iloc[i]
            labels = row.positive_labels.split(',')
            for lab in labels:
                if lab in chosen_ids:
                    ytids.append(row.name)
                    ytids_by_id_dict[lab].append(row.name)
                    break
            if (i+1) % 5000 == 0:
                print("Finished {:.2f}% -".format(100*(i+1)/len(df_ytid.index)), (i+1), "ytids of", len(df_ytid.index),)
        with open(pickle_path, 'wb') as handle:
            pickle.dump([ytids, ytids_by_id_dict], handle)
        print("Saved ytids to file")

    print("\nCounts per type:")
    ytids_by_id_dict_return = {}
    for k, v in ytids_by_id_dict.items():

        if len(v)>0:
            ytids_by_id_dict_return[k]=v
            print("Added", id2name(k), len(v))
        else:
            print("Skipping", id2name(k), len(v))
    print("")
    return(ytids, ytids_by_id_dict_return)

def download_yt_clip_and_save_wav(ytid, df_ytid):

    saved_wav_path = os.path.join(folder_wavs, '{}.wav'.format(ytid))
    if os.path.exists(saved_wav_path):
        print(ytid, "already saved - skipping")
        return

    ydl_opts['outtmpl'] ='{}.wav'.format(ytid)
    link = 'https://www.youtube.com/watch?v={}'.format(ytid)
    try:
        with youtube_dl.YoutubeDL(ydl_opts) as ydl:
            result = ydl.extract_info("{}".format(link))
            filename = ydl.prepare_filename(result)
    except:
        print("Couldn't download file", ytid, '- skipping')
        return
    data, _ = librosa.load(filename, sr=c['sr'])
    start = int(df_ytid.loc[ytid].start_seconds * c['sr'])
    end = int(df_ytid.loc[ytid].end_seconds * c['sr'])
    data_clipped = data[start:end]
    sf.write(saved_wav_path, data_clipped, samplerate=c['sr'])
    print('Saved','{}.wav'.format(ytid))
    os.remove(filename)

def download_and_save_all(ytids, df_ytid):
    for i, ytid in enumerate(ytids):
        download_yt_clip_and_save_wav(ytid, df_ytid)
        if (i + 1) % 10 == 0:
            print("\nFinished {:.2f}% -".format(100 * (i + 1) / len(ytids)), i + 1, "yt_audios of", len(ytids), "\n")

def convert_df_ytid_pkls_to_csv():

    pickle_path_1 = os.path.join(folder_audioset, '{}_ytids_and_ytids_dict.pkl'.format(bal))
    pickle_path_2 = os.path.join(folder_audioset, '{}_ytids_and_ytids_dict_music.pkl'.format(bal))
    pickle_path_3 = os.path.join(folder_audioset, '{}_ytids_and_ytids_dict_coughs.pkl'.format(bal))

    columns = ['label', 'fname']
    to_save = None
    for path in [pickle_path_1, pickle_path_2, pickle_path_3]:
        with open(path, 'rb') as handle:
            ytids, ytids_by_id_dict = pickle.load(handle)

        for id, ytid_list in ytids_by_id_dict.items():
            names = [id2name(id)] * len(ytid_list)
            ytid_list = [y+'.wav' for y in ytid_list]
            to_add = np.array([names,ytid_list]).T
            to_save = np.vstack((to_save,to_add)) if to_save is not None else to_add

    pd.DataFrame(to_save,columns=columns).to_csv(os.path.join(folder_audioset,'audioset_files.csv'),index=False)

def save_to_pkl(ytids_by_id_dict):

    for id, ytid_list in ytids_by_id_dict.items():

        name = id2name(id)
        pkl_name = 'audioset_{}.pkl'.format(name)
        pkl_path = os.path.join(folder_pkl, pkl_name)

        if os.path.exists(pkl_path):
            print("Pickle exists for:", pkl_name, "- skipping.")
            continue

        # Create pickle dictionary to store samples
        pickle_dict = {}
        for l in c['label_types']:
            pickle_dict[l] = []

        # Counter for this file
        pickle_counter = {}
        for obj in ["cnt", "time_sec", "skipped_cnt-too_short"]:
            pickle_counter[obj] = Counter()

        print('Processing', len(ytid_list), name, 'files')

        for ytid in ytid_list:
            try:
                y, _ = sf.read(os.path.join(folder_wavs, ytid + '.wav'))
            except:
                print("Couldn't read wav for ytid", ytid)
                continue

            name_with_domain = 'audioset_{}_{}'.format(name, ytid)
            sound_segs_per_file = 0
            cough_count = 0
            intervals = librosa.effects.split(y, top_db=20)
            intervals = combine_intervals(intervals)
            for i, (start_samps, end_samps) in enumerate(intervals):
                if end_samps - start_samps < .1 * c['sr']:
                    continue

                pickle_dict, pickle_counter, cough_count = process_samp(y, start_samps / c['sr'], end_samps / c['sr'],
                                                                        'sound', pickle_dict, pickle_counter,
                                                                        name_with_domain, name_with_domain, cough_count,
                                                                        suff='audioset_{}'.format(name))

                sound_segs_per_file += 1
            print('Finished', ytid, name, "segs in file:", sound_segs_per_file)

        print("sound cnt:", pickle_counter["cnt"]["sound"], "| Time (s):", pickle_counter["time_sec"]["sound"])
        print("Finished preprocessing on:", pkl_name)

        # Save pickle file and counter for this file
        with open(pkl_path, 'wb') as pkl_file:
            pickle.dump([pickle_dict, pickle_counter], pkl_file)

def flusense_textgrid_to_txt():

    folder = os.path.join(c['folder_raw'],'flusense')
    folder_labels = os.path.join(folder, 'labels_textgrid')

    labs = [f for f in os.listdir(folder_labels) if f.endswith('.TextGrid')]

    labs_all = []
    for lab in labs:

        name_no_ext = Path(lab).stem
        path = os.path.join(folder_labels, lab)
        path_new = os.path.join(folder, name_no_ext+'.label.txt')

        # if os.path.exists(path_new):
        #     continue

        tg = tgio.openTextgrid(path)
        t_name = tg.tierNameList[0]
        entryList = tg.tierDict[t_name].entryList

        with open(path_new, 'w') as handle:
            for entry in entryList:
                lab_cur = entry.label
                if lab_cur in c['label_convert'].keys():
                    lab_cur = c['label_convert'][lab_cur]

                labs_all += [lab_cur]
                handle.write('{}\t{}\t{}\n'.format(entry.start, entry.end, lab_cur))

    print(set(labs_all))

if __name__ == '__main__':

    # typ = 'sounds' #'sounds' #'music
    # assert(typ in ['music','sounds'])
    # chosen_classes_music = ['Music genre']
    # chosen_classes_sounds = ['Sneeze', 'Laughter', 'Baby cry, infant cry', 'Dishes, pots, and pans', 'Cutlery, silverware']
    # # chosen_classes_coughs = ['Cough', 'Throat clearing']
    #
    # chosen_classes = chosen_classes_music if typ=='music' else chosen_classes_sounds
    # # chosen_classes = chosen_classes_coughs
    #
    # df_ytid = get_df_YTID()
    # chosen_ids = get_all_ids(chosen_classes)
    #
    # ytids, ytids_by_id_dict = get_YTIDs_from_chosen_ids(df_ytid, chosen_ids, typ)

    # download_and_save_all(ytids,df_ytid)
    # save_to_pkl(ytids_by_id_dict)
    # convert_df_ytid_pkls_to_csv()
    # df = pd.read_csv(r'Z:\research\cough_count\data\raw\audioset\audioset_files.csv')
    flusense_textgrid_to_txt()


import os
import tempfile
import shutil

from diffman import Diffman
from hgapi import Repo

class DiffmanHg(Diffman):

    @property
    def scm_name(self):
        return 'hg'

    def diff(self, filepath):
        """ Computes the diff of the filepath
        """

        repo = Repo(self.config.path)
        thepatch = repo.hg_command("diff", filepath)

        return thepatch

    def patch(self, patch, thefile, content=None):
        # if there's no diff, there's no conflict
        if patch in ("", None):
            return True

        try:
            # write the file in a temporary file
            tmpfile = tempfile.mkstemp()
            os.write(tmpfile[0], patch)

            # create a temp directory
            tmpdir = tempfile.mkdtemp()
            tmprepo = Repo(tmpdir)

            # initialize a new hg repo
            tmprepo.hg_init()

            # copy the file to patch in the temp directory
            rel_path = os.path.relpath(thefile, self.config.path)
            self._dir_tree(rel_path, tmpdir)

            shutil.copy2(thefile, os.path.join(tmpdir, rel_path))

            # add the file to the temp hg repo
            tmprepo.hg_add('.')

            # commit the current working tree
            tmprepo.hg_commit('Baboon commit')

            # import the patch in the tmpfile hg repo
            try:
                tmprepo.hg_command('import', tmpfile[1], '--no-commit')
            except Exception:
                # an exception is raised if the command failed
                return False
        finally:
            try:
                os.remove(tmpfile[1])
                shutil.rmtree(tmpdir)
            except:
                """ There's at least one error during the clean of tmpfile
                files. It's a pity but not fatal."""
                pass

        return True

    def _dir_tree(self, rel_path, tmpdir):
        """ Create the same directory structure of rel_path to the tmpdir
        directory.
        """
        splitted = os.path.split(rel_path)
        aggr = []

        # Suppose the rel_path is equals to 'models/database/foo.js'.
        # aggr will be ['models', 'models/database', 'models/database/foo.js']
        for i, elem in enumerate(splitted):
            if i > 0:
                aggr.append(os.path.join(splitted[i - 1], splitted[i]))
            else:
                aggr.append(splitted[i])  # i == 0

        for cur in aggr:
            d = os.path.join(self.config.path, cur)
            if os.path.isdir(d):
                os.mkdir(os.path.join(tmpdir, cur))
            elif os.path.isfile(d):
                break


import cv2
import os
from numpy import *
patth='../../datasets/CASME_TIM10/'
to_patth='../../datasets/CASME_DETAIL_TIM10/'
emotionlisting=os.listdir(patth)
def multiScaleSharpen(img ,radius):
    h,w,chan = img.shape
    GaussBlue1 = zeros(img.shape,dtype = uint8)
    GaussBlue2 = zeros(img.shape, dtype=uint8)
    GaussBlue3 = zeros(img.shape, dtype=uint8)
    Dest_float_img = zeros(img.shape, dtype=float32)
    Dest_img = zeros(img.shape, dtype=uint8)
    w1 = 0.5
    w2 = 0.5
    w3 = 0.25
    GaussBlue1 = cv2.GaussianBlur(img,(radius,radius),1)
    GaussBlue2 = cv2.GaussianBlur(img,(radius*2-1,radius*2-1),2)
    GaussBlue3 = cv2.GaussianBlur(img,(radius*4-1,radius*4-1),4)
    for i in range(0,h):
        for j in range(0,w):
            for k in range(0,chan):
                Src = img.item(i,j,k)
                D1 = Src-GaussBlue1.item(i,j,k)
                D2 = GaussBlue1.item(i,j,k) - GaussBlue2.item(i,j,k)
                D3 = GaussBlue2.item(i,j,k) - GaussBlue3.item(i,j,k)
                if(D1 > 0):
                    sig = 1
                else:
                    sig = -1
                Dest_float_img.itemset((i,j,k),(1-w1*sig)*D1+w2*D2+w3*D3+Src)
    Dest_img = cv2.convertScaleAbs(Dest_float_img)
    return Dest_img

if __name__ == '__main__':
    for emotiondata in emotionlisting:
        negativepath = patth + emotiondata + "/"
        to_negativepath = to_patth + emotiondata + "/"
        directorylisting = os.listdir(negativepath)
        for video in directorylisting:
            videopath = negativepath + video
            to_videopath = to_negativepath + video
            os.mkdir(to_videopath)
            frames = []
            framelisting = os.listdir(videopath)
            framerange = [x for x in range(10)]
            for frame in framerange:
                imagepath = videopath + "/" + framelisting[frame]
                img = cv2.imread(imagepath)
                multiScaleSharpen_out = zeros(img.shape, dtype=uint8)
                multiScaleSharpen_out = multiScaleSharpen(img, 5)
                rotate_img_path = to_videopath + "/" + framelisting[frame]
                cv2.imwrite(rotate_img_path, multiScaleSharpen_out)


# BouhSlav Bot
# @Authors: cabinboy1031, lonelylittlelucas
# initialises the bot. All top level function calls and commands go here.
# Any new feature will be put into a class in a different file,
# When a feature is finished and tested it will be put here.
import discord.ext.commands
import discord
import asyncio
import sys
import signal

client = discord.Client()

# Setup and use of the given discord API key.
# key_file = open("./private/API_KEY.txt",'r')
# API_KEY = key_file.readlines()
# key_file.close()  # Always close your files when you dont need them anymore.
# TOKEN = API_KEY[0]

# Use command line arguments to get the API key
TOKEN = sys.argv[1]

BOT_PREFIX = ("!!")
commands = discord.ext.commands
client = commands.Bot(command_prefix=BOT_PREFIX)

role_filter = discord.Permissions
role_filter.administrator = True
admin_roles = []

@client.event
async def on_message(message):
    if message.author == client.user:
        return
    else:
        print('message revieved')
        print('user id')
        print(message.author.id)
        print('content')
        print(message.content)
        print("")
    await client.process_commands(message)

@client.command()
async def test_me(ctx):
    print("Backend systems are online")
    await ctx.send("I WORK FOR SOVIET RUSSIA! \
                    BASE COMMAND READY FOR OPERATION!")

@client.command()
async def cant_touch(ctx, *, message: str):
    if message == "this":
        await ctx.send("indeed, I cant")
    else:
        await ctx.send("but I *can* touch " + message)

async def change_status():
    await client.wait_until_ready()

    while not client.is_closed:
        current_status = "Slav Sim 3000"
        await client.change_presence(game=discord.Game(name=current_status))
        await asyncio.sleep(1600)

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')
    print('You may now give me input.')
    await client.change_presence(activity=discord.Game(name='with portals'))
    print("Current Guilds:")
    print(client.guilds)
    print("current role in guild:")

async def is_owner(ctx):
    return ctx.message.author.id == 206531711968739339\
        or ctx.message.author.id == 110578737203945472

@client.command()
@commands.check(is_owner)
async def change_do(ctx, *, message: str):
    if ctx.message.author.id == 206531711968739339 \
       or ctx.message.author.id == 110578737203945472:
        await client.change_presence(activity=discord.Game(name=message))
        print("status changed")
        await ctx.send("STATION CHANGED, COMMANDER!")
    else:
        await ctx.send("NO DISRESPECT,\
        BUT I DO NOT TAKE ORDERS FROM ONE OF YOUR STATUS")

@client.command(names="commands")
async def commands_(ctx):
    await ctx.send("```\n!!test_me: test contact with main base\n\
    !!cant_touch 'param': see what you can touch\n```")

@client.command(pass_context=True)
@commands.has_permissions(administrator=True)
async def kill(ctx):
    await ctx.send('shutting down')
    await client.close()

@kill.error
async def kill_error(ctx, error):
    if isinstance(error, commands.CheckFailure):
        await ctx.send("***ONE OF YOUR STATUS CANNOT DO SUCH A THING***")

async def check_role(user: discord.Member,
                     role: discord.Role):
    # @param user: role of the user
    # @param role: the role(s) that the guild mods want to check.
    for user_role in user.roles:
        if (user_role == role):
            return True
        else:
            continue
    return False

async def on_sigint():
    print("Client received SIGINT: Shutting down.")
    client.close()
    sys.exit(0)

def main():
    signal.signal(signal.SIGINT, on_sigint)
    client.loop.create_task(change_status())
    client.run(TOKEN)

main()


# -*- coding: utf-8 -*-
# Copyright 2009-2016 Noviat
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
import xlwt
import time
from datetime import datetime
from odoo import api, fields, models, _
from odoo.tools import float_is_zero, float_compare
from odoo.tools.misc import formatLang
from odoo.api import Environment

from odoo.report import report_sxw
from report_csv import report_csv
from utils import rowcol_to_cell, _render
from openerp.tools.translate import translate, _
import logging
_logger = logging.getLogger(__name__)

_column_sizes = [
    ('A', 18),
    ('B', 18),
    ('C', 18),
    ('D', 18),
    ('E', 18),
    ('F', 18),
    ('G', 18),
    ('H', 18),
    ('I', 18),
    ('J', 18),
    ('K', 18),
    ('L', 18),
    ('M', 18),
]

class efaktur_invoice_csv_parser(report_sxw.rml_parse):

    def __init__(self, cr, uid, name, context):
        super(efaktur_invoice_csv_parser, self).__init__(cr, uid, name, context=context)
        self.env = Environment(cr, uid, context)
        move_obj = self.env['account.invoice']
        self.context = context
        self.localcontext.update({
            'datetime': datetime,
            'get_name_npwp': self._get_name_npwp,
            'get_street_npwp': self._get_street_npwp,
            'amount_currency': self._amount_currency,
            'amount_currency_line': self._amount_currency_line,
        })

    def _get_name_npwp(self, partner_id, type):
        return partner_id.name.upper()
#         if partner_id:
#             part_ids = self.pool.get('res.partner').search(self.cr, self.uid, [('parent_id','=',partner_id.id), ('type','=',type)])
#             if part_ids:
#                 partner = self.pool.get('res.partner').browse(self.cr, self.uid, part_ids[0])
#                 return partner.name.upper()
#             else:
#                 return partner_id.name.upper()

    def _get_street_npwp(self, partner_id, type):
        address = self.env['res.partner'].browse(partner_id.id)
        partner_address = ''
        if address.child_ids:
            addr_npwp = False
            for addr_npwp in address.child_ids:
                if addr_npwp.type == 'npwp':
                    address = addr_npwp
            if addr_npwp:
                partner_address += address.street and address.street +'. ' or ''
                partner_address += address.street2 and address.street2 +'. ' or ''
                partner_address += address.blok and 'Blok '+ address.blok or ''
                partner_address += address.nomor and 'No. '+ address.nomor or ''
                partner_address += address.rt and address.rw and 'RT/RW: %s/%s' % (address.rt,address.rw) or ''
                partner_address += address.kelurahan_id and ', Kel. ' + address.kelurahan_id.name or ''
                partner_address += address.kecamatan_id_new and ', Kec. ' + address.kecamatan_id.name or ''
                partner_address += address.kabupaten_id and ', Kota/Kab. ' + address.kabupaten_id.name + ', ' or ''
                partner_address += address.city and address.city +'. ' or ''
                partner_address += address.zip and address.zip +'. ' or ''
                partner_address += address.country_id.name and address.country_id.name or ''
        if not partner_address:
            partner_address += address.street and address.street +'. ' or ''
            partner_address += address.street2 and address.street2 +'. ' or ''
            partner_address += address.blok and 'Blok '+ address.blok or ''
            partner_address += address.nomor and 'No. '+ address.nomor or ''
            partner_address += address.rt and address.rw and 'RT/RW: %s/%s' % (address.rt,address.rw) or ''
            partner_address += address.kelurahan_id and ', Kel. ' + address.kelurahan_id.name or ''
            partner_address += address.kecamatan_id_new and ', Kec. ' + address.kecamatan_id_new.name or ''
            partner_address += address.kabupaten_id and ', Kota/Kab. ' + address.kabupaten_id.name + ', ' or ''
            partner_address += address.city and address.city +'. ' or ''
            partner_address += address.zip and address.zip +'. ' or ''
            partner_address += address.country_id.name and address.country_id.name or ''
        return partner_address.upper()

    def _amount_currency(self, amount, inv):
        cur_obj = self.env['res.currency']
        amount = cur_obj.with_context({'date': inv.date_invoice or time.strftime('%Y-%m-%d')}).compute(amount, inv.company_id.currency_id.id, round=False)
        return amount

    def _amount_currency_line(self, amount, inv):
        cur_obj = self.env['res.currency']
        amount = cur_obj.with_context({'date': inv.invoice_id.date_invoice or time.strftime('%Y-%m-%d')}).compute(amount, inv.invoice_id.company_id.currency_id.id, round=False)
        return amount

class efaktur_account_invoice_csv(report_csv):
    column_sizes = [x[1] for x in _column_sizes]

    def generate_csv_report(self, _p, _xs, data, objects, wb):
        #print "==generate_csv_report==",objects.type
        if objects[0].type in ('out_invoice','out_refund'):
            report_name = _("Faktur Keluaran")
        else:
            report_name = _("Faktur Masukan")
        ws = wb.add_sheet(report_name[:31])
        ws.panes_frozen = True
        ws.remove_splits = True
        ws.portrait = 0  # Landscape
        ws.fit_width_to_pages = 1
        row_pos = 0

        # set print header/footer
        ws.header_str = self.xls_headers['standard']
        ws.footer_str = self.xls_footers['standard']
        c_row_cell_style = xlwt.easyxf(num_format_str='###0')
        # Column headers 1
        c_specs = [
                ('FK',  1, 0, 'text', 'FK'),
                ('KD_JENIS_TRANSAKSI',  1, 0, 'text', 'KD_JENIS_TRANSAKSI'),
                ('FG_PENGGANTI',  1, 0, 'text', 'FG_PENGGANTI'),
                ('NOMOR_FAKTUR',  1, 0, 'text', 'NOMOR_FAKTUR'),
                ('MASA_PAJAK',  1, 0, 'text', 'MASA_PAJAK'),
                ('TAHUN_PAJAK',  1, 0, 'text', 'TAHUN_PAJAK'),
                ('TANGGAL_FAKTUR',  1, 0, 'text', 'TANGGAL_FAKTUR'),
                ('NPWP',  1, 0, 'text', 'NPWP'),
                ('NAMA',  1, 0, 'text', 'NAMA'),
                ('ALAMAT_LENGKAP',  1, 0, 'text', 'ALAMAT_LENGKAP'),
                ('JUMLAH_DPP',  1, 0, 'text', 'JUMLAH_DPP'),
                ('JUMLAH_PPN',  1, 0, 'text', 'JUMLAH_PPN'),
                ('JUMLAH_PPNBM',  1, 0, 'text', 'JUMLAH_PPNBM'),
                ('ID_KETERANGAN_TAMBAHAN',  1, 0, 'text', 'ID_KETERANGAN_TAMBAHAN'),
                ('FG_UANG_MUKA',  1, 0, 'text', 'FG_UANG_MUKA'),
                ('UANG_MUKA_DPP',  1, 0, 'text', 'UANG_MUKA_DPP'),
                ('UANG_MUKA_PPN',  1, 0, 'text', 'UANG_MUKA_PPN'),
                ('UANG_MUKA_PPNBM',  1, 0, 'text', 'UANG_MUKA_PPNBM'),
                ('REFERENSI',  1, 0, 'text', 'REFERENSI'),
                ]
        row_data = self.xls_row_template(c_specs, [x[0] for x in c_specs])
        row_pos = self.xls_write_row(ws, row_pos, row_data, row_style=c_row_cell_style)
        # Column headers 2
        c_specs = [
                ('LT',  1, 0, 'text', 'LT'),
                ('NPWP',  1, 0, 'text', 'NPWP'),
                ('NAMA',  1, 0, 'text', 'NAMA'),
                ('JALAN',  1, 0, 'text', 'JALAN'),
                ('BLOK',  1, 0, 'text', 'BLOK'),
                ('NOMOR',  1, 0, 'text', 'NOMOR'),
                ('RT',  1, 0, 'text', 'RT'),
                ('RW',  1, 0, 'text', 'RW'),
                ('KECAMATAN',  1, 0, 'text', 'KECAMATAN'),
                ('KELURAHAN',  1, 0, 'text', 'KELURAHAN'),
                ('KABUPATEN',  1, 0, 'text', 'KABUPATEN'),
                ('PROPINSI',  1, 0, 'text', 'PROPINSI'),
                ('KODE_POS',  1, 0, 'text', 'KODE_POS'),
                ('NOMOR_TELEPON',  1, 0, 'text', 'NOMOR_TELEPON'),
                ]
        row_data = self.xls_row_template(c_specs, [x[0] for x in c_specs])
        row_pos = self.xls_write_row(ws, row_pos, row_data, row_style=c_row_cell_style)
        # Column headers 3
        c_specs = [
                ('OF',  1, 0, 'text', 'OF'),
                ('KODE_OBJEK',  1, 0, 'text', 'KODE_OBJEK'),
                ('NAMA',  1, 0, 'text', 'NAMA'),
                ('HARGA_SATUAN',  1, 0, 'text', 'HARGA_SATUAN'),
                ('JUMLAH_BARANG',  1, 0, 'text', 'JUMLAH_BARANG'),
                ('HARGA_TOTAL',  1, 0, 'text', 'HARGA_TOTAL'),
                ('DISKON',  1, 0, 'text', 'DISKON'),
                ('DPP',  1, 0, 'text', 'DPP'),
                ('PPN',  1, 0, 'text', 'PPN'),
                ('TARIF_PPNBM',  1, 0, 'text', 'TARIF_PPNBM'),
                ('PPNBM',  1, 0, 'text', 'PPNBM'),
                ]
        row_data = self.xls_row_template(c_specs, [x[0] for x in c_specs])
        row_pos = self.xls_write_row(ws, row_pos, row_data, row_style=c_row_cell_style)
        for inv in objects:
            print ("CANCELLL ", inv.is_cancel)
            if not inv.is_cancel:
                print ("AAAAAAAAAAAAA ", inv.is_cancel)
                c_specs = [
                        ('FK',  1, 0, 'text', 'FK'),
                        ('kd_jenis_transaksi',  1, 0, 'text', '01'),#x['faktur_pajak_no'] and x['faktur_pajak_no'][:2] or '01'
                        ('jenis_fp',  1, 0, 'text', '0'),#0=faktur pajak, 1=faktur pajak pengganti
                        ('nomor_faktur',  1, 0, 'text', inv.nomor_faktur_id and inv.nomor_faktur_id.number or ''),#010.000-15.10897639
                        ('masa_pajak',  1, 0, 'number', int(time.strftime('%m', time.strptime(inv.date_invoice,'%Y-%m-%d')))),
                        ('tahun_pajak',  1, 0, 'number', time.strftime('%Y', time.strptime(inv.date_invoice,'%Y-%m-%d'))),
                        ('tanggal_faktur',  1, 0, 'text', time.strftime('%d/%m/%Y', time.strptime(inv.date_invoice,'%Y-%m-%d'))),
                        ('npwp',  1, 0, 'text', inv.npwp_efaktur or '000000000000000'),#02.505.335.6-016.000
                        ('nama',  1, 0, 'text', _p.get_name_npwp(inv.partner_id, 'npwp') or ''),
                        ('alamat_lengkap',  1, 0, 'text', _p.get_street_npwp(inv.partner_id, 'npwp') or ''),
                        ('jumlah_dpp',  1, 0, 'number', int(inv.amount_untaxed)),
                        ('jumlah_ppn',  1, 0, 'number', int(inv.amount_tax)),
                        ('jumlah_ppnbm',  1, 0, 'number', 0),
                        ('id_keterangan_tambahan',  1, 0, 'text', ''),
                        ('fg_uang_muka',  1, 0, 'number', 0),
                        ('uang_muka_dpp',  1, 0, 'number', 0),
                        ('uang_muka_ppn',  1, 0, 'number', 0),
                        ('uang_muka_ppnbm',  1, 0, 'number', 0),
                        ('referensi',  1, 0, 'text', inv.partner_id and inv.partner_id.ktp or inv.origin or ''),
                    ]
            elif inv.is_cancel:
                print ("BBBBBBBBBBBBB ", inv.is_cancel)
                c_specs = [
                        ('FK',  1, 0, 'text', 'FK'),
                        ('kd_jenis_transaksi',  1, 0, 'text', '01'),#x['faktur_pajak_no'] and x['faktur_pajak_no'][:2] or '01'
                        ('jenis_fp',  1, 0, 'text', '1'),#0=faktur pajak, 1=faktur pajak pengganti
                        ('nomor_faktur',  1, 0, 'text', inv.nomor_faktur_id and inv.nomor_faktur_id.number or ''),#010.000-15.10897639
                        ('masa_pajak',  1, 0, 'number', int(time.strftime('%m', time.strptime(inv.date_invoice,'%Y-%m-%d')))),
                        ('tahun_pajak',  1, 0, 'number', time.strftime('%Y', time.strptime(inv.date_invoice,'%Y-%m-%d'))),
                        ('tanggal_faktur',  1, 0, 'text', time.strftime('%d/%m/%Y', time.strptime(inv.date_invoice,'%Y-%m-%d'))),
                        ('npwp',  1, 0, 'text', inv.npwp_efaktur or '000000000000000'),#02.505.335.6-016.000
                        ('nama',  1, 0, 'text', _p.get_name_npwp(inv.partner_id, 'npwp') or ''),
                        ('alamat_lengkap',  1, 0, 'text', _p.get_street_npwp(inv.partner_id, 'npwp') or ''),
                        ('jumlah_dpp',  1, 0, 'number', int(inv.amount_untaxed)),
                        ('jumlah_ppn',  1, 0, 'number', int(inv.amount_tax)),
                        ('jumlah_ppnbm',  1, 0, 'number', 0),
                        ('id_keterangan_tambahan',  1, 0, 'text', ''),
                        ('fg_uang_muka',  1, 0, 'number', 0),
                        ('uang_muka_dpp',  1, 0, 'number', 0),
                        ('uang_muka_ppn',  1, 0, 'number', 0),
                        ('uang_muka_ppnbm',  1, 0, 'number', 0),
                        ('referensi',  1, 0, 'text', inv.partner_id and inv.partner_id.ktp or inv.origin or ''),
                    ]

            row_data = self.xls_row_template(c_specs, [x[0] for x in c_specs])
            row_pos = self.xls_write_row(ws, row_pos, row_data, row_style=c_row_cell_style)
            #Faktur Company
            #===================================================================
            # c_specs = [
            #     ('FAPR',  1, 0, 'text', 'FAPR'),
            #     ('company_name',  1, 0, 'text', inv.partner_id.name),
            #     ('company_address',  1, 0, 'text', _p.get_street_npwp(inv.partner_id.id, 'npwp')),
            #     ]
            # row_data = self.xls_row_template(c_specs, [x[0] for x in c_specs])
            # row_pos = self.xls_write_row(ws, row_pos, row_data, row_style=c_row_cell_style)
            #===================================================================
            #print "===line.invoice_line_tax_ids==",sum(line.amount for line in self.tax_line_ids)
            for line in inv.invoice_line_ids:
                tax_amt_type = sum(tax.amount_type == 'percent' and tax.amount/100.0 or tax.amount for tax in line.invoice_line_tax_ids)
                c_specs = [
                    ('OF',  1, 0, 'text', 'OF'),
                    ('code_product',  1, 0, 'text', line.product_id.default_code or ''),
                    ('name_product',  1, 0, 'text', line.product_id.name or ''),
                    ('price_unit',  1, 0, 'number', int(_p.amount_currency_line(line.price_subtotal/line.quantity, line))),
                    ('quantity',  1, 0, 'number', line.quantity or 1),
                    ('price_unit_quantity',  1, 0, 'number', int(_p.amount_currency_line(line.price_subtotal, line))),
                    ('discount',  1, 0, 'number', line.discount or 0),
                    ('dpp_product',  1, 0, 'number', int(_p.amount_currency_line(line.price_subtotal, line))),
                    ('ppn_product',  1, 0, 'number', line.invoice_line_tax_ids and int(_p.amount_currency_line(round(tax_amt_type*line.price_subtotal,2),line))),
                    ('tarif_ppnbm',  1, 0, 'number', 0),
                    ('ppnbm',  1, 0, 'number', 0),
                ]
                row_data = self.xls_row_template(c_specs, [x[0] for x in c_specs])
                row_pos = self.xls_write_row(ws, row_pos, row_data, row_style=c_row_cell_style)

efaktur_account_invoice_csv('report.efaktur.account.invoice.csv',
              'account.invoice',
              parser=efaktur_invoice_csv_parser)


import json
import h5py
import os
import sys
import numpy as np

def parseConfigFile(json_path):
    data = json.load(open(json_path))
    data_dir = data['data_dir']
    target_dir = data['target_dir']
    training_files = [os.path.join(data_dir, x) for x in data['train_files']]
    validation_files = [os.path.join(data_dir, x) for x in data['validation_files']]
    testing_files = [os.path.join(data_dir, x) for x in data['test_files']]
    return target_dir, training_files, validation_files, testing_files
def buildDatasetFromShards(output_path, files):
    seq = list()
    meta = list()
    for file in files:
        with h5py.File(file,'r') as f:
            seq.append(f['sequences'][()])
            meta.append(f['metadata'][()])
    sequences = np.concatenate(seq, axis=0)
    metadata = np.concatenate(meta, axis=0)
    print("{} sequences in {}".format(sequences.shape[0], output_path))
    with h5py.File("{}.hdf5".format(output_path),'w') as f:
        f.create_dataset("sequences", data=sequences)
        f.create_dataset("metadata", data=metadata)

def main(config_path):
    data_dir, training_files, validation_files, testing_files = parseConfigFile(config_path)
    buildDatasetFromShards(os.path.join(data_dir, "train_data"), training_files)
    buildDatasetFromShards(os.path.join(data_dir, "validation_data"), validation_files)
    buildDatasetFromShards(os.path.join(data_dir, "test_data"), testing_files)
if __name__ == "__main__":
    if len(sys.argv) != 2:
        raise ValueError('Only arg is json file path')
    config_path = sys.argv[1]
    main(config_path)


#!/bin/env python
from __future__ import division, print_function
import sys
import healpy as hp
import numpy as np
import matplotlib.pyplot as mp
from qubic import *
from pysimulators import FitsArray
import time
import os

# External library needed (not part of qubic yet)
import spectroimlib_tmp as si

### NB: Some of the pathes here are on my computer. You will have to change them
TD = False
path = QubicCalibration().path
if(TD):
	print('Technological Demonstrator')
	os.system('\cp '+path+'/TD_CalQubic_HornArray_v4.fits '+path+'/CalQubic_HornArray_v5.fits')
	os.system('\cp '+path+'/TD_CalQubic_DetArray_v3.fits '+path+'/CalQubic_DetArray_v4.fits')
else:
	print('First Instrument')
	os.system('rm -f '+path+'/CalQubic_HornArray_v5.fits')
	os.system('rm -f '+path+'/CalQubic_DetArray_v4.fits')

inst = QubicInstrument()
clf()
subplot(2,1,1)
inst.horn.plot()
subplot(2,1,2)
inst.detector.plot()

######## Default configuration
### Sky
nside = 128
center = 0., -57.
center_gal = equ2gal(center[0], center[1])
dust_coeff = 1.39e-2
seed=None

### Detectors (for now using random pointing)
band = 150
TD = False
relative_bandwidth = 0.25
sz_ptg = 10.
nb_ptg = 1000
effective_duration = 30./365
ripples = False
noiseless = False

### Mapmaking
tol = 1e-3

### Number of sub-bands to build the TOD
nf_sub_build = 10
nf_sub_rec = 3

parameters = {'nside':nside, 'center':center, 'dust_coeff': dust_coeff,
				'band':band, 'relative_bandwidth':relative_bandwidth,
				'sz_ptg':sz_ptg, 'nb_ptg':nb_ptg, 'effective_duration':effective_duration,
				'tol': tol, 'ripples':ripples,
				'nf_sub_build':nf_sub_build,
				'nf_sub_rec': nf_sub_rec, 'noiseless':noiseless, 'seed':seed, 'TD':TD }

for k in parameters.keys(): print(k, parameters[k])

print('Creating input sky')
x0 = si.create_input_sky(parameters)

print('Creating pointing')
p = si.create_random_pointings(parameters['center'], parameters['nb_ptg'], parameters['sz_ptg'])

print('Creating TOD')
TOD = si.create_TOD(parameters, p, x0)

print('Doing Mapmaking on {} sub-map(s)'.format(nf_sub_rec))
maps_recon, cov, nus, nus_edge, maps_convolved = si.reconstruct_maps(TOD, parameters, p, x0=x0)
if int(parameters['nf_sub_rec'])==1: maps_recon=np.reshape(maps_recon, np.shape(maps_convolved))
cov = np.sum(cov, axis=0)
maxcov = np.max(cov)
unseen = cov < maxcov*0.1
diffmap = maps_convolved - maps_recon
maps_convolved[:,unseen,:] = hp.UNSEEN
maps_recon[:,unseen,:] = hp.UNSEEN
diffmap[:,unseen,:] = hp.UNSEEN
therms = np.std(diffmap[:,~unseen,:], axis = 1)

stokes = ['I', 'Q', 'U']
for istokes in [0,1,2]:
	figure(istokes)
	if istokes==0:
		xr=200
	else:
		xr=5
	for i in range(parameters['nf_sub_rec']):
		hp.gnomview(maps_convolved[i,:,istokes], rot=center_gal, reso=10,
			sub=(parameters['nf_sub_rec'],3,3*i+1), min=-xr, max=xr,
			title='Input '+stokes[istokes]+' SubFreq {}'.format(i))
		hp.gnomview(maps_recon[i,:,istokes], rot=center_gal, reso=10,
			sub=(parameters['nf_sub_rec'],3,3*i+2), min=-xr, max=xr,
			title='Output '+stokes[istokes]+' SubFreq {}'.format(i))
		hp.gnomview(diffmap[i,:,istokes], rot=center_gal, reso=10,
			sub=(parameters['nf_sub_rec'],3,3*i+3), min=-xr, max=xr,
			title='Residual '+stokes[istokes]+' SubFreq {}'.format(i))


# -*- coding: utf-8 -*-
import os, pandas, numpy, shutil, re, time
from lxml import objectify
from copy import copy
import spold2_utils, utils
from jinja2 import Environment, FileSystemLoader
from asteval import Interpreter

folder = os.path.join(utils.script_folder, 'modules', 'readers', 'documentation')
filename = 'spold2_fields.xlsx'
excel_columns_df = utils.read_excel(folder, filename, 'MD')
excel_columns_df.sort_values('tab order', inplace = True)
excel_columns = {}
for tab in list(excel_columns_df['tab']):
    if tab not in excel_columns:
        fields = excel_columns_df[excel_columns_df['tab'] == tab]
        fields = fields.sort_values('field order')
        excel_columns[tab] = tuple(fields['field'])
excel_columns_df = excel_columns_df.set_index(['tab', 'field']).sort_index(level=0)
def apply_options(x):
    if not utils.is_empty(x):
        interpreter = Interpreter()
        x = interpreter.eval(x, show_errors = False)
    return x
def apply_re(x):
    if not utils.is_empty(x):
        x = re.compile(x)
    return x
excel_columns_df['options'] = excel_columns_df['options'].apply(apply_options)
excel_columns_df['regular expression'] = excel_columns_df['regular expression'].apply(apply_re)
grouped = excel_columns_df.reset_index()
grouped = grouped[grouped['identity field']].groupby('tab')
MD_identity_fields = {}
for name, group in grouped:
    group = group.sort_values(by = 'field order')
    MD_identity_fields[name] = list(group['field'])
def build_MD(version, system_model, basepath = '', master_data_folder = ''):
    folder = utils.version_system_model_path(version, system_model, basepath = basepath)
    if master_data_folder == '':
        master_data_folder = os.path.join(folder, 'MasterData')
    MD_fields = utils.read_excel(os.path.join(utils.script_folder, 'modules',
        'release', 'documentation'), 'MasterData_fields.xlsx', 'fields')
    MD_tags = utils.read_excel(os.path.join(utils.script_folder, 'modules',
        'release', 'documentation'), 'MasterData_fields.xlsx', 'tags').set_index('file')
    properties = {}
    grouped = MD_fields.groupby('file')
    MD = {}
    read_me = [['filename', 'last modification']]
    for filename, group in grouped:
        print('reading master data from {}'.format(filename))
        if 'Exchange' in filename:
            properties[filename] = []
        df = []
        roots = []
        p = os.path.join(master_data_folder, '{}.xml'.format(filename))
        t = time.gmtime(os.path.getmtime(p))
        t = time.strftime('%Y-%m-%d %H:%M:%S', t)
        read_me.append(['{}.xml'.format(filename), t])
        with open(p, encoding = 'utf8') as f:
            roots.append(objectify.parse(f).getroot())
        if system_model != 'Undefined':
            #additional MD are supplied with calculation, in separate files
            p = os.path.join(os.path.dirname(master_data_folder), '{}.xml'.format(filename))
            if os.path.exists(p):
                with open(p, encoding = 'utf8') as f:
                    roots.append(objectify.parse(f).getroot())
        fields = list(zip(list(group['field type']), list(group['field name'])))
        for root in roots:
            if filename == 'Classifications':
                for child in root.iterchildren(tag = spold2_utils.tag_prefix + MD_tags.loc[filename, 'tag']):
                    for c in child.iterchildren(tag = spold2_utils.tag_prefix + 'classificationValue'):
                        to_add = {'classificationSystemId': child.get('id'),
                                  'classificationSystemName': child.name.text}
                        to_add = store_fields(fields, c, to_add = to_add)
                        df.append(copy(to_add))
            elif filename == 'Compartments':
                for child in root.iterchildren(tag = spold2_utils.tag_prefix + MD_tags.loc[filename, 'tag']):
                    for c in child.iterchildren(tag = spold2_utils.tag_prefix + 'subcompartment'):
                        to_add = {'compartmentId': child.get('id'),
                                  'compartmentName': child.name.text,
                                  'subcompartmentId': c.get('id'),
                                    'subcompartmentName': c.name.text,
                                    'comment': c.comment.text}
                        df.append(copy(to_add))
            else:
                if 'UsedUserMasterData' in root.tag:
                    tag = '{http://www.EcoInvent.org/UsedUserMasterData}' + MD_tags.loc[filename, 'tag']
                else:
                    tag = spold2_utils.tag_prefix + MD_tags.loc[filename, 'tag']
                for child in root.iterchildren(tag = tag):
                    to_add = store_fields(fields, child, to_add = {})
                    if hasattr(child, 'compartment'):
                        to_add['compartment'] = child.compartment.compartment.text
                        to_add['subcompartment'] = child.compartment.subcompartment.text
                    if hasattr(child, 'productInformation'):
                        content = {}
                        for cc in child.productInformation.iterchildren():
                            content[int(cc.get('index'))] = str(cc.text)
                        if len(content):
                            to_add['productInformation'] = ['']*(max(content.keys())+1)
                            for i, j in content.items():
                                to_add['productInformation'][i] = j
                            to_add['productInformation'] = [j for j in to_add['productInformation'] if not utils.is_empty(j)]
                            to_add['productInformation'] = '\n'.join(to_add['productInformation'])
                    else:
                        to_add['productInformation'] = ''
                    for p in child.iterchildren(tag = spold2_utils.tag_prefix + 'property'):
                        to_add_ = copy(to_add)
                        to_add_['propertyId'] = p.get('propertyId')
                        to_add_['amount'] = p.get('amount')
                        if utils.is_empty(to_add_['amount']):
                            to_add_['amount'] = 0.
                        else:
                            to_add_['amount'] = float(to_add_['amount'])
                        field = 'isDefiningValue'
                        if not utils.is_empty(p.get(field)):
                            to_add_[field] = p.get(field) == 'true'
                        else:
                            to_add_[field] = False
                        field = 'variableName'
                        if not utils.is_empty(p.get(field)):
                            to_add_[field] = p.get(field)
                        else:
                            to_add_[field] = ''
                        field = 'comment'
                        if hasattr(p, field) and not(utils.is_empty(p.comment.text)):
                            to_add_[field] = p.comment.text
                        else:
                            to_add_[field] = ''
                        properties[filename].append(copy(to_add_))
                    if filename == 'IntermediateExchanges':
                        for c in child.iterchildren(tag = spold2_utils.tag_prefix + 'classification'):
                            if str(c.classificationSystem) not in ['ISIC rev.4 ecoinvent']:
                                to_add[str(c.classificationSystem)] = str(c.classificationValue)
                        to_add['synonym'] = []
                        for c in child.iterchildren(tag = spold2_utils.tag_prefix + 'synonym'):
                            to_add['synonym'].append(str(c.text))
                        to_add['synonym'] = '\n'.join(to_add['synonym'])
                    df.append(copy(to_add))
        MD[filename] = utils.list_to_df(df)
        for col in list(group['field name']):
            if col not in MD[filename]:
                MD[filename][col] = ''
    for filename in properties:
        properties[filename] = utils.list_to_df(properties[filename])
        if 'id' not in properties[filename]:
            1/0
    MD, properties = join_info(MD, properties)
    filename = 'ExchangeActivityIndex.xml'
    filelist = utils.build_file_list(master_data_folder, 'xml')
    if filename in filelist:
        with open(os.path.join(master_data_folder, filename), encoding='utf8') as f:
            root = objectify.parse(f).getroot()
        df = []
        for exchangeActivityIndexEntry in root.iterchildren():
            for IO in ['input', 'output']:
                for o in getattr(exchangeActivityIndexEntry, IO).iterchildren():
                    to_add = {'id': exchangeActivityIndexEntry.get('validIntermediateExchangeId'),
                              'activityIndexEntryId': o.get('activityIndexEntryId'),
                                'input/output': IO}
                    df.append(to_add)
        df = utils.list_to_df(df)
        df = df.set_index('id')
        tab = 'IntermediateExchanges'
        df = df.join(MD[tab].set_index('id')[['name']])
        df = df.reset_index().rename(columns = {'id': 'intermediateExchangeId'})
        tab = 'ActivityIndex'
        df = df.set_index('activityIndexEntryId').join(MD[tab].set_index('id')[['activityName', 'geography',
                          'startDate', 'endDate']]).reset_index()
        MD['ExchangeActivityIndex'] = df.rename(columns = {'index': 'activityIndexEntryId'})

    MD = flag_duplicates_and_hidden(MD, system_model)
    MD = to_excel(os.path.join(folder, 'excel'), MD, read_me, properties)
    utils.pkl_dump(os.path.join(folder, 'pkl'), 'MD', MD)

def store_fields(fields, child, to_add = {}):
    for field_type, field in fields:
        if field_type == 'attribute':
            to_add[field] = child.get(field)
        else:
            if hasattr(child, field):
                to_add[field] = getattr(child, field).text
            else:
                to_add[field] = ''
    return to_add

def join_info(MD, properties):
    #add geography to ActivityIndex
    f = 'ActivityIndex'
    MD[f] = MD[f].set_index('geographyId')
    g = 'Geographies'
    t = MD[g].rename(columns = {'id': 'geographyId'})
    t = t.set_index('geographyId')[['shortname']]
    MD[f] = MD[f].join(t).reset_index()
    MD[f] = MD[f].rename(columns = {'shortname': 'geography'})

    #add activityName to ActivityIndex
    MD[f] = MD[f].set_index('activityNameId')
    g = 'ActivityNames'
    t = MD[g].rename(columns = {'id': 'activityNameId'})
    t = t.set_index('activityNameId')
    if 'productInformation' in MD[f]:
        del MD[f]['productInformation']
    MD[f] = MD[f].join(t).reset_index()
    MD[f] = MD[f].rename(columns = {'name': 'activityName'})

    #add system model to ActivityIndex
    MD[f] = MD[f].set_index('systemModelId')
    g = 'SystemModels'
    t = MD[g].rename(columns = {'id': 'systemModelId'})
    t = t.set_index('systemModelId')
    if 'productInformation' in MD[f]:
        del MD[f]['productInformation']
    MD[f] = MD[f].join(t).reset_index()
    MD[f] = MD[f].rename(columns = {'shortname': 'systemModelName'})
    MD[f] = MD[f].replace(to_replace = {'specialActivityType': spold2_utils.options['specialActivityType']})

    #arrange Classifications
    f = 'Classifications'
    MD[f] = MD[f].rename(columns = {'name': 'classificationValueName',
                                    'id': 'classificationValueId'})

    #arrange Sources
    f = 'Sources'
    excel_columns_df.loc[f, 'sourceType']['options']
    MD[f] = MD[f].replace(to_replace = {'sourceType': excel_columns_df.loc[f, 'sourceType']['options']})

    #arrange UnitConversions
    f = 'UnitConversions'
    MD[f]['factor'] = MD[f]['factor'].astype(float)
    df = MD[f].copy().rename(columns = {'unitFromName': 'unitToName',
                                        'unitToName': 'unitFromName'})
    df['factor'] = df['factor'].apply(lambda x: 1./x)
    MD[f] = pandas.concat([MD[f], df], sort = False)

    #arrange CAS
    for f in ['ElementaryExchanges', 'IntermediateExchanges']:
        MD[f]['casNumber'] = MD[f]['casNumber'].apply(utils.arrange_CAS)

    #arrange properties
    for f in properties:
        properties[f] = properties[f].set_index('propertyId')
        g = 'Properties'
        properties[f] = properties[f].join(MD[g].set_index('id')[
            ['name', 'unitName']], rsuffix = '_').reset_index()
        del properties[f]['unitName']
        properties[f] = properties[f].rename(columns = {
            'unitName_': 'unitName',
            'name_': 'propertyName',
            'index': 'propertyId'})

    return MD, properties

def to_excel(excel_folder, MD, read_me, properties = []):
    print('creating MasterData.xlsx in %s' % excel_folder)
    dfs = []
    for tab, columns in excel_columns.items():
        if 'prop.' not in tab:
            if tab == 'ExchangeActivityIndex' and tab not in MD:
                pass
            else:
                MD[tab] = utils.add_empty_columns(MD[tab], columns)
                dfs.append((MD[tab], tab, columns))
        else:
            new_tab = tab.replace(' prop.', '')
            properties[new_tab] = utils.add_empty_columns(properties[new_tab], excel_columns[tab])
            dfs.append((properties[new_tab], tab, excel_columns[tab]))
            MD[tab] = properties[new_tab].copy()
    filename = 'MasterData.xlsx'
    utils.dataframe_to_excel(excel_folder, filename, dfs, read_me = read_me, feedback = True)

    return MD

def make_version_df(basepath, version, system_model):
    '''used to prepare the mapping of master data between versions'''

    folder = utils.version_system_model_path(version, system_model, basepath = basepath)
    MD = utils.pkl_load(os.path.join(folder, 'pkl'), 'MD')
    info = [('ActivityNames', 'name'), ('ElementaryExchanges', 'name'),
            ('Geographies', 'shortname'), ('IntermediateExchanges', 'name')]
    df = []
    for name_type, field in info:
        df.extend([{'name type': name_type, version: name}
            for name in list(MD[name_type][field])])
    df = utils.list_to_df(df)

    #some activityNames only get generated after linking, so it is
    #necessary to look in the ao
    if system_model != 'Undefined':
        activityNames = set(df[df['name type'] == 'ActivityNames'][version])
        all_activityNames = copy(activityNames)
        folder = utils.version_system_model_path(version, system_model, basepath = basepath)
        ao = utils.pkl_load(os.path.join(folder, 'pkl'), 'ao')
        all_activityNames = all_activityNames.union(set(ao['activityName']))
        missing_activityNames = all_activityNames.difference(activityNames)
        to_add = [{'name type': 'ActivityNames', version: name}
            for name in list(missing_activityNames)]
        df = pandas.concat([df, utils.list_to_df(to_add)])
    return df

def MD_mapping(versions, system_model, basepath = '', force_update = False,
        known_change_folder = '', known_change_filename = ''):
    '''Yields a spreadsheet where the newest MD are compared to the previous MD'''
    folder = utils.version_system_model_path(versions[1], system_model, basepath = basepath)
    if not utils.is_empty(known_change_folder) and not utils.is_empty(known_change_filename):
        known_changes = utils.read_excel(known_change_folder, known_change_filename, 'renaming')
        known_changes, _ = utils.accelerate_df(known_changes, ['name type', 'previous'])
    else:
        known_changes = pandas.DataFrame()
    df = {}
    for version in versions:
        df[version] = make_version_df(basepath, version, system_model)
    d = []
    available = set(df[versions[1]][versions[1]])
    for i, sel in df[versions[0]].iterrows():
        key = tuple(sel[['name type', versions[0]]])
        sel = sel.to_dict()
        if key in known_changes:
            sel[versions[1]] = known_changes[key]['current']
        elif sel[versions[0]] in available:
            sel[versions[1]] = sel[versions[0]]
        else: pass
        d.append(sel)
    d = utils.list_to_df(d)
    d = pandas.concat([d, df[versions[1]]], sort = False)
    deleted = d[d[versions[1]].apply(utils.is_empty)]
    deleted = deleted.drop_duplicates(subset = ['name type', versions[0]])
    d = d[~d[versions[1]].apply(utils.is_empty)]
    d = d.drop_duplicates(subset = ['name type', versions[1]])
    d = pandas.concat([d, deleted], sort = False)
    d['status'] = d[versions].apply(utils.compare_2_values, axis = 1)
    columns = ['name type', versions[0], versions[1], 'status']
    filename = 'MD_mapping.xlsx'
    dfs = [(d, 'Sheet1', columns)]
    utils.dataframe_to_excel(os.path.join(folder, 'excel'), filename, dfs, feedback = True)
    return d

def MD_property_compare(versions, basepath, system_model):

    #opening both master data files from pkl
    MD = {}
    for version in versions:
        folder = os.path.join(basepath, version, system_model)
        pkl_folder = os.path.join(folder, 'pkl')
        MD[version] = utils.pkl_load(pkl_folder, 'MD')

    #opening the MD_mapping file from excel
    excel_folder = os.path.join(folder, 'excel')
    filename = 'MD_mapping.xlsx'
    MD_mapping = utils.read_excel(excel_folder, filename, 'Sheet1')
    MD_mapping = MD_mapping.set_index(['name type', versions[0]]).sort_index(level=0)

    #work on elementary and intermediate exchanges.  Operations are almost the same on both.
    filename = 'properties_compare_%s_%s.xlsx' % tuple(versions)
    writer = pandas.ExcelWriter(os.path.join(excel_folder, filename))
    for f in ['ElementaryExchanges', 'IntermediateExchanges']:

        #start with the properties in versions[0]
        df = MD[versions[0]][f + ' properties'].set_index('name')

        #join the name of the exchanges in versions[1]
        df = df.join(MD_mapping.loc[f])
        df = df.reset_index()
        df = df.rename(columns = {'level_0': 'name in %s' % versions[0],
                                  versions[1]: 'name',
                                'amount': 'amount in %s' % versions[0]})

        #delete some extra stuff
        for col in ['index', 'casNumber', 'comment', 'id']:
            if col in df:
                del df[col]

        #some exchanges might have been deleted, they have to be treated separately
        deleted = df[df['name'].apply(utils.is_empty)]

        #working on common exchanges first
        df = df[~df['name'].apply(utils.is_empty)]

        #join the property amount from versions[1]
        if 'Elementary' in f:
            indexes = ['name', 'compartment', 'subcompartment', 'propertyName']
        else:
            indexes = ['name', 'propertyName']
        df = df.set_index(indexes).sort_index(level=0)
        MD[versions[1]][f + ' properties'] = MD[versions[1]][f + ' properties'
            ].set_index(indexes).sort_index(level=0)
        df = df.join(MD[versions[1]][f + ' properties'][['amount']]).reset_index()

        #do some column renaming
        df = df.rename(columns = {'name': 'name in %s' % versions[1],
                                  'amount': 'amount in %s' % versions[1]})

        #prepare deleted to be merged with the rest
        if len(deleted) > 0:
            deleted['name'] = ''
            deleted = deleted.rename(columns = {'name': 'name in %s' % versions[1]})
            deleted['amount in %s' % versions[1]] = ''
            df = pandas.concat([df, deleted])

        #status column: updated, identical, added, deleted
        df['status'] = df['amount in %s' % versions[0]
            ] == df['amount in %s' % versions[1]]
        df.loc[df['amount in %s' % versions[1]].apply(utils.is_empty), 'status'] = 'deleted'
        df.loc[df['amount in %s' % versions[0]].apply(utils.is_empty), 'status'] = 'added'
        df.loc[df['status'] == True, 'status'] = 'identical'
        df.loc[df['status'] == False, 'status'] = 'updated'

        #save to excel
        columns = ['name in %s' % version for version in versions]
        if 'Elementary' in f:
            columns.extend(['compartment', 'subcompartment'])
        columns.extend(['propertyName', 'amount in %s' % versions[0],
                        'amount in %s' % versions[1], 'unitName', 'status'])
        df.to_excel(writer, f + ' prop.', columns = columns,
                    merge_cells = False, index = False)

    #save and close spreadsheet
    writer.save()
    writer.close()

    #this is for consultation by humans only, so the pkl version is not saved

def to_xml(MD, result_folder, source_folder):
    #join unitId and subcompartmentId to ElementaryExchanges
    tab = 'ElementaryExchanges'
    MD[tab] = MD[tab].set_index('unitName')
    df = MD['Units'].rename(columns = {'name': 'unitName', 'id': 'unitId'}).set_index('unitName')[['unitId']]
    MD[tab] = MD[tab].join(df).reset_index()
    df = MD['Compartments'].rename(columns = {'subcompartmentName': 'subcompartment',
        'compartmentName': 'compartment'}).set_index(['compartment', 'subcompartment']
        ).sort_index(level=0)[['subcompartmentId']]
    MD[tab] = MD[tab].set_index(
        ['compartment', 'subcompartment']).sort_index(level=0)
    MD[tab] = MD[tab].join(df).reset_index()

    #join unitId to IntermediateExchanges
    tab = 'IntermediateExchanges'
    MD[tab] = MD[tab].set_index('unitName')
    df = MD['Units'].rename(columns = {'name': 'unitName', 'id': 'unitId'}).set_index('unitName')[['unitId']]
    MD[tab] = MD[tab].join(df).reset_index()

    tabs = ['ActivityIndex', 'Compartments', 'ElementaryExchanges', 'IntermediateExchanges',
            'Geographies', 'ActivityNames', 'Companies', 'Sources', 'Units']
    for tab in tabs:
        for col in MD[tab]:
            MD[tab] = MD[tab].replace(to_replace = {col: {numpy.nan: ''}})
        all_entries = []
        if tab == 'Compartments':
            grouped = MD[tab].groupby('compartmentName')
            for name, group in grouped:
                to_add = {'name': name, 'id': group.iloc[0]['compartmentId']}
                subcompartments = []
                for i in range(len(group)):
                    sel = group.iloc[i]
                    subcompartments.append(dict(zip(['comment', 'compartmentId', 'compartmentName',
                                                     'id', 'name'], list(sel))))
                to_add['subcompartments'] = subcompartments
                all_entries.append(to_add)
        else:
            for i in range(len(MD[tab])):
                sel = MD[tab].iloc[i]
                to_add = dict(zip(list(sel.index), list(sel)))
                all_entries.append(to_add)
        template_name = '%s.xml' % tab
        template_path = r'C:\python\ecoinvent_python_scripts_3\templates\MasterData'
        env = Environment(loader=FileSystemLoader(template_path),
                          keep_trailing_newline = True,
                          lstrip_blocks = True,
                          trim_blocks = True)
        template = env.get_template(template_name)
        writer = open(os.path.join(result_folder, template_name), 'w', encoding = 'utf-8')
        output = template.render(**{'all_entries': all_entries})
        writer.write(output)
        writer.close()

    tabs = list(set(MD.keys()).difference(set(tabs)))
    tabs = [tab for tab in tabs if 'prop.' not in tab]
    for tab in tabs:
        source = os.path.join(source_folder, '%s.xml' % tab)
        destination = os.path.join(result_folder, '%s.xml' % tab)
        shutil.copy(source, destination)

def find_current_MD_path():
    folder = r'C://'
    folders, files = utils.folder_and_file_list(folder)
    found = False
    if 'Users' in folders:
        folder = os.path.join(folder, 'Users')
        folders, files = utils.folder_and_file_list(folder, add_path = True)
        for folder in folders:
            folder = os.path.join(folder, 'Documents', 'ecoinvent')
            if os.path.exists(folder):
                found = True
                break
            else:
                pass

    if not found:
        raise ValueError('not successful in finding the master data folder')
    else:
        folder = os.path.join(folder, 'EcoEditor', 'xml', 'MasterData', 'Production')

    return folder

def get_current_MD(force_update = False, remove_outdated = False):
    #find the age of the youngest current MD file
    master_data_folder = find_current_MD_path()
    filelist = utils.build_file_list(master_data_folder, extension = 'xml', add_path = True)
    youngest, t_MD = utils.find_youngest(filelist)
    #find the age of the pkl
    system_model = 'Undefined'
    version = 'current_MD'
    folder = utils.version_system_model_path(version, system_model, basepath = '')
    filelist = [os.path.join(folder, 'pkl', 'MD.pkl')]
    youngest, t_pkl = utils.find_youngest(filelist)
    if t_MD > t_pkl or force_update:
        #current MD more recent: time to update!
        build_MD(version, system_model, utils.db_basepath, master_data_folder = master_data_folder)
    folder = os.path.join(folder, 'pkl')
    MD = utils.pkl_load(folder, 'MD')
    if remove_outdated:
        MD = remove_outdated_datasets(MD)
    return MD

def flag_duplicates_and_hidden(MD, system_model):
    tab = 'ActivityIndex'
    grouped = MD[tab].groupby(['systemModelName', 'activityName', 'geography'])
    MD[tab] = []
    for name, group in grouped:
        group = group.copy()
        if len(group) > 1:
            group = group.sort_values(by='endDate', ascending = False)
            latest = group.iloc[0]['endDate']
            group['available'] = group['endDate'].apply(lambda x: x == latest)
        else:
            group['available'] = True
        MD[tab].append(group)
    MD[tab] = pandas.concat(MD[tab])
    MD[tab]['duplicate'] = False
    index = ['systemModelName', 'activityName', 'geography', 'startDate', 'endDate']
    try:
        grouped = MD[tab].groupby(index)
    except:
        1/0
    index = []
    for name, group in grouped:
        if name[0] == 'Undefined' and len(group) > 1:
            index.extend(list(group.index))
    MD[tab].loc[index, 'duplicate'] = True
    return MD

def set_MD_index_for_build_UMD(MD):
    for tab, fields in MD_identity_fields.items():
        if list(MD[tab].index.names) != fields:
            MD[tab] = MD[tab].reset_index().set_index(fields).sort_index(level=0)
    return MD

def remove_outdated_datasets(MD):
    MD['ActivityIndex'] = MD['ActivityIndex'][MD['ActivityIndex']['available']]
    return MD

if __name__ == '__main__':
    MD = get_current_MD(force_update = True)
#    MD = get_current_MD(force_update = False)