from __future__ import absolute_import
from __future__ import unicode_literals
import csv

from datetime import datetime

import argparse
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand, CommandError

from auditcare.utils.export import write_log_events, get_users_to_export
from corehq.apps.domain.models import Domain
from corehq.apps.users.models import WebUser

def valid_date(s):
    try:
        return datetime.strptime(s, "%Y-%m-%d")
    except ValueError:
        msg = "Not a valid date: '{0}'.".format(s)
        raise argparse.ArgumentTypeError(msg)

class Command(BaseCommand):
    help = """Generate request report"""

    def add_arguments(self, parser):
        parser.add_argument('filename', help="Output file path")
        parser.add_argument(
            '-d'
            '--domain',
            dest='domain',
            help="Limit logs to only this domain"
        )
        parser.add_argument(
            '-u',
            '--user',
            dest='user',
            help="Limit logs to only this user"
        )
        parser.add_argument(
            '-s',
            '--startdate',
            dest='start',
            type=valid_date,
            help="The start date - format YYYY-MM-DD",
        )
        parser.add_argument(
            '-e',
            '--enddate',
            dest='end',
            type=valid_date,
            help="The end date - format YYYY-MM-DD",
        )
        parser.add_argument(
            '--display-superuser',
            action='store_true',
            dest='display_superuser',
            default=False,
            help="Include superusers in report, otherwise 'Dimagi User'",
        )

    def handle(self, filename, **options):
        domain = options["domain"]
        user = options["user"]
        display_superuser = options["display_superuser"]

        dimagi_username = ""
        if not display_superuser:
            dimagi_username = "Dimagi Support"

        if not domain and not user:
            raise CommandError("Please provide one of 'domain' or 'user'")

        if domain:
            domain_object = Domain.get_by_name(domain)
            if not domain_object:
                raise CommandError("Domain not found")

        users, super_users = get_users_to_export(user, domain)

        with open(filename, 'wb') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(['Date', 'User', 'Domain', 'IP Address', 'Request Path'])
            for user in users:
                write_log_events(
                    writer, user, domain,
                    start_date=options['start'], end_date=options['end']
                )

            for user in super_users:
                write_log_events(
                    writer, user, domain,
                    override_user=dimagi_username,
                    start_date=options['start'], end_date=options['end']
                )


"""For each loop of estimation in CV, find the corresponding test samples and
pick them out.

"""
# Author: Muchenxuan Tong <mtong@appannie.com>

import sys
import os
import os.path
import pickle
from collections import defaultdict
from optparse import OptionParser
import pandas as pd

def parse_options():
    parser = OptionParser()
    parser.add_option("-c", "--cvdir", dest="cv_dir",
                      help="Required. The directory which contains cross-validation index.")
    (opts, args) = parser.parse_args()

    return (opts, args)

def main():
    (opts, args) = parse_options()
    cv_index_dir = opts.cv_dir
    est_dir = args[0]
    output_dir = args[1]

    cv_by_store = _read_and_group_cv_index_by_store(cv_index_dir)
    # In est_dir, each sub-dir is named with the cv loop number.
    #
    # Each sub-dir contains the regular estimation file, and we need to pick the
    # test samples out for each sub-dir.
    df_by_filename = defaultdict(list)
    for d in os.listdir(est_dir):
        d_full_path = os.path.join(est_dir, d)
        if os.path.isdir(d_full_path):
            print(d_full_path)
            kfold_index = int(d)
            df_by_filename_sub = _filter_and_gather_subdir(d_full_path, cv_by_store, kfold_index)
            # The corresponding est files in each sub-dir should have the same name
            # We use the fact to merge them.
            for (k, v) in df_by_filename_sub.iteritems():
                df_by_filename[k].append(v)

    # Concat the dfs and write into output file
    for (filename, v) in df_by_filename.iteritems():
        df = pd.concat(v)
        df.to_csv(os.path.join(output_dir, filename), index=False)

def _read_and_group_cv_index_by_store(cv_index_dir):
    """

    Arguments:
    - `cv_index_dir`:
    """
    pickle_names = filter(lambda s: s.endswith('pickle'), os.listdir(cv_index_dir))
    groupby_store = {}
    for name in pickle_names:
        store = name.split('_')[0]
        full_path = os.path.join(cv_index_dir, name)
        obj = pickle.load(open(full_path, 'r'))
        groupby_store[store] = obj
    return groupby_store

def _filter_and_gather_subdir(d, cv_by_store, kfold_index):
    est_files = filter(lambda s: s.endswith('.csv'), os.listdir(d))
    # The corresponding est files in each sub-dir should have the same name
    # We use the fact to merge them.
    df_by_filename = {}

    for f in est_files:
        (store, unit) = f.split('_')[:2]
        unit_key = 'downloads' if unit == 'Downloads' else 'sales'
        full_path = os.path.join(d, f)
        test_samples = set(map(int, cv_by_store[store][unit_key][kfold_index]))

        df = pd.read_csv(full_path)
        df = df[df['app_id'].isin(test_samples)]
        df_by_filename[f] = df

    return df_by_filename

if __name__ == '__main__':
    main()


import sqlite3
import os
import datetime
import time
from shutil import copyfile
from . import models
import calendar
from .models import Entities

class DatabaseHelper:

    DATA_DIR = os.path.expanduser('~/.liquidity/')
    DB_FILE = os.path.expanduser('~/.liquidity/liquidity.db')
    DB_FLAG_FILE = os.path.expanduser('~/.liquidity/.liquidity_db_exists')
    DB_SCHEMA_FILE = os.path.expanduser('~/.liquidity/liquidity_schema.sql')
    OFFSET_FILE = os.path.expanduser('~/.liquidity/.offset')

    def __init__(self):
        if not os.path.exists(self.DATA_DIR):
            os.mkdir(self.DATA_DIR)

        assert os.path.exists(self.DATA_DIR)
        self.conn = sqlite3.connect(self.DB_FILE)
        if not os.path.exists(self.DB_FLAG_FILE):
            self.create_db()
            print('\n\nCREATED LIQUIDITY TABLES\n\n')

        # self.open_database()
        self.transactions = self.get_transactions()
        self.load_offset()

    def open_database(self):
        self.conn = sqlite3.connect(self.DB_FILE)

    def create_db(self):
        copyfile('./app/db/schemas/liquidity_schema.sql', self.DATA_DIR + 'liquidity_schema.sql')
        assert os.path.exists(self.DB_SCHEMA_FILE)
        with open(self.DB_SCHEMA_FILE, 'r') as schema_file:
            queries = schema_file.read()
            self.conn.executescript(queries)
            self.conn.close()
            with open(self.DB_FLAG_FILE, 'w') as f:
                pass

    def get_current_month(self):
        return datetime.datetime.now().strftime('%B')

    def get_last_month(self):
        today = datetime.datetime.now()
        first = today.replace(day = 1)
        last_month = (first - datetime.timedelta(days = 1)).strftime('%B')

        return last_month

    def get_entities(self, type, month):
        self.open_database()

        query = '''SELECT
        id,
        e_name,
        e_value,
        e_type
        FROM  entities
        WHERE e_type = {type}
        '''.format(type = type)

        entities = []

        cursor = self.conn.execute(query)
        for row in cursor:
            entity = models.Entities()

            entity.id = int(row[0])
            entity.name = row[1]
            entity.value = float(row[2])
            entity.type = int(row[3])

            if entity.type == Entities.TYPE_EXPENSE or entity.type == Entities.TYPE_INCOME:
                entity.value =self.get_entity_monthly_value(entity, month)

            entities.append(entity)

        self.conn.close()
        return entities

    def get_entity_monthly_value(self, entity, month):
        transactions = self.transactions
        value = 0

        for transaction in transactions:
            t_month = datetime.datetime.fromtimestamp(transaction.timestamp).strftime('%B')
            if t_month == month:
                if entity.type == Entities.TYPE_INCOME:
                    if transaction.from_entity.type == Entities.TYPE_INCOME and entity.id == transaction.from_entity.id:
                        value += transaction.value
                elif entity.type == Entities.TYPE_EXPENSE:
                    if transaction.to_entity.type == Entities.TYPE_EXPENSE  and entity.id == transaction.to_entity.id:
                        value += transaction.value

        return value

    def add_entity(self, entity):
        self.open_database()

        query = '''INSERT INTO entities(
            e_name,
            e_value,
            e_type
        ) VALUES (
            "{name}",
            {value},
            {type}
        )
        '''.format(
            name = entity.name,
            value = entity.value,
            type = entity.type
        )

        self.conn.execute(query)
        self.conn.commit()

        self.conn.close()

    def update_entity(self, entity):
        self.open_database()

        query = '''UPDATE entities SET
            e_name = "{name}",
            e_value = {value},
            e_type = {type}
            WHERE id = {id}'''.format(
            name = entity.name,
            value = entity.value,
            type = entity.type,
            id = entity.id
        )

        self.conn.execute(query)
        self.conn.commit()
        self.conn.close()

    def delete_entity(self, entity):
        self.open_database()

        query = '''DELETE FROM entities WHERE id = {id}'''.format(
            id = entity.id
        )

        self.conn.execute(query)
        self.conn.commit()

        self.conn.close()
        pass

    def get_entity(self, id):
        self.open_database()

        query = '''SELECT
        id,
        e_name,
        e_value,
        e_type
        FROM  entities
        WHERE id = {id}
        '''.format(id = id)

        entities = []

        cursor = self.conn.execute(query)
        for row in cursor:
            entity = models.Entities()

            entity.id = int(row[0])
            entity.name = row[1]
            entity.value = float(row[2])
            entity.type = int(row[3])

            entities.append(entity)

        self.conn.close()
        if len(entities) > 0:
            return entities[0]
        else:
            return None

    def reset_monthly_entities(self):

        print('Backing up data...\n')

        today = datetime.datetime.now()
        first = today.replace(day = 1)
        last_month = (first - datetime.timedelta(days = 1)).strftime('%B')

        copyfile('db/databases/liquidity.db', 'db/databases/{}.db'.format(last_month))

        print('Resetting income and expense values...\n')

        query = '''UPDATE entities
        SET e_value = 0
        WHERE e_type = {} OR e_type = {}'''.format(
            models.Entities.TYPE_EXPENSE,
            models.Entities.TYPE_INCOME
        )

        self.open_database()
        self.conn.execute(query)
        self.conn.commit()
        self.conn.close()

    def refresh_transactions(self):
        self.transactions = self.get_transactions()

    def get_transactions(self):
        self.open_database()

        query = '''SELECT
        id,
        t_from_entity_id,
        t_to_entity_id,
        t_value,
        t_comment,
        t_timestamp
        FROM transactions
        ORDER BY t_timestamp
        '''

        transactions = []

        for row in self.conn.execute(query):
            transaction = models.Transactions()

            transaction.id = int(row[0])
            transaction.from_entity = self.get_entity(int(row[1]))
            transaction.to_entity = self.get_entity(int(row[2]))
            transaction.value = float(row[3])
            transaction.comment = row[4]
            transaction.timestamp = int(row[5])

            transactions.append(transaction)

        self.transactions = transactions
        return transactions

    def add_transaction(self, transaction):

        from_entity = transaction.from_entity
        to_entity = transaction.to_entity

        from_entity.value -= transaction.value
        to_entity.value += transaction.value

        self.update_entity(from_entity)
        self.update_entity(to_entity)

        query = '''INSERT INTO transactions(
            t_from_entity_id,
            t_to_entity_id,
            t_value,
            t_timestamp,
            t_comment
        ) VALUES (
            {from_e_id},
            {to_e_id},
            {value},
            {timestamp},
            "{comment}"
        )
        '''.format(
            from_e_id = transaction.from_entity.id,
            to_e_id = transaction.to_entity.id,
            value = transaction.value,
            timestamp = transaction.timestamp,
            comment = transaction.comment
        )

        try:
            self.open_database()
            self.conn.execute(query)
            self.conn.commit()
        except Exception as e:
            print(e)

        self.conn.close()

    def update_transaction(self, transaction):
        self.open_database()

        query = '''UPDATE transactions SET
            t_from_entity_id = {from_e_id},
            t_to_entity_id = {to_e_id},
            t_value = {value},
            t_timestamp = {timestamp},
            t_comment = "{comment}"
            WHERE id = {id}
        '''.format(
            from_e_id = transaction.from_entity.id,
            to_e_id = transaction.to_entity.id,
            value = transaction.value,
            timestamp = transaction.timestamp,
            comment = transaction.comment,
            id = transaction.id
        )

        self.conn.execute(query)
        self.conn.commit()

        self.conn.close()
        pass

    def get_transaction(self, id):
        t_id = int(id)

        for t in self.transactions:
            if int(t.id) == t_id:
                return t

    def delete_transaction(self, transaction):
        self.open_database()

        query = '''DELETE FROM transactions WHERE id = {id}'''.format(
            id = transaction.id
        )

        self.conn.execute(query)
        self.conn.commit()

        self.conn.close()
        pass

    #HOME FUNCTIONS
    def get_net_worth(self, month):
        assets = self.get_entities(models.Entities.TYPE_ASSET, month)
        debts = self.get_entities(models.Entities.TYPE_DEBT, month)

        assets_total = 0
        debts_total = 0

        for asset in assets:
            assets_total += asset.value

        for debt in debts:
            debts_total += debt.value

        return assets_total - debts_total

    def get_month_income(self, month):
        transactions = self.transactions
        current_month = month

        total_income = 0

        for transaction in transactions:
            t_month = datetime.datetime.fromtimestamp(transaction.timestamp).strftime('%B')
            if current_month == t_month:
                if transaction.from_entity.type == Entities.TYPE_INCOME:
                    total_income += transaction.value

        return total_income

    def get_month_expense(self, month):
        transactions = self.transactions
        current_month = month

        total_expense = 0

        for transaction in transactions:
            t_month = datetime.datetime.fromtimestamp(transaction.timestamp).strftime('%B')
            if current_month == t_month:
                if transaction.to_entity.type == Entities.TYPE_EXPENSE:
                    total_expense += transaction.value

        return total_expense

    def get_month_net(self, month):
        total_income = self.get_month_income(month)
        total_expense = self.get_month_expense(month)

        return total_income - total_expense

    def get_budget_dict(self, amount):
        dictionary = {
            'needs': float(amount) * 0.5,
            'wants': float(amount) * 0.2,
            'save': float(amount) * 0.3
        }

        return dictionary

    def get_spending_capacity(self, amount, month):
        spending_capacity = 0
        liquidity = amount
        expense = self.get_month_expense(month)

        spending_capacity = liquidity - expense
        return spending_capacity

    def get_daily_expense_limit(self, spending_capacity):
        return round(spending_capacity / 31, 2)

    def get_monthly_avg_income(self):
        incomes = []
        for i in range(1, 13):
            month = calendar.month_name[i]
            income = self.get_month_income(month)
            if income != 0:
                incomes.append(income)
        incomes.sort()
        try:
            return incomes[int(len(incomes)/2)] if len(incomes) % 2 != 0 else (incomes[int(len(incomes)/2)] + incomes[int(len(incomes)/2) -1 ])/2
        except IndexError:
            return 0

    def get_monthly_avg_expense(self):
        expenses = []
        for i in range(1, 13):
            month = calendar.month_name[i]
            expense = self.get_month_expense(month)
            if expense != 0:
                expenses.append(expense)
        expenses.sort()
        try:
            return expenses[int(len(expenses)/2)] if len(expenses) % 2 != 0 else (expenses[int(len(expenses)/2)] + expenses[int(len(expenses)/2) -1 ])/2
        except IndexError:
            return 0

    def get_monthly_avg_net(self):
        return self.get_monthly_avg_income() - self.get_monthly_avg_expense()

    def set_offset(self, days = 0):
        with open(self.OFFSET_FILE, 'w') as o_file:
            o_file.write(str(days))
        self.load_offset()

    def load_offset(self):
        try:
            with open(self.OFFSET_FILE, 'r') as o_file:
                self.offset = int(o_file.read())
        except:
            self.set_offset()

    def get_offset(self):
        return self.offset

    def get_daily_average_expense(self):
        expenses = []
        transactions = self.transactions

        for d in range(self.offset, datetime.datetime.now().day + 1):
            expense = 0
            for transaction in transactions:
                if datetime.datetime.fromtimestamp(transaction.timestamp).day == d:
                    if transaction.to_entity.type == Entities.TYPE_EXPENSE:
                        expense += transaction.value

            expenses.append(expense)

        expenses.sort()
        try:
            return expenses[int(len(expenses)/2)] if len(expenses) % 2 != 0 else (expenses[int(len(expenses)/2)] + expenses[int(len(expenses)/2) -1 ])/2
        except IndexError:
            return 0


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
pycopanbehave -- An adaptive network mode of behaviour selection in Python

Copyright (C) 2011--2016 Potsdam Institute for Climate Impact Research
Authors: Jonathan F. Donges <donges@pik-potsdam.de>,
         Carl-Friedrich Schleussner <schleussner@pik-potsdam.de>,
         Denis Engemann <denis.engemann@gmail.com>
URL:     <http://www.pik-potsdam.de/copan/software>

PLOT SCRIPTS
"""

#
#  Imports
#

import os.path as op
import os
import pickle

import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

#
#  Initializations
#

input_file = 'full_trans_smok.pkl'
# input_file='trans_smok_output_full.pkl'
with open(input_file) as w:
    data = pickle.load(w)

if not os.path.isdir('figures'):
    os.mkdir('figures')

to_plot = [1, 2, 3]
set_titles = False
smokers = data['no_of_smokers']
decim = 1
ci = 66
n_smokers = 1000.
start_time = 0.
n_xticks = [0, 200, 400, 600, 790]
ma_window_size = 30
keys = ['full', 'dyn', 'infl', 'mean_field']
mapping = dict(zip(keys, ['coupled', 'network', 'interaction', 'mean-field']))
colors_context = "#55247A", "#3368A5", "#D33C3E", '#37442A'
color_smoker = '#A44791'
color_nonsmoker = '#BECD00'
sns.set(style='ticks', context='poster', font_scale=1.4)
my_gray = '#6e6a6a'

plt.rcParams.update({k: my_gray for k in (
    'axes.labelcolor',
    'axes.edgecolor',
    'text.color',
    'xtick.color',
    'ytick.color'
)})

plt.rcParams.update({k: 2.2 for k in (
    'axes.linewidth',
    'ytick.major.width',
    'xtick.major.width'
)})

#
#  DEFINE PLOTTING FUNCTIONS
#

def moving_average(values, window, axis):
    """
    Add docstring!
    """
    weights = np.repeat(1.0, window) / window
    sma = np.apply_along_axis(
        lambda m: np.convolve(m, weights, mode='valid'), arr=values, axis=axis)
    return sma

def preprocess_array(X):
    """
    Add docstring!
    """
    return moving_average(np.transpose(X, (1, 2, 0)), ma_window_size, 1)

def rescale(x, axis, method='percent'):
    """
    Add docstring!
    """
    if method == 'divide':
        x = x / x[:, 0:2, :]
    elif method == 'percent':
        x = (x - x[:, 0:1, :]) / x[:, 0:1, :]
        # x = (x - x[:, 0:20, :].mean(axis=1)) / x[:, 0:20, :].mean(axis=1)
        x *= 100
    return x

#
#  MAIN PLOTTING SCRIPT
#

#  FIGURE 1
if 1 in to_plot:
    X = [np.asarray(smokers[k])[:1000, start_time:] for k in keys]
    X = preprocess_array(X)
    X /= n_smokers

    times = np.arange(X.shape[1])

    fig = plt.figure()
    for i, (key, color) in enumerate(zip(keys, colors_context)):
        sns.tsplot(X[::decim, :, i:i + 1], color=color,
                   condition=mapping[key])

    plt.xlabel('Time [AU]')
    plt.xticks(n_xticks)
    plt.xlim(0, max(n_xticks))
    plt.ylabel('Proportion of smokers')
    if set_titles:
        plt.title('Evolution of Smoking prevalence')
    sns.despine(offset=10, trim=True)
    plt.gca().set_xticklabels(np.linspace(0, 1, len(n_xticks)))
    plt.subplots_adjust(bottom=0.15)
    plt.show()
    fig.savefig(op.join('figures', 'n_smokers-%i.png' % ci), dpi=300)

####################
#  FIGURE 2
if 2 in to_plot:

    centrality = data['centrality']
    centrality_smoker = np.array([
        np.asarray(centrality['smoker'][k])[:1000, start_time:] for k in keys])
    centrality_nosmoker = np.array([
        np.asarray(centrality['non_smoker'][k])[:1000, start_time:]
        for k in keys])
    centrality_smoker = rescale(preprocess_array(centrality_smoker), axis=1)
    centrality_nosmoker = rescale(preprocess_array(centrality_nosmoker),
                                  axis=1)
    centrality_diff = centrality_smoker - centrality_nosmoker

    fig, axes = plt.subplots(1, 2, figsize=(12, 8), sharex=True, sharey=True)
    ax1, ax2 = axes.ravel()
    ax1.set_xlim(0, max(n_xticks))
    ax2.set_xlim(0, max(n_xticks))
    ax1.set_ylim(-40, 30)
    ax2.set_ylim(-40, 30)
    ax1.text(20, 27, 'a', weight='bold', fontsize=24)
    ax2.text(20, 27, 'b', weight='bold', fontsize=24)

    X = centrality_smoker
    for i, (key, color) in enumerate(zip(keys, colors_context)):
        sns.tsplot(X[::decim, :, i:i + 1],
                   color=color, condition=mapping[key], ax=ax1, ci=[95, ci],
                   legend=False)
    if set_titles:
        fig.suptitle('Evolution of eigenvector centrality', fontsize=32)

    plt.xticks(n_xticks)
    # if set_titles:
    ax1.set_title('smokers')
    ax1.set_xlabel('Time [AU]')
    ax1.set_ylabel('Relative change in centrality [percent]')
    ax1.set_xlim(0, max(n_xticks))

    X = centrality_nosmoker
    for i, (key, color) in enumerate(zip(keys, colors_context)):
        sns.tsplot(X[::decim, :, i:i + 1],
                   color=color, condition=mapping[key], ax=ax2, ci=[95, ci],
                   legend=True)

    plt.xticks(n_xticks)
    ax2.set_xticklabels(np.linspace(0, 1, len(n_xticks)))
    # if set_titles:
    ax2.set_title('non-smokers')
    ax2.set_xlabel('Time [AU]')
    ax2.set_xlim(0, max(n_xticks))

    sns.despine(offset=10, trim=True)
    plt.subplots_adjust(bottom=0.15, top=.85, right=.96, left=0.14, wspace=.27)
    plt.show()
    fig.savefig(op.join('figures', 'centrality_smokers--%i.png' % ci), dpi=300)

####################
#  FIGURE 3

X = np.array([np.asarray(data['conditional_prob'][k])[:1000, start_time:, :]
              for k in keys])
X = np.transpose(X, (-1, 1, 2, 0))
X = moving_average(X, ma_window_size, axis=2)

if 3 in to_plot:
    fig, axes = plt.subplots(2, 2, figsize=(16, 12), sharex=False,
                             sharey=False)
    axes = axes.ravel()
    n_yticks = [0, 200, 400, 700]
    for i, (key, color) in enumerate(zip(keys, colors_context)):
        ax = axes[i]
        ax.set_ylim(-100, 700)
        ax.text(50, 710, 'abcd'[i], weight='bold', fontsize=24)
        for degree, (x, subcolor) in enumerate(
                zip(X,
                    sns.light_palette(color, len(X))[::-1]), 1):
            sns.tsplot(rescale(x[::decim, :, i:i + 1] + 1, axis=1),
                       color=subcolor, condition='degree %i' % degree,
                       ax=ax, ci=ci)
        # plt.xticks(times, times[::1000])
        if i == 0:
            ax.set_ylabel('Relative change in CP [percent]', labelpad=10)
        ax.set_title(mapping[key])
        ax.set_xlim(0, max(n_xticks))
        ax.set_xticks(n_xticks)
        if i in (1, 3):
            ax.set_ylabel('')
            ax.set_yticks(n_yticks, ['', '', '', ''])
        if i in (0, 1):
            ax.set_xlabel('')
            ax.set_yticks(n_yticks, ['', '', '', ''])
        if i in (2, 3):
            ax.set_xlabel('Time [AU]')
        ax.set_xticklabels(np.linspace(0, 1, len(n_xticks)))

    if set_titles:
        fig.suptitle('Conditional probability of smoking', fontsize=32)
    sns.despine(offset=10, trim=True)
    plt.subplots_adjust(left=0.1, bottom=0.15, right=0.97, top=0.86,
                        wspace=.41, hspace=.41)
    fig.show()
    fig.savefig(op.join('figures', 'conditional_prob_all-%i.png' % ci),
                dpi=300)
# fig.savefig(op.join('figures', 'conditional_prob_all.png'))


import unittest

import lore.transformers
import numpy
import pandas
import datetime

class TestAreaCode(unittest.TestCase):
    def setUp(self):
        self.transformer = lore.transformers.AreaCode('phone')

    def test_phone_formats(self):
        values = pandas.DataFrame({
            'phone': [
                '12345678901',
                '+12345678901',
                '1(234)567-8901',
                '1 (234) 567-8901',
                '1.234.567.8901',
                '1-234-567-8901',
                '2345678901',
                '234.567.8901',
                '(234)5678901',
                '(234) 567-8901',
            ]
        })
        result = self.transformer.transform(values)
        self.assertEqual(result.tolist(), numpy.repeat('234', len(values)).tolist())

    def test_bad_data(self):
        values = pandas.DataFrame({
            'phone': [
                '1234567',
                '(123)4567',
                '',
                None,
                12345678901,
            ]
        })
        result = self.transformer.transform(values)
        self.assertEqual(result.tolist(), ['', '', '', None, ''])

class TestEmailDomain(unittest.TestCase):
    def setUp(self):
        self.transformer = lore.transformers.EmailDomain('email')

    def test_transform(self):
        values = pandas.DataFrame({
            'email': [
                'montana@instacart.com',
                'sue-bob+anne@instacart.com'
            ]
        })
        result = self.transformer.transform(values)
        self.assertEqual(result.tolist(), numpy.repeat('instacart.com', len(values)).tolist())

class TestNameFamilial(unittest.TestCase):
    def setUp(self):
        self.transformer = lore.transformers.NameFamilial('name')

    def test_transform(self):
        values = pandas.DataFrame({
            'name': [
                'mom',
                'Dad',
                'sue bob'
            ]
        })
        result = self.transformer.transform(values)
        self.assertEqual(result.tolist(), [True, True, False])

class TestDateTime(unittest.TestCase):
    def test_transform_day_of_week(self):
        transformer = lore.transformers.DateTime('test', 'dayofweek')
        data = pandas.DataFrame({'test': [datetime.datetime(2016, 12, 31), datetime.date(2017, 1, 1)]})
        transformed = transformer.transform(data)
        self.assertEqual(transformed.iloc[0] + 1, transformed.iloc[1])

class TestAge(unittest.TestCase):
    def test_transform_age(self):
        transformer = lore.transformers.Age('test', 'days')
        yesterday = datetime.datetime.now() - datetime.timedelta(days=1)

        data = pandas.DataFrame({'test': [datetime.datetime.now(), yesterday]})
        transformed = transformer.transform(data)
        self.assertEqual(transformed.astype(int).tolist(), [0, 1])

class TestNameAge(unittest.TestCase):
    def test_transform_name(self):
        transformer = lore.transformers.NameAge('test')

        data = pandas.DataFrame({'test': ['bob', 'Bob']})
        transformed = transformer.transform(data)
        self.assertTrue(transformed.iloc[0] > 0)
        self.assertEqual(transformed.iloc[0], transformed.iloc[1])

class TestNameSex(unittest.TestCase):
    def test_transform_name(self):
        transformer = lore.transformers.NameSex('test')

        data = pandas.DataFrame({'test': ['bob', 'Bob']})
        transformed = transformer.transform(data)
        self.assertTrue(transformed.iloc[0] > 0)
        self.assertEqual(transformed.iloc[0], transformed.iloc[1])

class TestNamePopulation(unittest.TestCase):
    def test_transform_name(self):
        transformer = lore.transformers.NamePopulation('test')

        data = pandas.DataFrame({'test': ['bob', 'Bob']})
        transformed = transformer.transform(data)
        self.assertTrue(transformed.iloc[0] > 0)
        self.assertEqual(transformed.iloc[0], transformed.iloc[1])

class TestStringLower(unittest.TestCase):
    def test_transform_name(self):
        transformer = lore.transformers.String('test', 'lower')

        data = pandas.DataFrame({'test': ['bob', 'Bob']})
        transformed = transformer.transform(data)
        self.assertEqual(transformed.iloc[0], 'bob')
        self.assertEqual(transformed.iloc[1], 'bob')


def solution(answers):
    answer = []
    '''
    cnt1 = 0
    cnt2 = 0
    cnt3 = 0
    '''

    scores = {1:0,2:0,3:0}
    ans2 = [2, 1, 2, 3, 2, 4, 2, 5]
    ans3 = [3, 3, 1, 1, 2, 2, 4, 4, 5, 5]
    for ind in range(len(answers)):
        #1번 수포자 정답확인
        if answers[ind]==(ind+1)%5 :
            scores[1] += 1

        #2번 수포자 정답확인
        if ans2[ind%8]==answers[ind] :
            scores[2] += 1

        #3번 수포자 정답확인
        if ans3[ind%10]==answers[ind] :
            scores[3] += 1

    scores = sorted(scores.items(),key=lambda x:x[1],reverse=True)

    for score in scores:
        if score[1] == scores[0][1]:
            answer.append(score[0])
        else:
            break

    return sorted(answer)


import pylab
import numpy
import pyfits
import scipy.ndimage as ndimage

img = "main_pah_feature_6_2_strength.fits"
image = pyfits.getdata(img, 0)[:,42:72]
header = pyfits.getheader(img, 0)
#pa = header['PA_RQST']
#image = ndimage.rotate(image, pa)
fig = pylab.figure()

ax = pylab.subplot(1,2,1)
im = pylab.imshow(image,vmin=10.**(-7.),interpolation="nearest")
ax2 = pylab.subplot(1,2,2)
header = pyfits.getheader(img, 0)
#pa = header['PA_RQST']
img2 = "main_pah_feature_7_7_strength.fits"
image2 = pyfits.getdata(img2, 0)[:,42:72]

#image2 = ndimage.rotate(image, pa)
im2 = pylab.imshow(image2,vmin=10.**(-7.),interpolation="nearest")

ax = pylab.subplot(1,3,1)
im = pylab.imshow(image,vmin=10.**(-7.),interpolation="nearest")
ax2 = pylab.subplot(1,3,2)
im3 = "main_pah_feature_11_3_strength.fits"
image3 = pyfits.getdata(img, 0)[:,42:72]
header = pyfits.getheader(img, 0)
#pa = header['PA_RQST']
#image = ndimage.rotate(image3, pa)
im3 = pylab.imshow(image3,vmin=10.**(-7.),interpolation="nearest")

#pylab.colorbar(im2,ax=ax2)
pylab.xlabel("x-ax")
pylab.ylabel("y-ax")

pylab.savefig("test.png")


import numpy as np
from astropy.io import fits
from scipy import integrate
import matplotlib.pyplot as plt

def blockshaped(arr, nrow, ncol):  # CONFIRMED
    h, w = arr.shape
    return arr.reshape(h // nrow, nrow, -1, ncol).swapaxes(1, 2).reshape(-1, nrow, ncol)

def rebin(data, n):
    rebinned = []
    for z in range(len(data)):
        subarrays = blockshaped(data[z, :, :], n, n)  # bin the data in groups of nxn (4x4) pixels
        # each pixel in the new, rebinned data cube is the mean of each 4x4 set of original pixels
        # reshaped = np.mean(np.mean(subarrays, axis=-1), axis=-1).reshape((int(len(data[0]) / 4.),
        #                                                                   int(len(data[0][0]) / 4.)))
        reshaped = n**2 * np.mean(np.mean(subarrays, axis=-1), axis=-1).reshape((int(len(data[0]) / n),
                                                                                int(len(data[0][0]) / n)))
        rebinned.append(reshaped)
    print('rebinned')
    return np.asarray(rebinned)

def compare(data, model, z_ax, inds_to_try2, v_sys, n):
    data_4 = rebin(data, n)
    ap_4 = rebin(model, n)

    for i in range(len(inds_to_try2)):
        print(inds_to_try2[i][0], inds_to_try2[i][1])
        plt.plot(z_ax, ap_4[:, inds_to_try2[i][1], inds_to_try2[i][0]], 'r+', label=r'Model')  # r-
        plt.plot(z_ax, data_4[:, inds_to_try2[i][1], inds_to_try2[i][0]], 'b+', label=r'Data')  # b:
        plt.axvline(x=v_sys, color='k', label=r'v$_{\text{sys}}$')
        # plt.title(str(inds_to_try2[i][0]) + ', ' + str(inds_to_try2[i][1]))  # ('no x,y offset')
        plt.legend()
        plt.xlabel(r'Frequency [GHz]')
        plt.ylabel(r'Flux Density [Jy/beam]')
        plt.show()
        plt.close()

if __name__ == "__main__":

    base = '/Users/jonathancohn/Documents/dyn_mod/'

    import pickle

    import dyn_model as dm
    params, priors = dm.par_dicts(base + 'param_files/ngc_3258bl_params.txt', q=False)  # get dicts of params and file names from parameter file

    ax_lab = [r'$\log_{10}$(M$_{\odot}$)', 'deg', 'deg', 'pixels', 'pixels', 'km/s', 'km/s', 'km/s', 'pc', 'pc', 'unitless', r'M$_{\odot}$/L$_{\odot}$']
    pars = ['mbh', 'inc', 'PAdisk', 'xloc', 'yloc', 'vsys', 'sig1', 'sig0', 'mu', 'r0', 'f', 'ml_ratio']
    for par in range(len(pars)):
        chains = 'emcee_out/flatchain_' + pars[par] + '_100_1_50.pkl'

        with open(base + chains, 'rb') as pk:
            u = pickle._Unpickler(pk)
            u.encoding = 'latin1'
            chain = u.load()

            if pars[par] == 'mbh':
                plt.hist(np.log10(chain), 100, color="k", histtype="step")  # axes[i]
                percs = np.percentile(np.log10(chain), [16., 50., 84.])
                plt.axvline(np.log10(params[pars[par]]), ls='-', color='k')
            else:
                plt.hist(chain, 100, color="k", histtype="step")  # axes[i]
                percs = np.percentile(chain, [16., 50., 84.])
                print(params[pars[par]])
                plt.axvline(params[pars[par]], ls='-', color='k')

            plt.axvline(percs[1], ls='-', color='b')  # axes[i]
            plt.axvline(percs[0], ls='--', color='b')  # axes[i]
            plt.axvline(percs[2], ls='--', color='b')  #
            plt.tick_params('both', labelsize=16)
            plt.xlabel(ax_lab[par])
            # plt.title("Dimension {0:d}".format(i))
            plt.title(pars[par] + ': ' + str(round(percs[1],4)) + ' (+'
                                     + str(round(percs[2] - percs[1], 2)) + ', -'
                                     + str(round(percs[1] - percs[0], 2)) + ')', fontsize=16)
            plt.show()
    print(oop)

    # hdu = fits.open(base + 'NGC_1332_newfiles/NGC1332_CO21_C3_MS_bri_20kms.pbcor.fits')
    hdu = fits.open(base + 'ngc_3258/ngc3258_CO21_bri.pbcor.fits')
    data_in = hdu[0].data[0]
    z_len = len(data_in)  # store the number of velocity slices in the data cube
    freq1 = float(hdu[0].header['CRVAL3'])
    f_step = float(hdu[0].header['CDELT3'])
    # f_0 = 2.305380000000e11  # 2.29369132e11  # intrinsic frequency of CO(2-1) line  # 2.30538e11
    f_0 = float(hdu[0].header['RESTFRQ'])
    hdu.close()
    freq_axis = np.arange(freq1, freq1 + (z_len * f_step), f_step)  # [bluest, less blue, ..., reddest]
    # freq_axis = freq_axis[:-1]
    # print(freq_axis[0], freq_axis[-1])  # 2.29909, 2.28777 [e11]
    v_sys = 2760.76  # 1562.2  # km/s
    z_ax = np.asarray([((f_0 - freq) / freq) * (3.*10**5) for freq in freq_axis])  # v_opt, km/s
    # print(z_ax)
    # print(oop)

    # hdu = fits.open(base + 'ngc1332_things/NGC_1332_fullsize_idl_n5_beam31_s1.fits')
    # idl_out = hdu[0].data
    # hdu.close()

    # hdu = fits.open(base + 'ngc1332_things/NGC_1332_fullsize_filtconv_n5_beam31_s1.fits')
    # filt_out = hdu[0].data
    # hdu.close()

    # hdu = fits.open(base + 'NGC_1332_fullsize_apconv_n5_beam31fwhm_s1_weightdiv_corrsig.fits')  # _zred
    # hdu = fits.open(base + 'NGC_1332_fullsize_apconv_n5_beam31fwhm_s1.fits')
    # hdu = fits.open(base+ 'NGC_1332_freqcube_summed_apconv_n5_beam31fwhm_s1.fits')
    # hdu = fits.open(base + 'NGC_1332_freqcube_summed_apconv_n5_beam31fwhm_spline_s1.fits')
    # hdu = fits.open(base + 'outputs/NGC_3258_general_7.42_2386000000.0_1.02_166.8_354.89_362.04_-0.46_4.75_2760.76_0.14_3.16_46.0__benlucycorr.fits')
    # hdu = fits.open(base + 'outputs/NGC_3258_general_7.42_3.16_2386000000.0_1.02_166.8_2760.76_-0.46_4.75_0.14_362.04_354.89_46.0__pcscale.fits')
    # hdu = fits.open(base + 'outputs/NGC_3258_general_7.42_2386000000.0_1.02_166.8_354.89_362.04_-0.46_4.75_2760.76_0.14_3.16_46.0__subcube.fits')
    # hdu = fits.open(base + 'outputs/NGC_3258_general_7.42_2386000000.0_1.02_166.8_354.89_362.04_-0.46_4.75_2760.76_0.14_3.16_46.0__subcube_ellmask_bl2.fits')
    hdu = fits.open(base + 'outputs/NGC_3258_general_7.42_2386000000.0_1.02_166.8_354.89_362.04_-0.46_4.75_2760.76_0.14_3.16_46.0__subcube_ellmask_bl2_noell.fits')
    # hdu = fits.open(base + 'outputs/ngc_3258_benlucy_outcube.fits')

    ap_out = hdu[0].data
    hdu.close()

    hdu = fits.open('/Users/jonathancohn/Documents/dyn_mod/outputs/NGC_3258_fitting_ellipse.fits')
    ell_fits = hdu[0].data
    hdu.close()

    plt.imshow(ell_fits, origin='lower')
    plt.colorbar()
    plt.show()

    print(ap_out.shape)
    print(data_in.shape)
    data_in = data_in[39:85, 310:410, 310:410]
    print(data_in.shape)

    data_in *= ell_fits
    ap_out *= ell_fits

    idx = 59  # 26
    idx = 20
    print(np.amin((ap_out[idx] - data_in[idx])/data_in[idx]), np.amax((ap_out[idx] - data_in[idx])/data_in[idx]),
          np.median((ap_out[idx] - data_in[idx])/data_in[idx]))
    plt.imshow((ap_out[idx] - data_in[idx])/data_in[idx], origin='lower')
    # plt.imshow((idl_out[26] - data_in[26])/data_in[26], origin='lower')
    plt.colorbar()
    # plt.xlim(700,750)
    # plt.ylim(720,750)
    # plt.plot(848, 720, 'w*')
    plt.show()
    # print(oop)
    plt.imshow(data_in[idx], origin='lower')
    plt.colorbar()
    # plt.plot(848, 720, 'w*')
    plt.show()

    data_4 = rebin(data_in, 4)
    # idl_4 = rebin(idl_out, 4)
    # filt_4 = rebin(filt_out)
    ap_4 = rebin(ap_out, 4)

    # CREATED ONCE
    # hdu = fits.PrimaryHDU(data_4)
    # hdul = fits.HDUList([hdu])
    # hdul.writeto(base + 'NGC_1332_newfiles/NGC1332_CO21_C3_MS_bri_20kms_bin4x4.pbcor.fits')

    '''
    # UNCOMMENT THIS LATER
    hdu = fits.PrimaryHDU(ap_4)
    hdul = fits.HDUList([hdu])
    hdul.writeto(base + 'outputs/NGC_3258_general_4x4binned.fits')
    # hdul.writeto(base + 'NGC_1332_freqcube_summed_apconv_n5_beam31fwhm_spline_s1_4x4binned.fits')
    # hdul.writeto(base + 'NGC_1332_freqcube_summed_apconv_n5_beam31fwhm_s1_4x4binned.fits')

    hdu = fits.PrimaryHDU(data_4 - ap_4)
    hdul = fits.HDUList([hdu])
    hdul.writeto(base + 'NGC_3258_data_minus_freqcube_4x4binned.fits')
    '''
    # hdul.writeto(base + 'NGC_1332_data_minus_freqcube_spline_4x4binned.fits')
    # print(oops)
    #hdu = fits.PrimaryHDU(data_4 - ap_4)
    #hdul = fits.HDUList([hdu])
    #hdul.writeto('data_minus_apconv_4x4_fixed.fits')
    # print(oop)
    # 212, 180 --> 212*4
    # x=2 --> x_0 = 2, 3
    # x=2, n=4 --> x_0 = 4,5,6,7 = (x-1)*n
    # x=4, n=4 --> x_0 = 12,13,14,15 = (x-1)*n:(x*n) = 12:16 YAY!
    # y
    # 3, 4 (n=2): [(3/2):(3+1)/2] 22 23 32 33
    # 0, 0 (n=2): [(0/2):(0/2)+1] 00, 01, 10, 11

    #plt.imshow(idl_4[23], origin='lower')
    #plt.colorbar()
    #plt.plot(212, 180, 'w*')
    #plt.show()
    print(ap_4.shape, data_4.shape)
    chi_sq_num = 0.
    for z in range(len(data_4)):
        chi_sq_num += np.sum((ap_4[z] - data_4[z]) ** 2)
    print(chi_sq_num)  # ~ -1160 for take2, ~7.2 for pcscale. Good?

    # inds_to_try2 = np.asarray([[212, 180], [159, 159], [165, 155], [155, 165], [160, 160], [170, 170], [155, 155], [165, 165]])
    # inds_to_try2 = np.asarray([[178, 178], [200, 172], [168, 155], [155, 168], [135, 140], [125, 150]])
    # inds_to_try2 = np.asarray([[89, 92], [95, 89], [92, 92], [91, 86]])  # red side, blue side, center top, center bot
    inds_to_try2 = np.asarray([[10,10], [10,15], [15,10]])

    for i in range(len(inds_to_try2)):
        print(inds_to_try2[i][0], inds_to_try2[i][1])

        '''
        data_full = []
        for ii in range((inds_to_try2[i][1])*4,((inds_to_try2[i][1]+1)*4)):
            for jj in range((inds_to_try2[i][0])*4, ((inds_to_try2[i][0]+1)*4)):
                data_full.append(data_in[:, ii, jj])
        data_full = np.asarray(data_full)
        # print(data_full.shape)  # 4x4 x 75 --> 16, 75

        data_stack = np.zeros(shape=(data_full[0].shape))
        for l in range(len(data_full)):
            # plt.plot(z_ax, data_full[l], 'k--')  # confirmed
            data_stack += data_full[l]
        # plt.plot(z_ax, data_stack, 'g--')  # confirmed identical to how Data is calculated
        '''
        plt.plot(z_ax[39:85], ap_4[:, inds_to_try2[i][1], inds_to_try2[i][0]], 'r-', label=r'Astropy conv')  # ap_out  # 0.01217/.00031 *
                 # / np.amax(ap_out[:, inds_to_try2[i][1], inds_to_try2[i][0]]), 'r-', label=r'Astropy conv')
        # plt.plot(z_ax, idl_out[:, inds_to_try2[i][1], inds_to_try2[i][0]], 'k--', label=r'IDL conv')  # idl_out
                 # / np.amax(idl_out[:, inds_to_try2[i][1], inds_to_try2[i][0]]), 'k--', label=r'IDL conv')
        # plt.plot(z_ax, filt_out[:, inds_to_try2[i][1], inds_to_try2[i][0]], 'b:', label=r'Scipy conv')
                 # / np.amax(filt_out[:, inds_to_try2[i][1], inds_to_try2[i][0]]), 'b:', label=r'Scipy conv')
        plt.plot(z_ax[39:85], data_4[:, inds_to_try2[i][1], inds_to_try2[i][0]], 'b:', label=r'Data')  # data_in
        plt.axvline(x=v_sys, color='k')
        plt.title(str(inds_to_try2[i][0]) + ', ' + str(inds_to_try2[i][1]))  # ('no x,y offset')
        plt.legend()
        plt.show()
        # base = '/Users/jonathancohn/Documents/dyn_mod/lps/sum_div_filtconv_'  # div_'
        # plt.savefig(base + 'newfiles_lp_s' + str(s) + '_' + str(inds_to_try2[i][0]) + '_' + str(inds_to_try2[i][1]))
        plt.close()
    print(oop)

    '''  #
    idl_filt = []
    idl_ap = []
    for z in range(len(idl_out)):
        print(z)
        idl_filt.append((idl_out[z] - filt_out[z]))
        idl_ap.append((idl_out[z] - ap_out[z]))
        if 35 < z < 39:
            plt.imshow(idl_filt[z], origin='lower')
            plt.colorbar()
            plt.show()

            plt.imshow(idl_ap[z], origin='lower')
            plt.colorbar()
            plt.show()

    idl_filt = np.asarray(idl_filt)
    idl_ap = np.asarray(idl_ap)

    idl_filt_coll = integrate.simps(idl_filt, axis=0)
    idl_ap_coll = integrate.simps(idl_ap, axis=0)

    plt.imshow(idl_filt_coll, origin='lower')
    plt.colorbar()
    plt.show()

    plt.imshow(idl_ap_coll, origin='lower')
    plt.colorbar()
    plt.show()
    print(oops)
    # '''  #

    obs3d = np.asarray([[1., 2., 3., 4.], [1., 2., 3., 4.], [5., 6., 7., 8.], [9., 10., 11., 12.]])
    s = 2
    print(obs3d.shape)

    '''
    subarrays = blockshaped(obs3d[:, :], 4, 4)  # bin the data in groups of 4x4 pixels
    # data[z, ::-1, :] flips the y-axis, which is stored in python in the reverse order (problem.png)

    # Take the mean along the first axis of each subarray in subarrays, then take the mean along the
    # other (s-length) axis, such that what remains is a 1d array of the means of the sxs subarrays. Then reshape
    # into the correct lengths
    reshaped = np.mean(np.mean(subarrays, axis=-1), axis=-1).reshape((int(len(data[0])/4.),
                                                                      int(len(data[0][0])/4.)))
    reshaped_m = np.mean(np.mean(subarrays_m, axis=-1), axis=-1).reshape((int(len(mask[0])/4.),
                                                                          int(len(mask[0][0])/4.)))
    rebinned.append(reshaped)
    rebinned_m.append(reshaped_m)
    print("Rebinning the cube done in {0} s".format(time.time() - t_rebin))  # 0.5 s
    '''

    intrinsic_cube = []  # np.zeros(shape=(len(z_ax), len(fluxes), len(fluxes[0])))
    # break each (s*len(realx), s*len(realy))-sized velocity slice of obs3d into an array comprised of sxs subarrays
    # i.e. break the 300s x 300s array into blocks of sxs arrays, so that I can take the means of each block
    subarrays = blockshaped(obs3d[:, :], s, s)

    print('sub:')
    print(subarrays)

    # Take the mean along the first (s-length) axis of each subarray in subarrays, then take the mean along the
    # other (s-length) axis, such that what remains is a 1d array of the means of the sxs subarrays. Then reshape
    # into the correct real x_pixel by real y_pixel lengths
    reshaped = s**2 * np.mean(np.mean(subarrays, axis=-1), axis=-1).reshape((2, 2))
    reshaped2 = np.sum(np.sum(subarrays, axis=-1), axis=-1).reshape((2, 2))
    print('reshape:')
    print(reshaped)

    print('reshaped2:')
    print(reshaped2)


import cv2
import os
import numpy as np

matching_results = {}
files = [f for f in os.listdir('.') if os.path.isfile(f)]
for file in files:
	if file.endswith(".ppm"):
		match = False
		image1 = cv2.imread(file)
		img1_gray = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)

		height, width = img1_gray.shape

		for file1 in files:
			if file1.endswith(".ppm") and file != file1:
				image2 = cv2.imread(file1)
				img2_gray = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
				# image_2 = image_resize(img2_gray, height = 800)

				height1, width1 = img2_gray.shape
				if height == height1 and width == width1:
					difference = img1_gray - img2_gray
					result = not np.any(difference)
					if result == True:
						matching_results[str(file1)] = True
					else:
						if str(file1) in matching_results:
							if matching_results[str(file1)] != True:
								matching_results[str(file1)] = False

for key, value in matching_results.items():
	if value == False:
		print('No match found for the image : ',key)
	else:
		print('Match found for the file : ', key)


#!/usr/bin/env python

import rospy
from nav_msgs.msg import Odometry
import tf
import numpy as np
import matplotlib.pyplot as plt
import time

start_time = 0.0
t_ = 0.0
roll_ = 0.0
pitch_ = 0.0

def callback(msg):
    # print('callback')

    global start_time
    global t_
    global roll_
    global pitch_

    e = tf.transformations.euler_from_quaternion((msg.pose.pose.orientation.x, msg.pose.pose.orientation.y, msg.pose.pose.orientation.z, msg.pose.pose.orientation.w))

    if start_time>0:
        t_ = time.time() - start_time
        roll_ = e[0]
        pitch_ = e[1]

def graph():
    print('graph_rpy')
    rospy.init_node('graph_rpy', anonymous=True)
    rospy.Subscriber("/imu_odometry", Odometry, callback)

    t = [0 for i in range(50)]
    roll = [0 for j in range(50)]
    pitch = [0 for k in range(50)]

    plt.ion()
    plt.figure()

    ### roll ###
    plt.subplot(2, 1, 1)
    # plt.title("roll")
    plt.xlabel("time[s]")
    plt.ylabel("roll[deg]")
    plt.ylim(-5, 5)
    plt.grid(True)
    li_r, = plt.plot(t, roll)

    ### pitch ###
    plt.subplot(2, 1, 2)
    # plt.title("pitch")
    plt.xlabel("time[s]")
    plt.ylabel("pitch[deg]")
    plt.ylim(-5, 5)
    plt.grid(True)
    li_p, = plt.plot(t, pitch)

    global start_time
    global t_
    global roll_
    global pitch_

    start_time = time.time()

    while not rospy.is_shutdown():
        # print('loop')

        t.append(t_)
        t.pop(0)
        roll.append(roll_/np.pi*180.0)
        roll.pop(0)
        pitch.append(pitch_/np.pi*180.0)
        pitch.pop(0)

        ### roll ###
        plt.subplot(2,1,1)
        li_r.set_xdata(t)
        li_r.set_ydata(roll)
        plt.xlim(min(t), max(t))
        # plt.draw()

        ### roll ###
        plt.subplot(2,1,2)
        li_p.set_xdata(t)
        li_p.set_ydata(pitch)
        plt.xlim(min(t), max(t))

        plt.draw()
        plt.pause(0.1)
    rospy.spin()

if __name__ == '__main__':
    graph()


from __future__ import print_function
import serial, time, threading
import serial.tools.list_ports
import sys, glob, math
# from ftdiManager import FTDIManager

class Statistics():
    dongleWakeup = 0
    dongleBytesDropped = 0
    radioPackagesSentOk = 0
    radioPackagesSentErr = 0
    radioPackagesReceiveOk = 0
    radioPackagesReceiveErr = 0
    def toString(self):
        statStr = "dongleWakeup ="+str(self.dongleWakeup)+"\n"
        statStr += "dongleBytesDropped ="+str(self.dongleBytesDropped)+"\n"
        statStr += "radioPackagesSentOk ="+str(self.radioPackagesSentOk)+"\n"
        statStr += "radioPackagesSentErr ="+str(self.radioPackagesSentErr)+"\n"
        statStr += "radioPackagesReceiveOk ="+str(self.radioPackagesReceiveOk)+"\n"
        statStr += "radioPackagesReceiveErr ="+str(self.radioPackagesReceiveErr)+"\n"
        return statStr

class Dongle():
    def __init__(self):
        self.ser = None
        self.stat = Statistics
        self.lock = threading.Lock()
        self.autoconnectThread = None
        pass

    def connect(self, portNr):
        self.lock.acquire()
        try:#actual BAUD rate on arduino 368 @8MHz => 8*10^6/(8*(8+1)) = 111111 (instead of 115200)
            if sys.platform.startswith('linux'):
                portList = glob.glob ('/dev/tty[A-Za-z]*')
                self.ser = serial.Serial(portList[portNr], 500000, timeout = 0.005)  # 0.015
            elif sys.platform.startswith('win32'):
                self.ser = serial.Serial(portNr, 500000, timeout = 0.015, xonxoff=False, rtscts=False, dsrdtr=False)
            elif sys.platform.startswith('cygwin'):
                print("Warning: Cygwin environment not yet tested")
                portList = glob.glob ('/dev/tty[A-Za-z]*')
                self.ser = serial.Serial(portList[portNr], 500000, timeout = 0.015)
            elif sys.platform.startswith('darwin'):
                #portsList = glob.glob('/dev/cu.usbserial-A602N31Z')
                portsList =glob.glob('/dev/cu.usb*')
                print("Warning: Mac OS X environment not yet tested")
                self.ser = serial.Serial(portsList[portNr], 500000, timeout = 0.015)
            # FTDIManager.setLatency(portNr, 1) #TODO: check that it is a dongle first ##########
            self.lock.release()
            time.sleep(2)
            return self.ping()
        except Exception: # serial.SerialException eller ValueError
            self.lock.release()
            print("Connection Exception ", sys.exc_info())
            return False

    class DongleConnectThread(threading.Thread):
        def __init__(self, dongle, period = 1):
            threading.Thread.__init__(self)
            self._dongle = dongle
            self._period = period
            self._connected = False

        def reconnect(self):
            self._connected = False

        def isConnected(self):
            return self._connected

        def run(self):
            while True:
                try:
                    if not self._connected:
                        if self._autoconnect():
                            self._connected = True
                    time.sleep(self._period)
                except Exception:
                    print("Exception in autoconnect thread! ", sys.exc_info())

        def _autoconnect(self):
            print("Trying to autoconnect dongle:")
            if self._dongle.ping() == True:
                print(" already connected")
                return True

            self._dongle.close() # close open port if any
            for port, _, _ in sorted(serial.tools.list_ports.comports()):
                print(" ",port, "", end='')
                res = self._dongle.connect(port)
                if res == True:
                    if self._dongle.ping() == True:
                        print("...success")
                        return True
                print("...failure")
                self._dongle.close()
            return False

    def autoconnectStart(self):
        if self.autoconnectThread == None:
            self.autoconnectThread= self.DongleConnectThread(self, period = 1)
            self.autoconnectThread.setName('Dongle Connect Thread')
            self.autoconnectThread.start()
        else:
            self.autoconnectThread.reconnect()

    def isConnected(self):
            return self.autoconnectThread.isConnected()

    def _isACK(self,reply):
        return (len(reply) != 0 and reply[0]==ord('A'))

    def _isSerialReady(self):
        if self.ser == None: return False
        try:
            self.ser.inWaiting()
            self.ser.write([])
            self.ser.read(size=0)
        except Exception:
            print("Dongle not ready! ", sys.exc_info())
            return False
        return True

    def wakeup(self):
        #if self.ser == None: return False
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.stat.dongleWakeup += 1
        for i in range(10):
            self.ser.write(bytes([0xff]))
        self.ser.flush()
        time.sleep(0.1)
        self.ser.read(self.ser.inWaiting())
        self.lock.release()
        return self.ping()

    def checkConnection(self, returnPing=True):
        if not self._isSerialReady():
            self.autoconnectStart()
            return False
        if returnPing:
            return self.ping()
        else:
            return True

    def ping(self):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        cTime = time.clock()
        self.ser.write(bytes([0xff]))
        self.ser.flush()
        reply = self.ser.read(1)
        lag = 1000*(time.clock()-cTime)
        if lag > 5:
            print("Dongle seems slow (%s ms), change latency timer to 1 ms in devicemanager..." % int(lag))
        self.lock.release()
        return self._isACK(reply)

    def setLed(self,val):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xfb]))
        self.ser.write(bytes([val]))
        self.ser.flush()
        reply = self.ser.read(1)
        self.lock.release()
        return self._isACK(reply)

    def setRGBLed(self,r,g,b):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xfa]))
        self.ser.write(bytes([r]))
        self.ser.write(bytes([g]))
        self.ser.write(bytes([b]))
        self.ser.flush()
        reply = self.ser.read(1)
        self.lock.release()
        return self._isACK(reply)

    def scanForModules(self):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xf4]))
        self.ser.flush()
        time.sleep(0.2)
        print("self.ser.inWaiting()=",self.ser.inWaiting())
        reply = self.ser.read(self.ser.inWaiting())
        print(reply)
        for x in reply:
            print(x)
        self.lock.release()

    def setDongleBuzzer(self, tone):
        if not self._isSerialReady(): return False
        tone = int(tone)
        b0 = int(tone & 0xff)
        b1 = int((tone >> 8) & 0xff)
        self.lock.acquire()
        self.ser.write(bytes([0xf9]))
        self.ser.write(bytes([b1]))
        self.ser.write(bytes([b0]))
        self.ser.flush()
        reply = self.ser.read(1)
        res = self._isACK(reply)
        self.lock.release()
        return res

    def getDongleTime(self):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xf8]))
        self.ser.flush()
        reply = self.ser.read(1)
        if (not self._isACK(reply)):
            self.lock.release()
            return -1
        b = self.ser.read(4)
        res = b[0] + (b[1]<<8) + (b[2]<<16) + (b[3]<<24)
        self.lock.release()
        return res

    def stopDongle(self):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xf7]))
        self.ser.flush()
        reply = self.ser.read(1)
        res = self._isACK(reply)
        self.lock.release()
        return res

    def writeRadioPacket(self, data):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        if not self.ser.inWaiting() == 0:
            self.stat.dongleBytesDropped += self.ser.inWaiting()
            print("Help!!!, throw away=", self.ser.read( self.ser.inWaiting() ), " Port latency must be lower (i.e 1ms)")
        buffer = []
        buffer.append(0xfe)
        buffer.append(len(data))
        for b in data:
            buffer.append(b)
        self.ser.write(bytes(buffer))
        self.ser.flush()
        reply = self.ser.read(1)
        success = self._isACK(reply)
        if (success):
            self.stat.radioPackagesSentOk +=1
        else:
            self.stat.radioPackagesSentErr +=1
            if not self.ser.inWaiting() == 0:
                self.stat.dongleBytesDropped += self.ser.inWaiting()
                data = self.ser.read( self.ser.inWaiting() )
                print("Help!!!, throw away=", data, " Port latency must be lower (i.e 1ms)")
                if len(data)!=0 and data[len(data)-1]==ord('A'):
                    success = True
        self.lock.release()
        return success

    def readPacket(self, nBytes, nTimeout=1):
        if not self._isSerialReady():
            print("not ready")
            return
        self.lock.acquire()
        for _ in range(nTimeout):
            packet = self.ser.read(nBytes)
            if len(packet) == nBytes: break
        if len(packet) == nBytes:
            self.stat.radioPackagesReceiveOk +=1
        else:
            # print("timeout")
            print("timeout, packet lenght = ", len(packet), "required length = ", nBytes)
            self.stat.radioPackagesReceiveErr +=1
        self.lock.release()
        return packet

    def close(self):
        if self.ser == None:
            return
        self.lock.acquire()
        self.ser.close()
        self.ser = None
        self.lock.release()

    def getStatistics(self):
        return self.stat


from socket import *
from threading import *

serverName = "127.0.0.1"
serverPort = 12006

clientSocket = socket(AF_INET, SOCK_STREAM)
clientSocket.connect((serverName,serverPort))

while 1 :
	sentence = bytes(input('Input lowercase sentence:'),'utf-8')

	clientSocket.send(sentence)
	if(sentence == b'bye'):
		break
	modifiedSentence = clientSocket.recv(1024)
	print ('From Server:', modifiedSentence)

clientSocket.close()


# 使用Scharr变换找到边缘幅度。相比其他方法有更好的旋转不变性

from skimage.filters import scharr, scharr_h, scharr_v
from skimage import io, img_as_float
from skimage.morphology import disk
from skimage.color import rgb2gray

image = io.imread('/home/qiao/PythonProjects/Scikit-image_On_CT/Test_Img/4.jpg')
# image = rgb2gray(image)
io.imshow(image)
io.show()

md = scharr(image)
io.imshow(md)
io.show()

md = scharr_h(image)
io.imshow(md)
io.show()

md = scharr_v(image)
io.imshow(md)
io.show()


import pandas as pd
from bs4 import BeautifulSoup
import requests
from datetime import date

urlfinancials = 'https://www.marketwatch.com/investing/stock/msft/financials'
urlbalancesheet = 'https://www.marketwatch.com/investing/stock/msft/financials/balance-sheet'

text_soup_financials = BeautifulSoup(
    requests.get(urlfinancials).text, "lxml")  # read in
text_soup_balancesheet = BeautifulSoup(
    requests.get(urlbalancesheet).text, "lxml")  # read in

# build lists for Income statement
titlesfinancials = text_soup_financials.findAll(
    'td', attrs={'class': 'overflow__cell'})
# print(titlesfinancials)
epslist = []
netincomelist = []
longtermdebtlist = []
interestexpenselist = []
ebitdalist = []

for title in titlesfinancials:
    if 'EPS (Basic)' in title.text:
        for td in title.findNextSiblings(attrs={'class': 'overflow__cell'}):
            epslist.append(td.text)
    if 'Net Income' in title.text:
        for td in title.findNextSiblings(attrs={'class': 'overflow__cell'}):
            netincomelist.append(td.text)
    if 'Interest Expense' in title.text:
        for td in title.findNextSiblings(attrs={'class': 'overflow__cell'}):
            interestexpenselist.append(td.text)
    if 'EBITDA' in title.text:
        for td in title.findNextSiblings(attrs={'class': 'overflow__cell'}):
            ebitdalist.append(td.text)

# # find the table headers for the Balance sheet
titlesbalancesheet = text_soup_balancesheet.findAll(
    'td', {'class': 'overflow__cell'})
equitylist = []
for title in titlesbalancesheet:
    if 'Total Shareholders\' Equity' in title.text:
        for td in title.findNextSiblings(attrs={'class': 'overflow__cell'}):
            equitylist.append(td.text)
    if 'Long-Term Debt' in title.text:
        for td in title.findNextSiblings(attrs={'class': 'overflow__cell'}):
            longtermdebtlist.append(td.text)

def get_element(list, element):
    try:
        return list[element]
    except:
        return '-'

# # get the data from the income statement lists
# # use helper function get_element
# eps = get_element(epslist, 4)
# epsGrowth = get_element(epslist, 4)
# netIncome = get_element(netincomelist, 4)
# shareholderEquity = get_element(equitylist, 4)
# roa = get_element(equitylist, 4)

# longtermDebt = get_element(longtermdebtlist, 4)
# interestExpense = get_element(interestexpenselist, 4)
# ebitda = get_element(ebitdalist, 4)

# load all the data into dataframe
fin_df = pd.DataFrame({'eps': epslist[:5], 'net Income': netincomelist[:5], 'shareholder Equity': equitylist[:5],
                       'longterm Debt': longtermdebtlist[:5], 'interest Expense': interestexpenselist[:5], 'ebitda': ebitdalist[:5]},
                      index=range(date.today().year-5, date.today().year))

fin_df.reset_index(inplace=True)
print(fin_df)
# return fin_df

# def get_element(list, element):
#     try:
#         return list[element]
#     except:
#         return '-'

# content = driver.page_source
# soup = BeautifulSoup(content)
# for a in soup.findAll('a',href=True, attrs={'class':'_31qSD5'}):
#     name=a.find('div', attrs={'class':'_3wU53n'})
#     price=a.find('div', attrs={'class':'_1vC4OE _2rQ-NK'})
#     rating=a.find('div', attrs={'class':'hGSR34 _2beYZw'})
#     products.append(name.text)
#     prices.append(price.text)
#     ratings.append(rating.text)
# df = pd.DataFrame({'Product Name':products,'Price':prices,'Rating':ratings})
# df.to_csv('products.csv', index=False, encoding='utf-8')


#!/usr/bin/env python
#encoding:utf-8
import numpy as np
import pandas as pd
import matplotlib as mpl
mpl.use('Agg')
from matplotlib.pyplot import *

import sys

#mpl.rcParams['font.family'] = ['sans-serif']
#mpl.rcParams['font.sans-serif'] = ['SimHei']
myfont = matplotlib.font_manager.FontProperties(fname='/usr/share/fonts/truetype/simhei.ttf')
begin_date = sys.argv[1]
end_date = sys.argv[2]
#begin_date = '2015-06-01'
#end_date = '2015-08-07'
SCALE = 20

figure(figsize=(20,12), dpi=200)
title(begin_date + u' ~ ' + end_date + u'  沪市每日融券余额及增减图', fontproperties=myfont, fontsize = 28)
#df = pd.read_csv('sh_rzrq_result.txt', index_col=0, parse_dates=True)
df = pd.read_csv('../../data/sh_rzrq_result_all.txt', index_col=0)
df = df.sort_index().loc[begin_date:end_date,'rqzj']

df1 = df.copy()
df1[df1.values < 0] = 0
df1[df1.values > SCALE] = SCALE
df1.plot(kind='bar', color='red', label=u'较前一日0-%s亿的增加量' %SCALE)

df2 = df.copy()
df2[df2.values > 0] = 0
df2[df2.values < -SCALE] = -SCALE
df2.plot(kind='bar', color='green', label=u'较前一日0-%s亿的减少量' %SCALE)

legend(loc='best')
ylim(-SCALE * 1.05, SCALE * 1.05)
#yticks([-SCALE, -SCALE*3/4, -SCALE/2, -SCALE/4, 0, SCALE/4, SCALE/2, SCALE*3/4, SCALE],
#       [r'$-20$', r'$-15$', r'$-10$', r'$-5$', r'$0$', r'$5$', r'$10$', r'$15$', r'$20$'], fontsize=18)
#ylabel(u'每日融券增减 (亿)', fontproperties=myfont, fontsize=18)
yticks([-SCALE, -SCALE/2, 0, SCALE/2, SCALE],
       [r'$0$', r'$25$', r'$50$', r'$75$', r'$100$'], fontsize=24)
ylabel(u'每日融券余额 (亿)', fontproperties=myfont, fontsize=24)
axhline(0,linestyle='-',linewidth=1, color='black')
grid(True)
ax1 = gca()
leg = ax1.get_legend()
ltext  = leg.get_texts()
setp(ltext, fontproperties=myfont, fontsize=18)
ax2 = ax1.twinx()

df7 = pd.read_csv('../../data/sh_rzrq_result_all.txt', index_col=0)
df7 = df7.sort_index().loc[begin_date:end_date,'rqye']
df7 = (df7 - 50) / 2.5
df7.plot(color="#EE00EE",linewidth=2.5,linestyle="-")

#xticks([])
x_min, x_max = ax2.get_xlim()
xlim(x_min - 1, x_max + 1)
ylim(-SCALE * 1.05, SCALE * 1.05)
#yticks([-SCALE, -SCALE*3/4, -SCALE/2, -SCALE/4, 0, SCALE/4, SCALE/2, SCALE*3/4, SCALE],
#       [r'$-20$', r'$-15$', r'$-10$', r'$-5$', r'$0$', r'$5$', r'$10$', r'$15$', r'$20$'], fontsize=18)
#yticks([-SCALE, -SCALE/2, 0, SCALE/2, SCALE],
#       [r'$0$', r'$25$', r'$50$', r'$75$', r'$100$'], fontsize=18)
#ylabel(u'每日融券余额 (亿)', fontproperties=myfont, fontsize=18)
yticks([-SCALE, -SCALE*3/4, -SCALE/2, -SCALE/4, 0, SCALE/4, SCALE/2, SCALE*3/4, SCALE],
       [r'$-20$', r'$-15$', r'$-10$', r'$-5$', r'$0$', r'$5$', r'$10$', r'$15$', r'$20$'], fontsize=24)
ylabel(u'每日融券增减 (亿)', fontproperties=myfont, fontsize=24)

savefig(u'../../pic/sh_rq_%s_Fig.jpg' %end_date, dpi=200)


import socket
import sys

class DataChannel():
	TYPE_HANDSHAKE = 0
	TYPE_CMD = 1
	TYPE_INFO = 2
	TYPE_ERR = 3
	TYPE_DISCONNECT = 4
	MAX_PACKET_SIZE = 1024
	TIMEOUT_S = 4
	HANDSHAKE = "1: A robot may not injure a human being or, through inaction, allow a human being to come to harm."
	PORT = 3011
	connected = False
	pairedAddress ='134.117.58.116'
	pairedPort = 3010
	registeredName = "Alarm Controller"
	gpSocket = 0

	def __init__(self):
		self.gpSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
		self.gpSocket.bind(("", self.PORT))
		self.gpSocket.setblocking(1)

	#generic accessors
	def getPairedAddress(self):
		return self.pairedAddress
	def getPairedPort(self):
		return self.pairedPort
	def getConnected(self):
		return self.connected
	def getLocalPort(self):
		return self.gpSocket.getsockname()[1]
	def getLocalAddress(self):
		return self.gpSocket.getsockname()[0]
	def getRegisteredName(self):
		return self.registeredName

	#close this data channel
	def close(self):
		self.gpSocket.close()
		self.connected = False

	#wait to receive a packet with a set timeout, in seconds
	#returns the data as well as source IP+port
	def rcvPacket(self, timeout):
		self.gpSocket.settimeout(timeout)
		data, addr = self.gpSocket.recvfrom(self.MAX_PACKET_SIZE)
		return [data, addr]

	#send data to IP
	def sendPacketByteArr(self, byteArr):
		if(self.connected):
			self.gpSocket.sendto(byteArr, (self.pairedAddress, self.pairedPort))
		else:
			print("Cannot send packet -- DataChannel not paired")

	#unpack a received packet according to our protocol
	def Unpack(self, packet):
		rawPacket = bytearray(packet)
		index = 1

		if (rawPacket[0] == self.TYPE_HANDSHAKE):
			handshakeKey = bytearray()
			deviceNameBytes = bytearray()
			#parse handshake key
			while(rawPacket[index] != 0):
				handshakeKey.append(rawPacket[index])
				index += 1

			#decode the bytes
			handshake = handshakeKey.decode("utf-8")
			index += 1

			#parse device name
			while(rawPacket[index] != 0):
				deviceNameBytes.append(rawPacket[index])
				index += 1

			#decode the bytes
			deviceName = deviceNameBytes.decode("utf-8")

			return [self.TYPE_HANDSHAKE, handshake, deviceName]

		if (rawPacket[0] == self.TYPE_CMD):
			commandKey = bytearray()
			extraInfo = bytearray()

			#parse the command key
			while(rawPacket[index] != 0x00):
				commandKey.append(rawPacket[index])
				index += 1

			#decode the bytes
			cmdKey = commandKey.decode("utf-8")
			index += 1

			#parse extra info
			while(rawPacket[index] != 0x00):
				extraInfo.append(rawPacket[index])
				index += 1

			#decode the bytes
			info = extraInfo.decode("utf-8")

			return [self.TYPE_CMD, cmdKey, info]

		if(rawPacket[0] == self.TYPE_INFO):
			infoBytes = bytearray()

			#parse the info string
			while(index < len(rawPacket)):
				infoBytes.append(rawPacket[index])
				index += 1
			#rawPacket.remove(0)
			#infoBytes.extend(rawPacket)

			#decode the bytes
			infoString = infoBytes.decode("utf-8")

			return [self.TYPE_INFO, infoString, 0x00]

		if(rawPacket[0] == self.TYPE_ERR):
			errBytes = bytearray()

			#parse the error message
			while(index < len(rawPacket)):
				errBytes.append(rawPacket[index])
				index += 1

			#decode the bytes
			errString = errBytes.decode("utf-8")

			return [self.TYPE_ERR, errString, 0x00]

		if(rawPacket[0] == self.TYPE_DISCONNECT):
			reasonBytes = bytearray()

			#parse the reason to disconnect
			while(index < len(rawPacket)):
				reasonBytes.append(rawPacket[index])
				index += 1

			#decode the bytes
			reasonString = reasonBytes.decode("utf-8")

			return [self.TYPE_DISCONNECT, reasonString, 0x00]

	#attempt a connection, returns True if successful, False if not
	def connect(self, sendAddress, sendPort, deviceName):
		sendBytes = bytearray()

		#create the handshake packet according to the defined protocol
		sendBytes.append(self.TYPE_HANDSHAKE)
		sendBytes.extend(self.HANDSHAKE.encode("utf-8"))
		sendBytes.append(0x00)
		sendBytes.extend(deviceName.encode("utf-8"))
		print(sendBytes)
		#send the handshake packet
		self.gpSocket.sendto(sendBytes, (sendAddress, sendPort))

		#wait 10 seconds for a response
		try:
			data, addr = self.rcvPacket(10)
		except socket.timeout:
			return False

		#if((data[0] == self.TYPE_HANDSHAKE) and (data[1] == 0x00)):
		self.pairedPort = addr[1]
		print(addr[1])
		self.pairedAddress = addr[0]
		print(addr[0])
		self.connected = True
		self.registeredName = deviceName
		return True
		#elif(data[0] == self.TYPE_ERR):
		#	print(self.Unpack(data)[1])
		#	return False
		#else:
		#	print("Invalid response to handshake\n")
		#	print(data)
		#	return False

	def disconnect(self, reason):
		#create the disconnect packet
		data = bytearray()
		data.append(self.TYPE_DISCONNECT)
		data.extend(reason.encode("utf-8"))
		#send the disconnect packet
		self.sendPacketByteArr(data)

		self.pairedPort = 0
		self.pairedAddress = 0
		self.connected = False
		self.registeredName = ""

	#This class is meant for modules, so it does not need to respond to a handshake
	#def respondHandshake(self, pairingAddress, pairingPort):
	#	self.connected = True
	#	self.pairedAddress = pairingAddress
	#	self.pairedPort = pairingPort
	#
	#	#create the handshake response
	#	data = bytearray()
	#	data.append(self.TYPE_HANDSHAKE)
	#	data.append(0x00)
	#	self.sendPacketByteArr(data)

	#send the command and extra info (should be formatted as string)
	def sendCmd(self, cmdKey, extraInfo):
		toSend = bytearray()
		toSend.append(self.TYPE_CMD)
		toSend.extend(cmdKey.encode("utf-8"))
		toSend.append(0x00)
		toSend.extend(extraInfo.encode("utf-8"))
		toSend.append(0x00)

		self.sendPacketByteArr(toSend)

	#send an info packet, info is a string
	def sendInfo(self, info):
		toSend = bytearray()
		toSend.append(self.TYPE_INFO)
		toSend.extend(info.encode("utf-8"))

		self.sendPacketByteArr(toSend)

	#send an error packet, errMsg is a string
	def sendErr(self, errMsg):
		toSend = bytearray()
		toSend.append(self.TYPE_ERR)
		toSend.extend(errMsg.encode("utf-8"))

		self.sendPacketByteArr(toSend)

#MAIN
#myChannel = DataChannel()

#connect(self, sendAddress, sendPort, deviceName)
#boolean = myChannel.connect  ("192.168.3.2", 3010, "AlarmClock")
#print(boolean)

#boolean = myChannel.connect("134.117.58.118", 3010, "AlarmClock")
#print(boolean)

#myChannel.sendInfo("THE ONLY THING YOU HAVE TO LOSE IS YOUR CHAINS")
#myChannel.sendCmd("req time", "")
#data, addr = myChannel.rcvPacket(10)
#print(data)
#opcode, info, extra = myChannel.Unpack(data)
#print(info)
#myChannel.sendCmd("RISE UP", "")
#myChannel.sendErr("STALIN DED")
#myChannel.disconnect("COMMUNISM DED")


#!/usr/bin/python
import os
from datetime import datetime
import mimetypes

# virtenv = os.environ['OPENSHIFT_PYTHON_DIR'] + '/virtenv/'
# virtualenv = os.path.join(virtenv, 'bin/activate_this.py')
# try:
#     execfile(virtualenv, dict(__file__=virtualenv))
# except IOError:
#     pass

#
# IMPORTANT: Put any additional includes below this line.  If placed above this
# line, it's possible required libraries won't be in your searchable path
#

def show_404_app(environ, start_response):
    ctype = 'text/html'
    response_headers = [('Content-Type', ctype)]
    start_response('404 Not Found', response_headers)
    response_body = '<title>Not found</title>
Page not found in our servers. Visit the main page.
'
    return response_body

def content_type(path):
    """Return a guess at the mime type for this path
    based on the file extension"""
    mime_type, discard = mimetypes.guess_type(path)
    if mime_type != None:
        return mime_type
    return "application/octet-stream"

def last_modified(path):
    return datetime.utcfromtimestamp(os.path.getmtime(path)).strftime('%a, %d %b %Y %H:%M:%S GMT')

def static_app(environ, start_response):
    """Serve static files"""
    mypath = os.path.dirname(os.path.realpath(__file__))

    path = environ['PATH_INFO']

    for char in ('..', '*', '!', '`', '$', '<', '>'):
        path = path.replace(char, '')

    path = mypath + '/static' + path

    if path.endswith(('py', 'pyc', 'sh')):
        return show_404_app(environ, start_response)
    elif path.endswith('/'):
        path += 'index.html'
    if os.path.exists(path):
        with open(path, 'rb') as f:
            content = f.read()
        ctype = content_type(path)
        headers = [('Content-Type', ctype), \
                   ('Last-Modified', last_modified(path)), \
                   ('Content-Length', str(len(content)))]
        start_response('200 OK', headers)
        # if ctype == 'application/pdf' and os.path.exists(mypath + '/../logs'):
        return [content]
    else:
        return show_404_app(environ, start_response)

# the main WSGI application
def application(environ, start_response):
    pi = environ['PATH_INFO']
    qs = environ['QUERY_STRING']
    return static_app(environ, start_response)

#
# Below for testing only
#
if __name__ == '__main__':
    from wsgiref.simple_server import make_server
    httpd = make_server('localhost', 8051, application)
    # Wait for a single request, serve it and quit.
    # httpd.handle_request()

    # Serve until process is killed
    httpd.serve_forever()


# -*- coding: utf-8 -*-
"""
Created on Tue Feb 20 11:56:30 2018

@author: Raluca Sandu
"""

import numpy as np
import dicom
import os
import matplotlib.pyplot as plt

#%%

def get_pixels_hu(scans):
    image = np.stack([s.pixel_array for s in scans])
    # Convert to int16 (from sometimes int16),
    # should be possible as values should always be low enough (<32k)
    image = image.astype(np.int16)

    # Set outside-of-scan pixels to 1
    # The intercept is usually -1024, so air is approximately 0
    image[image == -2000] = 0

    # Convert to Hounsfield units (HU)
    intercept = scans[0].RescaleIntercept
    slope = scans[0].RescaleSlope

    if slope != 1:
        image = slope * image.astype(np.float64)
        image = image.astype(np.int16)

    image += np.int16(intercept)

    return np.array(image, dtype=np.int16)

def read_dicom_dir(dirname):
    files = os.listdir(dirname)
    slices = [dicom.read_file(os.path.join(dirname, filename)) for filename in files]
    slices.sort(key = lambda x: int(x.InstanceNumber))
    return slices

#%%

rootdir = "\\\\cochlea.artorg.unibe.ch\IGT\Projects\LIVER\_Clinical_Data\Interventions\Bern\\2017\\17-08-25 "

for dirpath, dirnames, filenames in os.walk(rootdir):
    rootpath, studyName = os.path.split(dirpath)
    if studyName == 'IR Data':
        for dirname in dirnames:
            dirStudy = os.path.join(dirpath, dirname)
            print('dirstudy', dirStudy)
            for series in os.listdir(dirStudy):
                seriesPath = os.path.join(dirStudy, series)
                slices = read_dicom_dir(seriesPath)
                break
#                print('series',series)


# MGZF 蘑菇租房
import requests
from lxml import etree
import re
import csv
import pandas as pd
import time
import os

class Get_infor():
    def __init__(self):
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0',' WOW64) AppleWebKit/537.cipg (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"
        }
        self.URL = 'http://www.mgzf.com/'
        self.start_url = self.URL + 'list/pg1/'
        self.folder = './SH/'

        self.title_list=[]#标题列表
        self.address_list=[]#地址列表
        self.shape_list=[]#户型面积列表
        self.money_list=[]#价格列表
        self.href_list=[]#链接列表
        self.detail_list =[]#详细

    def run(self, useLocal=False):
        '''
        useLoal: use local html file
        '''
        isExists = os.path.exists(self.folder)
        if not isExists:  # 目录不存在，则创建
            os.makedirs(self.folder)
        if useLocal == True:
            # use local html file
            try:
                with open(self.folder+'urlList.html', 'r', encoding='utf-8') as f:
                    html = ''
                    for line in f.readlines():
                        html += line
            except:
                pass
        else:
            html = self.getHTMLText(self.start_url)
            with open(self.folder + 'urlList.html','w',encoding='utf-8') as f:
                f.write(html)
        pattern = re.compile('<span data-v-68579cc8>共([\s\S]*?)页</span>')
        allPage = re.findall(pattern, html)
        print("allPage: " + allPage[0])
        for page in range(1, int(allPage[0])+1):
            url = re.sub(r'pg\d', 'pg'+ str(page), self.start_url)
            self.parseUrl(page, url, useLocal)

        #字典中的key值即为csv中列名
        dataframe=pd.DataFrame({
            '标题':self.title_list,   '详细':self.detail_list,
            '地址':self.address_list, '房型':self.shape_list,
            '价格':self.money_list,   '链接':self.href_list})
        # 将DataFrame存储为csv,index表示是否显示行名，default=True
        dataframe.to_csv(self.folder+"./蘑菇网.csv",index=False,sep=',', encoding='utf_8_sig')

    def parseUrl(self, page, url, useLocal):
        print('parse url: ' + url)
        folder = self.folder +'Detail/'
        isExists = os.path.exists(folder)
        if not isExists:  # 目录不存在，则创建
            os.makedirs(folder)
        if useLocal == True:
            # use local html file
            with open(folder + str(page)+'.html', 'r', encoding='utf-8') as f:
                html = ''
                for line in f.readlines():
                    html += line
            html = etree.HTML(html)
        else:
            # save to local
            with open(folder + str(page)+'.html', 'w', encoding='utf-8') as f:
                f.write(self.getHTMLText(url))
            html = self.getHTMLText(url)
            html = etree.HTML(html)

        # use xpath
        ele = html.xpath('//*[@id="__layout"]/div/div/div[1]/div[5]/div[1]/div[2]/div')
        for e in ele:
            la = e.xpath('.//a')
            print(len(la))
            for i in la:
                title = i.xpath('.//div/div[2]/h1/span//text()')
                shape = i.xpath('.//div/div[2]/h2[1]//text()')
                detail = i.xpath('.//div/div[2]/h2[2]//text()')
                address = i.xpath('.//div/div[2]/p//text()')
                money = i.xpath('.//div/div[3]/p[1]/span//text()')
                self.title_list.append(title[0])
                self.shape_list.append(shape[0])
                if len(detail) > 0:
                    self.detail_list.append(detail[0])
                else:
                    self.detail_list.append('')
                a = i.xpath('.//@href')
                self.href_list.append(a[0])
                self.address_list.append(address[0])
                self.money_list.append(money[0])
        if useLocal == False:
            time.sleep(0.5)   # 延时

def test():
    # TODO
    # use Selenium (don't need add Cookie)
    work = Get_infor()
    work.run(useLocal=True)
    print('OK')

if __name__=='__main__':
    start = time.time()
    test()
    end = time.time()
    print("耗时： {}".format(end-start))


from pyserini.search import SimpleSearcher, querybuilder
import json
import sys
from BaselineSearcher import BaselineSearcher
import tensorflow_hub as hub
import numpy as np
import matplotlib.pyplot as plt
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

class USESearcher:

    def __init__(self, index_path):
        self.baseline_searcher = BaselineSearcher(index_path)
        self.name = "USESearcher"
        self.model = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4") # https://tfhub.dev/google/universal-sentence-encoder-large/5

    def get_name(self):
        return self.name

    def search(self, query, max_amount = 10):
        before_rerank_amount = 100
        hits = self.baseline_searcher.search(query)[:before_rerank_amount]

        arg_corpus = []
        for i, hit in enumerate(hits):
            arg = self.get_argument(hit.docid)
            text = arg['conclusion'] + ' ' + arg['premise']
            arg_corpus.append(text)

        embeddings = self.model(arg_corpus)
        query_embedding = self.model([query])[0]

        cos_scores = []
        for i, emb in enumerate(embeddings):
            similarity = np.inner(query_embedding, emb)
            cos_scores.append(similarity)

        for i, hit in enumerate(hits):
            hit.score = cos_scores[i] * hit.score

        hits.sort(key=lambda x: x.score, reverse=True)
        return hits[:max_amount]

    def get_argument(self, id):
        return self.baseline_searcher.get_argument(id)


def pascal_with_memoization(row, column):
    """
    >>> pascal_with_memoization(3, 2)
    3
    >>> pascal_with_memoization(3, 3)
    1
    >>> pascal_with_memoization(5,4)
    5
    """
    cache = {}
    for row_indx in range(0, row + 1):
        cache.setdefault(row_indx, []).insert(0, 1)
    for col_indx in range(1, column + 1):
        cache.setdefault(0, []).insert(col_indx, 0)
    def pascal(row, column):
        if column == 0:
            return cache[row][column]
        elif row  == 0:
            return  cache[row][column]
        else:
            value_1 = 0
            value_2 = 0
            lst = cache[row-1]
            if len(lst) > column:
                value_1 = cache[row-1][column]
                value_2 = cache[row-1][column-1]
            elif len(lst) > column - 1:
                value_1 = pascal(row-1, column)
                value_2 = cache[row-1][column-1]
                cache.setdefault(row-1, []).insert(column, value_1)
            else:
                value_1 = pascal(row-1, column)
                value_2 = pascal(row-1, column-1)
            return value_1 + value_2
    return pascal(row, column)


#encoding: UTF-8
#Adrian E. Tellez Lopez
#Tomar 2 colores primarios y sacar la mezcla

def main():
    color1 = raw_input("Pon el primer color primario")
    color2 = raw_input("Pon el segundo color primario")
    combina = mezclarColor (color1, color2)
    print ("La combinacion de", color1, "y", color2, "es", combina)

def mezclarColor (color1, color2):
    Color1 = color1.upper()
    Color2 = color2.upper()
    if Color1 == "AZUL":
        if Color2 == "AMARILLO":
            mezcla = "verde"
        elif Color2 == "ROJO":
            mezcla = "morado"

    elif Color1 == Color2:
        return color1

    elif Color1 == "ROJO":
        if Color2 == "AMARILLO":
            mezcla = "naranja"
        elif Color2 == "AZUL":
            return "morado"

    elif Color1 == "AMARILLO":
        if Color2 == "ROJO":
            mezcla = "naranja"
        elif Color2 == "AZUL":
            mezcla = "verde"
    else:
        return "Error: pon un color primario"

    return mezcla

main()


#!/usr/bin/python3

import collections
import json

import locale

from lxml import html
from lxml.cssselect import CSSSelector

import requests
import re

locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')

class FrozenDict(collections.Mapping):
    """An immutable dict, see http://stackoverflow.com/a/2704866"""

    def __init__(self, *args, **kwargs):
        self._d = dict(*args, **kwargs)
        self._hash = None

    def __iter__(self):
        return iter(self._d)

    def __len__(self):
        return len(self._d)

    def __getitem__(self, key):
        return self._d[key]

    def __hash__(self):
        # It would have been simpler and maybe more obvious to
        # use hash(tuple(sorted(self._d.iteritems()))) from this discussion
        # so far, but this solution is O(n). I don't know what kind of
        # n we are going to run into, but sometimes it's hard to resist the
        # urge to optimize when it will gain improved algorithmic performance.
        if self._hash is None:
            self._hash = 0
            for pair in self .iteritems():
                self._hash ^= hash(pair)
        return self._hash

def parse_product(element):
    select_name = CSSSelector('[itemprop=name]')
    select_url = CSSSelector('[itemprop=url]')
    select_price = CSSSelector('[itemprop=price]')
    select_desc = CSSSelector('.product_descr p')

    name = select_name(element)
    url = select_url(element)
    price = select_price(element)
    desc = select_desc(element)

    assert len(name) == 1
    assert len(desc) == 1
    assert len(price) == 1
    assert len(url) == 1

    if (len(name) != 1 or len(desc) != 1 or len(price) != 1 or len(url) != 1):
        return None

    return (name[0].text.strip(), {
        'url': url[0].get('href'),
        'desc': re.sub(re.compile(r"\r\s*"), "\n", desc[0].text.strip()),
        'price': locale.atof(price[0].get('content')),
    })

def get_products():
    page = requests.get("https://routerboard.com")
    root = html.fromstring(page.content)

    select_products = CSSSelector(".product_entry:not(.hideBox):not(.hist)")

    products = select_products(root)

    return {name: details for name, details in (
        parse_product(x) for x in products)}

def format_product(name, product):
    return "%s: %s\n  %s\n  %s" % (
        name,
        locale.currency(product['price']),
        product['desc'].replace("\n", "\n  "),
        product['url']
    )

if __name__ == '__main__':
    settings = json.loads(open("settings.json").read())

    db_old = None
    try:
        db_file = open(settings['db_file'], 'r')
        db_old = json.load(db_file)
    except (json.JSONDecodeError, FileNotFoundError):
        pass

    db_new = get_products()
    db_file = open(settings['db_file'], 'w')

    if db_old is not None:
        for name in (db_new.keys() - db_old.keys()):
            print("+ %s\n" % format_product(name, db_new[name]))
        for name in (db_old.keys() - db_new.keys()):
            print("- %s\n" % format_product(name, db_old[name]))

    json.dump(db_new, db_file, sort_keys=True, indent=4)

    db_file.close()


import fcntl
import os.path
import os
from pathlib import Path
from signal import Signals # pylint: disable=no-name-in-module
import subprocess
import sys
from queue import Queue, Empty
from threading import Thread
from tempfile import NamedTemporaryFile

HR = "-" * 40
SR = "*" * 120

def run(arguments, timeout=30, verb="Выполняем", **kwargs):
    print(f"{verb}: {' '.join(arguments)}")
    try:
        result = subprocess.run(arguments, timeout=timeout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
        print(HR)
    except subprocess.TimeoutExpired as e:
        print("ОШИБКА: Время выполнения истекло")
        print(f"Вывод программы:\n{HR}\n{e.stdout}\n{HR}")
        if e.stderr:
            print(f"{e.stderr}\n{HR}")
        raise
    print("\n--- stdout", HR)
    if isinstance(result.stdout, bytes):
        result.stdout = result.stdout.decode()
    sys.stdout.write(result.stdout)
    if isinstance(result.stderr, (bytes, str)):
        print("\n--- stderr", HR)
        if isinstance(result.stderr, str):
            print(f"{result.stderr}\n{HR}")
        else:
            #sys.stdout.buffer.write(result.stderr)
            print(f"\n{HR}")
    else:
        print("\n", HR)
    if result.returncode != 0:
        print(f"Выполнение завершилось с кодом: {result.returncode}")
    return result

class Executable:
    def __init__(self, *args):
        self.args = list(args)

    def run_test(self, name, arguments=[], inputs=[], output=False, input=None):
        return run_test(name, executable=self.args, arguments=arguments, inputs=inputs, output=output, input=input)
    def expect_failure(self, name, arguments=[], inputs=[], output=False, code=None, input=None):
        return expect_failure(name, executable=self.args, arguments=arguments, inputs=inputs, output=output, code=code, input=input)
    def expect_success(self, name, arguments=[], inputs=[], output=False, input=None):
        return expect_success(name, executable=self.args, arguments=arguments, inputs=inputs, output=output, input=input)

def run_test(name, executable, arguments=[], inputs=[], output=False, input=None):
    tempfiles = []
    run_arguments = list(executable)
    run_arguments += [str(arg) for arg in arguments]
    for i in inputs:
        tf = NamedTemporaryFile("wb")
        tf.write(i.encode())
        tf.flush()
        tempfiles.append(tf)
        run_arguments.append(tf.name)

    if output:
        tf = NamedTemporaryFile("r")
        tempfiles.append(tf)
        run_arguments.append(tf.name)

    print_arguments = list(executable)
    print_arguments += [str(arg) for arg in arguments]
    print_arguments += [f"входной-файл-{i}" for i, _ in enumerate(inputs)]
    if output:
        print_arguments.append("выходной-файл")

    print(f"""{HR}
=== Запуск теста: {name}. ===
Командная строка: |{' '.join(print_arguments)}|
""")
    for n, i in enumerate(inputs):
        print(f"""входной-файл-{n}:
{HR}
{i}
{HR}
""")
    if input:
        print(f"""Стандартный ввод (первые 100 символов):
{HR}
{input[:100]}
{HR}
""")
    result = run(run_arguments, input=input)
    if output:
        try:
            tf = tempfiles[-1]
            tf.seek(0)
            result.output_file = tf.read(64*1024*1024)
        except Empty:
            result.output_file = None
        print(f"""В выходной файл записано:
{HR}
{result.output_file}
{HR}
""")
    return result

def prepare(name=None):
    if name and Path(name).exists():
        return Executable(name)
    if Path("./a.out").exists():
        return Executable("./a.out")
    if Path("./main").exists():
        return Executable("./main")

    print("Компиляция завершилась с ошибкой, файл ./a.out не найден")
    return

def expect_failure(name, arguments=[], inputs=[], output=False, code=None, executable=[], input=None):
    result = run_test(name, executable=executable, arguments=arguments, inputs=inputs, output=output, input=input)
    if result.returncode == 0:
        print(f"ОШИБКА {SR}\nПрограмма завершилась успешно (с кодом 0), но должна была вернуть ненулевой код ошибки")
        return False
    elif result.returncode < 0:
        print(f"ОШИБКА {SR}\nПрограмма упала с исключением {Signals(-result.returncode).name}")
        return False
    elif code and result.returncode != code:
        print(f"ОШИБКА {SR}\nПрограмма завершилась с кодом ошибки {result.returncode}, а требовался код {code}")
        return False
    elif result.returncode == 154:
        print(f"ОШИБКА {SR}\nПрограмма завершилась с зарезервированным кодом ошибки {result.returncode}. Ваша программа не должна возвращать этот код.")
        return False
    else:
        print(f"Программа завершилась с ненулевым кодом ошибки {result.returncode}, что и требовалось по условию теста.")
    return result

def expect_success(name, arguments=[], inputs=[], output=False, executable=[], input=None):
    result = run_test(name, executable=executable, arguments=arguments, inputs=inputs, output=output, input=input)
    if result.returncode > 0:
        print(f"ОШИБКА {SR}\nПрограмма завершилась с кодом ошибки {result.returncode} ({result.returncode - 256}), но должна была завершиться с нулевым кодом")
        return False
    if result.returncode < 0:
        print(f"ОШИБКА {SR}\nПрограмма упала с исключением {Signals(-result.returncode).name}")
        return False
    print("Программа завершилась успешно")
    return result


import cv2
import numpy as np
import os
import pandas as pd
from configparser import ConfigParser
from generator import AugmentedImageSequence
from models.keras import ModelFactory
from tensorflow.keras import backend as kb

def get_output_layer(model, layer_name):
    # get the symbolic outputs of each "key" layer (we gave them unique names).
    layer_dict = dict([(layer.name, layer) for layer in model.layers])
    layer = layer_dict[layer_name]
    return layer

def create_cam(df_g, output_dir, image_source_dir, model, generator, class_names, targets=None):
    """
    Create a CAM overlay image for the input image

    :param targets: producs CAMs only for these labels. if left empty, CAMs are generated for all labels
    :param df_g: pandas.DataFrame with file paths
    :param output_dir: str
    :param image_source_dir: str
    :param model: keras model
    :param generator: generator.AugmentedImageSequence
    :param class_names: list of str
    """
    file_name = df_g["Path"]
    print(f"**process image: {file_name}**")

    # draw cam with labels
    img_ori = cv2.imread(filename=os.path.join(image_source_dir, file_name))
    """
    label = df_g["label"]
    if label == "Infiltrate":
        label = "Infiltration"
    """
    # label = "Lung Opacity"
    if targets is None:
        targets = class_names

    img_transformed = generator.load_image(file_name)

    # CAM overlay
    # Get the 1024 input weights to the softmax.
    class_weights = model.layers[-1].get_weights()[0]
    # print(class_weights.shape)
    # print(index)
    # print(class_weights[..., index].shape)
    final_conv_layer = get_output_layer(model, "bn")
    get_output = kb.function([model.layers[0].input], [final_conv_layer.output, model.layers[-1].output])
    [conv_outputs, predictions] = get_output([np.array([img_transformed])])
    conv_outputs = conv_outputs[0, :, :, :]

    for label in targets:
        # print(f"process label: {label}")
        index = class_names.index(label)
        output_file = f"{file_name.split('/')[-1].split('.')[-2]}_{label}.{file_name.split('.')[-1]}"
        output_path = os.path.join(output_dir, output_file)

        # Create the class activation map.
        cam = np.zeros(dtype=np.float32, shape=(conv_outputs.shape[:2]))
        for i, w in enumerate(class_weights[..., index]):
            cam += w * conv_outputs[:, :, i]
        # print(f"predictions: {predictions}")
        cam /= np.max(cam)
        # cam[np.where(cam < 0.2)] = 0
        cam = cv2.resize(cam, img_ori.shape[:2][::-1])  # flip image dimensions, see https://stackoverflow.com/questions/21248245/opencv-image-resize-flips-dimensions

        heatmap = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET)
        heatmap[np.where(cam < 0.2)] = 0
        img = heatmap * 0.5 + img_ori

        # add label & predicted probability

        cv2.putText(img, text=label + f": {str(predictions[...,index])}", org=(5, 20), fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                    fontScale=0.8, color=(0, 0, 255), thickness=1)
        cv2.imwrite(output_path, img)

def main():
    # parser config
    config_file = "./config.ini"
    cp = ConfigParser()
    cp.read(config_file)

    # default config
    output_dir = cp["DEFAULT"].get("output_dir")
    base_model_name = cp["DEFAULT"].get("base_model_name")
    class_names = cp["DEFAULT"].get("class_names").split(",")
    image_source_dir = cp["DEFAULT"].get("image_source_dir")
    image_dimension = cp["TRAIN"].getint("image_dimension")

    # parse weights file path
    output_weights_name = cp["TRAIN"].get("output_weights_name")
    weights_path = os.path.join(output_dir, output_weights_name)
    best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}")

    # CAM config
    # cam_list_file = cp["CAM"].get("cam_list_file")
    cam_folder = cp["CAM"].get("cam_folder")
    use_best_weights = cp["CAM"].getboolean("use_best_weights")

    print("** load model **")
    if use_best_weights:
        print("** use best weights **")
        model_weights_path = best_weights_path
    else:
        print("** use last weights **")
        model_weights_path = weights_path

    model_factory = ModelFactory()
    model = model_factory.get_model(
        class_names,
        model_name=base_model_name,
        use_base_weights=False,
        weights_path=model_weights_path)

    # print(model.summary())

    print("read contents of cam folder")
    cam_files = [f for f in os.listdir(cam_folder) if os.path.isfile(os.path.join(cam_folder, f))]
    df_images = pd.DataFrame(cam_files)
    df_images.columns = ["Path"]

    print("create a generator for loading transformed images")
    cam_sequence = AugmentedImageSequence(
        dataset_csv_file=os.path.join(output_dir, "valid.csv"),  # variable must be passed, but is not used for CAMs
        class_names=class_names,
        source_image_dir=cam_folder,
        batch_size=1,
        target_size=(image_dimension, image_dimension),
        augmenter=None,
        steps=1,
        shuffle_on_epoch_end=False,
    )

    image_output_dir = os.path.join(output_dir, "cam_output")
    if not os.path.isdir(image_output_dir):
        os.makedirs(image_output_dir)

    print("create CAM")
    df_images.apply(
        lambda g: create_cam(
            df_g=g,
            output_dir=image_output_dir,
            image_source_dir=cam_folder,
            model=model,
            generator=cam_sequence,
            class_names=class_names,
            # targets=["Lung Lesion"]
        ),
        axis=1,
    )

if __name__ == "__main__":
    main()