import os
import re
import pickle
import random
import inspect
import argparse
import subprocess
import torch
from torch import optim
from logging import getLogger

from .logger import create_logger
from .loader import AVAILABLE_ATTR

FALSY_STRINGS = {'off', 'false', '0'}
TRUTHY_STRINGS = {'on', 'true', '1'}

MODELS_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'models')

logger = getLogger()

def initialize_exp(params):
    """
    Experiment initialization.
    """
    # dump parameters
    params.dump_path = get_dump_path(params)
    pickle.dump(params, open(os.path.join(params.dump_path, 'params.pkl'), 'wb'))

    # create a logger
    logger = create_logger(os.path.join(params.dump_path, 'train.log'))
    logger.info('============ Initialized logger ============')
    logger.info('\n'.join('%s: %s' % (k, str(v)) for k, v
                          in sorted(dict(vars(params)).items())))
    return logger

def bool_flag(s):
    """
    Parse boolean arguments from the command line.
    """
    if s.lower() in FALSY_STRINGS:
        return False
    elif s.lower() in TRUTHY_STRINGS:
        return True
    else:
        raise argparse.ArgumentTypeError("invalid value for a boolean flag. use 0 or 1")

def attr_flag(s):
    """
    Parse attributes parameters.
    """
    if s == "*":
        return s
    attr = s.split(',')
    assert len(attr) == len(set(attr))
    attributes = []
    for x in attr:
        if '.' not in x:
            attributes.append((x, 2))
        else:
            split = x.split('.')
            assert len(split) == 2 and len(split[0]) > 0
            assert split[1].isdigit() and int(split[1]) >= 2
            attributes.append((split[0], int(split[1])))
    return sorted(attributes, key=lambda x: (x[1], x[0]))

def check_attr(params):
    """
    Check attributes validy.
    """
    if params.attr == '*':
        params.attr = attr_flag(','.join(AVAILABLE_ATTR))
    else:
        assert all(name in AVAILABLE_ATTR and n_cat >= 2 for name, n_cat in params.attr)
    params.n_attr = sum([n_cat for _, n_cat in params.attr])

def get_optimizer(model, s):
    """
    Parse optimizer parameters.
    Input should be of the form:
        - "sgd,lr=0.01"
        - "adagrad,lr=0.1,lr_decay=0.05"
    """
    if "," in s:
        method = s[:s.find(',')]
        optim_params = {}
        for x in s[s.find(',') + 1:].split(','):
            split = x.split('=')
            assert len(split) == 2
            assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None
            optim_params[split[0]] = float(split[1])
    else:
        method = s
        optim_params = {}

    if method == 'adadelta':
        optim_fn = optim.Adadelta
    elif method == 'adagrad':
        optim_fn = optim.Adagrad
    elif method == 'adam':
        optim_fn = optim.Adam
        optim_params['betas'] = (optim_params.get('beta1', 0.5), optim_params.get('beta2', 0.999))
        optim_params.pop('beta1', None)
        optim_params.pop('beta2', None)
    elif method == 'adamax':
        optim_fn = optim.Adamax
    elif method == 'asgd':
        optim_fn = optim.ASGD
    elif method == 'rmsprop':
        optim_fn = optim.RMSprop
    elif method == 'rprop':
        optim_fn = optim.Rprop
    elif method == 'sgd':
        optim_fn = optim.SGD
        assert 'lr' in optim_params
    else:
        raise Exception('Unknown optimization method: "%s"' % method)

    # check that we give good parameters to the optimizer
    expected_args = inspect.getargspec(optim_fn.__init__)[0]
    assert expected_args[:2] == ['self', 'params']
    if not all(k in expected_args[2:] for k in optim_params.keys()):
        raise Exception('Unexpected parameters: expected "%s", got "%s"' % (
            str(expected_args[2:]), str(optim_params.keys())))

    return optim_fn(model.parameters(), **optim_params)

def clip_grad_norm(parameters, max_norm, norm_type=2):
    """Clips gradient norm of an iterable of parameters.
    The norm is computed over all gradients together, as if they were
    concatenated into a single vector. Gradients are modified in-place.
    Arguments:
        parameters (Iterable[Variable]): an iterable of Variables that will have
            gradients normalized
        max_norm (float or int): max norm of the gradients
        norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for infinity norm.
    """
    parameters = list(parameters)
    max_norm = float(max_norm)
    norm_type = float(norm_type)
    if norm_type == float('inf'):
        total_norm = max(p.grad.data.abs().max() for p in parameters)
    else:
        total_norm = 0
        for p in parameters:
            param_norm = p.grad.data.norm(norm_type)
            total_norm += param_norm ** norm_type
        total_norm = total_norm ** (1. / norm_type)
    clip_coef = max_norm / (total_norm + 1e-6)
    if clip_coef >= 1:
        return
    for p in parameters:
        p.grad.data.mul_(clip_coef)

def get_dump_path(params):
    """
    Create a directory to store the experiment.
    """
    assert os.path.isdir(MODELS_PATH)

    # create the sweep path if it does not exist
    sweep_path = os.path.join(MODELS_PATH, params.name)
    if not os.path.exists(sweep_path):
        subprocess.Popen("mkdir %s" % sweep_path, shell=True).wait()

    # create a random name for the experiment
    chars = 'abcdefghijklmnopqrstuvwxyz0123456789'
    while True:
        exp_id = ''.join(random.choice(chars) for _ in range(10))
        dump_path = os.path.join(MODELS_PATH, params.name, exp_id)
        if not os.path.isdir(dump_path):
            break

    # create the dump folder
    if not os.path.isdir(dump_path):
        subprocess.Popen("mkdir %s" % dump_path, shell=True).wait()
    return dump_path

def reload_model(model, to_reload, attributes=None):
    """
    Reload a previously trained model.
    """
    # reload the model
    assert os.path.isfile(to_reload)
    to_reload = torch.load(to_reload)

    # check parameters sizes
    model_params = set(model.state_dict().keys())
    to_reload_params = set(to_reload.state_dict().keys())
    assert model_params == to_reload_params, (model_params - to_reload_params,
                                              to_reload_params - model_params)

    # check attributes
    attributes = [] if attributes is None else attributes
    for k in attributes:
        if getattr(model, k, None) is None:
            raise Exception('Attribute "%s" not found in the current model' % k)
        if getattr(to_reload, k, None) is None:
            raise Exception('Attribute "%s" not found in the model to reload' % k)
        if getattr(model, k) != getattr(to_reload, k):
            raise Exception('Attribute "%s" differs between the current model (%s) '
                            'and the one to reload (%s)'
                            % (k, str(getattr(model, k)), str(getattr(to_reload, k))))

    # copy saved parameters
    for k in model.state_dict().keys():
        if model.state_dict()[k].size() != to_reload.state_dict()[k].size():
            raise Exception("Expected tensor {} of size {}, but got {}".format(
                k, model.state_dict()[k].size(),
                to_reload.state_dict()[k].size()
            ))
        model.state_dict()[k].copy_(to_reload.state_dict()[k])

def print_accuracies(values):
    """
    Pretty plot of accuracies.
    """
    assert all(len(x) == 2 for x in values)
    for name, value in values:
        logger.info('{:<20}: {:>6}'.format(name, '%.3f%%' % (100 * value)))
    logger.info('')

def get_lambda(l, params):
    """
    Compute discriminators' lambdas.
    """
    s = params.lambda_schedule
    if s == 0:
        return l
    else:
        return l * float(min(params.n_total_iter, s)) / s


# -*- coding: utf-8 -*-
from plone import api
from plone.app.event.interfaces import IEventSettings
from plone.registry.interfaces import IRegistry
from zope.component import getUtility
import datetime
import os

from imio.helpers.content import create, richtextval, add_image

data_path = os.path.join(os.path.dirname(__file__), "data")

def get_path(filename):
    return os.path.join(data_path, filename)

def post_install(context):
    """Post install script."""
    if context.readDataFile("cpskindemo_default.txt") is None:
        return

    portal = context.getSite()
    # frontpage = api.content.get('/front-page')
    # frontpage.processForm()
    # Edit frontpage to have explain text on how use demo site
    add_events(portal)
    add_news(portal)
    add_folders(portal)
    add_album(portal)
    add_users(portal)
    add_document(portal)
    # add_directory(portal)

def add_events(portal):
    """Add some demo events."""
    timezone = "Europe/Brussels"
    reg = getUtility(IRegistry)
    settings = reg.forInterface(IEventSettings, prefix="plone.app.event")
    if not settings.portal_timezone:
        settings.portal_timezone = timezone
    now = datetime.datetime.now()
    tomorrow = datetime.datetime.today() + datetime.timedelta(days=1)
    events = [
        {
            "cont": "/evenements",
            "type": "Event",
            "title": "Atelier photo",
            "attrs": {
                "description": "Participer à un atelier photo",
                "start": datetime.datetime(now.year, now.month, now.day, 18),
                "end": datetime.datetime(now.year, now.month, now.day, 21),
                "timezone": timezone,
                "hiddenTags": set([u"a-la-une",]),
            },
            "functions": [(add_image, [], {"filepath": get_path("atelierphoto.jpg")})],
            "trans": ["publish_and_hide"],
        },
        {
            "cont": "/evenements",
            "type": "Event",
            "title": "Concert",
            "attrs": {
                "description": "Participer à notre concert caritatif",
                "start": datetime.datetime(
                    tomorrow.year, tomorrow.month, tomorrow.day, 21
                ),
                "end": datetime.datetime(
                    tomorrow.year, tomorrow.month, tomorrow.day, 23
                ),
                "timezone": timezone,
            },
            "functions": [(add_image, [], {"filepath": get_path("concert.jpg")})],
            "trans": ["publish_and_hide"],
        },
        {
            "cont": "/evenements",
            "type": "Event",
            "title": "Marché aux fleurs",
            "attrs": {
                "description": "Vener découvrir notre marché aux fleurs",
                "start": tomorrow,
                "end": tomorrow + datetime.timedelta(weeks=1),
                "timezone": timezone,
            },
            "functions": [
                (add_image, [], {"filepath": get_path("marcheauxfleurs.jpg")})
            ],
            "trans": ["publish_and_hide"],
        },
        {
            "cont": "/evenements",
            "type": "Event",
            "title": "Carnaval",
            "attrs": {
                "description": "Venez fêter le carnaval",
                "start": datetime.datetime(
                    tomorrow.year, tomorrow.month, tomorrow.day, 15
                ),
                "end": datetime.datetime(
                    tomorrow.year, tomorrow.month, tomorrow.day, 18
                ),
                "timezone": timezone,
            },
            "functions": [(add_image, [], {"filepath": get_path("sorciere.jpg")})],
            "trans": ["publish_and_hide"],
        },
    ]
    create(events)

def add_news(portal):
    news = [
        {
            "cont": "/actualites",
            "type": "News Item",
            "title": "Nouvelle brasserie",
            "attrs": {
                "description": "Une nouvelle brasserie va ouvrir ses portes près de chez vous",
                "text": richtextval(
                    "Bonjour,

Une nouvelle brasserie va ouvrir ses portes près de "
                    "chez vous"
                ),
                "hiddenTags": set([u"a-la-une",]),
            },
            "functions": [(add_image, [], {"filepath": get_path("brasserie.jpg")})],
            "trans": ["publish_and_hide"],
        },
        {
            "cont": "/actualites",
            "type": "News Item",
            "title": "Météo",
            "attrs": {
                "description": "Attention à la météo de ces prochains jours",
                "text": richtextval(
                    "Bonjour,

Faites attention à la météo de ces prochains jours"
                ),
            },
            "functions": [(add_image, [], {"filepath": get_path("meteo.jpg")})],
            "trans": ["publish_and_hide"],
        },
    ]
    create(news)

def add_document(portal):
    document = [
        {
            "cont": "/",
            "type": "Document",
            "title": "Mot clés",
            "attrs": {
                "iamTags": set([u"jeune", u"entrepreneur", u"nouvel habitant"]),
                "isearchTags": set([u"démarches administrative"]),
            },
        },
        {
            "cont": 110,
            "type": "Document",
            "title": "Le collège communal",
            "attrs": {"description": "Présentation collège communal",},
            "trans": ["publish_and_show"],
        },
    ]
    create(document, globl=True)

def add_folders(portal):

    folders = [
        {
            "cid": 100,
            "cont": "/ma-commune",
            "type": "Folder",
            "title": u"Vie politique",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 110,
            "cont": 100,
            "type": "Folder",
            "title": u"Collège communal",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 111,
            "cont": 100,
            "type": "Folder",
            "title": u"Conseil communal",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 200,
            "cont": "/ma-commune",
            "type": "Folder",
            "title": u"Services communaux",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 210,
            "cont": 200,
            "type": "Folder",
            "title": u"Population-Etat civil",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 220,
            "cont": 200,
            "type": "Folder",
            "title": u"Informatique",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 230,
            "cont": 200,
            "type": "Folder",
            "title": u"Heures d'ouverture",
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 240,
            "cont": 200,
            "type": "Folder",
            "title": u"Autres services",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 250,
            "cont": 240,
            "type": "Folder",
            "title": u"CPAS",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 260,
            "cont": 250,
            "type": "Folder",
            "title": u"Album photos",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 300,
            "cont": "/loisirs",
            "type": "Folder",
            "title": u"Sports",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 310,
            "cont": 300,
            "type": "Folder",
            "title": u"Piscine communale",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 320,
            "cont": 300,
            "type": "Folder",
            "title": u"Annuaire des clubs sportifs",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 400,
            "cont": "/loisirs",
            "type": "Folder",
            "title": u"Folklores",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 410,
            "cont": 400,
            "type": "Folder",
            "title": u"Carnaval",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 415,
            "cont": 400,
            "type": "Folder",
            "title": u"Images folklores",
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 420,
            "cont": 400,
            "type": "Folder",
            "title": u"Marché de Noël",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 500,
            "cont": "/loisirs",
            "type": "Folder",
            "title": u"Tourisme",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 510,
            "cont": 500,
            "type": "Folder",
            "title": u"Barrage",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 600,
            "cont": "/economie",
            "type": "Folder",
            "title": u"L'entreprenariat",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 610,
            "cont": 600,
            "type": "Folder",
            "title": u"CSAM",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 620,
            "cont": 600,
            "type": "Folder",
            "title": u"EId",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 700,
            "cont": "/economie",
            "type": "Folder",
            "title": u"Zonings",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 710,
            "cont": 700,
            "type": "Folder",
            "title": u"Industriels",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 720,
            "cont": 700,
            "type": "Folder",
            "title": u"Port",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 800,
            "cont": "/je-suis",
            "type": "Folder",
            "title": u"Jeune",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 810,
            "cont": "/je-suis",
            "type": "Folder",
            "title": u"Entrepreneur",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 820,
            "cont": "/je-suis",
            "type": "Folder",
            "title": u"Nouvel habitant",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 900,
            "cont": "/je-trouve",
            "type": "Folder",
            "title": u"Démarches administratives",
            "trans": ["publish_and_show"],
        },
        {
            "cid": 910,
            "cont": "/je-trouve",
            "type": "Folder",
            "title": u"Taxes",
            "trans": ["publish_and_show"],
        },
    ]

    create(folders, globl=True)

def add_leadimage_from_file(obj, file_name):
    data_path = os.path.join(os.path.dirname(__file__), "data")
    file_path = os.path.join(data_path, file_name)
    if not obj.hasObject(file_name):
        from plone.namedfile.file import NamedBlobImage

        namedblobimage = NamedBlobImage(
            data=open(file_path, "r").read(), filename=unicode(file_name)
        )
        image = api.content.create(
            type="Image",
            title=file_name,
            image=namedblobimage,
            container=api.portal.get(),
            language="fr",
        )
        image.setTitle(file_name)
        image.reindexObject()
        setattr(obj, "image", namedblobimage)

def add_news_image_from_file(obj, file_name):
    if not obj.hasObject(file_name):
        # with deterity image
        add_leadimage_from_file(obj, file_name)

def add_image_from_file(container, file_name):
    data_path = os.path.join(os.path.dirname(__file__), "data")
    filePath = os.path.join(data_path, file_name)
    if not container.hasObject(file_name):
        # with deterity image
        from plone.namedfile.file import NamedBlobImage

        namedblobimage = NamedBlobImage(
            data=open(filePath, "r").read(), filename=unicode(file_name)
        )
        image = api.content.create(
            type="Image",
            title=file_name,
            image=namedblobimage,
            container=container,
            language="fr",
        )
        image.setTitle(file_name)
        image.reindexObject()

def add_album(portal):
    objects = [
        {
            "cid": 10,
            "cont": "/",
            "type": "Folder",
            "title": "Album",
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 15,
            "cont": 10,
            "type": "Image",
            "title": "Moto",
            "functions": [(add_image, [], {"filepath": get_path("moto.jpg")})],
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 20,
            "cont": 10,
            "type": "Image",
            "title": "Météo",
            "functions": [(add_image, [], {"filepath": get_path("meteo.jpg")})],
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 25,
            "cont": 110,
            "type": "Image",
            "title": "Collège",
            "functions": [(add_image, [], {"filepath": get_path("college.png")})],
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 30,
            "cont": 415,
            "type": "Image",
            "title": "fille en rouge",
            "functions": [(add_image, [], {"filepath": get_path("fille-rouge.jpg")})],
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 31,
            "cont": 415,
            "type": "Image",
            "title": "carnaval plume",
            "functions": [
                (add_image, [], {"filepath": get_path("carnaval-plume.jpg")})
            ],
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 32,
            "cont": 415,
            "type": "Image",
            "title": "masque carnanval",
            "functions": [
                (add_image, [], {"filepath": get_path("masque-carnaval.jpg")})
            ],
            "trans": ["publish_and_hide"],
        },
        {
            "cid": 33,
            "cont": 415,
            "type": "Image",
            "title": "Sorcière",
            "functions": [(add_image, [], {"filepath": get_path("sorciere.jpg")})],
            "trans": ["publish_and_hide"],
        },
    ]
    cids = create(objects, globl=True)
    cids[10].setLayout("galleryview")

def add_users(portal):
    pass

def add_directory(portal):
    position_types = [{"name": u"Directeur", "token": u"directeur"}]
    organization_types = [
        {"name": u"Communal", "token": u"communal"},
        {"name": u"Privé", "token": u"prive"},
    ]
    organization_levels = [
        {"name": u"Intérieur", "token": u"interieur"},
        {"name": u"Exttérieur", "token": u"exterieur"},
    ]

    params = {
        "position_types": position_types,
        "organization_types": organization_types,
        "organization_levels": organization_levels,
    }
    sportif = api.content.create(
        container=portal,
        type="directory",
        id="annuaire-club-sportifs",
        title=u"Annuaire des clubs sportifs",
        **params
    )

    params = {
        "title": u"Imio",
        "organization_type": u"communal",
    }
    organization1 = api.content.create(
        container=sportif, type="organization", id="imio", **params
    )

    params = {
        "position_types": position_types,
        "organization_types": organization_types,
        "organization_levels": organization_levels,
    }
    fonctionnaires = api.content.create(
        container=portal,
        type="directory",
        id="annuaire-des-fonctionnaires",
        title="Annuaire des fonctionnaires",
        **params
    )

    person = api.content.create(container=fonctionnaires, type="person", id="foobar")
    person.firstname = u"Foo"
    person.lastname = u"Bar"
    person.gender = u"F"
    person.street = u"Zoning Industriel"
    person.number = u"34"
    person.zip_code = u"5190"
    person.city = u"Mornimont"

    params = {
        "title": u"Anciens fonctionnaires",
        "organization_type": u"prive",
    }
    organization2 = api.content.create(
        container=fonctionnaires, type="organization", id="organization2", **params
    )


import collections

class TreeNode(object):
    def __init__(self, data=0, left=None, right=None):
        self.data = data
        self.left, self.right = left, right

    def __repr__(self):
        return str(self.data) + " " + str(self.left) + " " + str(self.right)

gpos = 0

def delete_nodes_to_make_forests(root, delete_nodes):
    def update_forests(nd, pos):
        if nd and nd.data in delete_nodes and pos not in forest:
            return
        forest[pos].append(nd.data if nd and nd.data not in delete_nodes else None)

    def build_forests(node, pos):
        global gpos
        update_forests(node, pos)
        if not node:
            return
        for nei in (node.left, node.right):
            if node.data in delete_nodes:
                if nei:
                    gpos += 1
                    build_forests(nei, gpos)
            else:
                build_forests(nei, pos)

    forest = collections.defaultdict(list)
    build_forests(root, gpos)

    print (forest.keys())
    for key in forest:
        print (forest[key])

if __name__ == "__main__":
    tree = TreeNode(1, TreeNode(2, TreeNode(4), TreeNode(5, TreeNode(8, TreeNode(9), TreeNode(10)), TreeNode(12))), TreeNode(3, TreeNode(6), TreeNode(7)))
    print (tree)
    delete_nodes_to_make_forests(tree, [1, 5])
    #delete_nodes_to_make_forests(tree, [12])


from typing import *
import numpy as np
import pandas as pd

import time
from collections import namedtuple
import threading
import math
import warnings

from .spectral_clustering import spectral_clustering
from .flow_clustering import flow_clustering
from .GraphLocal import GraphLocal
from .triangleclusters import triangleclusters

def ncp_experiment(ncpdata,R,func,method_stats):
    if ncpdata.input_stats:
        input_stats = ncpdata.graph.set_scores(R)
        for F in ncpdata.set_funcs: # build the list of keys for set_funcs
            input_stats.update(F(ncpdata.graph, R))
        input_stats = {"input_" + str(key):value for key,value in input_stats.items() } # add input prefix
    else:
        input_stats = {}

    start = time.time()
    S = func(ncpdata.graph, R)
    dt = time.time() - start

    if len(S) > 0:
        output_stats = ncpdata.graph.set_scores(S)
        for F in ncpdata.set_funcs: # build the list of keys for set_funcs
            output_stats.update(F(ncpdata.graph, S))
        output_stats = {"output_" + str(key):value for key,value in output_stats.items() } # add output prefix

        method_stats['methodfunc']  = func
        method_stats['time'] = dt
        return [ncpdata.record(**input_stats, **output_stats, **method_stats)]
    else:
        return [] # nothing to return

def ncp_node_worker(ncpdata,sets,func,timeout_ncp,methodname):
    start = time.time()
    setno = 0
    for R in sets:
        #print("setno = %i"%(setno))
        setno += 1

        method_stats = {'input_set_type': 'node', 'input_set_params':R[0]}
        ncpdata.results.extend(ncp_experiment(ncpdata, R, func, method_stats))

        end = time.time()
        if end - start > timeout_ncp:
            break

# todo avoid code duplication
def ncp_neighborhood_worker(ncpdata,sets,func,timeout_ncp,methodname):
    start = time.time()
    setno = 0
    for R in sets:
        #print("setno = %i"%(setno))
        setno += 1

        R = R.copy() # duplicate so we don't keep extra weird data around
        node = R[0]
        R.extend(ncpdata.graph.neighbors(R[0]))
        method_stats = {'input_set_type': 'neighborhood', 'input_set_params':node}

        ncpdata.results.extend(ncp_experiment(ncpdata, R, func, method_stats))

        end = time.time()
        if end - start > timeout_ncp:
            break

# todo avoid code duplication
def ncp_set_worker(ncpdata,setnos,func,timeout_ncp,methodname):
    start = time.time()
    setno = 0
    for id in setnos:
        #print("setno = %i"%(setno))
        setno += 1
        R = ncpdata.sets[id]
        #R = R.copy() # duplicate so we don't keep extra weird data around
        method_stats = {'input_set_type': 'set', 'input_set_params':id}

        ncpdata.results.extend(ncp_experiment(ncpdata, R, func, method_stats))

        end = time.time()
        if end - start > timeout_ncp:
            break

class NCPData:
    def __init__(self, graph, setfuncs=[], input_stats=True, do_largest_component=True):
        if do_largest_component:
            self.graph = graph.largest_component()
        else:
            self.graph = graph

        # Todo - have "largest_component" return a graph for the largest component
        self.input_stats = input_stats
        self.set_funcs = setfuncs

        standard_fields = self.graph.set_scores([0])
        for F in self.set_funcs: # build the list of keys for set_funcs
            standard_fields.update(F(self.graph, [0]))
        if self.input_stats:
            result_fields = ["input_" + field for field in standard_fields.keys()]
        result_fields.extend(["output_" + str(field) for field in standard_fields.keys()])
        result_fields.extend(["methodfunc", "input_set_type", "input_set_params", "time"])
        self.record = namedtuple("NCPDataRecord", field_names=result_fields)
        self.neighborhood_cond = None
        self.reset_records()
        self.default_method = None
        self.method_names = {} # This stores human readable and usable names for methods

    def reset_records(self):
        self.results = []
        self.sets = []
        self.method_names = {}

    """ Generate nodes at random from the set of locally minimal seeds according
    to a vertex-based feature. By default, that feature is neighborhood conductance,
    following

        Vertex neighborhoods, low conductance cuts, and good seeds for local community methods
        DF Gleich and C Seshahdri, KDD 2012.

    But, by providing any vector of data for each node in the graph,
    this code can be extended. Note that the conductance of each vertex
    neighborhood in the graph is calculated the first time and cached,
    which could cause a long initialization.

    @param ratio The number of nodes to pick (if > 1.0) or the fraction
    of total nodes to pick (if <= 1.0) (no default)

    @param mindegree the minimum degree of a node to pick (default 5)

    @param feature The vector of data to use to produce the locally
    minimial seeds(default None, which means to use neighborhood conductance)

    @param strict (default True) if we use a strict minimum in the definition
    of a local minimum.
    """
    def random_localmin_nodes(self, ratio, mindegree=5, feature=None, strict=True):
        if feature is None:
            # then we use conductance
            if self.neighborhood_cond is None:
                self.neighborhood_cond = triangleclusters(self.graph)[0]
            feature = self.neighborhood_cond
        # find the set of minimal verts
        minverts = self.graph.local_extrema(feature, strict)[0]
        # filter by mindegree
        minverts = [v for v in minverts if self.graph.d[v] >= mindegree]

        if 0 < ratio <= 1:
            n_nodes = min(np.ceil(ratio*len(minverts)),len(minverts))
            n_nodes = int(n_nodes)
        elif ratio > 1.0: # this is a number of nodes
            n_nodes = min(int(ratio), len(minverts))
        else:
            raise(ValueError("the ratio parameter must be positive"))

        return np.random.choice(minverts, size=n_nodes, replace=False)

    def random_nodes(self, ratio):
        n = self.graph._num_vertices
        if 0 < ratio <= 1:
            n_nodes = min(np.ceil(ratio*n),n)
            n_nodes = int(n_nodes)
        elif ratio > 1.0: # this is a number of nodes
            n_nodes = int(ratio)
        else:
            raise(ValueError("the ratio parameter must be positive"))

        return np.random.choice(np.arange(0,n), size=n_nodes, replace=False)

    """ Decode and return the input set to a particular experiment. """
    def input_set(self, j):
        result = self.results[j] # todo check for validity
        if result.input_set_type=="node":
            return [result.input_set_params]
        elif result.input_set_type=="neighborhood":
            R = self.graph.neighbors(result.input_set_params)
            R.append(result.input_set_params)
            return R
        elif result.input_set_type=="set":
            return self.sets[result.input_set_params].copy()
        else:
            raise(ValueError("the input_set_type is unrecognized"))

    def output_set(self, j):
        result = self.results[j] # todo check for validity
        func = result.methodfunc
        R = self.input_set(j)
        return func(self.graph, R)

    def _check_method(self, method, name):
        if method == None:
            method = self.default_method
            if method == None:
                raise(ValueError("NCP call with method=None, but no default method specified"))
        if method in self.method_names:
            # in this case, we already have a name...
            if name is not None and name is not self.method_names[method]:
                warnings.warn("Duplicate name given to NCP method\n" +
                              "original name was: %s\n"%(self.method_names[method]) +
                              "new name attempted: %s\n"%(name), stacklevel=3)
        else:
            # we need to name the method
            if name is None:
                # need to figure out a name, let's use something boring.
                self.method_names[method] = "method-%i"%(len(self.method_names))
            else:
                self.method_names[method] = name

        return method

    def add_random_node_samples(self, ratio=0.3, timeout=1000, nthreads=4, method=None, methodname=None):
        method = self._check_method(method, methodname)
        nodes = self.random_nodes(ratio)

        threads = []
        threadnodes = np.array_split(nodes, nthreads)

        for i in range(nthreads):
            sets = [ [j] for j in threadnodes[i] ] # make a set of sets
            t = threading.Thread(target=ncp_node_worker,args=(self, sets, method, timeout, methodname))
            threads.append(t)
            t.start()
        for t in threads:
            t.join()

    def add_random_neighborhood_samples(self, ratio=0.3, timeout=1000, nthreads=4, method=None, methodname=None):
        method = self._check_method(method, methodname)
        nodes = self.random_nodes(ratio)

        threads = []
        threadnodes = np.array_split(nodes, nthreads)

        for i in range(nthreads):
            sets = [ [j] for j in threadnodes[i] ] # make a set of sets
            t = threading.Thread(target=ncp_neighborhood_worker,args=(self, sets, method, timeout, methodname))
            threads.append(t)
            t.start()
        for t in threads:
            t.join()

    """ Add expansions of all the locally minimal seeds.

    This routine follows ideas from:

        Vertex neighborhoods, low conductance cuts, and good seeds for local community methods
        DF Gleich and C Seshahdri, KDD 2012.

    which grow sets of out locally minimal seed vertices. These are
    vertices whose neighborhood conductance (in that paper) were better than all
    of the other neighbors. By default, this routine will use the same
    type of expansion.

    @param feature (default None, which means to use neighborhood conductance)
    @param strict, ratio, mindegree (see random_localmin_nodes)
    @param neighborhods this determinds if we seed based on the vertex (False)
    or the vertex neighborhood (True) (default True, following Gleich & Seshadhri)
    @param timeout, nthreads, method, methodname (see add_random_node samples)
    """
    def add_localmin_samples(self,
            feature=None, strict=True, ratio=1.0, mindegree=5, # localmin parameters
            neighborhoods=True, # use neighborhood expansion
            timeout=1000, nthreads=4, method=None, methodname=None):

        method = self._check_method(method, methodname)
        nodes = self.random_localmin_nodes(ratio,
            feature=feature, strict=strict, mindegree=mindegree)

        threads = []
        threadnodes = np.array_split(nodes, nthreads)

        for i in range(nthreads):
            sets = [ [j] for j in threadnodes[i] ] # make a set of sets
            if neighborhoods:
                t = threading.Thread(target=ncp_neighborhood_worker,args=(self, sets, method, timeout, methodname))
            else:
                t = threading.Thread(target=ncp_node_worker,args=(self, sets, method, timeout, methodname))
            threads.append(t)
            t.start()
        for t in threads:
            t.join()

    def add_random_neighborhood_samples(self, ratio=0.3, timeout=1000, nthreads=4, method=None, methodname=None):
        method = self._check_method(method, methodname)
        nodes = self.random_nodes(ratio)

        threads = []
        threadnodes = np.array_split(nodes, nthreads)

        for i in range(nthreads):
            sets = [ [j] for j in threadnodes[i] ] # make a set of sets
            t = threading.Thread(target=ncp_neighborhood_worker,args=(self, sets, method, timeout, methodname))
            threads.append(t)
            t.start()
        for t in threads:
            t.join()

    def add_set_samples(self, sets, nthreads=4, method=None, methodname=None, timeout=1000):
        method = self._check_method(method, methodname)
        threads = []
        startset = len(self.sets)
        self.sets.extend(sets)
        endset = len(self.sets)

        setnos = np.array_split(range(startset,endset), nthreads) # set numbers

        """
        results_id = len(self.results)
        self.results.append([])
        self.method_names.append(methodname)
        """

        #self.results[methodname] = []

        for i in range(nthreads):
            t = threading.Thread(target=ncp_set_worker,args=(self, setnos[i], method, timeout, methodname))
            threads.append(t)
            t.start()
        for t in threads:
            t.join()

    def as_data_frame(self):
        """
        DF = pd.DataFrame.from_records([], columns=self.record._fields)
        for methodname in self.results.keys():
            df = pd.DataFrame.from_records(self.results[methodname], columns=self.record._fields)
            df.rename(columns={'method':'methodfunc'}, inplace=True)
            df["method"] = methodname
            DF = DF.append(df,ignore_index=True)
        return DF
        """
        df = pd.DataFrame.from_records(self.results, columns=self.record._fields)
        df.rename(columns={'method':'methodfunc'}, inplace=True)
        df["method"] = df["methodfunc"].map(self.method_names)
        return df

    def approxPageRank(self,
                       gamma: float = 0.01/0.99,
                       rholist: List[float] = [1.0e-10,1.0e-8,1.0e-7,1.0e-6,1.0e-5,1.0e-4],
                       ratio: float = 0.3,
                       nthreads: int = 4,
                       timeout: float = 1000):
        #self.reset_records("approxPageRank")
        alpha = 1.0-1.0/(1.0+gamma)
        funcs = {lambda G,R: spectral_clustering(G,R,alpha=alpha,rho=rho,method="acl")[0]:'acl;rho=%.0e'%(rho)
                    for rho in rholist}
        for func in funcs.keys():
            self.add_random_node_samples(method=func,methodname=funcs[func],ratio=ratio,nthreads=nthreads,timeout=timeout/len(funcs))

    def l1reg(self,
              gamma: float = 0.01/0.99,
              rholist: List[float] = [1.0e-10,1.0e-8,1.0e-7,1.0e-6,1.0e-5,1.0e-4],
              ratio: float = 0.3,
              nthreads: int = 4,
              timeout: float = 1000):
        #self.reset_records("l1reg")
        alpha = 1.0-1.0/(1.0+gamma)
        funcs = {lambda G,R: spectral_clustering(G,R,alpha=alpha,rho=rho,method="l1reg")[0]:'l1reg;rho=%.0e'%(rho)
                    for rho in rholist}
        for func in funcs.keys():
            self.add_random_node_samples(method=func,methodname=funcs[func],ratio=ratio,nthreads=nthreads,timeout=timeout/len(funcs))

    def crd(self,
            U: int = 3,
            h: int = 10,
            w: int = 2,
            ratio: float = 0.3,
            nthreads: int = 4,
            timeout: float = 1000):
        #self.reset_records("crd")
        func = lambda G,R: flow_clustering(G,R,w=w, U=U, h=h,method="crd")[0]
        self.add_random_neighborhood_samples(method=func,methodname="crd",
                ratio=ratio,nthreads=nthreads,timeout=timeout/2)
        self.add_random_node_samples(method=func,methodname="crd",
                ratio=ratio,nthreads=nthreads,timeout=timeout/2)
        return self

    def mqi(self,
            ratio: float = 0.3,
            nthreads: int = 4,
            timeout: float = 1000):
        #self.reset_records("mqi")
        func = lambda G,R: flow_clustering(G,R,method="mqi")[0]
        self.add_random_neighborhood_samples(ratio=ratio,nthreads=nthreads,timeout=timeout,
                method=func,methodname="mqi")


import sys
import binascii
import json
import bisect
import logging as log

from rangeset import RangeSet

import idaapi

#
# ScratchABit API and code
#

START = 0
END = 1
PROPS = 2
BYTES = 3
FLAGS = 4

from .defs import *

arch_id = None

def str_area(area):
    if not area:
        return "Area(None)"
    return "Area(0x%x-0x%x, %s)" % (area[START], area[END], area[PROPS])

def area_props(area):
    return area[PROPS]

class Function:

    def __init__(self, start, end=None):
        self.ranges = RangeSet()
        self.start = start
        self.end = end

    def add_insn(self, addr, sz):
        self.ranges.add((addr, addr + sz))

    def add_range(self, start, end):
        self.ranges.add((start, end))

    def get_ranges(self):
        return self.ranges.to_list()

    def get_end(self):
        if self.end is not None:
            return self.end
        bounds = self.ranges.bounds()
        if bounds:
            return bounds[1]

    def get_end_method(self):
        if self.end is not None:
            bounds = self.ranges.bounds()
            addr = "?"
            if bounds:
                addr = "0x%x" % (self.ranges.bounds()[1] - 1)
            return "as set by loader (detected: %s)" % addr
        return "as detected"

class AddressSpace:
    UNK = 0
    CODE = 0x01
    CODE_CONT = 0x02
    DATA = 0x04
    DATA_CONT = 0x08
    STR = 0x10  # Continuation is DATA_CONT
    ALT_CODE = 0x20  # Alternative code encoding, e.g. Thumb on ARM, cont. is CODE_CONT
    FILL = 0x40  # Filler/alignment bytes
    FUNC = 0x80  # Can appear with CODE, meaning this instruction belongs to a function

    def __init__(self):
        self.area_list = []
        # List of subareas and binary search index for it
        self.subarea_list = []
        self.subarea_search = []
        # Map from referenced addresses to their properties. Among them:
        # "args":
        # Properties of instruction's args; at the very least, this should
        # differentiate between literal numeric values and addresses/offsets/pointers
        # to other objects
        # "comm":
        # Comment
        # "label"
        # Label
        # "xref":
        # Cross-reference records
        # "fun_s", "fun_e"
        # Function start and beyond-end addresses, map to Function object
        self.addr_map = {}
        # Map from label to its address
        self.labels_rev = {}
        # Problem spots which automatic control/data flow couldn't resolve
        self.issues = {}
        # Cached last accessed area
        self.last_area = None
        # Cached function start addresses
        self.func_starts = None
        # Map from func_starts's indexes to function objects
        self.func_starts_arr = []
        # True during loading stage, False during UI interaction stage
        self.is_loading = False
        # Was area flags/content changed (and thus require saving)?
        self.changed = False

    # Memory Area API

    def add_area(self, start, end, props):
        log.debug("add_area(%08x, %08x, %s)", start, end, props)
        sz = end - start + 1
        bytes = bytearray(sz)
        flags = bytearray(sz)
        a = (start, end, props, bytes, flags)
        self.area_list.append(a)
        # Area list should be sorted. Assume it's short and just resort it each time.
        self.area_list.sort()
        return a

    def get_areas(self):
        return self.area_list

    def area_no(self, area):
        return self.area_list.index(area)

    def addr2area(self, addr):
        if self.last_area:
            a = self.last_area
            if a[0] <= addr <= a[1]:
                return (addr - a[0], a)
        for a in self.area_list:
            if a[0] <= addr <= a[1]:
                self.last_area = a
                return (addr - a[0], a)
        return (None, None)

    def min_addr(self):
        return self.area_list[0][START]

    def max_addr(self):
        return self.area_list[-1][END]

    # Return next address in the address space, or None
    def next_addr(self, addr):
        offset, area = self.addr2area(addr)
        if addr != area[END]:
            return addr + 1
        i = self.area_no(area) + 1
        if i == len(self.area_list):
            return None
        return self.area_list[i][START]

    def is_exec(self, addr):
        off, area = self.addr2area(addr)
        if not area:
            return False
        return "X" in area[PROPS]["access"]

    # Binary Data API

    def load_content(self, file, addr, sz=None):
        off, area = self.addr2area(addr)
        to = off + sz if sz else None
        file.readinto(memoryview(area[BYTES])[off:to])

    def is_valid_addr(self, addr):
        off, area = self.addr2area(addr)
        return area is not None

    def get_byte(self, addr):
        off, area = self.addr2area(addr)
        if area is None:
            raise InvalidAddrException(addr)
        return area[BYTES][off]

    def set_byte(self, addr, val):
        self.changed = True
        off, area = self.addr2area(addr)
        if area is None:
            raise InvalidAddrException(addr)
        area[BYTES][off] = val & 0xff

    def get_bytes(self, addr, sz):
        off, area = self.addr2area(addr)
        if area is None:
            raise InvalidAddrException(addr)
        return area[BYTES][off:off + sz]

    def get_data(self, addr, sz):
        # TODO: address size
        if sz == 4:
            sym = self.get_addr_prop(addr, "sym")
            if sym is not None:
                return sym

        off, area = self.addr2area(addr)
        val = 0
        for i in range(sz):
            val = val | (area[BYTES][off + i] << 8 * i)
        return val

    def set_data(self, addr, data, sz):
        self.changed = True
        off, area = self.addr2area(addr)
        val = 0
        for i in range(sz):
            area[BYTES][off + i] = data & 0xff
            data >>= 8

    # Convenience function for plugins
    def memcpy(self, dst, src, sz):
        for i in range(sz):
            b = self.get_byte(src)
            self.set_byte(dst, b)
            src += 1
            dst += 1

    # Binary Data Flags API

    def get_flags(self, addr, mask=~(FUNC | ALT_CODE)):
        off, area = self.addr2area(addr)
        if area is None:
            raise InvalidAddrException(addr)
        return area[FLAGS][off] & mask

    def get_unit_size(self, addr):
        off, area = self.addr2area(addr)
        flags = area[FLAGS]
        sz = 1
        if flags[off] & ~(self.FUNC | self.ALT_CODE) == self.CODE:
            f = self.CODE_CONT
        elif flags[off] in (self.DATA, self.STR):
            f = self.DATA_CONT
        elif flags[off] == self.FILL:
            f = self.FILL
        else:
            return 1
        off += 1

        try:
            while flags[off] == f:
                off += 1
                sz += 1
        except IndexError:
            pass

        return sz

    # Taking an offset inside unit, return offset to the beginning of unit
    @classmethod
    def adjust_offset_reverse(cls, off, area):
        flags = area[FLAGS]
        if flags[off] == cls.FILL:
            while off > 0:
                if flags[off] != cls.FILL:
                    off += 1
                    break
                off -= 1
            return off

        while off > 0:
            if flags[off] in (cls.CODE_CONT, cls.DATA_CONT):
                off -= 1
            else:
                break
        return off

    def adjust_addr_reverse(self, addr):
        off, area = self.addr2area(addr)
        if area is None:
            return None
        return self.adjust_offset_reverse(off, area) + area[START]

    def set_flags(self, addr, sz, head_fl, rest_fl=0):
        self.changed = True
        off, area = self.addr2area(addr)
        flags = area[FLAGS]
        flags[off] = head_fl
        off += 1
        for i in range(sz - 1):
            flags[off + i] = rest_fl

    def update_flags(self, addr, and_mask, or_mask):
        off, area = self.addr2area(addr)
        if area is None:
            raise InvalidAddrException(addr)
        flags = area[FLAGS]
        flags[off] = (flags[off] & and_mask) | or_mask
        self.changed = True

    def make_undefined(self, addr, sz):
        self.set_flags(addr, sz, self.UNK, self.UNK)

    def make_code(self, addr, sz, extra_flags=0):
        self.changed = True
        off, area = self.addr2area(addr)
        area_byte_flags = area[FLAGS]
        area_byte_flags[off] |= self.CODE | extra_flags
        for i in range(sz - 1):
            area_byte_flags[off + 1 + i] |= self.CODE_CONT

    def make_alt_code(self, addr):
        self.update_flags(addr, 0xff, self.ALT_CODE)

    # Mark instructions in given range as belonging to function
    def mark_func_bytes(self, addr, sz):
        self.changed = True
        off, area = self.addr2area(addr)
        area_byte_flags = area[FLAGS]
        for i in range(sz):
            fl = area_byte_flags[off + i]
            assert fl in (self.CODE, self.CODE | self.ALT_CODE, self.CODE_CONT)
            if fl == self.CODE:
                area_byte_flags[off + i] |= self.FUNC

    def make_data(self, addr, sz):
        self.changed = True
        off, area = self.addr2area(addr)
        area_byte_flags = area[FLAGS]
        area_byte_flags[off] |= self.DATA
        for i in range(sz - 1):
            area_byte_flags[off + 1 + i] |= self.DATA_CONT

    def make_data_array(self, addr, sz, num_items, prefix=""):
        # Make a data array. First-class arrays are not supported so far,
        # so just mark data units sequentially
        self.append_comment(addr, "%sArray, num %s: %d" % (prefix, "bytes" if sz == 1 else "items", num_items))
        for i in range(num_items):
            self.make_data(addr, sz)
            addr += sz

    def make_filler(self, addr, sz):
        self.set_flags(addr, sz, self.FILL, self.FILL)

    # Address properties API

    def set_addr_prop(self, addr, prop, val):
        self.changed = True
        self.addr_map.setdefault(addr, {})[prop] = val

    def get_addr_prop(self, addr, prop, default=None):
        return self.addr_map.get(addr, {}).get(prop, default)

    def get_addr_prop_dict(self, addr):
        return self.addr_map.get(addr, {})

    # Label API

    def get_default_label_prefix(self, ea):
        try:
            fl = self.get_flags(ea)
        except InvalidAddrException:
            return "invalid_"
        if fl == self.CODE:
            prefix = "loc_"
        elif fl & self.DATA:
            prefix = "dat_"
        else:
            prefix = "unk_"
        return prefix

    def get_default_label(self, ea):
        prefix = self.get_default_label_prefix(ea)
        return "%s%08x" % (prefix, ea)

    def make_label(self, prefix, ea):
        l = self.get_addr_prop(ea, "label")
        if isinstance(l, str):
            # If it's real label, don't change it
            return
        if not prefix:
            prefix = self.get_default_label_prefix(ea)
        l = "%s%08x" % (prefix, ea)
        self.set_addr_prop(ea, "label", l)
        self.labels_rev[l] = ea

    # auto_label will change its prefix automatically based on
    # type of data it points.
    def make_auto_label(self, ea):
        if self.get_addr_prop(ea, "label"):
            return
        self.set_addr_prop(ea, "label", ea)
        self.labels_rev[ea] = ea

    # Delete a label, only if it's auto
    def del_auto_label(self, ea):
        label = self.get_addr_prop(ea, "label")
        if not label or isinstance(label, str):
            return
        self.set_addr_prop(ea, "label", None)
        del self.labels_rev[ea]

    def get_label(self, ea):
        label = self.get_addr_prop(ea, "label")
        if isinstance(label, int):
            return "%s%08x" % (self.get_default_label_prefix(ea), label)
        return label

    def set_label(self, ea, label):
        # Make sure the label can be actually visible - create an area for it if none
        off, area = self.addr2area(ea)
        if area is None:
            self.add_area(ea, ea, {"name": "autocreated to host %s label" % label})
        if self.is_loading:
            existing = self.get_addr_prop(ea, "label")
            if existing is not None and not isinstance(existing, int):
                log.warn("Duplicate label for %x: %s (existing: %s)" % (ea, label, existing))
                self.append_comment(ea, "Another label: " + label)
                return
        self.set_addr_prop(ea, "label", label)
        self.labels_rev[label] = ea

    def make_unique_label(self, ea, label):
        existing = self.get_label(ea)
        if existing == label:
            return label
        cnt = 0
        while True:
            l = label
            if cnt > 0:
                l += "__%d" % cnt
            if l not in self.labels_rev:
                self.set_label(ea, l)
                if self.is_loading and cnt > 0:
                    self.append_comment(ea, "Original label: " + label)
                return l
            cnt += 1

    def get_label_list(self):
        return sorted([x if isinstance(x, str) else self.get_default_label(x) for x in self.labels_rev.keys()])

    def resolve_label(self, label):
        if label in self.labels_rev:
            return self.labels_rev[label]
        try:
            ea = int(label.split("_", 1)[1], 16)
        except:
            return None
        if ea in self.labels_rev and self.get_default_label(ea) == label:
            return ea

    def label_exists(self, label):
        return label in self.labels_rev

    # Comment API

    def get_comment(self, ea):
        comm = self.get_addr_prop(ea, "comm")
        return comm

    def set_comment(self, ea, comm):
        self.set_addr_prop(ea, "comm", comm)

    def append_comment(self, ea, comm):
        existing = self.get_comment(ea)
        if existing is not None:
            comm = existing + "\n" + comm
        self.set_addr_prop(ea, "comm", comm)

    # (Pseudo)instruction Argument Properties API

    def set_arg_prop(self, ea, arg_no, prop, prop_val):
        arg_props = self.get_addr_prop(ea, "args", {})
        if arg_no not in arg_props:
            arg_props[arg_no] = {}
        props = arg_props[arg_no]
        props[prop] = prop_val
        self.set_addr_prop(ea, "args", arg_props)

    def get_arg_prop(self, ea, arg_no, prop):
        arg_props = self.get_addr_prop(ea, "args", {})
        return arg_props.get(arg_no, {}).get(prop)

    def get_arg_prop_dict(self, ea, arg_no):
        arg_props = self.get_addr_prop(ea, "args", {})
        return arg_props.get(arg_no, {})

    def make_arg_offset(self, insn_addr, arg_no, ref_addr):
        # Convert an immediate argument to an offset one
        # insn_addr - address of (pseudo)instruction
        # arg_no - argument no. of instruction
        # ref_addr - value of the argument (i.e. address it refers to)
        old_subtype = self.get_arg_prop(insn_addr, arg_no, "subtype")
        if old_subtype and old_subtype != IMM_ADDR:
            # Preserve old numeric value subtype to unconvert back to it
            # if need.
            self.set_arg_prop(insn_addr, arg_no, "num_subtype", old_subtype)

        self.set_arg_prop(insn_addr, arg_no, "subtype", IMM_ADDR)

        if isinstance(ref_addr, str):
            # Symbolic address
            # TODO: this works only for "dd" virtual instruction
            self.set_addr_prop(insn_addr, "sym", ref_addr)
            return

        label = self.get_label(ref_addr)
        if not label:
            self.make_auto_label(ref_addr)
        self.add_xref(insn_addr, ref_addr, idaapi.dr_O)

    def unmake_arg_offset(self, insn_addr, arg_no, ref_addr):
        # Convert offset argument to normal immediate value
        old_subtype = self.get_arg_prop(insn_addr, arg_no, "num_subtype")
        self.set_arg_prop(insn_addr, arg_no, "subtype", old_subtype)
        self.del_xref(insn_addr, ref_addr, idaapi.dr_O)
        # If this was last xref, and label is automatic, kill it too
        if not self.get_xrefs(ref_addr):
            self.del_auto_label(ref_addr)

    def is_arg_offset(self, insn_addr, arg_no):
        old_subtype = self.get_arg_prop(insn_addr, arg_no, "subtype")
        return old_subtype == IMM_ADDR

    # Xref API

    def add_xref(self, from_ea, to_ea, type):
        xrefs = self.get_addr_prop(to_ea, "xrefs", {})
        xrefs[from_ea] = type
        self.set_addr_prop(to_ea, "xrefs", xrefs)

    def del_xref(self, from_ea, to_ea, type):
        xrefs = self.get_addr_prop(to_ea, "xrefs", {})
        del xrefs[from_ea]
        self.set_addr_prop(to_ea, "xrefs", xrefs)

    def get_xrefs(self, ea):
        xrefs = self.get_addr_prop(ea, "xrefs", None)
        return xrefs

    # Functions API

    def make_func(self, from_ea, to_ea_excl=None):
        f = self.get_addr_prop(from_ea, "fun_s")
        if f is not None:
            return f
        f = Function(from_ea, to_ea_excl)
        self.set_addr_prop(from_ea, "fun_s", f)

        if to_ea_excl is not None:
            self.set_addr_prop(to_ea_excl, "fun_e", f)
        # Reset cache
        self.func_starts = None
        return f

    def is_func(self, ea):
        return self.get_addr_prop(ea, "fun_s") is not None

    # If ea is start of function, return Function object
    def get_func_start(self, ea):
        return self.get_addr_prop(ea, "fun_s")

    # If ea is end of function, return Function object
    def get_func_end(self, ea):
        return self.get_addr_prop(ea, "fun_e")

    def set_func_end(self, func, ea):
        self.set_addr_prop(ea, "fun_e", func)

    # Look up function containing address
    def lookup_func(self, ea):
        # TODO: cache func ranges, use binary search instead
        if self.func_starts is None:
            self.func_starts = []
            self.func_starts_arr = []
            for start, props in sorted(self.addr_map.items()):
                func = props.get("fun_s")
                if func:
                    self.func_starts.append(start)
                    self.func_starts_arr.append(func)

        i = bisect.bisect_right(self.func_starts, ea)
        if i:
            func = self.func_starts_arr[i - 1]
            end = func.get_end()
            if end and func.start <= ea < end:
                return func
        return None

    # Get all functions
    def iter_funcs(self):
        for addr, props in self.addr_map.items():
            func = props.get("fun_s")
            if func:
                yield (addr, func)

    def get_func_list(self):
        return sorted([self.get_label(addr) for addr, f in self.iter_funcs()])

    # Memory Subarea API

    def add_subarea(self, start, end, name):
        log.debug("add_subarea(%x, %x, %s)", start, end, name)
        self.subarea_list.append((start, end, name))
        self.subarea_search.append(start)

    # Call this once all add_subarea() calls were made
    def finish_subareas(self):
        self.subarea_list.sort()
        self.subarea_search.sort()

    # Look up subarea containing address
    def lookup_subarea(self, ea):
        i = bisect.bisect_right(self.subarea_search, ea)
        if i:
            area = self.subarea_list[i - 1]
            if area[0] <= ea <= area[1]:
                return area
        return None

    # Issues API

    def add_issue(self, ea, descr):
        self.issues[ea] = descr

    def get_issues(self):
        res = []
        for ea in sorted(self.issues.keys()):
            res.append((ea, self.issues[ea]))
        return res

    # Persistence API

    def save_area(self, stream, area):
        stream.write("%08x %08x\n" % (area[START], area[END]))
        flags = area[FLAGS]
        i = 0
        while True:
            chunk = flags[i:i + 32]
            if not chunk:
                break
            stream.write(str(binascii.hexlify(chunk), 'utf-8') + "\n")
            i += 32
        stream.write("\n")

    def save_areas(self, stream):
        for a in self.area_list:
            self.save_area(stream, a)

    def save_addr_props(self, prefix):
        areas = self.area_list
        area_i = 0
        stream = open(prefix + ".%08x" % areas[area_i][START], "w")
        area_end = areas[area_i][END]
        stream.write("header:\n")
        stream.write(" version: 1.0\n")
        for addr, props in sorted(self.addr_map.items()):
            # If entry has just fun_e data, skip it. As fun_e is set
            # on an address past the last byte of func, this address
            # also may not belong to any section, so skipping it
            # to start with is helpful.
            if len(props) == 1 and "fun_e" in props:
                continue

            if addr > area_end:
                stream.close()
                area_i += 1
                try:
                    while addr < areas[area_i][START]:
                        area_i += 1
                    while addr > areas[area_i][END]:
                        area_i += 1
                except IndexError:
                    return
                assert addr >= areas[area_i][START], "%x vs %x" % (addr, areas[area_i][START])
                stream = open(prefix + ".%08x" % areas[area_i][START], "w")
                #stream.write("addr=%x area_end=%x\n" % (addr, area_end))
                area_end = areas[area_i][END]
                stream.write("header:\n")
                stream.write(" version: 1.0\n")
            stream.write("0x%08x:\n" % addr)
            fl = self.get_flags(addr)
            stream.write(" f: %s %02x\n" % (flag2char(fl), fl))
            label = props.get("label")
            arg_props = props.get("args")
            comm = props.get("comm")
            xrefs = props.get("xrefs")
            func = props.get("fun_s")
            if label is not None:
                if label == addr:
                    stream.write(" l:\n")
                else:
                    stream.write(" l: %s\n" % label)
            if arg_props is not None:
                arg_props_header = False
                for arg_no, data in sorted(arg_props.items()):
                    data = {k: v for k, v in data.items() if v is not None}
                    if data:
                        if not arg_props_header:
                            stream.write(" args:\n")
                            arg_props_header = True
                        stream.write("  %s: %r\n" % (arg_no, data))
                    #for k, v in sorted(data.items()):
                    #    stream.write("   %s: %s\n" % (k, v))
            if comm is not None:
                stream.write(" cmnt: %r\n" % comm)

            if func is not None:
                if func.end is not None:
                    stream.write(" fn_end: 0x%08x\n" % func.end)
                else:
                    stream.write(" fn_end: '?'\n")
                stream.write(" fn_ranges: [")
                first = True
                for r in func.get_ranges():
                    if not first:
                        stream.write(", ")
                    stream.write("[0x%08x,0x%08x]" % r)
                    first = False
                stream.write("]\n")

            if xrefs:
                stream.write(" x:\n" % xrefs)
                for from_addr in sorted(xrefs.keys()):
                    stream.write(" - 0x%08x: %s\n" % (from_addr, xrefs[from_addr]))

    def load_addr_props(self, stream):
        l = stream.readline()
        assert l == "header:\n"
        l = stream.readline()
        assert l == " version: 1.0\n"
        l = stream.readline()
        while l:
            assert l.endswith(":\n")
            addr = int(l[:-2], 0)
            props = self.addr_map.get(addr, {})
            l = stream.readline()
            while l and l[0] == " ":
                key, val = [x.strip() for x in l.split(":", 1)]
                l = None

                if key == "l":
                    if not val:
                        val = addr
                    props["label"] = val
                    self.labels_rev[val] = addr
                elif key == "cmnt":
                    props["comm"] = val[1:-1].replace("\\n", "\n")
                elif key == "fn_end":
                    if val == "'?'":
                        end = None
                    else:
                        end = int(val, 0)
                    f = Function(addr, end)
                    props["fun_s"] = f
                    # Handled by finish_func() below
                    #if end is not None:
                    #    self.addr_map[end] = {"fun_e": f}
                elif key == "fn_ranges":
                    if val != "[]":
                        assert val.startswith("[[") and val.endswith("]]"), val
                        val = val[2:-2]
                        f = props["fun_s"]
                        for r in val.split("], ["):
                            r = [int(x, 0) for x in r.split(",")]
                            f.add_range(*r)
                        # Now, call finish func to set func end address, either from
                        # fn_end or fn_ranges
                        finish_func(f)

                elif key == "args":
                    arg_props = {}
                    while True:
                        l = stream.readline()
                        if not l or not l.startswith("  "):
                            break
                        arg_no, data = [x.strip() for x in l.split(":", 1)]
                        assert data[0] == "{" and data[-1] == "}"
                        data = data[1:-1]
                        vals = {}
                        for pair in data.split(","):
                            seq = [x.strip() for x in pair.split(":", 1)]
                            for x in seq:
                                assert x[0] == "'" and x[-1] == "'", x
                            k, v = [x[1:-1] for x in seq]
                            vals[k] = v
                        arg_props[int(arg_no)] = vals
                    props["args"] = arg_props

                elif key == "x":
                    xrefs = {}
                    while True:
                        l = stream.readline()
                        if not l or not l.startswith(" - "):
                            break
                        key, val = [x.strip() for x in l[3:].split(":", 1)]
                        xrefs[int(key, 0)] = val
                    assert xrefs
                    props["xrefs"] = xrefs

                if l is None:
                    l = stream.readline()

            self.addr_map[addr] = props

    def load_area(self, stream, area):
        l = stream.readline()
        vals = [int(v, 16) for v in l.split()]
        assert area[START] == vals[0] and area[END] == vals[1]
        flags = area[FLAGS]
        i = 0
        while True:
            l = stream.readline().rstrip()
            if not l:
                break
            l = binascii.unhexlify(l)
            flags[i:i + len(l)] = l
            i += len(l)

    def load_areas(self, stream):
        for a in self.area_list:
            self.load_area(stream, a)

    # Hack for idaapi interfacing
    # TODO: should go to "Analysis" object
    def analisys_stack_push(self, ea, flow_flag=idaapi.fl_JN):
        global analisys_stack_branches, analisys_stack_calls
        global analisys_stack_returns, analysis_current_func
        # If we know something is func (e.g. from loader), jump
        # to it means tail-call.
        if flow_flag == idaapi.fl_RET_FROM_CALL:
            analisys_stack_returns.append((ea, analysis_current_func))
        elif flow_flag == idaapi.fl_CN or self.is_func(ea):
            analisys_stack_calls.append(ea)
        else:
            analisys_stack_branches.append(ea)

ADDRESS_SPACE = AddressSpace()
_processor = None
def set_processor(p):
    global _processor
    _processor = p
    idaapi.set_processor(p)

def set_arch_id(id):
    global arch_id
    arch_id = id

analisys_stack_calls = []
analisys_stack_returns = []
analisys_stack_branches = []
analysis_current_func = None

def add_entrypoint(ea, as_func=True):
    if as_func:
        ADDRESS_SPACE.make_func(ea, None)
        analisys_stack_calls.append(ea)
    else:
        analisys_stack_branches.append(ea)

def init_cmd(ea):
    _processor.cmd.ea = ea
    _processor.cmd.size = 0
    _processor.cmd.disasm = None

def finish_func(f):
    if f:
        log.info("Function %s (0x%x) ranges: %s" % (ADDRESS_SPACE.get_label(f.start), f.start, f.ranges.str(hex)))
        end = f.get_end()
        if end is not None:
            ADDRESS_SPACE.set_func_end(f, end)

def analyze(callback=None):
    global analysis_current_func

    if callback is None:
        callback = lambda cnt: None

    cnt = 0
    limit = 1000000
    analysis_current_func = None
    while limit:
        if analisys_stack_branches:
            ea = analisys_stack_branches.pop()
            try:
                fl = ADDRESS_SPACE.get_flags(ea, ~ADDRESS_SPACE.ALT_CODE)
            except InvalidAddrException:
                log.warn("Branch outside address space detected: 0x%x" % ea)
                continue

            if fl == ADDRESS_SPACE.CODE | ADDRESS_SPACE.FUNC:
                fun = ADDRESS_SPACE.get_func_start(ea)
                if fun:
                    log.warn("Jump to (or flow into) a function at 0x%x detected" % ea)

            if analysis_current_func:
                if fl == ADDRESS_SPACE.CODE | ADDRESS_SPACE.FUNC:
                    continue
                if fl not in (ADDRESS_SPACE.CODE, ADDRESS_SPACE.UNK):
                    log.warn("Unexpected flags 0x%x at 0x%x while tracing code branch, skipping it", fl, ea)
                    ADDRESS_SPACE.add_issue(ea, "Jump/flow into non-code")
                    continue
            else:
                if fl != ADDRESS_SPACE.UNK:
                    if fl != ADDRESS_SPACE.CODE:
                        ADDRESS_SPACE.add_issue(ea, "Jump/flow into non-code")
                    continue
        elif analisys_stack_calls:
            finish_func(analysis_current_func)
            analysis_current_func = None
            ea = analisys_stack_calls.pop()
            fun = ADDRESS_SPACE.get_func_start(ea)
            if fun.get_ranges():
                continue
            log.info("Starting analysis of function 0x%x" % ea)
            analysis_current_func = ADDRESS_SPACE.make_func(ea)
        elif analisys_stack_returns:
            ea, analysis_current_func = analisys_stack_returns.pop()
            #log.debug("Restarting analysis of call return at 0x%x (fl=%x)", ea, ADDRESS_SPACE.get_flags(ea, 0xff))
            analisys_stack_branches.append(ea)
            continue
        else:
            finish_func(analysis_current_func)
            break
        init_cmd(ea)
        try:
            insn_sz = _processor.ana()
        except InvalidAddrException:
            # Ran out of memory area, just continue
            # with the rest of paths
            continue
#        print("size: %d" % insn_sz, _processor.cmd)
        if insn_sz:
            if not _processor.emu():
                assert False
            if analysis_current_func:
                analysis_current_func.add_insn(ea, insn_sz)
                ADDRESS_SPACE.make_code(ea, insn_sz, ADDRESS_SPACE.FUNC)
            else:
                ADDRESS_SPACE.make_code(ea, insn_sz)
            _processor.out()
#            print("%08x %s" % (_processor.cmd.ea, _processor.cmd.disasm))
#            print("---------")
            limit -= 1
            cnt += 1
            if cnt % 1000 == 0:
                callback(cnt)
#    if not analisys_stack:
#        print("Analisys finished")

class Model:

    def __init__(self, target_addr=0, target_subno=0):
        self._lines = []
        self._cnt = 0
        self._subcnt = 0
        self._last_addr = -1
        self._addr2line = {}
        self.AS = None
        self.target_addr = target_addr
        self.target_subno = target_subno
        self.target_addr_lineno_0 = -1
        self.target_addr_lineno = -1
        self.target_addr_lineno_real = -1

    def lines(self):
        return self._lines

    def add_object(self, addr, line):
        if addr != self._last_addr:
            self._last_addr = addr
            self._subcnt = 0
        if addr == self.target_addr:
            if self._subcnt == 0:
                # Contains first line related to the given addr
                self.target_addr_lineno_0 = self._cnt
            if self._subcnt == self.target_subno:
                # Contains line no. target_subno related to the given addr
                self.target_addr_lineno = self._cnt
            if not line.virtual:
                # Contains line where actual instr/data/unknown bytes are
                # rendered (vs labels/xrefs/etc.)
                self.target_addr_lineno_real = self._cnt
        self._lines.append(line)
        self._addr2line[(addr, self._subcnt)] = self._cnt
        line.subno = self._subcnt
        if not line.virtual:
            # Line of "real" disasm object
            self._addr2line[(addr, -1)] = self._cnt
        self._cnt += 1
        self._subcnt += 1

    def addr2line_no(self, addr, subno=-1):
        return self._addr2line.get((addr, subno))

    def undefine_unit(self, addr):
        sz = self.AS.get_unit_size(addr)
        self.AS.make_undefined(addr, sz)

def data_sz2mnem(sz):
    s = {1: "db", 2: "dw", 4: "dd"}[sz]
    return idaapi.fillstr(s, idaapi.DEFAULT_WIDTH)

class DisasmObj:

    # Size of "leader fields" in disasm window - address, raw bytes, etc.
    # May be set by MVC controller
    LEADER_SIZE = 9

    # Default indent for a line
    indent = " " * idaapi.DEFAULT_INDENT

    # Default operand positions list is empty and set on class level
    # to save memory. To be overriden on object level.
    arg_pos = ()

    # If False, this object corresponds to real bytes in input binary stream
    # If True, doesn't correspond to bytes in memory: labels, etc.
    virtual = True

    # Textual comment to append
    comment = ""

    # Instance variable expected to be set on each instance:
    # ea =
    # size =
    # subno =  # relative no. of several lines corresponding to the same ea

    def render(self):
        # Render object as a string, set it as .cache, and return it
        pass

    def get_operand_addr(self):
        # Get "the most addressful" operand
        # This for example will be called when Enter is pressed
        # not on a specific instruction operand, so this should
        # return value of the operand which contains an address
        # (or the "most suitable" of them if there're few).
        return None

    def __len__(self):
        # Each object should return real character len as display on the screen.
        # Should be fast - called on each cursor movement.
        try:
            return self.LEADER_SIZE + len(self.indent) + len(self.cache)
        except AttributeError:
            return self.LEADER_SIZE + len(self.indent) + len(self.render())

    def content_len(self):
        return len(self) - (self.LEADER_SIZE + len(self.indent))

class Instruction(idaapi.insn_t, DisasmObj):

    virtual = False

    def render(self):
        _processor.cmd = self
        _processor.out()
        s = self.disasm + self.comment
        self.cache = s
        return s

    def get_operand_addr(self):
        # Assumes RISC design where only one operand can be address
        mem = imm = None
        for o in self._operands:
            if o.flags & idaapi.OF_SHOW:
                if o.type == idaapi.o_near:
                    # Jumps have priority
                    return o
                if o.type == idaapi.o_mem:
                    mem = o
                elif o.type == idaapi.o_imm:
                    imm = o
        if mem:
            return mem
        return imm

class Data(DisasmObj):

    virtual = False

    def __init__(self, ea, sz, val):
        self.ea = ea
        self.size = sz
        self.val = val

    def render(self):
        subtype = ADDRESS_SPACE.get_arg_prop(self.ea, 0, "subtype")
        if subtype == IMM_ADDR:
            label = self.val
            if not isinstance(label, str):
                label = ADDRESS_SPACE.get_label(label)
            s = "%s%s" % (data_sz2mnem(self.size), label)
        else:
            s = "%s0x%x" % (data_sz2mnem(self.size), self.val)
        s += self.comment
        self.cache = s
        return s

    def get_operand_addr(self):
        o = idaapi.op_t(0)
        o.value = self.val
        o.addr = self.val
        o.type = idaapi.o_imm
        return o

class String(DisasmObj):

    virtual = False

    def __init__(self, ea, sz, val):
        self.ea = ea
        self.size = sz
        self.val = val

    def render(self):
        s = "%s%s" % (data_sz2mnem(1), repr(self.val).replace("\\x00", "\\0"))
        s += self.comment
        self.cache = s
        return s

class Fill(DisasmObj):

    virtual = False

    def __init__(self, ea, sz):
        self.ea = ea
        self.size = sz
        self.cache = idaapi.fillstr(".fill", idaapi.DEFAULT_WIDTH) + str(sz)

    def render(self):
        return self.cache

class Unknown(DisasmObj):

    virtual = False
    size = 1

    def __init__(self, ea, val):
        self.ea = ea
        self.val = val

    def render(self):
        ch = ""
        if 0x20 <= self.val <= 0x7e:
            ch = " ; '%s'" % chr(self.val)
        s = "%s0x%02x%s" % (idaapi.fillstr("unk", idaapi.DEFAULT_WIDTH), self.val, ch)
        s += self.comment
        self.cache = s
        return s

class Label(DisasmObj):

    indent = ""

    def __init__(self, ea):
        self.ea = ea

    def render(self):
        label = ADDRESS_SPACE.get_label(self.ea)
        s = "%s:" % label
        self.cache = s
        return s

class Xref(DisasmObj):

    indent = ""

    def __init__(self, ea, from_addr, type):
        self.ea = ea
        self.from_addr = from_addr
        self.type = type

    def render(self):
        func = ADDRESS_SPACE.lookup_func(self.from_addr)
        extra = ""
        if func:
            extra = ADDRESS_SPACE.get_label(func.start)
            off = self.from_addr - func.start
            if off != 0:
                extra += "+0x%x" % off
            extra = " (%s)" % extra
        s = (" " * idaapi.DEFAULT_XREF_INDENT) + "; xref: %s 0x%x" % (self.type, self.from_addr) + extra
        self.cache = s
        return s

    def get_operand_addr(self):
        o = idaapi.op_t(0)
        o.addr = self.from_addr
        return o

class Literal(DisasmObj):

    indent = ""

    def __init__(self, ea, str):
        self.ea = ea
        self.cache = str

    def render(self):
        return self.cache

# Separate types to differentiate content
class AreaWrapper(Literal):
    pass

# Separate types to differentiate content
class FunctionWrapper(Literal):
    pass

def render():
    model = Model()
    render_partial(model, 0, 0, 1000000)
    return model

# How much bytes may a single disasm object (i.e. a line) occupy
MAX_UNIT_SIZE = 4

def render_partial_around(addr, subno, context_lines):
    log.debug("render_partial_around(%x, %d)", addr, subno)
    off, area = ADDRESS_SPACE.addr2area(addr)
    if area is None:
        return None
    back = context_lines * MAX_UNIT_SIZE
    off -= back
    if off < 0:
        area_no = ADDRESS_SPACE.area_no(area) - 1
        while area_no >= 0:
            area = ADDRESS_SPACE.area_list[area_no]
            sz = area[1] - area[0] + 1
            off += sz
            if off >= 0:
                break
            area_no -= 1
        if off < 0:
            # Reached beginning of address space, just set as such
            off = 0
    assert off >= 0
    log.debug("render_partial_around: off=0x%x, %s", off, str_area(area))
    off = ADDRESS_SPACE.adjust_offset_reverse(off, area)
    log.debug("render_partial_around adjusted: off=0x%x, %s", off, str_area(area))
    model = Model(addr, subno)
    render_partial(model, ADDRESS_SPACE.area_list.index(area), off, context_lines, addr)
    log.debug("render_partial_around model done, lines: %d", len(model.lines()))
    assert model.target_addr_lineno_0 >= 0
    if model.target_addr_lineno == -1:
        # If we couldn't find exact subno, use 0th subno of that addr
        # TODO: maybe should be last subno, because if we couldn't find
        # exact one, it was ~ last and removed, so current last is "closer"
        # to it.
        model.target_addr_lineno = model.target_addr_lineno_0
    return model

def render_from(model, addr, num_lines):
    off, area = ADDRESS_SPACE.addr2area(addr)
    if area is None:
        return None
    return render_partial(model, ADDRESS_SPACE.area_list.index(area), off, num_lines)

def render_partial(model, area_no, offset, num_lines, target_addr=-1):
    model.AS = ADDRESS_SPACE
    start = True
    #for a in ADDRESS_SPACE.area_list:
    while area_no < len(ADDRESS_SPACE.area_list):
        a = ADDRESS_SPACE.area_list[area_no]
        area_no += 1
        i = 0
        if start:
            i = offset
            start = False
        if i == 0:
            model.add_object(a[START], AreaWrapper(a[START], "; Start of 0x%x area (%s)" % (a[START], a[PROPS].get("name", "noname"))))
        bytes = a[BYTES]
        flags = a[FLAGS]
        areasize = len(bytes)
        while i < areasize:
            addr = a[START] + i
            # If we didn't yet reach target address, compensate for
            # the following decrement of num_lines. The logic is:
            # render all lines up to target_addr, and then num_lines past it.
            if target_addr >= 0 and addr < target_addr:
                num_lines += 1

            props = ADDRESS_SPACE.get_addr_prop_dict(addr)
            func = props.get("fun_s")
            if func:
                model.add_object(addr, FunctionWrapper(addr, "; Start of function '%s'" % ADDRESS_SPACE.get_label(func.start)))

            xrefs = props.get("xrefs")
            if xrefs:
                for from_addr in sorted(xrefs.keys()):
                    model.add_object(addr, Xref(addr, from_addr, xrefs[from_addr]))

            label = props.get("label")
            if label:
                model.add_object(addr, Label(addr))

            f = flags[i] & 0x7f
            if f == AddressSpace.UNK:
                out = Unknown(addr, bytes[i])
                sz = 1
                i += 1
            elif f & AddressSpace.DATA:
                sz = 1
                j = i + 1
                while j < areasize and flags[j] & AddressSpace.DATA_CONT:
                    sz += 1
                    j += 1
                assert sz <= 4
                out = Data(addr, sz, ADDRESS_SPACE.get_data(addr, sz))
                i += sz
            elif f == AddressSpace.STR:
                str = chr(bytes[i])
                sz = 1
                j = i + 1
                while j < areasize and flags[j] == AddressSpace.DATA_CONT:
                    str += chr(bytes[j])
                    sz += 1
                    j += 1
                out = String(addr, sz, str)
                i += sz
            elif f == AddressSpace.FILL:
                sz = 1
                j = i + 1
                while j < areasize and flags[j] == AddressSpace.FILL:
                    sz += 1
                    j += 1
                out = Fill(addr, sz)
                i += sz
            elif f & ~AddressSpace.ALT_CODE == AddressSpace.CODE:
                out = Instruction(addr)
                _processor.cmd = out
                sz = _processor.ana()
                _processor.out()
                i += sz
            else:
                out = Literal(addr, "; UNEXPECTED value: %02x flags: %02x" % (bytes[i], f))
                sz = 1
                i += 1
                # Uncomment below for fail fast-and-soon
                #assert 0, "@%08x flags=%x" % (addr, f)

            comm = props.get("comm")
            if comm:
                comm_indent = " " * (out.content_len() + len(out.indent) + 2)
                out.comment = "  ; " + comm.split("\n", 1)[0]

            model.add_object(addr, out)
            #sys.stdout.write(out + "\n")

            if comm:
                for comm_l in comm.split("\n")[1:]:
                    comm_obj = Literal(addr, "; " + comm_l)
                    comm_obj.indent = comm_indent
                    model.add_object(addr, comm_obj)

            next_addr = addr + sz
            next_props = ADDRESS_SPACE.get_addr_prop_dict(next_addr)
            func_end = next_props.get("fun_e")
            if func_end:
                model.add_object(addr, FunctionWrapper(addr, "; End of function '%s' (%s)" % (
                    ADDRESS_SPACE.get_label(func_end.start), func_end.get_end_method()
                )))

            num_lines -= 1
            if not num_lines:
                return next_addr

        model.add_object(a[END], AreaWrapper(a[END], "; End of 0x%x area (%s)" % (a[START], a[PROPS].get("name", "noname"))))

def flag2char(f):
    if f == AddressSpace.UNK:
        return "."
    elif f == AddressSpace.CODE:
        return "C"
    elif f == AddressSpace.CODE | AddressSpace.ALT_CODE:
        return "T"
    elif f & ~AddressSpace.ALT_CODE == AddressSpace.CODE | AddressSpace.FUNC:
        return "F"
    elif f == AddressSpace.CODE_CONT:
        return "c"
    elif f == AddressSpace.DATA:
        return "D"
    elif f == AddressSpace.DATA_CONT:
        return "d"
    elif f == AddressSpace.STR:
        return "A"
    elif f == AddressSpace.FILL:
        return "-"
    else:
        return "X"

def print_address_map():
    for a in ADDRESS_SPACE.area_list:
        for i in range(len(a[FLAGS])):
            if i % 128 == 0:
                sys.stdout.write("\n")
                sys.stdout.write("%08x " % (a[START] + i))
            sys.stdout.write(flag2char(a[FLAGS][i]))
        sys.stdout.write("\n")

idaapi.set_address_space(ADDRESS_SPACE)