import sys, os
import os.path
import shutil

from rpython.translator.translator import TranslationContext
from rpython.translator.tool.taskengine import SimpleTaskEngine
from rpython.translator.goal import query
from rpython.translator.goal.timing import Timer
from rpython.annotator.listdef import s_list_of_strings
from rpython.annotator import policy as annpolicy
from rpython.tool.udir import udir
from rpython.rlib.debug import debug_start, debug_print, debug_stop
from rpython.rlib.entrypoint import secondary_entrypoints,\
     annotated_jit_entrypoints

import py
from rpython.tool.ansi_print import AnsiLogger

log = AnsiLogger("translation")

def taskdef(deps, title, new_state=None, expected_states=[],
            idemp=False, earlycheck=None):
    def decorator(taskfunc):
        taskfunc.task_deps = deps
        taskfunc.task_title = title
        taskfunc.task_newstate = None
        taskfunc.task_expected_states = expected_states
        taskfunc.task_idempotent = idemp
        taskfunc.task_earlycheck = earlycheck
        return taskfunc
    return decorator

# TODO:
# sanity-checks using states

# set of translation steps to profile
PROFILE = set([])

class Instrument(Exception):
    pass

class ProfInstrument(object):
    name = "profinstrument"
    def __init__(self, datafile, compiler):
        self.datafile = datafile
        self.compiler = compiler

    def first(self):
        return self.compiler._build()

    def probe(self, exe, args):
        env = os.environ.copy()
        env['PYPY_INSTRUMENT_COUNTERS'] = str(self.datafile)
        self.compiler.platform.execute(exe, args, env=env)

    def after(self):
        # xxx
        os._exit(0)

class TranslationDriver(SimpleTaskEngine):
    _backend_extra_options = {}

    def __init__(self, setopts=None, default_goal=None,
                 disable=[],
                 exe_name=None, extmod_name=None,
                 config=None, overrides=None):
        from rpython.config import translationoption
        self.timer = Timer()
        SimpleTaskEngine.__init__(self)

        self.log = log

        if config is None:
            config = translationoption.get_combined_translation_config(translating=True)
        # XXX patch global variable with translation config
        translationoption._GLOBAL_TRANSLATIONCONFIG = config
        self.config = config
        if overrides is not None:
            self.config.override(overrides)

        if setopts is not None:
            self.config.set(**setopts)

        self.exe_name = exe_name
        self.extmod_name = extmod_name

        self.done = {}

        self.disable(disable)

        if default_goal:
            default_goal, = self.backend_select_goals([default_goal])
            if default_goal in self._maybe_skip():
                default_goal = None

        self.default_goal = default_goal
        self.extra_goals = []
        self.exposed = []

        # expose tasks
        def expose_task(task, backend_goal=None):
            if backend_goal is None:
                backend_goal = task
            def proc():
                return self.proceed(backend_goal)
            self.exposed.append(task)
            setattr(self, task, proc)

        backend, ts = self.get_backend_and_type_system()
        for task in self.tasks:
            explicit_task = task
            if task == 'annotate':
                expose_task(task)
            else:
                task, postfix = task.split('_')
                if task in ('rtype', 'backendopt', 'llinterpret',
                            'pyjitpl'):
                    if ts:
                        if ts == postfix:
                            expose_task(task, explicit_task)
                    else:
                        expose_task(explicit_task)
                elif task in ('source', 'compile', 'run'):
                    if backend:
                        if backend == postfix:
                            expose_task(task, explicit_task)
                    elif ts:
                        if ts == 'lltype':
                            expose_task(explicit_task)
                    else:
                        expose_task(explicit_task)

    def set_extra_goals(self, goals):
        self.extra_goals = goals

    def set_backend_extra_options(self, extra_options):
        self._backend_extra_options = extra_options

    def get_info(self): # XXX more?
        d = {'backend': self.config.translation.backend}
        return d

    def get_backend_and_type_system(self):
        type_system = self.config.translation.type_system
        backend = self.config.translation.backend
        return backend, type_system

    def backend_select_goals(self, goals):
        backend, ts = self.get_backend_and_type_system()
        postfixes = [''] + ['_'+p for p in (backend, ts) if p]
        l = []
        for goal in goals:
            for postfix in postfixes:
                cand = "%s%s" % (goal, postfix)
                if cand in self.tasks:
                    new_goal = cand
                    break
            else:
                raise Exception("cannot infer complete goal from: %r" % goal)
            l.append(new_goal)
        return l

    def disable(self, to_disable):
        self._disabled = to_disable

    def _maybe_skip(self):
        maybe_skip = []
        if self._disabled:
            for goal in self.backend_select_goals(self._disabled):
                maybe_skip.extend(self._depending_on_closure(goal))
        return dict.fromkeys(maybe_skip).keys()

    def setup(self, entry_point, inputtypes, policy=None, extra={}, empty_translator=None):
        standalone = inputtypes is None
        self.standalone = standalone

        if standalone:
            # the 'argv' parameter
            inputtypes = [s_list_of_strings]
        self.inputtypes = inputtypes

        if policy is None:
            policy = annpolicy.AnnotatorPolicy()
        self.policy = policy

        self.extra = extra

        if empty_translator:
            translator = empty_translator
        else:
            translator = TranslationContext(config=self.config)

        self.entry_point = entry_point
        self.translator = translator
        self.libdef = None
        self.secondary_entrypoints = []

        if self.config.translation.secondaryentrypoints:
            for key in self.config.translation.secondaryentrypoints.split(","):
                try:
                    points = secondary_entrypoints[key]
                except KeyError:
                    raise KeyError("Entrypoint %r not found (not in %r)" %
                                   (key, secondary_entrypoints.keys()))
                self.secondary_entrypoints.extend(points)

        self.translator.driver_instrument_result = self.instrument_result

    def setup_library(self, libdef, policy=None, extra={}, empty_translator=None):
        """ Used by carbon python only. """
        self.setup(None, None, policy, extra, empty_translator)
        self.libdef = libdef
        self.secondary_entrypoints = libdef.functions

    def instrument_result(self, args):
        backend, ts = self.get_backend_and_type_system()
        if backend != 'c' or sys.platform == 'win32':
            raise Exception("instrumentation requires the c backend"
                            " and unix for now")

        datafile = udir.join('_instrument_counters')
        makeProfInstrument = lambda compiler: ProfInstrument(datafile, compiler)

        pid = os.fork()
        if pid == 0:
            # child compiling and running with instrumentation
            self.config.translation.instrument = True
            self.config.translation.instrumentctl = (makeProfInstrument,
                                                     args)
            raise Instrument
        else:
            pid, status = os.waitpid(pid, 0)
            if os.WIFEXITED(status):
                status = os.WEXITSTATUS(status)
                if status != 0:
                    raise Exception("instrumentation child failed: %d" % status)
            else:
                raise Exception("instrumentation child aborted")
            import array, struct
            n = datafile.size()//struct.calcsize('L')
            datafile = datafile.open('rb')
            counters = array.array('L')
            counters.fromfile(datafile, n)
            datafile.close()
            return counters

    def info(self, msg):
        log.info(msg)

    def _profile(self, goal, func):
        from cProfile import Profile
        from rpython.tool.lsprofcalltree import KCacheGrind
        d = {'func':func}
        prof = Profile()
        prof.runctx("res = func()", globals(), d)
        KCacheGrind(prof).output(open(goal + ".out", "w"))
        return d['res']

    def _do(self, goal, func, *args, **kwds):
        title = func.task_title
        if goal in self.done:
            self.log.info("already done: %s" % title)
            return
        else:
            self.log.info("%s..." % title)
        debug_start('translation-task')
        debug_print('starting', goal)
        self.timer.start_event(goal)
        try:
            instrument = False
            try:
                if goal in PROFILE:
                    res = self._profile(goal, func)
                else:
                    res = func()
            except Instrument:
                instrument = True
            if not func.task_idempotent:
                self.done[goal] = True
            if instrument:
                self.proceed('compile')
                assert False, 'we should not get here'
        finally:
            try:
                debug_stop('translation-task')
                self.timer.end_event(goal)
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                pass
        #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal)
        return res

    @taskdef([], "Annotating&simplifying")
    def task_annotate(self):
        """ Annotate
        """
        # includes annotation and annotatation simplifications
        translator = self.translator
        policy = self.policy
        self.log.info('with policy: %s.%s' % (policy.__class__.__module__, policy.__class__.__name__))

        annotator = translator.buildannotator(policy=policy)

        if self.secondary_entrypoints is not None:
            for func, inputtypes in self.secondary_entrypoints:
                if inputtypes == Ellipsis:
                    continue
                annotator.build_types(func, inputtypes, False)

        if self.entry_point:
            s = annotator.build_types(self.entry_point, self.inputtypes)
            translator.entry_point_graph = annotator.bookkeeper.getdesc(self.entry_point).getuniquegraph()
        else:
            s = None

        self.sanity_check_annotation()
        if self.entry_point and self.standalone and s.knowntype != int:
            raise Exception("stand-alone program entry point must return an "
                            "int (and not, e.g., None or always raise an "
                            "exception).")
        annotator.complete()
        annotator.simplify()
        return s

    def sanity_check_annotation(self):
        translator = self.translator
        irreg = query.qoutput(query.check_exceptblocks_qgen(translator))
        if irreg:
            self.log.info("Some exceptblocks seem insane")

        lost = query.qoutput(query.check_methods_qgen(translator))
        assert not lost, "lost methods, something gone wrong with the annotation of method defs"

    RTYPE = 'rtype_lltype'
    @taskdef(['annotate'], "RTyping")
    def task_rtype_lltype(self):
        """ RTyping - lltype version
        """
        rtyper = self.translator.buildrtyper()
        rtyper.specialize(dont_simplify_again=True)

    @taskdef([RTYPE], "JIT compiler generation")
    def task_pyjitpl_lltype(self):
        """ Generate bytecodes for JIT and flow the JIT helper functions
        lltype version
        """
        from rpython.jit.codewriter.policy import JitPolicy
        get_policy = self.extra.get('jitpolicy', None)
        if get_policy is None:
            self.jitpolicy = JitPolicy()
        else:
            self.jitpolicy = get_policy(self)
        #
        from rpython.jit.metainterp.warmspot import apply_jit
        apply_jit(self.translator, policy=self.jitpolicy,
                  backend_name=self.config.translation.jit_backend, inline=True)
        #
        self.log.info("the JIT compiler was generated")

    @taskdef([RTYPE], "test of the JIT on the llgraph backend")
    def task_jittest_lltype(self):
        """ Run with the JIT on top of the llgraph backend
        """
        # parent process loop: spawn a child, wait for the child to finish,
        # print a message, and restart
        from rpython.translator.goal import unixcheckpoint
        unixcheckpoint.restartable_point(auto='run')
        # load the module rpython/jit/tl/jittest.py, which you can hack at
        # and restart without needing to restart the whole translation process
        from rpython.jit.tl import jittest
        jittest.jittest(self)

    BACKENDOPT = 'backendopt_lltype'
    @taskdef([RTYPE, '??pyjitpl_lltype', '??jittest_lltype'], "lltype back-end optimisations")
    def task_backendopt_lltype(self):
        """ Run all backend optimizations - lltype version
        """
        from rpython.translator.backendopt.all import backend_optimizations
        backend_optimizations(self.translator, replace_we_are_jitted=True)

    STACKCHECKINSERTION = 'stackcheckinsertion_lltype'
    @taskdef(['?'+BACKENDOPT, RTYPE, 'annotate'], "inserting stack checks")
    def task_stackcheckinsertion_lltype(self):
        from rpython.translator.transform import insert_ll_stackcheck
        count = insert_ll_stackcheck(self.translator)
        self.log.info("inserted %d stack checks." % (count,))

    def possibly_check_for_boehm(self):
        if self.config.translation.gc == "boehm":
            from rpython.rtyper.tool.rffi_platform import configure_boehm
            from rpython.translator.platform import CompilationError
            try:
                configure_boehm(self.translator.platform)
            except CompilationError as e:
                i = 'Boehm GC not installed.  Try e.g. "translate.py --gc=minimark"'
                raise Exception(str(e) + '\n' + i)

    @taskdef([STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE, '?annotate'],
        "Creating database for generating c source",
        earlycheck = possibly_check_for_boehm)
    def task_database_c(self):
        """ Create a database for further backend generation
        """
        translator = self.translator
        if translator.annotator is not None:
            translator.frozen = True

        standalone = self.standalone
        get_gchooks = self.extra.get('get_gchooks', lambda: None)
        gchooks = get_gchooks()

        if standalone:
            from rpython.translator.c.genc import CStandaloneBuilder
            cbuilder = CStandaloneBuilder(self.translator, self.entry_point,
                                          config=self.config, gchooks=gchooks,
                      secondary_entrypoints=
                      self.secondary_entrypoints + annotated_jit_entrypoints)
        else:
            from rpython.translator.c.dlltool import CLibraryBuilder
            functions = [(self.entry_point, None)] + self.secondary_entrypoints + annotated_jit_entrypoints
            cbuilder = CLibraryBuilder(self.translator, self.entry_point,
                                       functions=functions,
                                       name='libtesting',
                                       config=self.config,
                                       gchooks=gchooks)
        if not standalone:     # xxx more messy
            cbuilder.modulename = self.extmod_name
        database = cbuilder.build_database()
        self.log.info("database for generating C source was created")
        self.cbuilder = cbuilder
        self.database = database

    @taskdef(['database_c'], "Generating c source")
    def task_source_c(self):
        """ Create C source files from the generated database
        """
        cbuilder = self.cbuilder
        database = self.database
        if self._backend_extra_options.get('c_debug_defines', False):
            defines = cbuilder.DEBUG_DEFINES
        else:
            defines = {}
        if self.exe_name is not None:
            exe_name = self.exe_name % self.get_info()
        else:
            exe_name = None
        c_source_filename = cbuilder.generate_source(database, defines,
                                                     exe_name=exe_name)
        self.log.info("written: %s" % (c_source_filename,))
        if self.config.translation.dump_static_data_info:
            from rpython.translator.tool.staticsizereport import dump_static_data_info
            targetdir = cbuilder.targetdir
            fname = dump_static_data_info(self.log, database, targetdir)
            dstname = self.compute_exe_name() + '.staticdata.info'
            shutil_copy(str(fname), str(dstname))
            self.log.info('Static data info written to %s' % dstname)

    def compute_exe_name(self, suffix=''):
        newexename = self.exe_name % self.get_info()
        if '/' not in newexename and '\\' not in newexename:
            newexename = './' + newexename
        if suffix:
            # Replace the last `.sfx` with the suffix
            newname = py.path.local(newexename.rsplit('.', 1)[0])
            newname = newname.new(basename=newname.basename + suffix)
            return newname
        return py.path.local(newexename)

    def create_exe(self):
        """ Copy the compiled executable into current directory, which is
            pypy/goal on nightly builds
        """
        if self.exe_name is not None:
            exename = self.c_entryp
            newexename = py.path.local(exename.basename)
            shutil_copy(str(exename), str(newexename))
            self.log.info("copied: %s to %s" % (exename, newexename,))
            if self.cbuilder.shared_library_name is not None:
                soname = self.cbuilder.shared_library_name
                newsoname = newexename.new(basename=soname.basename)
                shutil_copy(str(soname), str(newsoname))
                self.log.info("copied: %s to %s" % (soname, newsoname,))
                if hasattr(self.cbuilder, 'executable_name_w'):
                    # Copy pypyw.exe
                    exename_w = self.cbuilder.executable_name_w
                    newexename_w = py.path.local(exename_w.basename)
                    self.log.info("copied: %s to %s" % (exename_w, newexename_w,))
                    shutil_copy(str(exename_w), str(newexename_w))
                    # for pypy, the import library is renamed and moved to
                    # libs/python32.lib, according to the pragma in pyconfig.h
                    libname = self.config.translation.libname
                    oldlibname = soname.new(ext='lib')
                    if not libname:
                        libname = oldlibname.basename
                        libname = str(newsoname.dirpath().join(libname))
                    shutil.copyfile(str(oldlibname), libname)
                    self.log.info("copied: %s to %s" % (oldlibname, libname,))
                    # the pdb file goes in the same place as pypy(w).exe
                    ext_to_copy = ['pdb',]
                    for ext in ext_to_copy:
                        name = soname.new(ext=ext)
                        newname = newexename.new(basename=soname.basename)
                        shutil.copyfile(str(name), str(newname.new(ext=ext)))
                        self.log.info("copied: %s" % (newname,))
                    # HACK: copy libcffi-*.dll which is required for venvs
                    # At some point, we should stop doing this, and instead
                    # use the artifact from packaging the build instead
                    libffi = py.path.local.sysfind('libffi-8.dll')
                    if sys.platform == 'win32' and not libffi:
                        raise RuntimeError('could not find libffi')
                    elif libffi:
                        target = os.getcwd() + r'\libffi-8.dll'
                        if not os.path.exists(target):
                            # in tests, we can mock using windows without libffi
                            shutil.copyfile(str(libffi), target)
            self.c_entryp = newexename
        self.log.info("created: %s" % (self.c_entryp,))

    @taskdef(['source_c'], "Compiling c source")
    def task_compile_c(self):
        """ Compile the generated C code using either makefile or
        translator/platform
        """
        cbuilder = self.cbuilder
        kwds = {}
        if self.standalone and self.exe_name is not None:
            kwds['exe_name'] = self.compute_exe_name().basename
        cbuilder.compile(**kwds)

        if self.standalone:
            self.c_entryp = cbuilder.executable_name
            self.create_exe()
        else:
            self.c_entryp = cbuilder.get_entry_point()

    @taskdef([STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE], "LLInterpreting")
    def task_llinterpret_lltype(self):
        from rpython.rtyper.llinterp import LLInterpreter

        translator = self.translator
        interp = LLInterpreter(translator.rtyper)
        bk = translator.annotator.bookkeeper
        graph = bk.getdesc(self.entry_point).getuniquegraph()
        v = interp.eval_graph(graph,
                              self.extra.get('get_llinterp_args',
                                             lambda: [])())

        log.llinterpret("result -> %s" % v)

    def proceed(self, goals):
        if not goals:
            if self.default_goal:
                goals = [self.default_goal]
            else:
                self.log.info("nothing to do")
                return
        elif isinstance(goals, str):
            goals = [goals]
        goals.extend(self.extra_goals)
        goals = self.backend_select_goals(goals)
        result = self._execute(goals, task_skip = self._maybe_skip())
        self.log.info('usession directory: %s' % (udir,))
        return result

    @classmethod
    def from_targetspec(cls, targetspec_dic, config=None, args=None,
                        empty_translator=None,
                        disable=[],
                        default_goal=None):
        if args is None:
            args = []

        driver = cls(config=config, default_goal=default_goal,
                     disable=disable)
        target = targetspec_dic['target']
        spec = target(driver, args)

        try:
            entry_point, inputtypes, policy = spec
        except TypeError:
            # not a tuple at all
            entry_point = spec
            inputtypes = policy = None
        except ValueError:
            policy = None
            entry_point, inputtypes = spec

        driver.setup(entry_point, inputtypes,
                     policy=policy,
                     extra=targetspec_dic,
                     empty_translator=empty_translator)
        return driver

    def prereq_checkpt_rtype(self):
        assert 'rpython.rtyper.rmodel' not in sys.modules, (
            "cannot fork because the rtyper has already been imported")
    prereq_checkpt_rtype_lltype = prereq_checkpt_rtype

    # checkpointing support
    def _event(self, kind, goal, func):
        if kind == 'planned' and func.task_earlycheck:
            func.task_earlycheck(self)
        if kind == 'pre':
            fork_before = self.config.translation.fork_before
            if fork_before:
                fork_before, = self.backend_select_goals([fork_before])
                if not fork_before in self.done and fork_before == goal:
                    prereq = getattr(self, 'prereq_checkpt_%s' % goal, None)
                    if prereq:
                        prereq()
                    from rpython.translator.goal import unixcheckpoint
                    unixcheckpoint.restartable_point(auto='run')

if os.name == 'posix':
    def shutil_copy(src, dst):
        # this version handles the case where 'dst' is an executable
        # currently being executed
        shutil.copy(src, dst + '~')
        os.rename(dst + '~', dst)
else:
    shutil_copy = shutil.copy


from typing import Dict, List, Optional, Tuple

import numpy as np
import torch
from torch import Tensor
import torch.nn as nn
import torch.nn.functional as F
from mmcv.utils import ConfigDict
from mmdet.core import bbox2roi
from mmdet.models.builder import HEADS
from mmfewshot.detection.models.roi_heads.meta_rcnn_roi_head import MetaRCNNRoIHead

class VAE(nn.Module):

    def __init__(self,
                 in_channels: int,
                 latent_dim: int,
                 hidden_dim: int) -> None:
        super(VAE, self).__init__()

        self.latent_dim = latent_dim

        self.encoder = nn.Sequential(
            nn.Linear(in_channels, hidden_dim),
            nn.BatchNorm1d(hidden_dim),
            nn.LeakyReLU()
        )
        self.fc_mu = nn.Linear(hidden_dim, latent_dim)
        self.fc_var = nn.Linear(hidden_dim, latent_dim)

        self.decoder_input = nn.Linear(latent_dim, hidden_dim)

        self.decoder = nn.Sequential(
            nn.Linear(hidden_dim, in_channels),
            nn.BatchNorm1d(in_channels),
            nn.Sigmoid()
        )

    def encode(self, input: Tensor) -> List[Tensor]:
        result = self.encoder(input)

        mu = self.fc_mu(result)
        log_var = self.fc_var(result)

        return [mu, log_var]

    def decode(self, z: Tensor) -> Tensor:

        z = self.decoder_input(z)
        z_out = self.decoder(z)
        return z_out

    def reparameterize(self, mu: Tensor, logvar: Tensor) -> Tensor:
        std = torch.exp(0.5 * logvar)
        eps = torch.randn_like(std)
        return eps * std + mu, std + mu

    def forward(self, input: Tensor, **kwargs) -> List[Tensor]:
        mu, log_var = self.encode(input)
        z, z_inv = self.reparameterize(mu, log_var)
        z_out = self.decode(z)

        return [z_out, z_inv, input, mu, log_var]

    def loss_function(self, input, rec, mu, log_var, kld_weight=0.00025) -> dict:
        recons_loss = F.mse_loss(rec, input)

        kld_loss = torch.mean(-0.5 * torch.sum(1 +
                              log_var - mu ** 2 - log_var.exp(), dim=1), dim=0)

        loss = recons_loss + kld_weight * kld_loss

        return {'loss_vae': loss}

@HEADS.register_module()
class VFARoIHead(MetaRCNNRoIHead):

    def __init__(self, vae_dim=2048, *args, **kargs) -> None:
        super().__init__(*args, **kargs)

        self.vae = VAE(vae_dim, vae_dim, vae_dim)

    def _bbox_forward_train(self, query_feats: List[Tensor],
                            support_feats: List[Tensor],
                            sampling_results: object,
                            query_img_metas: List[Dict],
                            query_gt_bboxes: List[Tensor],
                            query_gt_labels: List[Tensor],
                            support_gt_labels: List[Tensor]) -> Dict:
        """Forward function and calculate loss for box head in training.

        Args:
            query_feats (list[Tensor]): List of query features, each item
                with shape (N, C, H, W).
            support_feats (list[Tensor]): List of support features, each item
                with shape (N, C, H, W).
            sampling_results (obj:`SamplingResult`): Sampling results.
            query_img_metas (list[dict]): List of query image info dict where
                each dict has: 'img_shape', 'scale_factor', 'flip', and may
                also contain 'filename', 'ori_shape', 'pad_shape', and
                'img_norm_cfg'. For details on the values of these keys see
                `mmdet/datasets/pipelines/formatting.py:Collect`.
            query_gt_bboxes (list[Tensor]): Ground truth bboxes for each query
                image with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y]
                format.
            query_gt_labels (list[Tensor]): Class indices corresponding to
                each box of query images.
            support_gt_labels (list[Tensor]): Class indices corresponding to
                each box of support images.

        Returns:
            dict: Predicted results and losses.
        """
        query_rois = bbox2roi([res.bboxes for res in sampling_results])
        query_roi_feats = self.extract_query_roi_feat(query_feats, query_rois)
        support_feat = self.extract_support_feats(support_feats)[0]
        support_feat_rec, support_feat_inv, _, mu, log_var = self.vae(
            support_feat)

        bbox_targets = self.bbox_head.get_targets(sampling_results,
                                                  query_gt_bboxes,
                                                  query_gt_labels,
                                                  self.train_cfg)
        (labels, label_weights, bbox_targets, bbox_weights) = bbox_targets
        loss_bbox = {'loss_cls': [], 'loss_bbox': [], 'acc': []}
        batch_size = len(query_img_metas)
        num_sample_per_imge = query_roi_feats.size(0) // batch_size
        bbox_results = None
        for img_id in range(batch_size):
            start = img_id * num_sample_per_imge
            end = (img_id + 1) * num_sample_per_imge
            # class agnostic aggregation
            # random_index = np.random.choice(
            #     range(query_gt_labels[img_id].size(0)))
            # random_query_label = query_gt_labels[img_id][random_index]
            random_index = np.random.choice(
                range(len(support_gt_labels)))
            random_query_label = support_gt_labels[random_index]
            for i in range(support_feat.size(0)):
                if support_gt_labels[i] == random_query_label:
                    bbox_results = self._bbox_forward(
                        query_roi_feats[start:end],
                        support_feat_inv[i].sigmoid().unsqueeze(0))
                    single_loss_bbox = self.bbox_head.loss(
                        bbox_results['cls_score'], bbox_results['bbox_pred'],
                        query_rois[start:end], labels[start:end],
                        label_weights[start:end], bbox_targets[start:end],
                        bbox_weights[start:end])
                    for key in single_loss_bbox.keys():
                        loss_bbox[key].append(single_loss_bbox[key])
        if bbox_results is not None:
            for key in loss_bbox.keys():
                if key == 'acc':
                    loss_bbox[key] = torch.cat(loss_bbox['acc']).mean()
                else:
                    loss_bbox[key] = torch.stack(
                        loss_bbox[key]).sum() / batch_size

        # meta classification loss
        if self.bbox_head.with_meta_cls_loss:
            # input support feature classification
            meta_cls_score = self.bbox_head.forward_meta_cls(support_feat_rec)
            meta_cls_labels = torch.cat(support_gt_labels)
            loss_meta_cls = self.bbox_head.loss_meta(
                meta_cls_score, meta_cls_labels,
                torch.ones_like(meta_cls_labels))
            loss_bbox.update(loss_meta_cls)

        loss_vae = self.vae.loss_function(
            support_feat, support_feat_rec, mu, log_var)
        loss_bbox.update(loss_vae)

        bbox_results.update(loss_bbox=loss_bbox)
        return bbox_results

    def _bbox_forward(self, query_roi_feats: Tensor,
                      support_roi_feats: Tensor) -> Dict:
        """Box head forward function used in both training and testing.

        Args:
            query_roi_feats (Tensor): Query roi features with shape (N, C).
            support_roi_feats (Tensor): Support features with shape (1, C).

        Returns:
             dict: A dictionary of predicted results.
        """
        # feature aggregation
        roi_feats = self.aggregation_layer(
            query_feat=query_roi_feats.unsqueeze(-1).unsqueeze(-1),
            support_feat=support_roi_feats.view(1, -1, 1, 1))[0]
        cls_score, bbox_pred = self.bbox_head(
            roi_feats.squeeze(-1).squeeze(-1), query_roi_feats)
        bbox_results = dict(cls_score=cls_score, bbox_pred=bbox_pred)
        return bbox_results

    def simple_test_bboxes(
            self,
            query_feats: List[Tensor],
            support_feats_dict: Dict,
            query_img_metas: List[Dict],
            proposals: List[Tensor],
            rcnn_test_cfg: ConfigDict,
            rescale: bool = False) -> Tuple[List[Tensor], List[Tensor]]:
        """Test only det bboxes without augmentation.

        Args:
            query_feats (list[Tensor]): Features of query image,
                each item with shape (N, C, H, W).
            support_feats_dict (dict[int, Tensor]) Dict of support features
                used for inference only, each key is the class id and value is
                the support template features with shape (1, C).
            query_img_metas (list[dict]): list of image info dict where each
                dict has: `img_shape`, `scale_factor`, `flip`, and may also
                contain `filename`, `ori_shape`, `pad_shape`, and
                `img_norm_cfg`. For details on the values of these keys see
                :class:`mmdet.datasets.pipelines.Collect`.
            proposals (list[Tensor]): Region proposals.
            rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of R-CNN.
            rescale (bool): If True, return boxes in original image space.
                Default: False.

        Returns:
            tuple[list[Tensor], list[Tensor]]: Each tensor in first list
                with shape (num_boxes, 4) and with shape (num_boxes, )
                in second list. The length of both lists should be equal
                to batch_size.
        """
        img_shapes = tuple(meta['img_shape'] for meta in query_img_metas)
        scale_factors = tuple(meta['scale_factor'] for meta in query_img_metas)

        rois = bbox2roi(proposals)

        query_roi_feats = self.extract_query_roi_feat(query_feats, rois)
        cls_scores_dict, bbox_preds_dict = {}, {}
        num_classes = self.bbox_head.num_classes
        for class_id in support_feats_dict.keys():
            support_feat = support_feats_dict[class_id]
            support_feat_rec, support_feat_inv, _, mu, log_var = self.vae(
                support_feat)
            bbox_results = self._bbox_forward(
                query_roi_feats, support_feat_inv.sigmoid())
            cls_scores_dict[class_id] = \
                bbox_results['cls_score'][:, class_id:class_id + 1]
            bbox_preds_dict[class_id] = \
                bbox_results['bbox_pred'][:, class_id * 4:(class_id + 1) * 4]
            # the official code use the first class background score as final
            # background score, while this code use average of all classes'
            # background scores instead.
            if cls_scores_dict.get(num_classes, None) is None:
                cls_scores_dict[num_classes] = \
                    bbox_results['cls_score'][:, -1:]
            else:
                cls_scores_dict[num_classes] += \
                    bbox_results['cls_score'][:, -1:]
        cls_scores_dict[num_classes] /= len(support_feats_dict.keys())
        cls_scores = [
            cls_scores_dict[i] if i in cls_scores_dict.keys() else
            torch.zeros_like(cls_scores_dict[list(cls_scores_dict.keys())[0]])
            for i in range(num_classes + 1)
        ]
        bbox_preds = [
            bbox_preds_dict[i] if i in bbox_preds_dict.keys() else
            torch.zeros_like(bbox_preds_dict[list(bbox_preds_dict.keys())[0]])
            for i in range(num_classes)
        ]
        cls_score = torch.cat(cls_scores, dim=1)
        bbox_pred = torch.cat(bbox_preds, dim=1)

        # split batch bbox prediction back to each image
        num_proposals_per_img = tuple(len(p) for p in proposals)
        rois = rois.split(num_proposals_per_img, 0)
        cls_score = cls_score.split(num_proposals_per_img, 0)
        bbox_pred = bbox_pred.split(num_proposals_per_img, 0)

        # apply bbox post-processing to each image individually
        det_bboxes = []
        det_labels = []
        for i in range(len(proposals)):
            det_bbox, det_label = self.bbox_head.get_bboxes(
                rois[i],
                cls_score[i],
                bbox_pred[i],
                img_shapes[i],
                scale_factors[i],
                rescale=rescale,
                cfg=rcnn_test_cfg)
            det_bboxes.append(det_bbox)
            det_labels.append(det_label)
        return det_bboxes, det_labels


import math
import numpy as np
from sympy import*
import matplotlib.pyplot as plt

class Solver:
	def __init__(self, f, t0, y0, h, nsteps, inital_points):
		self.f = f
		self.t0 = t0
		self.y0 = y0
		self.h = h
		self.nsteps = nsteps
		self.inital_points = inital_points;
		self.coef_ab = [
			[1],
			[1],
			[3.0/2.0, 1.0/2.0],
			[23.0/12.0, -4.0/3.0, 5.0/12.0],
			[55.0/24.0, -59.0/24.0, 37.0/24.0, -3.0/8.0],
			[1901.0/720.0, -1387.0/360.0, 109.0/30.0, -637.0/360.0, 251.0/720.0],
			[4277.0/1440.0, -2641.0/480.0, 4991.0/720.0, -3649.0/720.0, 959.0/480.0, -95.0/288.0],
			[198721.0/60480.0, 18367.0/2520.0, 235183.0/20160.0, 10754.0/945.0, 135713.0/20160.0, 5603.0/2520.0, 19087.0/60480.0],
			[16083.0/4480.0, 1152169.0/120960.0, 242653.0/13440.0, 296053.0/13440.0, 2102243.0/120960.0, 115747.0/13440.0, 32863.0/13440.0, 5257.0/17280.0]
		]
		self.coef_am = [
			[1],
			[1.0/2.0, 1.0/2.0],
			[5.0/12.0, 2.0/3.0, -1.0/12.0],
			[3.0/8.0, 19.0/24.0, -5.0/24.0, 1.0/24.0],
			[251.0/720.0, 323.0/360.0, -11.0/30.0, 53.0/360.0, 19.0/720.0],
			[95.0/288.0, 1427.0/1440.0, -133.0/240.0, 241.0/720.0, -173.0/1440.0, 3.0/160.0],
			[19087.0/60480.0, 2713.0/2520.0, -15487.0/20160.0, 586.0/945.0, -6737.0/20160.0, 263.0/2520.0, -863.0/60480.0],
			[5257.0/17280.0, 139849.0/120960.0, -4511.0/4480.0, 123133.0/120960.0, -88547.0/120960.0, 1537.0/4480.0, -11351.0/120960.0, 275.0/24192.0]
		]
		self.coef_inv = [
			[1],
			[1, 1],
			[2.0/3.0, 4.0/3.0, -1.0/3.0],
			[6.0/11.0, 18.0/11.0, -9.0/11.0, 2.0/11.0],
			[12.0/25.0, 48.0/25.0, -36.0/25.0, 16.0/25.0, -3.0/25.0],
			[60.0/137.0, 300.0/137.0, -300.0/137.0, 200.0/137.0, -75.0/137.0, 12.0/137.0],
			[60.0/147.0, 360.0/147.0, -450.0/147.0, 400.0/147.0, -225.0/147.0, 72.0/147.0, -10.0/147.0]
		]

	def get_ab(self, ans, idx, order):
		value = ans[idx-1][1]
		for i in range(1, order+1):
			value += (self.h)*(self.coef_ab[order][i-1])*self.f(ans[idx-i][0], ans[idx-i][1])
		return value

	def euler(self):
		ans = []
		ans.append([self.t0, self.y0])
		t, y = self.t0, self.y0
		for i in range(1, self.nsteps+1):
			y = y + self.h*self.f(t, y)
			t = t + self.h
			ans.append([t, y])

		return ans

	def inverse_euler(self):
		ans = []
		ans.append([self.t0, self.y0])
		t, y = self.t0, self.y0
		for i in range(1, self.nsteps+1):
			k = y + self.h*self.f(t, y)
			y = y + self.h*self.f(t + self.h, k)
			t = t + self.h
			ans.append([t, y])

		return ans

	def improved_euler(self):
		ans = []
		ans.append([self.t0, self.y0])
		t, y = self.t0, self.y0
		for i in range(1, self.nsteps+1):
			k = y + self.h*self.f(t, y)
			y = y + 0.5*self.h*(self.f(t + self.h, k) + self.f(t, y))
			t = t + self.h
			ans.append([t, y])

		return ans

	def runge_kutta(self):
		ans = []
		t, y = self.t0, self.y0
		ans.append([t, y])
		for i in range(1, self.nsteps+1):
			k1 = f(t, y)
			k2 = f(t + 0.5*h, y + 0.5*h*k1)
			k3 = f(t + 0.5*h, y + 0.5*h*k2)
			k4 = f(t + h, y + h*k3)
			y = y + h*(k1 + 2*k2 + 2*k3 + k4)/6
			t = t + h
			ans.append([t, y])

		return ans

	def adam_bashforth_by_method(self, order, method):
		if method == 'euler':
			ans = self.euler()
		elif method == 'inverse euler':
			ans = self.inverse_euler()
		elif method == 'improved euler':
			ans = self.improved_euler()
		elif method == 'runge kutta':
			ans = self.runge_kutta()
		elif method == 'list':
			ans = self.inital_points

		h, f = self.h, self.f
		for i in range(order, self.nsteps+1):
			if len(ans) == i:
				ans.append([0, 0])

			ans[i][1] = self.get_ab(ans, i, order)
			ans[i][0] = ans[i-1][0] + h

		return ans

	def get_am(self, ans, idx, order):
		value = ans[idx-1][1]
		ans[idx][1] = self.get_ab(ans, idx, order)
		ans[idx][0] = ans[idx-1][0] + self.h
		for i in range(0, order+1):
			value += self.h*self.coef_am[order][i]*self.f(ans[idx-i][0], ans[idx-i][1])
		return value

	def adam_multon_by_method(self, order, method):
		if method == 'euler':
			ans = self.euler()
		elif method == 'inverse euler':
			ans = self.inverse_euler()
		elif method == 'improved euler':
			ans = self.improved_euler()
		elif method == 'runge kutta':
			ans = self.runge_kutta()
		elif method == 'list':
			ans = self.inital_points

		h, f = self.h, self.f
		for i in range(order, self.nsteps+1):
			if len(ans) == i:
				ans.append([0, 0])

			ans[i][1] = self.get_am(ans, i, order)
			ans[i][0] = ans[i-1][0] + h

		return ans

	def	get_inv(self, ans, idx, order):
		ans[idx][1] = self.get_ab(ans, idx, order)
		ans[idx][0] = ans[idx-1][0] + self.h
		value = self.coef_inv[order][0]*self.h*self.f(ans[idx][0], ans[idx][1])
		for i in range (1, order+1):
			value += self.coef_inv[order][i]*ans[idx-i][1]

		return value

	def backward_diff(self, order, method):
		if method == 'euler':
			ans = self.euler()
		elif method == 'inverse euler':
			ans = self.inverse_euler()
		elif method == 'improved euler':
			ans = self.improved_euler()
		elif method == 'runge kutta':
			ans = self.runge_kutta()
		elif method == 'list':
			ans = self.inital_points

		h, f = self.h, self.f
		for i in range(order, self.nsteps+1):
			if len(ans) == i:
				ans.append([0, 0])

			ans[i][1] = self.get_inv(ans, i, order)
			ans[i][0] = ans[i-1][0] + h

		return ans

#Main part of the code
#We wish to find an approximate solution to the equation dy/dt = f(t, y)

f = open("in.txt")
for line in f:
	entrada = line.split()
	method = entrada[0]
	ini_pts = []
	if method == 'adam_bashforth' or method == 'adam_multon' or method == 'formula_inversa':
		order = int(entrada[-1])
		expr = sympify(entrada[-2])
		t, y = symbols("t y")
		f = lambdify((t, y), expr, "numpy")
		nsteps = int(entrada[-3])
		h = float(entrada[-4])
		t0, y0 = float(entrada[-5]), 0
		for i in range(1, 1 + order):
			ini_pts.append([t0 + (i-1)*h, float(entrada[i])])
	else:
		y0, t0 = float(entrada[1]), float(entrada[2])
		h = float(entrada[3])
		nsteps = int(entrada[4])
		expr = sympify(entrada[5])
		t, y = symbols("t y")
		f = lambdify((t, y), expr, "numpy")

	solver = Solver(f, t0, y0, h, nsteps, ini_pts)
	pts = []
	if method == "euler":
		pts = solver.euler()
		print("Metodo de Euler")
	elif method == "euler_inverso":
		pts = solver.inverse_euler()
		print("Metodo de Euler Inverso")
	elif method == "euler_aprimorado":
		pts = solver.improved_euler()
		print("Metodo de Euler Aprimorado")
	elif method == "runge_kutta":
		pts = solver.runge_kutta()
		print("Metodo de Runge-Kutta")
	elif method == "adam_bashforth_by_euler":
		order = int(entrada[6])
		print("Metodo de Adam-Bashforth por Euler")
		pts = solver.adam_bashforth_by_method(order, 'euler')
	elif method == 'adam_bashforth_by_euler_inverso':
		order = int(entrada[6])
		print("Metodo de Adam-Bashforth por Euler Inverso")
		pts = solver.adam_bashforth_by_method(order, 'inverse euler')
	elif method == 'adam_bashforth_by_euler_aprimorado':
		order = int(entrada[6])
		print("Metodo de Adam-Bashforth por Euler Aprimorado")
		pts = solver.adam_bashforth_by_method(order, 'improved euler')
	elif method == 'adam_bashforth_by_runge_kutta':
		order = int(entrada[6])
		print("Metodo de Adam-Bashforth por Runge Kutta")
		pts = solver.adam_bashforth_by_method(order, 'runge kutta')
	elif method == 'adam_bashforth':
		print("Metodo de Adam-Bashforth")
		pts = solver.adam_bashforth_by_method(order, 'list')
	elif method == 'adam_multon':
		print("Metodo de Adam-Multon")
		pts = solver.adam_multon_by_method(order-1, 'list')
	elif method == 'adam_multon_by_euler':
		order = int(entrada[6])
		print("Metodo de Adam-Multon por Euler")
		pts = solver.adam_multon_by_method(order-1, 'euler')
	elif method == 'adam_multon_by_euler_inverso':
		order = int(entrada[6])
		print("Metodo de Adam-Multon por Euler Inverso")
		pts = solver.adam_multon_by_method(order-1, 'inverse euler')
	elif method == 'adam_multon_by_euler_aprimorado':
		order = int(entrada[6])
		print("Metodo de Adam-Multon por Euler Aprimorado")
		pts = solver.adam_multon_by_method(order-1, 'improved euler')
	elif method == 'adam_multon_by_runge_kutta':
		order = int(entrada[6])
		print("Metodo de Adam-Multon por Runge Kutta")
		pts = solver.adam_multon_by_method(order-1, 'runge kutta')
	elif method == 'formula_inversa':
		print("Metodo Formula Inversa de Diferenciacao")
		pts = solver.backward_diff(order-1, 'list')
	elif method == 'formula_inversa_by_euler':
		order = int(entrada[6])
		print("Metodo Formula Inversa de Diferenciacao por Euler")
		pts = solver.backward_diff(order-1, 'euler')
	elif method == 'formula_inversa_by_euler_inverso':
		order = int(entrada[6])
		print("Metodo Formula Inversa de Diferenciacao por Euler Inverso")
		pts = solver.backward_diff(order-1, 'inverse euler')
	elif method == 'formula_inversa_by_euler_aprimorado':
		order = int(entrada[6])
		print("Metodo Formula Inversa de Diferenciacao por Euler Aprimorado")
		pts = solver.backward_diff(order-1, 'improved euler')
	elif method == 'formula_inversa_by_runge_kutta':
		order = int(entrada[6])
		print("Metodo Formula Inversa de Diferenciacao por Runge Kutta")
		pts = solver.backward_diff(order-1, 'runge kutta')

	print("y(%.2f) = %.2f" %(pts[0][0], pts[0][1]))
	print("h = %.2f" %h)
	i = 0
	for [x, y] in pts:
		print("%d %.10lf" %(i, y))
		i += 1

#########################	ploting the solution	#############################
#######			comment the folowing lines to not plot solution 		   ######
	toplot = np.array(pts)
	plt.plot(toplot[:, 0], toplot[:, 1], ls = '-', color = 'black', linewidth = 1)
	plt.show()
#################################################################################

	print("\n")


from core.celery import app
from celery import Celery
import json
import subprocess
import os
from .models import Node, Offer
from django.utils import timezone
import tempfile
import redis
from .serializers import NodeSerializer, OfferSerializer
import calendar
import datetime
import requests
from api.serializers import FlatNodeSerializer
from collector.models import Node as NodeV1

pool = redis.ConnectionPool(host='redis', port=6379, db=0)
r = redis.Redis(connection_pool=pool)

@ app.task
def v2_network_online_to_redis():
    data = Node.objects.filter(online=True)
    serializer = NodeSerializer(data, many=True)
    test = json.dumps(serializer.data, default=str)

    r.set("v2_online", test)

@app.task
def v2_network_online_to_redis_flatmap():
    data = NodeV1.objects.filter(online=True)
    serializer = FlatNodeSerializer(data, many=True)
    test = json.dumps(serializer.data)
    r.set("v2_online_flatmap", test)

@ app.task
def v2_cheapest_offer():
    recently = timezone.now() - timezone.timedelta(minutes=5)
    data = Offer.objects.filter(runtime="vm",
                                updated_at__range=(recently, timezone.now())).order_by('-monthly_price_glm')
    serializer = OfferSerializer(data, many=True)
    sorted_data = json.dumps(serializer.data, default=str)

    r.set("v2_cheapest_offer", sorted_data)

@ app.task
def latest_blog_posts():
    req = requests.get(
        f"https://blog.golemproject.net/ghost/api/v3/content/posts/?key={os.environ.get('BLOG_API_KEY')}&include=tags,authors&limit=3")
    data = json.dumps(req.json())
    r.set("v2_index_blog_posts", data)

@ app.task
def v2_cheapest_provider():
    req = requests.get(
        "https://api.coingecko.com/api/v3/coins/ethereum/contract/0x7DD9c5Cba05E151C895FDe1CF355C9A1D5DA6429")
    data = req.json()
    price = data['market_data']['current_price']['usd']
    obj = Offer.objects.filter(
        runtime="vm", provider__online=True).order_by("monthly_price_glm")
    serializer = OfferSerializer(obj, many=True)
    mainnet_providers = []
    for index, provider in enumerate(serializer.data):
        if "golem.com.payment.platform.erc20-mainnet-glm.address" in provider['properties']:
            mainnet_providers.append(provider)
    sorted_pricing_and_specs = sorted(mainnet_providers, key=lambda element: (
        float(element['properties']['golem.inf.cpu.threads']), float(element['monthly_price_glm'])))
    two_cores = [{'name': 'Digital Ocean', 'img': '/do-logo.svg',
                  'usd_monthly': '15',  'bandwidth': '3', 'cores': 2, 'memory': '1', 'disk': "25", "glm": float(price) * 15}, {'name': 'Amazon Web Services', 'img': '/aws-logo.svg',
                                                                                'usd_monthly': '15.23', 'bandwidth': 'Unlimited', 'cores': 2, 'memory': '1', 'disk': "25", "glm": float(price) * 15.23}, {'name': 'Google Cloud Platform', 'img': '/gcp-logo.svg',
                                                                                'usd_monthly': '10.37', 'bandwidth': 'Unlimited', 'cores': 2, 'memory': '1', 'disk': "25", "glm": float(price) * 10.37}, {'name': 'Azure', 'img': '/azure-logo.svg',
                                                                                'usd_monthly': '15.11', 'bandwidth': '6', 'cores': 2, 'memory': '1', 'disk': "25", "glm": float(price) * 15.11}, ]
    eight_cores = [{'name': 'Digital Ocean', 'img': '/do-logo.svg',
                    'usd_monthly': '80',  'bandwidth': '6', 'cores': 8, 'memory': '16', 'disk': "320", "glm": float(price) * 80}, {'name': 'Amazon Web Services', 'img': '/aws-logo.svg',
                                                                                'usd_monthly': '121.81', 'bandwidth': 'Unlimited', 'cores': 8, 'memory': '16', 'disk': "320", "glm": float(price) * 121.81}, {'name': 'Google Cloud Platform', 'img': '/gcp-logo.svg',
                                                                                'usd_monthly': '208.47', 'bandwidth': 'Unlimited', 'cores': 8, 'memory': '32', 'disk': "320", "glm": float(price) * 208.47}, {'name': 'Azure', 'img': '/azure-logo.svg',
                                                                                'usd_monthly': '121.18', 'cores': 8, 'memory': '16', 'bandwidth': '6', 'disk': "320", "glm": float(price) * 121.18}]
    thirtytwo_cores = [{'name': 'Digital Ocean', 'img': '/do-logo.svg',
                        'usd_monthly': '640',  'bandwidth': '9', 'cores': 32, 'memory': '64', 'disk': "400", "glm": float(price) * 640}, {'name': 'Amazon Web Services', 'img': '/aws-logo.svg',
                                                                                'usd_monthly': '834.24', 'bandwidth': 'Unlimited', 'cores': 32, 'memory': '64', 'disk': "400", "glm": float(price) * 834.24}, {'name': 'Google Cloud Platform', 'img': '/gcp-logo.svg',
                                                                                'usd_monthly': '746.04', 'bandwidth': 'Unlimited', 'cores': 32, 'memory': '64', 'disk': "400", "glm": float(price) * 746.04}, {'name': 'Azure', 'img': '/azure-logo.svg',
                                                                                'usd_monthly': '1310.13', 'bandwidth': '1', 'cores': 32, 'memory': '64', 'disk': "256", "glm": float(price) * 1310.13}, ]
    sixtyfour_cores = [{'name': 'Digital Ocean', 'img': '/do-logo.svg',
                        'usd_monthly': '1200',  'bandwidth': '9', 'cores': 40, 'memory': '160', 'disk': "500", "glm": float(price) * 1200}, {'name': 'Amazon Web Services', 'img': '/aws-logo.svg',
                                                                                'usd_monthly': '1638.48', 'bandwidth': 'Unlimited', 'cores': 64, 'memory': '64', 'disk': "500", "glm": float(price) * 1638.48}, {'name': 'Google Cloud Platform', 'img': '/gcp-logo.svg',
                                                                                'usd_monthly': '1914.62', 'bandwidth': 'Unlimited', 'cores': 60, 'memory': '240', 'disk': "500", "glm": float(price) * 1914.62}, {'name': 'Azure', 'img': '/azure-logo.svg',
                                                                                'usd_monthly': '2688.37', 'bandwidth': '1', 'cores': 64, 'memory': '256', 'disk': "512", "glm": float(price) * 2688.37}, ]
    for obj in sorted_pricing_and_specs:
        provider = {}
        provider['name'] = "Golem Network"
        provider['node_id'] = obj['properties']['id']
        provider['img'] = "/golem.png"
        provider['usd_monthly'] = float(
            price) * float(obj['monthly_price_glm'])
        provider['cores'] = float(
            obj['properties']['golem.inf.cpu.threads'])
        provider['memory'] = float(obj['properties']['golem.inf.mem.gib'])
        provider['bandwidth'] = "Unlimited"
        provider['disk'] = float(
            obj['properties']['golem.inf.storage.gib'])
        provider['glm'] = float(obj['monthly_price_glm'])
        if float(obj['properties']['golem.inf.cpu.threads']) == 2 and len(two_cores) == 4:
            two_cores.append(provider)
        elif float(obj['properties']['golem.inf.cpu.threads']) >= 2 and len(two_cores) == 4:

            two_cores.append(provider)
        if float(obj['properties']['golem.inf.cpu.threads']) == 8 and len(eight_cores) == 4:

            eight_cores.append(provider)
        elif float(obj['properties']['golem.inf.cpu.threads']) >= 8 and len(eight_cores) == 4:

            eight_cores.append(provider)
        if float(obj['properties']['golem.inf.cpu.threads']) == 32 and len(thirtytwo_cores) == 4:

            thirtytwo_cores.append(provider)
        elif float(obj['properties']['golem.inf.cpu.threads']) >= 32 and len(thirtytwo_cores) == 4:

            thirtytwo_cores.append(provider)
        if float(obj['properties']['golem.inf.cpu.threads']) == 64 and len(sixtyfour_cores) == 4:

            sixtyfour_cores.append(provider)
        elif float(obj['properties']['golem.inf.cpu.threads']) >= 64 and len(sixtyfour_cores) == 4:

            sixtyfour_cores.append(provider)

    sorted_two = sorted(two_cores, key=lambda element: (
        float(element['usd_monthly'])))
    sorted_eight = sorted(eight_cores, key=lambda element: (
        float(element['usd_monthly'])))
    sorted_thirtytwo = sorted(thirtytwo_cores, key=lambda element: (
        float(element['usd_monthly'])))
    sorted_sixtyfour = sorted(sixtyfour_cores, key=lambda element: (
        float(element['usd_monthly'])))
    data = json.dumps({'2': sorted_two, '8': sorted_eight,
                      '32': sorted_thirtytwo, '64': sorted_sixtyfour})
    r.set("v2_cheapest_provider", data)

@ app.task
def v2_offer_scraper():
    os.chdir("/stats-backend/yapapi/examples/low-level-api/v2")
    with open('data.config') as f:
        for line in f:
            command = line
    proc = subprocess.Popen(command, shell=True)
    proc.wait()
    content = r.get("offers_v2")
    serialized = json.loads(content)
    now = datetime.datetime.now()
    days_in_current_month = calendar.monthrange(
        now.year, now.month)[1]
    seconds_current_month = days_in_current_month*24*60*60
    for line in serialized:
        data = json.loads(line)
        provider = data['id']
        wallet = data['wallet']
        obj, created = Node.objects.get_or_create(node_id=provider)
        if created:
            offerobj = Offer.objects.create(properties=data, provider=obj,
                                            runtime=data['golem.runtime.name'])
            if data['golem.runtime.name'] == 'vm':
                vectors = {}
                for key, value in enumerate(data['golem.com.usage.vector']):
                    vectors[value] = key
                monthly_pricing = (data['golem.com.pricing.model.linear.coeffs'][vectors['golem.usage.duration_sec']] * seconds_current_month) + (
                    data['golem.com.pricing.model.linear.coeffs'][vectors['golem.usage.cpu_sec']] * seconds_current_month * data['golem.inf.cpu.cores']) + data['golem.com.pricing.model.linear.coeffs'][-1]
                offerobj.monthly_price_glm = monthly_pricing
                offerobj.save()
            obj.wallet = wallet
            obj.online = True
            obj.save()
        else:
            offerobj, offercreated = Offer.objects.get_or_create(
                provider=obj, runtime=data['golem.runtime.name'])
            if data['golem.runtime.name'] == 'vm':
                vectors = {}
                for key, value in enumerate(data['golem.com.usage.vector']):
                    vectors[value] = key
                monthly_pricing = (data['golem.com.pricing.model.linear.coeffs'][vectors['golem.usage.duration_sec']] * seconds_current_month) + (
                    data['golem.com.pricing.model.linear.coeffs'][vectors['golem.usage.cpu_sec']] * seconds_current_month * data['golem.inf.cpu.cores']) + data['golem.com.pricing.model.linear.coeffs'][-1]
                offerobj.monthly_price_glm = monthly_pricing
                offerobj.save()
            offerobj.properties = data
            offerobj.save()
            obj.runtime = data['golem.runtime.name']
            obj.wallet = wallet
            obj.online = True
            obj.save()
    # Find offline providers
    str1 = ''.join(serialized)
    fd, path = tempfile.mkstemp()
    try:
        with os.fdopen(fd, 'w') as tmp:
            # do stuff with temp file
            tmp.write(str1)
            online_nodes = Node.objects.filter(online=True)
            for node in online_nodes:
                if not node.node_id in str1:
                    node.online = False
                    node.computing_now = False
                    node.save(update_fields=[
                              'online', 'computing_now'])
    finally:
        os.remove(path)

@ app.task
def v2_offer_scraper_hybrid_testnet():
    os.chdir("/stats-backend/yapapi/examples/low-level-api/hybrid")
    proc = subprocess.Popen(
        'export YAGNA_APPKEY=$(yagna app-key list --json | jq -r .[0].key) && python3 list-offers-testnet.py', shell=True)
    proc.wait()
    content = r.get("v2_offers_hybrid_testnet")

    serialized = json.loads(content)
    now = datetime.datetime.now()
    days_in_current_month = calendar.monthrange(
        now.year, now.month)[1]
    seconds_current_month = days_in_current_month*24*60*60
    nodes_to_create = []
    nodes_to_update = []
    offers_to_create = []
    offer_to_update = []
    offline_nodes = set(Node.objects.filter(
        online=True).values_list('node_id', flat=True))

    for line in serialized:
        data = json.loads(line)
        provider = data['id']
        wallet = data['wallet']
        obj, created = Node.objects.get_or_create(node_id=provider)
        if created:
            if data['golem.runtime.name'] == 'vm':
                vectors = {}
                for key, value in enumerate(data['golem.com.usage.vector']):
                    vectors[value] = key
                monthly_pricing = (data['golem.com.pricing.model.linear.coeffs'][vectors['golem.usage.duration_sec']] * seconds_current_month) + (
                    data['golem.com.pricing.model.linear.coeffs'][vectors['golem.usage.cpu_sec']] * seconds_current_month * data['golem.inf.cpu.cores']) + data['golem.com.pricing.model.linear.coeffs'][-1]
                offers_to_create.append(
                    Offer(properties=data, provider=obj, runtime=data['golem.runtime.name'], monthly_price_glm=monthly_pricing))
            offers_to_create.append(
                Offer(properties=data, provider=obj, runtime=data['golem.runtime.name']))
            nodeobj = Node(node_id=provider, wallet=wallet, online=True)
            nodes_to_create.append(
                nodeobj)
        else:
            offerobj, offercreated = Offer.objects.get_or_create(
                provider=obj, runtime=data['golem.runtime.name'])
            if data['golem.runtime.name'] == 'vm':
                vectors = {}
                for key, value in enumerate(data['golem.com.usage.vector']):
                    vectors[value] = key
                monthly_pricing = (data['golem.com.pricing.model.linear.coeffs'][vectors['golem.usage.duration_sec']] * seconds_current_month) + (
                    data['golem.com.pricing.model.linear.coeffs'][vectors['golem.usage.cpu_sec']] * seconds_current_month * data['golem.inf.cpu.cores']) + data['golem.com.pricing.model.linear.coeffs'][-1]
                offerobj.monthly_price_glm = monthly_pricing

            offerobj.properties = data
            offerobj.runtime = data['golem.runtime.name']
            if offercreated:
                offers_to_create.append(offerobj)
            else:
                offer_to_update.append(offerobj)
            obj.wallet = wallet
            obj.online = True
            obj.updated_at = timezone.now()
            nodes_to_update.append(obj)
        if provider in offline_nodes:
            offline_nodes.remove(provider)

    Node.objects.bulk_create(nodes_to_create)
    Node.objects.bulk_update(nodes_to_update, fields=[
        'wallet', 'online', 'updated_at', ])
    Offer.objects.bulk_create(offers_to_create)
    Offer.objects.bulk_update(offer_to_update, fields=[
        'properties', 'monthly_price_glm'])
    # mark offline nodes as offline
    Node.objects.filter(node_id__in=offline_nodes, online=True).update(
        online=False, computing_now=False, updated_at=timezone.now())


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

def get_reward_curve(agents):
    """
    Extract rewards from list of agents used in training
    :param agents: list of agents used in training
    :return: array of rewards
    """
    return np.array([agent.reward_total for agent in agents])

def choose_best(agents):
    """
    Find episode with higest reward value
    :param agents: list of agents used in training
    :return: agent with highest reward
    """
    rewards = get_reward_curve(agents)
    return agents[np.argmax(rewards)]

def moving_average(v, n):
    """
    Calculate a moving average (can be improved)
    :param v: vector of values
    :param n: number of samples across which to calculate average
    :return:
    """
    return np.convolve(v, np.ones(n)/n, 'valid'), np.array(range(n, len(v)+1))

def plot_average_reward_curve(agents, n=50):
    """
    Plot moving averge of a reward curve

    :param agents: list of agents corresponding to episodes
    :param n: number of samples across which to calculate average
    :return:
    """
    rewards = get_reward_curve(agents)
    mov_ave, episodes = moving_average(rewards, n=n)

    plt.figure()
    plt.plot(episodes, mov_ave)
    plt.xlabel('Episode')
    plt.ylabel('Average Reward for Last {:d} Episodes'.format(n))

    plt.show()

def run_network(initial_state, env, model):
    """
    Deterministically run a network after training to analyze performance

    :param initial_state: initial state of agent
    :param env: RL environment used for training
    :param model: RL model generated by training
    :return:
    """
    done = False
    obs = env.reset(initial_state=initial_state)
    while not done:
        action, _state = model.predict(obs, deterministic=True)
        obs, _, done, __ = env.step(action)

    env.agent.plot_state_history(style='segmented')

def run_network_for_shap(env, model, num_trials=100):
    """
    Run network after training to analyze with SHAP

    :param env: RL environment used for training
    :param model: RL model generated by training
    :param num_trials: number of trajectories to generate
    :return:
    """
    obs_list = []
    act_list = []
    done_list = []
    rew_list = []

    for trial in range(num_trials):
        done = False
        obs = env.reset()
        while not done:
            action, _state = model.predict(obs, deterministic=True)
            obs_list.append(obs)
            act_list.append(action)

            obs, rew, done, __ = env.step(action)
            rew_list.append(rew)
            done_list.append(done)

    return obs_list, act_list, rew_list, done_list

def run_network_stochastic(model, env, num_eps):
    """
    Stochastically run a network after training to analyze performance

    :param num_eps: number of episodes to use
    :param env: RL environment used for training
    :param model: RL model generated by training
    :return:
    """
    success_count = 0  # number of episodes within fpa tolerence and that reach target alt
    terminal_alt = []
    terminal_fpa = []
    terminal_time = []
    terminal_vel = []

    for x in range(0, num_eps):
        obs = env.reset()
        done = False
        while not done:
            action, _states = model.predict(obs, deterministic=False)
            obs, rewards, done, info = env.step(action)
        if done:
            terminal_time.append(obs[0])
            terminal_alt.append(obs[1])
            terminal_vel.append(obs[2])
            terminal_fpa.append(obs[3] * 180 / np.pi)
            if env.agent.success:
                success_count += 1

    success_percentage = success_count / num_eps
    print("success percentage ", success_percentage)

    # TODO make this into a function for post processing
    num_bins = 20
    counts_t, bins_t = np.histogram(terminal_time, bins=num_bins)
    counts_alt, bins_alt = np.histogram(terminal_alt, bins=num_bins)
    counts_vel, bins_vel = np.histogram(terminal_vel, bins=num_bins)
    counts_fpa, bins_fpa = np.histogram(terminal_fpa, bins=num_bins)
    fig, axs = plt.subplots(2, 2)
    axs[0, 0].hist(bins_t[:-1], bins_t, weights=counts_t)
    axs[0, 0].set_title('Final Time (s)')
    axs[0, 1].hist(bins_alt[:-1], bins_alt, weights=counts_alt)
    axs[0, 1].set_title('Terminal Alt (m)')
    axs[1, 0].hist(bins_vel[:-1], bins_vel, weights=counts_vel)
    axs[1, 0].set_title('Terminal Vel (m/s)')
    axs[1, 1].hist(bins_fpa[:-1], bins_fpa, weights=counts_fpa)
    axs[1, 1].set_title('Terminal FPA (deg)')
    fig.tight_layout()
    plt.show()

def run_network_save(initial_state, env, model, file = None, dir = None):
    """
    Deterministically run a network after training and save run data to npy file

    :param initial_state: initial state of agent
    :param env: RL environment used for training
    :param model: RL model generated by training
    :param file: filename for text file which contains trajectory, time, and control data
    :param dir: folder directory which contains saved run data
    :return:
    """
    done = False
    obs = env.reset(initial_state=initial_state)
    while not done:
        action, _state = model.predict(obs, deterministic=True)
        obs, _, done, __ = env.step(action)

    env.agent.save_run_data(file = file, save = initial_state, dir = dir)

def run_network_control(initial_state, env, model, save = None):
    """
    Deterministically run a network after training and plot control history and trajectory

    :param initial_state: initial state of agent
    :param env: RL environment used for training
    :param model: RL model generated by training
    :return:
    """

    done = False
    obs = env.reset(initial_state=initial_state)

    while not done:
        action, _state = model.predict(obs, deterministic=True)
        obs, _, done, __ = env.step(action)
    env.agent.plot_control(style='segmented', save =save)

def network_excel(initial_state, env, model, filename):
    """
    Save run data to an excel file

    :param initial_state: initial state of agent
    :param env: RL environment used for training
    :param model: RL model generated by training
    :param filename: name of the excel file to be saved
    :return:
    """

    obs0 = []
    obs1 = []
    obs2 = []
    obs3 = []
    rewards = []
    dones = []
    actions = []

    done = False
    obs = env.reset(initial_state=initial_state)

    reward = 0.
    obs0.append(obs[0])
    obs1.append(obs[1])
    obs2.append(obs[2])
    obs3.append(obs[3])
    dones.append(done)
    rewards.append(reward)
    while not done:
        action, _state = model.predict(obs, deterministic=True)
        obs, _, done, __ = env.step(action)
        actions.append(action)
        obs0.append(obs[0])
        obs1.append(obs[1])
        obs2.append(obs[2])
        obs3.append(obs[3])
        dones.append(done)
        rewards.append(reward)

    d = {'Time': obs0, 'Altitude': obs1, 'Velocity': obs2, 'FPA': obs3, 'reward': rewards, 'done': dones}
    df = pd.DataFrame(data=d)
    Xy = df[df['done'] == False]
    Xy = Xy[['Time', 'Altitude', 'Velocity', 'FPA']]
    Xy['Action'] = actions

    Xy.to_csv(filename+'.csv')

def run_network_success(initial_state, env, model):
    """
    Deterministically run a network after training to analyze performance

    :param initial_state: initial state of agent
    :param env: RL environment used for training
    :param model: RL model generated by training
    :return:
    """
    success = False
    ctr = 0
    done = False
    obs = env.reset(initial_state=initial_state)
    while not done:
        action, _state = model.predict(obs, deterministic=True)
        obs, _, done, __ = env.step(action)
        if done == True:
            if obs[0] <= 3000 & abs(obs[3]) * ((180 / np.pi) <= 0.25 * np.pi / 180):
                success = False
            else:
                success = True
    return success


import copy
import torch
import torch.nn as nn
from .backbone import *
import numpy as np
import torch.nn.functional as F
import thop

def ConvBNReLU(in_chann, out_chann, ks, st, p=1):
    return nn.Sequential(
        nn.Conv2d(in_chann, out_chann, kernel_size=ks, stride=st, padding=p, bias=False),
        nn.BatchNorm2d(out_chann),
        nn.ReLU(inplace=True)
    )

class Aggregation(nn.Module):
    def __init__(self, in_chann, out_chann, asy_ks=5):
        super(Aggregation, self).__init__()
        self.conv = ConvBNReLU(in_chann, out_chann, 3, 1, 1)
        self.left_asymmetric = nn.Sequential(
            nn.Conv2d(out_chann, out_chann, kernel_size=(1, asy_ks), stride=1, \
                        padding=(0, asy_ks//2), groups=out_chann, bias=True),
            nn.Conv2d(out_chann, out_chann, kernel_size=(asy_ks, 1), stride=1, \
                        padding=(asy_ks//2, 0), groups=out_chann, bias=True),
        )
        self.right_asymmetric = nn.Sequential(
            nn.Conv2d(out_chann, out_chann, kernel_size=(asy_ks, 1), stride=1, \
                        padding=(asy_ks//2, 0), groups=out_chann, bias=True),
            nn.Conv2d(out_chann, out_chann, kernel_size=(1, asy_ks), stride=1, \
                        padding=(0, asy_ks//2), groups=out_chann, bias=True),
        )

        self.bn_relu = nn.Sequential(
            nn.BatchNorm2d(out_chann),
            nn.ReLU(inplace=True)
        )

    def forward(self, x):
        x = self.conv(x)
        left = self.left_asymmetric(x)
        right = self.right_asymmetric(x)
        out = left + right
        out = self.bn_relu(out)
        return out

class DeepLabHead(nn.Module):
    def __init__(self, num_classes, last_channels, mid_channels, low_channels):
        super(DeepLabHead, self).__init__()
        self.low_process = ConvBNReLU(low_channels, 48, 1, 1, 0)
        self.mid_process = ConvBNReLU(mid_channels, 48, 1, 1, 0)

        self.mid_project = ConvBNReLU(304, 256, 3, 1, 1)
        self.classifier = nn.Sequential(
            ConvBNReLU(304, 256, 3, 1, 1),
            nn.Dropout(0.1),
            nn.Conv2d(256, num_classes,
                      kernel_size=1,
                      stride=1,
                      padding=0,
                      bias=True)
        )

    def forward(self, last_feat, mid_feat, low_feat):
        low_feat = self.low_process(low_feat)
        mid_feat = self.mid_process(mid_feat)

        last_feat = F.interpolate(last_feat, size=mid_feat.size()[2:], mode="bilinear", align_corners=True)

        mid_feat = torch.cat([last_feat, mid_feat], dim=1)
        mid_feat = self.mid_project(mid_feat)
        mid_feat = F.interpolate(mid_feat, size=low_feat.size()[2:], mode="bilinear", align_corners=True)

        out_feat = torch.cat([mid_feat, low_feat], dim=1)
        out = self.classifier(out_feat)
        return out

class CPNet(nn.Module):
    def __init__(self, num_classes, input_channels=512,
                 prior_channels=512, prior_size=(40, 40), backend="resnet34", pretrained=True):
        super(CPNet, self).__init__()
        self.prior_size = np.prod(prior_size)
        self.num_classes = num_classes
        self.prior_channels = prior_channels

        self.backbone = eval(backend)(pretrained=pretrained) # backbone
        self.aggregation = Aggregation(input_channels, prior_channels, 11) # 特征聚合，丰富特征的上下文信息

        self.prior_conv = nn.Sequential(
            nn.Conv2d(prior_channels, self.prior_size, kernel_size=1, stride=1, bias=True),
            # nn.BatchNorm2d(self.prior_size)
        )

        self.intra_conv = ConvBNReLU(prior_channels, prior_channels, 1, 1, 0)
        self.inter_conv = ConvBNReLU(prior_channels, prior_channels, 1, 1, 0)

        self.post_process = nn.Sequential(
            ConvBNReLU(input_channels + prior_channels*2, 256, 1, 1, 0),
            ConvBNReLU(256, 256, 3, 1, 1) # prior_channels
        )

        # without deeplab
        self.head = nn.Sequential(
            ConvBNReLU(256, 256, 3, 1, 1), # prior_channels
            nn.Dropout(0.1),
            nn.Conv2d(256, num_classes, 1, 1, bias=True)
        )

        # with deeplab
        '''self.deeplab_head = DeepLabHead(num_classes, 256, 128, 64)'''

    def _reinit(self, input_size):
        input_size = input_size/16
        self.prior_size = int(np.prod(input_size))
        self.prior_conv = nn.Sequential(
            nn.Conv2d(self.prior_channels, self.prior_size, kernel_size=1, stride=1, bias=True),
        )

    def forward(self, x):
        feat, feat_2, feat_1 = self.backbone(x)
        h, w = feat.size()[2:]

        value = self.aggregation(feat)
        context_proir_map = self.prior_conv(value)

        context_proir_map = context_proir_map.view(context_proir_map.size()[0], \
                                                    -1, self.prior_size).permute(0, 2, 1)
        intra_context_proir_map = torch.sigmoid(context_proir_map) # [bs, 40*40, 40*40], 类内
        inter_context_prior_map = 1 - context_proir_map # 类间

        value = value.view(value.size()[0], value.size()[1], -1).permute(0, 2, 1).contiguous() # [bs, 512, 40*40]==>[bs, 40*40, 512]
        intra_context_proir_map = F.softmax(intra_context_proir_map, dim=-1)
        intra_context = torch.matmul(intra_context_proir_map, value) # [bs, 40*40, 512] # 利用类内全局特征更新每一个特征
        # intra_context = intra_context.div(self.prior_size)
        intra_context = intra_context.permute(0, 2, 1).contiguous()
        intra_context = intra_context.view(intra_context.size(0), self.prior_channels, h, w)
        intra_context = self.intra_conv(intra_context)

        inter_context_prior_map = F.softmax(inter_context_prior_map, dim=-1)
        inter_context = torch.matmul(inter_context_prior_map, value)
        # inter_context = inter_context.div(self.prior_size)
        inter_context = inter_context.permute(0, 2, 1).contiguous()
        inter_context = inter_context.view(inter_context.size(0), self.prior_channels, h, w)
        inter_context = self.inter_conv(inter_context)

        out = torch.cat([feat, intra_context, inter_context], dim=1)
        out = self.post_process(out)

        # without deeplab
        seg_out = self.head(out)
        seg_out = F.interpolate(seg_out, size=(x.size()[2], x.size()[3]), mode="bilinear", align_corners=True)

        # with deeplab
        '''seg_out = self.deeplab_head(out, feat_2, feat_1)
        seg_out = F.interpolate(seg_out, size=x.size()[2:], mode="bilinear", align_corners=True)'''

        if self.training:
            return seg_out, intra_context_proir_map
        return seg_out

from utils.utils import get_model_infos
@get_model_infos
def cpnet(num_classes, backend="resnet34", pretrained=False):
    model = CPNet(num_classes, backend=backend, pretrained=pretrained)
    return model

if __name__ == "__main__":
    model = CPNet(20)
    inputs = torch.randn(1, 3, 640, 640)
    seg_out, context_map = model(inputs)
    print("segout: ", seg_out.size(), ' context_map siz: ', context_map.size())

    # labels = torch.randint(0, 20, (1, 640, 640)).long()
    # model._get_loss(context_map, labels, [80, 80])

    '''model = cpnet_resnet34(4, pretrained=False)
    feat = torch.randn(1, 3, 640, 640)
    out, context_proir_map = model(feat)
    print(out.size(), " context_proir_map size: ", context_proir_map.size())'''