import pytz
import logging
from datetime import datetime, timedelta

from odoo import fields, models, api, _
from odoo.exceptions import UserError, ValidationError

_logger = logging.getLogger(__name__)

class MrpDistributeTimesheetLine(models.TransientModel):
    _name = 'mrp.distribute.timesheet.line'
    _description = 'Represents a sample of the distribution'

    project_id = fields.Many2one(
        'project.project',
        string='Project',
    )
    production_id = fields.Many2one(
        'mrp.production',
        string='Production',
    )
    start_time = fields.Datetime()
    end_time = fields.Datetime()

    def _prepare_analytic_line(self, name):
        diff = self.end_time - self.start_time
        hours = diff.total_seconds() / 3600
        vals = {
            'name': name,
            'project_id': self.project_id.id,
            'production_id': self.production_id.id,
            'date_time': self.start_time,
            'unit_amount': hours,
        }
        return vals

class MrpDistributeTimesheet(models.TransientModel):
    _name = 'mrp.distribute.timesheet'
    _description = 'Distribute working time along multiple production orders'

    @api.model
    def _default_date_time(self):
        def ceil_dt(dt, delta):
            return dt + (datetime.min - dt) % delta

        return ceil_dt(fields.Datetime.now(), timedelta(minutes=-15))

    @api.model
    def _default_reason(self):
        return self.env.ref('mrp_timesheet_distribution.layout_and_wiring')

    production_ids = fields.Many2many(
        'mrp.production',
        string='Production Orders',
        readonly=True,
    )
    reason_id = fields.Many2one(
        'mrp.distribute.timesheet.reason',
        string='Reason',
        required=True,
        default=_default_reason
    )
    custom_reason = fields.Char(string='Other Reason')
    date_time = fields.Datetime(
        default=_default_date_time,
        required=True,
    )
    unit_amount = fields.Float(
        'Quantity',
        default=0.0,
        required=True,
    )
    timesheet_line_ids = fields.Many2many(
        'mrp.distribute.timesheet.line',
        string='Timesheet Lines',
        compute="_compute_timesheet_line_ids",
    )

    exclude_time = fields.Boolean()
    excluded_start_time = fields.Datetime()
    excluded_end_time = fields.Datetime()

    @api.model
    def default_get(self, fields):
        rec = super().default_get(fields)
        active_ids = self._context.get('active_ids')
        active_model = self._context.get('active_model')

        if active_model == 'mrp.production' and active_ids:
            production_ids = self.env['mrp.production'].browse(active_ids)
            rec.update({'production_ids': production_ids.ids})
        return rec

    @api.multi
    def action_reopen(self):
        return self._reopen()

    @api.multi
    def _reopen(self, id=False):
        return {
            'type': 'ir.actions.act_window',
            'view_mode': 'form',
            'view_type': 'form',
            'res_id': id or self.id,
            'res_model': self._name,
            'target': 'new',
            'context': {
                'default_model': self._name,
            },
        }

    @api.multi
    def action_distribute(self):
        self._do_distribute()

    @api.multi
    def action_distribute_continue(self):
        self._do_distribute()
        mrp_distribute_timesheet_id = self.create(
            {
                "date_time": self.timesheet_line_ids[-1].end_time,
                "reason_id": self.reason_id.id,
                "exclude_time": self.exclude_time,
                "production_ids": [(6, 0, self.production_ids.ids)],
            }
        )
        mrp_distribute_timesheet_id.onchange_date_time()
        return self._reopen(mrp_distribute_timesheet_id.id)

    @api.onchange('date_time')
    def onchange_date_time(self):
        # Convert datetime into user timezone to manipulate hours and minutes
        tz = self.env.context.get('tz') or self.env.user.tz
        date_time_tz = pytz.timezone(tz).normalize(
            pytz.utc.localize(self.date_time)
        )

        resource_calendar_id = self.env.user.resource_calendar_id
        attendance_ids = resource_calendar_id.attendance_ids.filtered(
            lambda r: r.dayofweek == str(self.date_time.weekday())
        )

        st = et = False
        for attendance_id in attendance_ids:
            if attendance_id.day_period == 'morning':
                hour, minute = divmod(float(attendance_id.hour_to) * 60, 60)
                st = date_time_tz.replace(
                    hour=round(hour), minute=round(minute), second=0
                )
            if attendance_id.day_period == 'afternoon':
                hour, minute = divmod(float(attendance_id.hour_from) * 60, 60)
                et = date_time_tz.replace(
                    hour=round(hour), minute=round(minute), second=0
                )

        # Set start and end time in user timezone
        if not st or not et:
            st = date_time_tz.replace(hour=12, minute=0, second=0)
            et = date_time_tz.replace(hour=13, minute=30, second=0)

        # Convert back data to UTC since all datetime data must be
        # stored without timezone info (means UTC)
        self.excluded_start_time = pytz.utc.normalize(st).replace(tzinfo=None)
        self.excluded_end_time = pytz.utc.normalize(et).replace(tzinfo=None)

    @api.depends(
        'date_time', 'unit_amount', 'exclude_time', 'excluded_start_time',
        'excluded_end_time'
    )
    def _compute_timesheet_line_ids(self):
        self.timesheet_line_ids.unlink()
        if self.production_ids and self.date_time and self.unit_amount:
            start = self.date_time
            end = start + timedelta(hours=self.unit_amount)

            if self.exclude_time and self.excluded_end_time > start >= self.excluded_start_time:
                start = self.excluded_end_time
                end = start + timedelta(hours=self.unit_amount)
                self._generate_timesheet_interval(start, end)
            elif self.exclude_time and end > self.excluded_start_time and start < self.excluded_end_time:
                start_delta = self.excluded_start_time - start
                self._generate_timesheet_interval(start, start + start_delta)
                self._generate_timesheet_interval(
                    self.excluded_end_time, self.excluded_end_time +
                    timedelta(hours=self.unit_amount) - start_delta
                )
            else:
                self._generate_timesheet_interval(start, end)

    def _generate_timesheet_interval(self, start, end):
        diff = (end - start) / len(self.production_ids)
        i = 0
        line_ids = self.env['mrp.distribute.timesheet.line']
        for production_id in self.production_ids:
            vals = {
                'start_time': i * diff + start,
                'end_time': (i + 1) * diff + start,
                'project_id': production_id.project_id.id,
                'production_id': production_id.id,
            }
            line_ids += self.env['mrp.distribute.timesheet.line'].create(vals)
            i += 1
        self.timesheet_line_ids += line_ids

    @api.model
    def _get_or_create_task(self, project_id, name):
        time_tracking_type = self.env.ref(
            'project_identification.time_tracking_type'
        )
        stage_done = self.env.ref(
            'project_task_default_stage.project_tt_deployment'
        )
        task_id = self.env['project.task'].search(
            [
                ('project_id', '=', project_id.id),
                ('name', '=', name),
            ],
            limit=1
        )
        if not task_id:
            vals = {
                'project_id': project_id.id,
                'type_id': time_tracking_type.id,
                'stage_id': stage_done.id,
                'name': name,
            }
            task_id = self.env['project.task'].create(vals)
        return task_id

    def _do_distribute(self):
        if not self.timesheet_line_ids:
            raise ValidationError(
                _('The number of timesheet lines cannot be 0.')
            )

        for line_id in self.timesheet_line_ids:
            name = self.reason_id.name
            if self.reason_id == self.env.ref(
                'mrp_timesheet_distribution.other'
            ):
                name = self.custom_reason

            vals_line = line_id._prepare_analytic_line(name)
            if not 'task_id' in vals_line:
                vals_line['task_id'] = self._get_or_create_task(
                    line_id.project_id, name
                ).id

            self.env['account.analytic.line'].create(vals_line)


from odoo import models, fields, api, tools
from datetime import datetime, timedelta
import json

# ***************** CREAR CIUDADES **********************

class create_city(models.TransientModel):
    _name = 'medievol.create_city'

    def _default_player(self):
        jugador = self.env['res.partner'].browse(self._context.get('active_id'))
        print(jugador.name)
        return jugador

    player = fields.Many2one('res.partner', default=_default_player, readonly=True)
    region = fields.Many2one('medievol.region')
    state = fields.Selection([('i', "Elegir región"), ('f', "Finalizar región")], default='i')
    nombre = fields.Char()

    @api.multi
    def back(self):
        if self.state == 'f':
            self.state = 'i'

        return {
            'type': 'ir.actions.act_window',
            'name': 'Creación de ciudad',
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'target': 'new'
        }

    @api.multi
    def next(self):
        if self.state == 'i':
            self.state = 'f'

        return {
            'type': 'ir.actions.act_window',
            'name': 'Creación de ciudad',
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'target': 'new'
        }

    @api.multi
    def new(self):
        c_template = self.env.ref('medievol.city1')
        c = self.env['medievol.city'].create({
            'name': self.nombre,
            'image': c_template.image,
            'poblation': 100,
            'defense': 50,
            'quarry_id': self.env.ref('medievol.quarry1').id,
            'mine_id': self.env.ref('medievol.mine1').id,
            'farm_id': self.env.ref('medievol.farm1').id,
            'castle_id': self.env.ref('medievol.castle1').id,
            'infirmary_id': self.env.ref('medievol.infirmary0').id,
            'wall_id': self.env.ref('medievol.wall0').id,
            'barracks_id': self.env.ref('medievol.barracks0').id,
            'treasury_id': self.env.ref('medievol.treasury0').id,
            'forge_id': self.env.ref('medievol.forge0').id,
            'player_id': self.player.id,
            'region_id': self.region.id
        })
        for i in [1, 2, 3, 4]:
            if i < 4:
                p = self.env['medievol.means_city'].create({
                    'cantidad': 100,
                    'city_id': c.id,
                    'means_id': self.env.ref('medievol.means' + str(i)).id
                })
            else:
                p = self.env['medievol.means_city'].create({
                    'cantidad': 50,
                    'city_id': c.id,
                    'means_id': self.env.ref('medievol.means' + str(i)).id
                })

# ***************** CREAR GUERRAS *********************************

class type_soldiers(models.TransientModel):
    _name = 'medievol.type_soldiers'

    brujo = fields.Many2one('medievol.create_wars')
    tipo = fields.Many2one('medievol.soldiers')
    quantity = fields.Float()

class create_wars(models.TransientModel):
    _name = 'medievol.create_wars'

    def _default_city(self):
        return self.env['medievol.city'].browse(self._context.get('active_id'))

    def _default_player(self):
        player = self.env['medievol.city'].browse(self._context.get('active_id')).player_id
        return player
    def _get_date(self):
        date = datetime.now()
        return fields.Datetime.to_string(date)

    def _get_date_fin(self):
        date = datetime.now() + timedelta(hours=4)
        return fields.Datetime.to_string(date)

    city = fields.Many2one('medievol.city', default=_default_city, readonly=True)
    player = fields.Many2one('res.partner', default=_default_player, readonly=True)
    fecha_ini = fields.Datetime(default=_get_date)
    fecha_fin = fields.Datetime(default=_get_date_fin)
    soldiers_wars = fields.Many2many('medievol.soldiers_wars')

    tipo = fields.Many2one('medievol.soldiers')
    def _get_attack_domain(self):
        c = self._context.get('citys')
        if c:
          c = json.loads(c)
          return [('id','in',c)]

    city_attack = fields.Many2one('medievol.city', domain=_get_attack_domain)

    wiz_soldiers = fields.One2many('medievol.type_soldiers', 'brujo')
    quantity = fields.Float()
    state = fields.Selection([('i', "Selección Soldados"), ('c', "Ciudad a Atacar"), ('f', "Finalizar")],
                             default='i')

    @api.onchange('city')
    def _onchange_soldiers(self):
        for f in self:
            return {
                'domain': {'tipo': [('id', 'in', f.city.soldiers_city_id.soldiers_id.ids)]}
            }

    @api.multi
    def send_soldiers(self):
        for f in self:
            self.env['medievol.type_soldiers'].create({
                'brujo': f.id,
                'tipo': f.tipo.id,
                'quantity': f.quantity
            })
        return {
            'type': 'ir.actions.act_window',
            'name': 'Creación de Batallas',
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'target': 'new'
        }

    @api.multi
    def exit(self):
        return

    @api.multi
    def next(self):
        if self.state == 'i':
            self.state = 'c'
        elif self.state == 'c':
            self.state = 'f'

        return {
            'type': 'ir.actions.act_window',
            'name': 'Creación de Batallas',
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'context': {'citys': json.dumps(self.env['medievol.city'].search([]).filtered(lambda r: r.id != self.city.id).ids)},
            'target': 'new'
        }

    @api.multi
    def back(self):
        if self.state == 'f':
            self.state = 'c'
        elif self.state == 'c':
            self.state = 'i'

        return {
            'type': 'ir.actions.act_window',
            'name': 'Creación de Batallas',
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'target': 'new'
        }

    @api.multi
    def new_battle(self):
        for f in self:
            soldados_cantidad = f.wiz_soldiers
            for soldiers in soldados_cantidad:
                batalla = self.env['medievol.wars'].create({})

                self.env['medievol.soldiers_wars'].create({
                    'cantidad': soldiers.quantity,
                    'soldiers_id': soldiers.tipo.id,
                    'city_id': f.city.id,
                    'city_attack_id': f.city_attack.id,
                    'batalla_id': batalla.id,
                    'player_id': f.player.id
                })

# ***************** CREAR SOLDADOS **********************

class create_soldiers(models.TransientModel):
    _name = 'medievol.create_soldiers'

    def _default_city(self):
        city = self.env['medievol.city'].browse(self._context.get('city_id'))
        return city

    city = fields.Many2one('medievol.city', default=_default_city, readonly=True)
    soldiers = fields.Many2one('medievol.soldiers')
    quantity = fields.Float()
    comprobar = fields.Boolean()
    cfoodsoldier = fields.Integer(readonly=True)
    cgoldsoldier = fields.Integer(readonly=True)
    cfoodcity = fields.Float(readonly=True)
    cgoldcity = fields.Float(readonly=True)

    state = fields.Selection([('i', "Selección Soldados"), ('c', "Gastos de Producción"), ('f', "Finalizar")],
                             default='i')

    @api.multi
    def new(self):
        c = self.env['medievol.recruit_soldiers'].create({
            'cantidad': self.quantity,
            'city_id': self.city.id,
            'soldiers_id': self.soldiers.id
        })
        encontrado = False
        soldados_ciudad = self.city.soldiers_city_id.search([])
        if len(soldados_ciudad) > 0:
            for sol in soldados_ciudad:
                if c.soldiers_id == sol.soldiers_id:
                    sol.cantidad = sol.cantidad + c.cantidad
                    encontrado = True

        if encontrado == False:
            soldier_new = self.env['medievol.soldiers_city'].create({
                'cantidad': c.cantidad,
                'city_id': c.city_id.id,
                'soldiers_id': c.soldiers_id.id
            })

        granja = self.env['medievol.farm'].search([])[0]
        tesoreria = self.env['medievol.treasury'].search([])[0]
        recursos = self.city.means_city_id
        for r in recursos:
            if r.means_id == granja.means_id:
                r.cantidad = r.cantidad - self.cfoodsoldier
            elif r.means_id == tesoreria.means_id:
                r.cantidad = r.cantidad - self.cgoldsoldier

    @api.multi
    def back(self):
        if self.state == 'f':
            self.state = 'c'
        elif self.state == 'c':
            self.state = 'i'

        return {
            'type': 'ir.actions.act_window',
            'name': 'Creación de Soldados',
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'target': 'new'
        }

    @api.multi
    def next(self):
        if self.state == 'i':
            self.state = 'c'
            soldado = self.soldiers
            self.cfoodsoldier = soldado.food * self.quantity
            self.cgoldsoldier = soldado.gold * self.quantity
            granja = self.env['medievol.farm'].search([])[0]
            tesoreria = self.env['medievol.treasury'].search([])[0]
            recursos = self.city.means_city_id
            for r in recursos:
                if r.means_id == granja.means_id:
                    self.cfoodcity = r.cantidad
                elif r.means_id == tesoreria.means_id:
                    self.cgoldcity = r.cantidad
            cantidadfood = self.cfoodcity - self.cfoodsoldier
            cantidadgold = self.cgoldcity - self.cgoldsoldier
            if cantidadfood >= 0 and cantidadgold >= 0:
                self.comprobar = True
            else:
                self.comprobar = False

        elif self.state == 'c':
            if self.comprobar:
                self.state = 'f'

        return {
            'type': 'ir.actions.act_window',
            'name': 'Creación de Soldados',
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'target': 'new'
        }


# Import keras.
import keras as kr
import tensorflow as tf
from keras.models import Sequential
from keras.datasets import mnist
from keras.models import Sequential, load_model
from keras.layers.core import Dense, Dropout, Activation
from keras.utils import np_utils
from keras.models import load_model
import sklearn.preprocessing as pre
import sys
import gzip
import numpy as np
import cv2
import matplotlib.pyplot as plt
import PIL
from PIL import Image, ImageDraw, ImageTk
import tkinter as tk
import os.path

## Function to build the model, need to have a folder called dataset on same level as the script
## .gz files must be located in here

## Canvas all taken from https://www.youtube.com/watch?v=OdDCsxfI8S0
width = 200
height = 200
center = height//2
white = (255, 255, 255)
green = (0,128,0)
ModelCreated = False

def save():
    filename = "image.png"
    image1.save(filename)

def paint(event):
    # python_green = "#476042"
    x1, y1 = (event.x - 1), (event.y - 1)
    x2, y2 = (event.x + 1), (event.y + 1)
    cv.create_oval(x1, y1, x2, y2, fill="black",width=5)
    draw.line([x1, y1, x2, y2],fill="black",width=5)

##############################################################################################

def buildModel():

    model = kr.models.Sequential()

    ## https://machinelearningmastery.com/handwritten-digit-recognition-using-convolutional-neural-networks-python-keras/
    ## Used this reference for dropout values as I was unsure why this was used
    model.add(Dense(512, input_shape=(784,)))
    model.add(Activation('relu'))
    model.add(Dropout(0.2))

    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(Dropout(0.2))

    model.add(Dense(10))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

    with gzip.open('dataset/train-images-idx3-ubyte.gz', 'rb') as f:
        train_img = f.read()

    with gzip.open('dataset/train-labels-idx1-ubyte.gz', 'rb') as f:
        train_lbl = f.read()

    train_img =  np.array(list(train_img[16:])).reshape(60000, 28, 28).astype(np.uint8)/ 255.0
    train_lbl =  np.array(list(train_lbl[ 8:])).astype(np.uint8)

    encoder = pre.LabelBinarizer()
    encoder.fit(train_lbl)
    outputs = encoder.transform(train_lbl)
    inputs = train_img.reshape(60000, 784)

    # Train the model with our inputs(Images) and outputs (Labels)
    #print("Building neural network - May take a few mins!")
    model.fit(inputs, outputs, epochs=10, batch_size=128)
    model.save('Mnist')

def CompareImage(imageFile):
    with gzip.open('dataset/train-labels-idx1-ubyte.gz', 'rb') as f:
        train_lbl = f.read()
    train_lbl =  np.array(list(train_lbl[ 8:])).astype(np.uint8)

    New_model = load_model('Mnist')
    print("According to my network your number is: ")
    encoder = pre.LabelBinarizer()
    encoder.fit(train_lbl)
    print(encoder.inverse_transform(New_model.predict(imageFile)))

def convertImage(imagefile):

    ## Really good reference for this : http://www.appstate.edu/~marshallst/GLY3455/lectures/9_Image_Processing.pdf

    ## Convert to greyscale
    im = Image.open(imagefile).convert('L')

    ## Make sure image is resized
    im= im.resize((28, 28), Image.BICUBIC)

    ## Convert to list
    im = list(im.getdata())

    # Currently everything is in bytes 0 - 255 , we want to make this 0-1
    im = [(255 - x) * 1.0 / 255.0 for x in im]

    ## need to reshape for our model, expects an array of length 1-D array of size 784
    im =  np.array(list(im)).reshape(1,784)

    print("Image successfully converted! Sending To model")

    ## Send the ready array to our build model function
    CompareImage(im)

def print_menu():

    print("-" * 15 , "Welcome to Keiths Digit Recognition Script" , 15 * "-")
    print("A. Create Model (Must do this first) " + "Model Created: " + str(ModelCreated))
    print("B. Select your own image")
    print("C. Draw your digit")
    print("D. Exit")

loop=True

## Check if model is created

while loop:
         ## While loop which will keep going until loop = False
    if os.path.isfile('Mnist'):
        ModelCreated = True

    print_menu()    ## Displays menu
    choice = input("Enter your choice [A-C]: ")
    print(choice)

    if choice == 'A':
        print("Creating Model")
        buildModel()
    elif choice == 'B':
        userInput = input("Please enter file name/path: ")
        convertImage(userInput)
    elif choice=='C':
        # Canvas taken from https://www.youtube.com/watch?v=OdDCsxfI8S0
        print("Creating canvas (X canvas off when finished and select option one and enter 'image.png')")
        root = tk.Tk()

        # Tkinter create a canvas to draw on
        cv = tk.Canvas(root, width=width, height=height, bg='white')
        cv.pack()

        # PIL create an empty image and draw object to draw on
        # memory only, not visible
        image1 = PIL.Image.new("RGB", (width, height), white)
        draw = ImageDraw.Draw(image1)

        # do the Tkinter canvas drawings (visible)
        # cv.create_line([0, center, width, center], fill='green')

        cv.pack()
        cv.bind("<B1-Motion>", paint)

        # do the PIL image/draw (in memory) drawings
        # draw.line([0, center, width, center], green)

        # PIL image can be saved as .png .jpg .gif or .bmp file (among others)
        # filename = "my_drawing.png"
        # image1.save(filename)
        button=tk.Button(text="save",command=save)
        button.pack()
        root.mainloop()
    elif choice=='D':
        print("Exit")

        ## You can add your code or functions here
        loop=False # This will make the while loop to end as not value of loop is set to False
    else:
        # Any integer inputs other than values 1-5 we print an error message
        print("Wrong option selection. Enter any key to try again..")


import logging
from typing import Dict, List
import torch
from torch import nn

from detectron2.config import configurable
from detectron2.utils.events import get_event_storage

from detectron2.modeling import META_ARCH_REGISTRY
from typing import Any
from .meta_one_stage_detector import MetaProposalNetwork
from sylph.modeling.code_generator.build import build_code_generator
from detectron2.modeling.meta_arch import GeneralizedRCNN
from detectron2.layers.batch_norm import FrozenBatchNorm2d
logger = logging.getLogger(__name__)

__all__ = ["FewShotGeneralizedRCNN", "FewShotDetector"]

@META_ARCH_REGISTRY.register()
class FewShotGeneralizedRCNN(MetaProposalNetwork, GeneralizedRCNN):
    """
    Generalized R-CNN. Any models that contains the following three components:
    1. Per-image feature extraction (aka backbone)
    2. Region proposal generation
    3. Per-region feature extraction and prediction

    Inherits both Faster RCNN and a MetaProposalNetwork
    """
    @configurable
    def __init__(
        self,
        *,
        cfg,
        episodic_learning: bool,
        code_generator: nn.Module, # add code generator, optional
        **kwargs,
    ):
        """
        Args:
            backbone: a backbone module, must follow detectron2's backbone interface
            proposal_generator: a module that generates proposals using backbone features
            roi_heads: a ROI head that performs per-region computation
            pixel_mean, pixel_std: list or tuple with #channels element, representing
                the per-channel mean and std to be used to normalize the input image
            input_format: describe the meaning of channels of input. Needed by visualization
            vis_period: the period to run visualization. Set to 0 to disable.
        """
        GeneralizedRCNN.__init__(self, **kwargs)
        self.cfg = cfg
        self.episodic_learning = episodic_learning
        self.code_generator = code_generator
        self.in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES
        if self.episodic_learning:
            assert self.code_generator is not None

        self._freeze_parameters(cfg)

    def _freeze_proposal_heads(self, cfg):
        if cfg.MODEL.PROPOSAL_GENERATOR.FREEZE:
            for _, p in self.proposal_generator.named_parameters():
                p.requires_grad = False
            # convert the batch norm to frozen batch norm
            FrozenBatchNorm2d.convert_frozen_batchnorm(self.proposal_generator)
            logger.info("froze proposal_generator heads parameters")

    def _freeze_roi_heads(self, cfg):
        if cfg.MODEL.ROI_HEADS.FREEZE:
            for _, p in self.roi_heads.named_parameters():
                p.requires_grad = False
            # convert the batch norm to frozen batch norm
            FrozenBatchNorm2d.convert_frozen_batchnorm(self.roi_heads)
            logger.info("froze roi heads parameters")

    def _freeze_parameters(self, cfg):
        """
        Freeze parameters from BACKBONE, PROPOSAL_GENERATOR, and META_LEARN.CODE_GENERATOR
        """
        self._freeze_backbone_parameters(cfg)
        self._freeze_detector_head(cfg) # proposal_generator
        self._freeze_code_generator(cfg)
        self._freeze_roi_heads(cfg)
        self._freeze_proposal_heads(cfg)

    @classmethod
    def from_config(cls, cfg):
        ret = GeneralizedRCNN.from_config(cfg)
        # backbone = build_backbone(cfg)
        episodic_learning = cfg.MODEL.META_LEARN.EPISODIC_LEARNING
        input_shape = ret["backbone"].output_shape()
        in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES
        input_shapes = [input_shape[f] for f in in_features]
        strides = [input_shape[f].stride for f in in_features]
        code_generator = ( #TODO: add strides for
            build_code_generator(cfg, feature_channels=input_shapes[0].channels, feature_levels=len(in_features), strides=strides)
            if episodic_learning
            else None
        )
        ret["cfg"] = cfg
        ret["episodic_learning"] = episodic_learning
        ret["code_generator"] = code_generator
        return ret

    def forward_base_detector(self, batched_inputs: List[Dict[str, torch.Tensor]]): # rename the forward to forward_base_detector
        """
        Args:
            batched_inputs: a list, batched outputs of :class:`DatasetMapper` .
                Each item in the list contains the inputs for one image.
                For now, each item in the list is a dict that contains:

                * image: Tensor, image in (C, H, W) format.
                * instances (optional): groundtruth :class:`Instances`
                * proposals (optional): :class:`Instances`, precomputed proposals.

                Other information that's included in the original dicts, such as:

                * "height", "width" (int): the output resolution of the model, used in inference.
                  See :meth:`postprocess` for details.

        Returns:
            list[dict]:
                Each dict is the output for one input image.
                The dict contains one key "instances" whose value is a :class:`Instances`.
                The :class:`Instances` object has the following keys:
                "pred_boxes", "pred_classes", "scores", "pred_masks", "pred_keypoints"
        """
        assert not self.episodic_learning
        if not self.training:
            return GeneralizedRCNN.inference(self, batched_inputs)

        images = self.preprocess_image(batched_inputs)
        if "instances" in batched_inputs[0]:
            gt_instances = [x["instances"].to(self.device) for x in batched_inputs]
        else:
            gt_instances = None

        features = self.backbone(images.tensor)

        if self.proposal_generator is not None:
            proposals, proposal_losses = self.proposal_generator(images, features, gt_instances)
        else:
            assert "proposals" in batched_inputs[0]
            proposals = [x["proposals"].to(self.device) for x in batched_inputs]
            proposal_losses = {}

        _, detector_losses = self.roi_heads(images, features, proposals, gt_instances)
        if self.vis_period > 0:
            storage = get_event_storage()
            if storage.iter % self.vis_period == 0:
                self.visualize_training(batched_inputs, proposals)

        losses = {}
        losses.update(detector_losses)
        losses.update(proposal_losses)
        return losses

    def forward_few_shot_detector_training(self, batched_inputs: List[Dict[str, Any]]):
        assert self.training
        assert "support_set" in batched_inputs[0]
        assert "query_set" in batched_inputs[0]
        # 1. Separate batched inputs to batched support set and query set

        # bs * SHOT
        batched_inputs_support_set = [
            record for x in batched_inputs for record in x["support_set"]
        ]
        # bs
        batched_inputs_support_set_targets = [
            x["support_set_target"] for x in batched_inputs
        ]
        batched_inputs_support_set_targets = self._put_to_device(batched_inputs_support_set_targets)

        support_set_gt_instances =[x["instances"].to(self.device) for x in batched_inputs_support_set]
        # check the batch norm info, need to use all modules
        for name, module in self.backbone.named_children():
            if isinstance(module, FrozenBatchNorm2d):
                print(f"module {name}, statistics: {module.running_mean,module.running_var,module.weight,module.bias}")
        # bs * QUERY_SHOT
        batched_inputs_query_set = [
            record for x in batched_inputs for record in x["query_set"]
        ]

        # 2. Extract features
        support_set_images_lst = self.convert_batched_inputs_to_image_list(batched_inputs_support_set)
        support_set_images_feature = self._extract_backbone_features(support_set_images_lst.tensor)

        query_set_images_lst = self.convert_batched_inputs_to_image_list(batched_inputs_query_set)
        query_set_images_feature = self._extract_backbone_features(query_set_images_lst.tensor)

        # filter gt_instances
        query_set_gt_instances = self._get_gt(
            batched_inputs_query_set, support_set_targets=batched_inputs_support_set_targets
        )
        # CHANGE START
        if self.proposal_generator is not None:
            # TODO: use gt_instances before filtering in meta-learning if retraining
            proposals, proposal_losses = self.proposal_generator(query_set_images_lst, query_set_images_feature, query_set_gt_instances)
            # print(f"proposal here: {proposal_losses}")
        else:
            assert "proposals" in batched_inputs[0]
            proposals = [x["proposals"].to(self.device) for x in batched_inputs]
            proposal_losses = {}

        # Get support set class codes
        support_set_class_codes = self.code_generator([support_set_images_feature[f] for f in self.in_features], support_set_gt_instances)

        # print(f"support_set_class_codes: {support_set_class_codes}")
        # Generate detection
        _, detector_losses = self.roi_heads(query_set_images_lst, query_set_images_feature, proposals, query_set_gt_instances, support_set_class_codes, batched_inputs_support_set_targets)
        losses = {}
        losses.update(detector_losses)
        losses.update(proposal_losses)
        if "snnl" in support_set_class_codes:
            losses.update(
                {"loss_snnl": support_set_class_codes["snnl"]}
            )
        return losses

    def forward_instances(self, batched_inputs: List[Dict[str, Any]], class_codes: Dict[str, torch.Tensor], do_postprocess: bool =True):
        """
        Inputs is support set of length 1. Used only for inference. Generate class codes for all given support sets in batched_inputs
        Args:
            batched_inputs: a list of images as dict from a normal data loader.
            class_codes: a dict with "cls_conv" and "cls_bias" as torch tensor
        Returns:
            support_set_class_codes: a list of dict output from code generator
        """
        assert self.episodic_learning
        assert not self.training
        assert class_codes is not None

        images_lst = self.convert_batched_inputs_to_image_list(batched_inputs)
        images_features = self.backbone(images_lst.tensor)

        if self.proposal_generator is not None:
            proposals, _ = self.proposal_generator(images_lst, images_features, None)
        else:
            assert "proposals" in batched_inputs[0]
            proposals = [x["proposals"].to(self.device) for x in batched_inputs]
        results, _ = self.roi_heads(images=images_lst, features=images_features, proposals=proposals, targets=None, class_codes=class_codes, class_codes_target=None)

        if do_postprocess:
            assert not torch.jit.is_scripting(), "Scripting is not supported for postprocess."
            return GeneralizedRCNN._postprocess(results, batched_inputs, images_lst.image_sizes)
        else:
            return results

    def forward(self, batched_inputs: List[Dict[str, Any]]):
        """
        Forward for base detector's training and inference and meta-learning's training stage.
        Args:
            Same as in :class:`GeneralizedRCNN.forward`

        Returns:
            list[dict]:
                Each dict is the output for one input image.
                The dict contains one key "proposals" whose value is a
                :class:`Instances` with keys "proposal_boxes" and "objectness_logits".
        """

        if not self.episodic_learning:
            return self.forward_base_detector(batched_inputs)
        # episodic learning
        if self.training:
            return self.forward_few_shot_detector_training(batched_inputs)
        else:
            raise NotImplementedError(
                "Episodic learning inferrence for image and features is not supported in forward."
            )

@META_ARCH_REGISTRY.register()
class FewShotDetector(FewShotGeneralizedRCNN):
    """
    Meta Arach for few-shot detection, four forward types:
    1. training model, includes both pretraining and meta-learning training (TODO: pretraining can be done in other code base)
    2. run_tupe = "meta_learn_test_support": takes batched input from support set data loader, return class codes as dict
    3. run_tupe = "meta_learn_test_instance": takes batched inputs from query set data loader and class codes, return processed result
    4. run_type = None: a normal base detector inference
    """

    def forward(
        self, batched_inputs, class_code=None, run_type=None
    ):
        # logics for pretraining and meta-learning training
        if self.training:
            return super().forward(batched_inputs)
        # logics for testing
        if run_type is None:
            processed_results = super().forward(batched_inputs)
            return processed_results
        if run_type == "meta_learn_test_support":
            return self.forward_class_code(batched_inputs)
        if run_type == "meta_learn_test_instance":
            # class code could be None here if eval with pretrained class code
            return self.forward_instances(batched_inputs, class_code)
        raise NotImplementedError(f"not support this forward type: {run_type}, class_code: {class_code}")

    def forward_class_code(self, batched_inputs):
        assert not self.training, "Not for training"
        return super().forward_class_code(batched_inputs)

    def forward_instances(self, batched_inputs, class_codes):
        assert not self.training, "Not for training"
        return super().forward_instances(
            batched_inputs, class_codes
        )  # proposal and losses


import keras.models

import sys
import simplejson as json

model = keras.models.load_model('./modeli/prt10.43.hdf5')

#"vreme" , "temperatura", "vlaznost", "pritisak", "brzina", "oblacnost", "dan u nedelji" , "mesec"

args = sys.argv
path_json = args[1]

with open(path_json) as json_file:
    data = json.load(json_file)
    vreme = data["vreme"]
    temperatura = data["temperatura"]
    vlaznost = data["vlaznost"]
    pritisak = data["pritisak"]
    brzina = data["brzina"]
    oblacnost = data["oblacnost"]
    dan_u_nedelji = data["dan u nedelji"]
    mesec = data["mesec"]
    ulaz_u_mrezu= [[vreme, temperatura, vlaznost, pritisak, brzina, oblacnost, dan_u_nedelji, mesec]]

y_predicted = model.predict(ulaz_u_mrezu)

predvidjeno_cestica = {"PM10": str(y_predicted[0][0])}
with open('./izlazi/predvidjanje.txt', 'w') as f:
    json.dump(predvidjeno_cestica, f, ensure_ascii=False)


import os
import requests
import time
from typing import Any
from .core.enums import Chain
from .core.base import Web3Connector
from . import log

__all__ = [
    "Etherscan",
    "Etherscanner",
]

class ResponseParser:

    @staticmethod
    def parse(response: requests.Response) -> Any:
        content = response.json()
        result = content["result"]
        if "status" in content.keys():
            status = bool(int(content["status"]))
            message = content["message"]
            assert status, f"{result} -- {message}"
        else:
            raise ValueError(f"failed to get status from response {content}")
        return result

class Etherscan:

    _api_key: str
    _base_url: str
    _retry_wait_seconds: float = 1.001 # retry after this seconds
    _max_retries: int = 5

    __api_key_env_var_map = {
        Chain.ETHEREUM: "ETHERSCAN_API_KEY",
        Chain.ARBITRUM: "ARBISCAN_API_KEY",
        Chain.OPTIMISM: "OPTIMISTIC_SCAN_API_KEY",
        Chain.BINANCE: "BSCSCAN_API_KEY",
    }
    __base_url_map = {
        Chain.ETHEREUM: "https://api.etherscan.io/api?",
        Chain.ARBITRUM: "https://api.arbiscan.io/api?",
        Chain.OPTIMISM: "https://optimistic.etherscan.io/api?",
        Chain.BINANCE: "https://api.bscscan.com/api?",
    }

    def __init__(self, chain: Chain) -> None:
        api_key_env_var = self.__api_key_env_var_map[chain]
        self._api_key = os.environ[api_key_env_var]
        self._base_url = self.__base_url_map[chain]

    def get(self, **kw):
        kw["apikey"] = self._api_key
        url = self._base_url + "&".join([f"{k}={v}" for k, v in kw.items()])

        retries = 0
        while retries < self._max_retries:
            try:
                r = requests.get(url, headers={"User-Agent": ""})
                return ResponseParser.parse(r)
            except Exception as e:
                print(f"{url} failed with error:\n{e}")
                print(f"waiting for {self._retry_wait_seconds} seconds...")
                time.sleep(self._retry_wait_seconds)
                retries += 1

    def get_block_number_by_timestamp(self, timestamp: int) -> int:
        kw = dict(
            module = "block",
            action = "getblocknobytime",
            timestamp = timestamp,
            closest = "before",
            apikey = self._api_key,
        )
        return int(self.get(**kw))

class Etherscanner(Web3Connector):

    _scan: Etherscan
    _chain: Chain

    def init_scan(self, chain: Chain):
        self._scan = Etherscan(chain)

    @property
    def scan(self) -> Etherscan:
        return self._scan

    def get_abi(self, addr: str) -> list:
        """ Get abi from contract address.
        """
        return self.scan.get(module="contract", action="getabi", address=addr)


import json

from backend.util.mapper import to_dict

class Question:
    def __init__(self, question: str, options: dict, answer: str, vote: str, dataset_id: int, analysis: str, id=None):
        self.id = id
        self.question = question
        self.options = options
        self.answer = answer
        self.vote = vote
        self.dataset_id = dataset_id
        self.analysis = analysis
        self.notes = []

    def with_notes(self, notes: list):
        self.notes = notes

    def to_dict(self):
        _dict = to_dict(self)
        _dict.pop('notes', None)
        return _dict

    def to_string(self):
        options = '\n'.join([f"{key}. {self.options[key]}" for key in self.options])
        vote_answer = self.vote.split(' ')[0]
        alter_answers = self.answer if self.answer == vote_answer else f"{self.answer} or {vote_answer}"
        return f"Question: {self.question}\n" \
               f"{options}\n" \
               f"Possible answers: {alter_answers}\n"

    @staticmethod
    def parse(row):
        if row is None:
            return None
        return Question(
            row[1],
            json.loads(row[2]),
            row[3],
            row[4],
            row[5],
            row[6],
            row[0]
        )


# pylint: disable=abstract-method
"""
Input schema for Kemux.

This is the schema that is used to validate incoming messages.
"""

import dataclasses
import types

import dateutil.parser
import faust
import faust.models.fields

import kemux.data.schema.base

# pylint: disable=protected-access
@dataclasses.dataclass
class InputSchema(kemux.data.schema.base.Schema):
    """
    InputSchema Class

    The schema that is used to validate incoming messages.
    """

    @classmethod
    def construct_input_record_class(cls) -> None:
        """
        Factory used to construct the faust.Record subclass that is used to accept and validate incoming messages.

        Raises:
            ValueError: If a validator is not a valid callable.
        """

        class InputRecord(
            faust.Record,
            serializer='json',
            date_parser=dateutil.parser.parse
        ):
            """
            InputRecord Class

            The faust.Record that is used to accept and validate incoming messages.
            """

            def validate_message(self) -> None:
                """
                Validate the message using validators defined by the user.
                These validators follow the following naming pattern: "_<field_name>_validator"

                Raises:
                    ValueError: If a validator is not a valid callable.
                """

                message_data = self.__dict__  # pylint: disable=eval-used
                for field in cls.fields:
                    validator_name = f'_{field}_validator'
                    validator = getattr(
                        self.__class__,
                        validator_name
                    )
                    if not isinstance(validator, types.FunctionType):
                        raise ValueError(f'Validator: {validator_name} is not callable')
                    actual_field_value = message_data.get(field)
                    validator(actual_field_value)

            def to_dict(self) -> dict:
                """
                Convert the record to a dict.

                Returns:
                    dict: The input record as a dict.
                """

                return {
                    field: self.__dict__.get(field)
                    for field in cls.fields
                }

        for field_name, field_type in cls.fields.items():
            InputRecord.__annotations__[field_name] = field_type
            setattr(
                InputRecord,
                field_name,
                faust.models.fields.FieldDescriptor(
                    required=True,
                    exclude=False,
                    default=None,
                    type=field_type
                )
            )

        implemented_validators = [
            getattr(cls, field)
            for field in cls.__dict__
            if field.endswith('_validator')
        ]
        for validator in implemented_validators:
            if not isinstance(validator, types.FunctionType):
                raise ValueError(f'Validator: {validator} is not callable')
            setattr(InputRecord, validator.__name__, validator)

        cls.record_class = InputRecord  # type: ignore

    @classmethod
    def asdict(cls) -> dict[str, type]:
        """
        Convert the nested faust.Record subclass to a dict.

        Returns:
            dict: The nested faust.Record subclass as a dict.
        """

        return cls.record_class.asdict()


import pycurl
from urllib import parse

class LexofficeUpload:
    """
    A class for uploading invoice documents
    """

    def __init__(self, apiToken: str) -> None:
        self.apiUrl = 'https://api.lexoffice.io/v1/files'
        self.apiToken = apiToken

    def fileUpload(self, tmpFile, fileName: str):
        """
        Upload a file to a Lexoffice Account
        """

        tmpFile.seek(0)  # Go back to beginning of file to read from tmpFile after writing

        try:
            c = pycurl.Curl()
            c.setopt(c.URL, self.apiUrl)
            c.setopt(c.POST, 1)
            c.setopt(c.HTTPPOST, [
                ("file", (
                     c.FORM_FILE, tmpFile.name,
                     c.FORM_FILENAME, parse.quote(fileName)
                 )
                 ),
                ("type", "voucher")
            ])
            c.setopt(pycurl.HTTPHEADER, [
                f"Authorization: Bearer {self.apiToken}",
                "Content-Type: multipart/form-data",
                "Accept: application/json"
            ])
            response = c.perform_rs()
            c.close()
            return response

        except Exception as e:
            exit("Error: File upload failed\n\n" + str(e))


# The data updating script
# Brings in query from the file social.sql
# Working from file means the SQL code can be developed and tested
# directly within the Postgres environment and psycopg2 creates access
# to that code from within Python

# Unclear what context would require this psycopg2 setup with no real
# Python value-add. Seems a shell script could run the command
# 'psql socialmedia < social.sql' on a regular basis. Seems psycopg2
# is more useful when there is some additional manipulation of the query
# needed that is not easily done in shell

import psycopg2
print('.. ', end='')
conn = psycopg2.connect(dbname='socialmedia')
c = conn.cursor()
with open('social.sql', 'r') as q:
    query = q.read().replace('\n', ' ')
c.execute(query)
conn.commit()
print('The daily_update table is refreshed ..')
conn.close()


import os
import tweepy

api_handle = None

def get_authorization():
    TWITTER_API_KEY = os.getenv('TWITTER_API_KEY', None)
    TWITTER_API_SECRET = os.getenv('TWITTER_API_SECRET', None)

    auth = tweepy.AppAuthHandler(TWITTER_API_KEY, TWITTER_API_SECRET)
    return auth

def get_api():
    global api_handle

    if api_handle:
        return api_handle

    print("Connecting to twitter.")
    auth = get_authorization()
    api = tweepy.API(auth, wait_on_rate_limit=True,
                     wait_on_rate_limit_notify=True)
    if (not api):
        raise Exception("Cannot connect to twitter using provided credentials")

    api_handle = api
    return api


from datetime import datetime
import os.path
import vcr
from csep.utils.comcat import search

HOST = 'webservices.rm.ingv.it'

def get_datadir():
    root_dir = os.path.dirname(os.path.abspath(__file__))
    data_dir = os.path.join(root_dir, 'artifacts', 'BSI')
    return data_dir

def test_search():
    datadir = get_datadir()
    tape_file = os.path.join(datadir, 'vcr_search.yaml')
    with vcr.use_cassette(tape_file):
        # L'Aquila
        eventlist = search(starttime=datetime(2009, 4, 6, 0, 0, 0),
                           endtime=datetime(2009, 4, 7, 0, 0, 0),
                           minmagnitude=5.5, host=HOST, limit=15000, offset=0)
        event = eventlist[0]
        assert event.id == 1895389

def test_summary():
    datadir = get_datadir()
    tape_file = os.path.join(datadir, 'vcr_summary.yaml')
    with vcr.use_cassette(tape_file):
        eventlist = search(starttime=datetime(2009, 4, 6, 0, 0, 0),
                           endtime=datetime(2009, 4, 7, 0, 0, 0),
                           minmagnitude=5.5, host=HOST, limit=15000, offset=0)
        event = eventlist[0]
        cmp = '1895389 2009-04-06 01:32:40.400000 (42.342,13.380) 8.3 km M6.1'
        assert str(event) == cmp
        assert event.id == 1895389
        assert event.time == datetime(2009, 4, 6, 1, 32, 40, 400000)
        assert event.latitude == 42.342
        assert event.longitude == 13.380
        assert event.depth == 8.3
        assert event.magnitude == 6.1


import pytest
import six

from pymarketstore import jsonrpc

from unittest.mock import patch
import importlib
importlib.reload(jsonrpc)

@patch.object(jsonrpc, 'requests')
def test_jsonrpc(requests):
    requests.Session().post.return_value = 'dummy_data'

    cli = jsonrpc.MsgpackRpcClient('http://localhost:5993/rcp')
    result = cli._rpc_request('DataService.Query', a=1)
    assert result == 'dummy_data'
    resp = {
        'jsonrpc': '2.0',
        'id': 1,
        'result': {'ok': True},
    }
    assert cli._rpc_response(resp)['ok']

    del resp['result']
    resp['error'] = {
        'message': 'Error',
        'data': 'something',
    }
    with pytest.raises(Exception) as e:
        cli._rpc_response(resp)
    assert 'Error: something' in str(e)


import logging
import shutil
from datetime import datetime, timedelta
from itertools import chain
from pathlib import Path
from zipfile import ZipFile

logger = logging.getLogger(__name__)

MAX_STORAGE_DAYS = 90
FREE_SPACE_PERCENT = 10

def get_file_datetime(path):
    year, month, day = path.parent.parts[-3:]
    return datetime(int(year), int(month), int(day))

def free_storage_memory():
    total, used, free = shutil.disk_usage('./storage')
    if free / total * 100 >= FREE_SPACE_PERCENT:
        logger.info("You don't need to archive and move files. Space of storage is more than 10%")
        return

    patterns = ("*.wav", "*.mp3")
    current_date = datetime.now()
    iterable = chain.from_iterable(Path("./storage").rglob(pattern) for pattern in patterns)

    storage_dir = None
    with ZipFile("archive.zip", mode="w") as archive:
        for path in sorted(iterable, key=get_file_datetime):
            file_date = get_file_datetime(path)

            if not storage_dir:
                storage_dir = path.parent
            if storage_dir != path.parent:
                break

            if file_date + timedelta(days=MAX_STORAGE_DAYS) < current_date:
                archive.write(path, path.name)
                logger.info(f"Added {path} file to {archive.filename}")

    if not storage_dir:
        logger.warning("Storage is empty. Exit")
        return

    if archive.namelist():
        year, month, day = storage_dir.parts[-3:]
        archive_path = Path("./archive") / year / month / day
        archive_path.mkdir(parents=True, exist_ok=True)

        shutil.move(archive.filename, archive_path)
        logger.info(f"Moved {archive.filename} to {archive_path}")

        shutil.rmtree(storage_dir)
        logger.info(f"Removed {storage_dir} directory")

        free_storage_memory()

def main():
    # noinspection PyArgumentList
    logging.basicConfig(
        level=logging.DEBUG,
        format="%(asctime)s - %(levelname)s - %(message)s",
        handlers=[
            logging.FileHandler("all.log", mode="a"),
            logging.StreamHandler()
        ]
    )

    try:
        free_storage_memory()
    except Exception as exc:
        logger.exception(exc)

if __name__ == '__main__':
    main()


import os

from pypcd import pypcd
from rich.progress import track

class Bin2Pcd:
    def __init__(
        self,
        input,
        input_dims,
        output,
    ):
        self.input = input
        self.input_dims = input_dims
        self.output = output

        self.file_list = self.get_file_list()

        # make output dir
        if not os.path.exists(self.output):
            os.makedirs(self.output)

    def get_file_list(self):
        if os.path.isdir(self.input):
            file_list = os.listdir(self.input)
            file_list = [os.path.join(self.input, file_name) for file_name in file_list]
        else:
            file_list = [self.input]

        # filter file_list with .bin suffix
        file_list = [file_name for file_name in file_list if file_name[-4:] == ".bin"]

        return file_list

    def run(self):
        for bin_file in track(self.file_list):
            pcd_file = bin_file.split("/")[-1][:-4] + ".pcd"
            pcd_file = os.path.join(self.output, pcd_file)

            # check file valid by check file size
            if os.path.getsize(bin_file) == 0:
                print("file size is 0, skip: ", bin_file)
                continue

            pc = pypcd.PointCloud.from_bin(bin_file, format=self.input_dims)
            pc.save_pcd(pcd_file, compression="binary_compressed")
            print("save pcd file to: ", pcd_file)


from pathlib import Path
import os
import subprocess
import time
from urllib.parse import unquote_plus

import httpx

from api.settings import API_TOKEN

# this assumes API server is running at :5000 and that worker is also running

DELAY = 1  # seconds
MAX_RETRIES = 240

API_URL = "http://localhost:5000"
# API_URL = "http://localhost:8080"
# API_URL = "http://localhost:8080/southeastssa"

OUT_DIR = Path("/tmp/api")

if not OUT_DIR.exists():
    os.makedirs(OUT_DIR)

def poll_until_done(job_id, current=0, max=MAX_RETRIES):
    r = httpx.get(f"{API_URL}/api/reports/status/{job_id}?token={API_TOKEN}")

    if r.status_code != 200:
        raise Exception(f"Error processing request (HTTP {r.status_code}): {r.text}")

    json = r.json()
    status = json.get("status")
    progress = json.get("progress")
    message = json.get("message")
    errors = json.get("errors", [])
    task = json.get("task")

    if status == "success":
        return task, json["result"], errors

    if status == "failed":
        print(f"Failed: {json['detail']}")
        return task, None, errors

    print(f"Progress: {progress}, message: {message}, errors: {errors}")

    current += 1
    if current == max:
        print("Max retries hit, stopping...")
        return task, None, None

    time.sleep(DELAY)

    return poll_until_done(job_id, current=current, max=max)

def test_upload_file(filename):
    files = {"file": open(filename, "rb")}
    r = httpx.post(
        f"{API_URL}/api/upload?token={API_TOKEN}",
        files=files,
    )

    if r.status_code != 200:
        raise Exception(f"Error processing request (HTTP {r.status_code}): {r.text}")

    json = r.json()

    job_id = json.get("job")

    if job_id is None:
        raise Exception(json)

    return poll_until_done(job_id)

def test_create_report(uuid, datasets, field, name):
    r = httpx.post(
        f"{API_URL}/api/report?token={API_TOKEN}",
        data={"uuid": uuid, "datasets": datasets, "field": field, "name": name},
    )

    if r.status_code != 200:
        raise Exception(f"Error processing request (HTTP {r.status_code}): {r.text}")

    json = r.json()

    job_id = json.get("job")

    if job_id is None:
        raise Exception(json)

    return poll_until_done(job_id)

def download_file(url):
    r = httpx.get(f"{API_URL}{url}")

    attachment_filename = r.headers["content-disposition"].split("filename*=utf-8''")[1]
    filename = OUT_DIR / unquote_plus(attachment_filename)

    with open(filename, "wb") as out:
        out.write(r.read())

    return filename

if __name__ == "__main__":

    # name, filename = [
    #     "Balduina atropurpurea",
    #     "examples/Balduina_pop_resiliency_final.zip",
    # ]
    # name, filename = [
    #     "Rabbitsfoot",
    #     "examples/Rabbitsfott_resilience_final_SECAS_only.zip",
    # ]
    name, filename = ["Test species", "examples/SingleTest.zip"]

    task, result, errors = test_upload_file(filename)
    print(f"----------------\ntask: {task}\nresult: {result}\nerrors: {errors}\n")

    if result is not None:
        uuid = result["uuid"]

        if result.get("count") == 1:
            field = None
        else:
            # arbitrarily pick first field available
            field = list(result["fields"].keys())[0]

        # include all present datasets in analysis
        datasets = ",".join(
            dataset
            for dataset, present in result["available_datasets"].items()
            if present
        )

    task, result, errors = test_create_report(uuid, datasets, field, name)
    print(f"----------------\ntask: {task}\nresult: {result}\nerrors: {errors}\n")

    outfilename = download_file(result)

    subprocess.run(["open", outfilename])


#!/usr/bin/env python

"""
This scrapes Streak for the Cash information from ESPN and writes it to an Excel spreadsheet.
It uses BeautifulSoup 4 and xlsxwriter.

In order to use this, you will need to download bs4, lxml, and xlsxwriter.

Prop array format: [Sport, League, Prop, Active Picks %, Winner, Winner %, Loser, Loser %]

Command format: python props.py [number of days]
"""

import urllib.request
import re
import xlsxwriter
import operator
import collections
import sys
import ast
import os
from datetime import date, timedelta, datetime
from bs4 import BeautifulSoup

def _format_url(date):
    """Format ESPN link to scrape records from."""
    link = ['http://streak.espn.go.com/en/entry?date=' + date]
    print(date)
    return link[0]

def scrape_props(espn_page, allProps):
    """Scrape ESPN's pages for data."""
    url = urllib.request.urlopen(espn_page)
    soup = BeautifulSoup(url.read(), 'lxml')

    props = soup.find_all('div', attrs={'class': 'matchup-container'})

    for prop in props:
        propArray = []

        time = prop.find('span', {'class': 'startTime'})['data-locktime']
        time = time[:-4]
        format_time = datetime.strptime(time, '%B %d, %Y %I:%M:%S %p')

        sport = prop.find('div', {'class': 'sport-description'})
        if sport:
            propArray.append(sport.text)
        else:
            propArray.append('Adhoc')

        title = prop.find(
            'div', {'class': ['gamequestion', 'left']}).text.split(': ')
        propArray.append(title[0])
        propArray.append(title[1])

        overall_percentage = prop.find('div', {'class': 'progress-bar'})
        propArray.append(overall_percentage['title'].split(' ')[3])

        percentages = prop.find_all('span', {'class': 'wpw'})

        info = prop.find_all('span', {'class': 'winner'})
        temp = info[0].parent.find_all('span', {'id': 'oppAddlText'})
        [rec.extract() for rec in temp]

        temp = info[1].parent.find_all('span', {'id': 'oppAddlText'})
        [rec.extract() for rec in temp]

        if info[0].contents[0].name == 'img':
            propArray.append(info[0].parent.get_text())
            propArray.append(percentages[0].text)
            propArray.append(info[1].parent.get_text()[1:])
            propArray.append(percentages[1].text)
        else:
            propArray.append(info[1].parent.get_text())
            propArray.append(percentages[1].text)
            propArray.append(info[0].parent.get_text()[1:])
            propArray.append(percentages[0].text)

        # Prop array format:
        # [Sport, League, Prop, Active Picks %, Winner, Winner %, Loser, Loser %]
        allProps[format_time.date()].append(propArray)

def write_to_excel(allProps):
    wb = xlsxwriter.Workbook('Streak.xlsx')

    ws = wb.add_worksheet('All Props')
    #ws.set_column(0, 0, 2.29)
    #ws.set_column(1, 1, 14.14)
    #ws.set_column(5, 5, 2.29)

    date_format = wb.add_format({'num_format': 'mm/dd/yy'})
    format_percentage = wb.add_format({'num_format': '0.0"%"'})
    format_percentage2 = wb.add_format({'num_format': '0.00"%"'})

    i = 0
    for date in allProps:
        for prop in allProps[date]:
            ws.write(i, 0, date, date_format)
            ws.write(i, 1, prop[0])
            ws.write(i, 2, prop[1])
            ws.write(i, 3, prop[2])
            ws.write(i, 4, float(prop[3][:-1]), format_percentage2)
            ws.write(i, 5, prop[4])
            ws.write(i, 6, float(prop[5][:-1]), format_percentage)
            ws.write(i, 7, prop[6])
            ws.write(i, 8, float(prop[7][:-1]), format_percentage)
            i += 1
    wb.close()

def main(argv):
    allProps = collections.OrderedDict()

    for x in range(0, int(argv[0])):
        newDate = date.today() - timedelta(days=x + 1)
        allProps[newDate] = []
        scrape_props(_format_url(newDate.strftime('%Y%m%d')), allProps)

    write_to_excel(allProps)

if __name__ == '__main__':
    import time
    start = time.time()
    main(sys.argv[1:])
    print(time.time() - start, 'seconds')


''' Common pricing methods corresponding to Interest rate Instruments '''
import datetime as dt
#from collections import OrderedDict
import json
import os
import scipy as sci
import numpy as np
import pandas as pd
# import interest_rate_base as intbase
import interest_rate_dates as intdate
import interest_rate_discount_lorimier as intdisc
import curve_constructor as cb
import pyfi_filter as fi_filter

class curve_builder_lorimier(cb.curve_builder):
    ''' Class constructing applying LORIMEIR mmethod:
    methods: (3): Hilbert Space Smoothing Spline
    '''

    def __init__(self, options, alpha=0, dbg=False):
        ''' Constructor Depends options dictionary && dataframe of
            -  zero coupon yields and maturities

        '''
        if isinstance(options, str) and os.path.exists(options):
            with open(options, "r") as fp:
                init_options = json.load(fp)
            fp.close()
        elif isinstance(options, dict):
            init_options = options.copy()
        else:
            raise ValueError("Faulty -- options specification ")

        self.df = None
        self.alpha = alpha
        self.determine_instruments(init_options, init_options['data']['file'], dbg=dbg)
        super().__init__(init_options, method=3, dbg=dbg)
        self.calc_exact_method0()

    def determine_instruments(self, options, filename, dbg=False):
        ''' Loads Data Frame of acceptable instruments '''
        if 'instruments' not in options.keys() and isinstance(filename, str) and\
                os.path.exists(filename):
            df = pd.read_csv(filename, index_col=0)
        elif 'instruments' in options.keys():
            if dbg:
                print("instruments already exist")
        else:
            raise ValueError("No Instruments Found")

        if isinstance(df, pd.DataFrame) and all(df.shape) > 0:
            self.df = df.copy()
            if self.df.shape[0] < 1:
                raise ValueError("Data Frame contains only excluded items")

            # TODO Apply repeated filter concept here
            if 'filter' in options.keys() and isinstance(options['filter'], dict):
                obj = self.determine_filter_obj(options['filter'], dbg)

                ind_inc = obj.include()
                self.df = self.df[ind_inc]
                self.df['excluded'] = False
            elif 'filter' in options.keys() and isinstance(options['filter'], list):
                for itm in options['filter']:
                    obj = self.determine_filter_obj(itm, dbg)

                    ind_inc = obj.include()
                    self.df = self.df[ind_inc]
                    self.df['excluded'] = False
        else:
            raise ValueError("Empty Data Frame")

        options["instruments"] = {}

        for cnt, itm in enumerate(self.df.iterrows()):
            name = ("PRINC_STRIP" if itm[1]['Description'].find("STRIPPED PRIN") >= 0 else
                    "TREAS_BILL")

            day_count = ("30_360" if itm[1]['Description'].find("STRIPPED PRIN") >= 0 else
                         "act_360")
            name = name + str(cnt+1)
            if "Price" in self.df.columns:
                price = itm[1]['Price']

            if "maturity_date" in self.df.columns:
                maturity_date = dt.datetime.strptime(itm[1]['maturity_date'], "%m/%d/%Y")
                maturity_date = dt.datetime.strftime(maturity_date,
                                                     options["control"]["date_format"])

            options["instruments"][name] = {"type": "ZERO COUPON", "price": price,
                                            "date": maturity_date, "day_count": day_count}
    def determine_filter_obj(self, item, dbg=False):
        ''' Calculates Filter object '''
        if isinstance(item, dict) and "operator" in item.keys():
            obj = fi_filter.fixed_income_filter_logical(item, self.df, dbg=dbg)
        elif isinstance(item, dict) and "type" in item.keys() and\
                item['type'].upper() == "NUMERIC":
            obj = fi_filter.fixed_income_filter_numeric(item, self.df, dbg=dbg)
        elif isinstance(item, dict) and "type" in item.keys() and\
                item['type'].upper() == "TEXT":
            obj = fi_filter.fixed_income_filter(item, self.df, dbg=dbg)
        elif isinstance(item, dict) and "type" in item.keys() and\
                item['type'].upper() == "REPEATED_MATURITY":
            obj = fi_filter.repeated_maturity_filter(item, self.df, dbg=dbg)
            if dbg:
                print("Repeated Count %d" % (obj.get_gt_one_count()))
        else:
            if dbg:
                print(item)
            raise ValueError("Faulty Item")
        return obj

    def build_arrays(self):
        ''' Constructs DataFrame elements '''
        instrument_cnt = len(self.instruments)
        if self.cf_matrix is None:
            self.cf_matrix = np.zeros([instrument_cnt+1, instrument_cnt+1])
            self.cf_prices = np.zeros([instrument_cnt+1])

            for i, zero in enumerate(self.instruments.values()):
                self.cf_prices[i+1] = self.alpha*zero.get_maturity()*zero.get_yield()
                self.cf_matrix[0][i+1] = zero.get_maturity()
                self.cf_matrix[i+1][0] = self.alpha*zero.get_maturity()

            for i, zero_i in enumerate(self.instruments.keys()):
                for j, zero_j in enumerate(self.instruments.keys()):
                    self.cf_matrix[i+1][j+1] = self.lorimier_dot_prod(zero_i, zero_j)

        if self.results is None: # # instruments X column count
            if "columns" in self.options['control'].keys():
                rows = len(self.options['control']["columns"])
                mtrx = np.zeros([instrument_cnt, rows])
                names_sorted = sorted(self.names, key=lambda x: self.names[x])
                self.results = pd.DataFrame(mtrx, index=names_sorted,
                                            columns=self.options['control']['columns'])

            if self.dbg:
                print("Construction NP array shape %i length %i" % (self.results.shape[0],
                                                                    self.results.shape[1]))

    def load_cf_results(self):
        ''' Loads results elemt upto (but not including) zeros '''
        for key, zero in self.instruments.items():
            self.load_data_row(position=key, rate=zero.get_price(),
                               date=zero.get_maturity(),
                               typ='ZERO')

    def load_cf_matrices(self):
        ''' loads cash matrix elements (self.cf_matrix '''

    def calc_exact_method0(self):
        ''' implements eaxct method with complete NxN matrix '''
        res = False
        if isinstance(self.cf_matrix, np.ndarray) and all(self.cf_matrix.shape) > 0 and\
                isinstance(self.cf_prices, np.ndarray) and\
                self.cf_prices.shape[0] == self.cf_matrix.shape[0] == self.cf_matrix.shape[1]:
            dates_dict = self.options.copy()
            dates_dict['parameters'] = {}
            dates_dict['parameters']['alpha'] = self.alpha

            dates, _ = intdate.calc_schedule_list(
                self.cf_dates.keys(), self.options)

            beta = sci.linalg.solve(self.cf_matrix, self.cf_prices)
            if self.dbg:
                print(beta)
            dates_dict['parameters']['beta'] = beta.copy()
            dates_dict['parameters']['tau'] = []
            for itm in self.instruments.values():
                dates_dict['parameters']['tau'].append(itm.get_maturity())

            dates_dict['parameters']['dates'] = intdate.generate_schedule_dict(
                start=self.options['start_date'], period='S', count=dates,
                convention=self.options['control']['convention'],
                date_adjust=self.options['control']['date_adjust'])

            prices = [bnd.get_price() for bnd in self.instruments.values()
                      if isinstance(bnd, cb.intrate.fi_instrument)]

            self.zeros = intdisc.discount_calculator_lorimier(
                prices, data_type=3, dates=dates_dict, dbg=self.dbg)

            mapping_dict = {"zero": "zero", "maturity": "maturity", "date_diff": "date_diff",
                            "forward": "forward", "yield": "yield_hat"}

            override = bool("control" in self.options.keys() and "override" in
                            self.options['control'].keys() and
                            int(self.options['control']['override']) > 0)

            self.apply_zeros(mapping_dict, override)
            res = True
        else:
            if self.dbg:
                print("Warning -- zeros NOT calculated")
        return res

    def lorimier_dot_prod(self, h_i, h_j):
        ''' Calculates dot product of h_i, h_j  based on lorimeir definition'''
        v1 = self.instruments[h_i].get_maturity()
        v2 = self.instruments[h_j].get_maturity()

        res = v1*v2 + 0.5*min(v1, v2)**2*max(v1, v2) - min(v1, v2)**3/6.
        res = (self.alpha*res + (1.0 if v1 == v2 else 0.0))
        if self.dbg:
            print(h_i, h_j, v1, v2)

        return res


"""
Test for import machinery
"""
import unittest
import sys
import textwrap
import subprocess
import os
from PyInstaller.lib.modulegraph import modulegraph

class TestNativeImport (unittest.TestCase):
    # The tests check that Python's import statement
    # works as these tests expect.

    def importModule(self, name):
        if '.' in name:
            script = textwrap.dedent("""\
                try:
                    import %s
                except ImportError:
                    import %s
                print (%s.__name__)
            """) %(name, name.rsplit('.', 1)[0], name)
        else:
            script = textwrap.dedent("""\
                import %s
                print (%s.__name__)
            """) %(name, name)

        p = subprocess.Popen(
            [sys.executable, '-c', script],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            cwd=os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                'testpkg-relimport'),
            encoding='utf8',
        )
        data = p.communicate()[0]
        data = data.strip()

        if data.endswith(' refs]'):
            # with --with-pydebug builds
            data = data.rsplit('\n', 1)[0].strip()

        sts = p.wait()

        if sts != 0:
            print (data)
        self.assertEqual(sts, 0)
        return data

    def testRootModule(self):
        m = self.importModule('mod')
        self.assertEqual(m, 'mod')

    def testRootPkg(self):
        m = self.importModule('pkg')
        self.assertEqual(m, 'pkg')

    def testSubModule(self):
        m = self.importModule('pkg.mod')
        self.assertEqual(m, 'pkg.mod')

    # python3 always has __future__.absolute_import
    def testOldStyle(self):
        m = self.importModule('pkg.oldstyle.mod')
        self.assertEqual(m, 'mod')

    def testNewStyle(self):
        m = self.importModule('pkg.toplevel.mod')
        self.assertEqual(m, 'mod')

    def testRelativeImport(self):
        m = self.importModule('pkg.relative.mod')
        self.assertEqual(m, 'pkg.mod')

        m = self.importModule('pkg.subpkg.relative.mod')
        self.assertEqual(m, 'pkg.mod')

        m = self.importModule('pkg.subpkg.mod2.mod')
        self.assertEqual(m, 'pkg.sub2.mod')

        m = self.importModule('pkg.subpkg.relative2')
        self.assertEqual(m, 'pkg.subpkg.relative2')

class TestModuleGraphImport (unittest.TestCase):
    if not hasattr(unittest.TestCase, 'assertIsInstance'):
        def assertIsInstance(self, value, types):
            if not isinstance(value, types):
                self.fail("%r is not an instance of %r"%(value, types))

    def setUp(self):
        root = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                'testpkg-relimport')
        self.mf = modulegraph.ModuleGraph(path=[ root ] + sys.path)
        #self.mf.debug = 999
        self.script_name = os.path.join(root, 'script.py')
        self.mf.add_script(self.script_name)

    def testGraphStructure(self):

        # 1. Script to imported modules
        n = self.mf.find_node(self.script_name)
        self.assertIsInstance(n, modulegraph.Script)

        imported = ('mod', 'pkg', 'pkg.mod', 'pkg.oldstyle',
            'pkg.relative', 'pkg.toplevel', 'pkg.subpkg.relative',
            'pkg.subpkg.relative2', 'pkg.subpkg.mod2')

        for nm in imported:
            n2 = self.mf.find_node(nm)
            ed = self.mf.edgeData(n, n2)
            self.assertIsInstance(ed, modulegraph.DependencyInfo)
            self.assertEqual(ed, modulegraph.DependencyInfo(
                fromlist=False, conditional=False, function=False, tryexcept=False))

        refs = self.mf.outgoing(n)
        self.assertEqual(set(refs), set(self.mf.find_node(nm) for nm in imported))

        refs = list(self.mf.incoming(n))
        # The script is a toplevel item and is therefore referred to from the graph root (aka 'None')
        # FIXME fails since PyInstaller skips edges pointing to the current
        # graph, see change 49c725e9f5a79b65923b8e1bfdd794f0f6f7c4bf
        #self.assertEqual(refs, [None])

        # 2. 'mod'
        n = self.mf.find_node('mod')
        self.assertIsInstance(n, modulegraph.SourceModule)
        refs = list(self.mf.outgoing(n))
        self.assertEqual(refs, [])

        #refs = list(self.mf.incoming(n))
        #self.assertEquals(refs, [])

        # 3. 'pkg'
        n = self.mf.find_node('pkg')
        self.assertIsInstance(n, modulegraph.Package)
        refs = list(self.mf.outgoing(n))
        self.maxDiff = None
        self.assertEqual(refs, [n])

        #refs = list(self.mf.incoming(n))
        #self.assertEquals(refs, [])

        # 4. pkg.mod
        n = self.mf.find_node('pkg.mod')
        self.assertIsInstance(n, modulegraph.SourceModule)
        refs = set(self.mf.outgoing(n))
        self.assertEqual(refs, set([self.mf.find_node('pkg')]))
        ed = self.mf.edgeData(n, self.mf.find_node('pkg'))
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=False, conditional=False, function=False, tryexcept=False))

        # 5. pkg.oldstyle
        n = self.mf.find_node('pkg.oldstyle')
        self.assertIsInstance(n, modulegraph.SourceModule)
        refs = set(self.mf.outgoing(n))
        n2 = self.mf.find_node('mod')
        self.assertEqual(refs, set([self.mf.find_node('pkg'), n2]))
        ed = self.mf.edgeData(n, n2)
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=False, conditional=False, function=False, tryexcept=False))

        # 6. pkg.relative
        n = self.mf.find_node('pkg.relative')
        self.assertIsInstance(n, modulegraph.SourceModule)
        refs = set(self.mf.outgoing(n))
        self.assertEqual(refs, set([self.mf.find_node('__future__'), self.mf.find_node('pkg'), self.mf.find_node('pkg.mod')]))

        ed = self.mf.edgeData(n, self.mf.find_node('pkg.mod'))
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=True, conditional=False, function=False, tryexcept=False))

        ed = self.mf.edgeData(n, self.mf.find_node('__future__'))
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=False, conditional=False, function=False, tryexcept=False))

        #ed = self.mf.edgeData(n, self.mf.find_node('__future__.absolute_import'))
        #self.assertIsInstance(ed, modulegraph.DependencyInfo)
        #self.assertEqual(ed, modulegraph.DependencyInfo(
            #fromlist=True, conditional=False, function=False, tryexcept=False))

        # 7. pkg.toplevel
        n = self.mf.find_node('pkg.toplevel')
        self.assertIsInstance(n, modulegraph.SourceModule)
        refs = set(self.mf.outgoing(n))
        self.assertEqual(refs, set([self.mf.find_node('__future__'), self.mf.find_node('pkg'), self.mf.find_node('mod')]))

        ed = self.mf.edgeData(n, self.mf.find_node('mod'))
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=False, conditional=False, function=False, tryexcept=False))

        ed = self.mf.edgeData(n, self.mf.find_node('__future__'))
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=False, conditional=False, function=False, tryexcept=False))

        #ed = self.mf.edgeData(n, self.mf.find_node('__future__.absolute_import'))
        #self.assertIsInstance(ed, modulegraph.DependencyInfo)
        #self.assertEqual(ed, modulegraph.DependencyInfo(
            #fromlist=True, conditional=False, function=False, tryexcept=False))

        # 8. pkg.subpkg
        n = self.mf.find_node('pkg.subpkg')
        self.assertIsInstance(n, modulegraph.Package)
        refs = set(self.mf.outgoing(n))
        self.assertEqual(refs, set([self.mf.find_node('pkg')]))

        ed = self.mf.edgeData(n, self.mf.find_node('pkg'))
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=False, conditional=False, function=False, tryexcept=False))

        # 9. pkg.subpkg.relative
        n = self.mf.find_node('pkg.subpkg.relative')
        self.assertIsInstance(n, modulegraph.SourceModule)
        refs = set(self.mf.outgoing(n))
        self.assertEqual(refs, set([self.mf.find_node('__future__'), self.mf.find_node('pkg'), self.mf.find_node('pkg.subpkg'), self.mf.find_node('pkg.mod')]))

        ed = self.mf.edgeData(n, self.mf.find_node('pkg.subpkg'))
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=False, conditional=False, function=False, tryexcept=False))

        ed = self.mf.edgeData(n, self.mf.find_node('pkg.mod'))
        self.assertIsInstance(ed, modulegraph.DependencyInfo)
        self.assertEqual(ed, modulegraph.DependencyInfo(
            fromlist=True, conditional=False, function=False, tryexcept=False))

        # 10. pkg.subpkg.relative2
        n = self.mf.find_node('pkg.subpkg.relative2')
        self.assertIsInstance(n, modulegraph.SourceModule)
        refs = set(self.mf.outgoing(n))
        self.assertEqual(refs, set([self.mf.find_node('pkg.subpkg'), self.mf.find_node('pkg.relimport'), self.mf.find_node('__future__')]))

        # 10. pkg.subpkg.mod2
        n = self.mf.find_node('pkg.subpkg.mod2')
        self.assertIsInstance(n, modulegraph.SourceModule)
        refs = set(self.mf.outgoing(n))
        self.assertEqual(refs, set([
            self.mf.find_node('__future__'),
            self.mf.find_node('pkg.subpkg'),
            self.mf.find_node('pkg.sub2.mod'),
            self.mf.find_node('pkg.sub2'),
        ]))

    def testRootModule(self):
        node = self.mf.find_node('mod')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(node.identifier, 'mod')

    def testRootPkg(self):
        node = self.mf.find_node('pkg')
        self.assertIsInstance(node, modulegraph.Package)
        self.assertEqual(node.identifier, 'pkg')

    def testSubModule(self):
        node = self.mf.find_node('pkg.mod')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(node.identifier, 'pkg.mod')

    def testOldStyle(self):
        node = self.mf.find_node('pkg.oldstyle')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(node.identifier, 'pkg.oldstyle')
        sub = [ n for n in self.mf.get_edges(node)[0] if n.identifier != '__future__' ][0]
        self.assertEqual(sub.identifier, 'mod')

    def testNewStyle(self):
        node = self.mf.find_node('pkg.toplevel')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(node.identifier, 'pkg.toplevel')
        sub = [ n for n in self.mf.get_edges(node)[0] if not n.identifier.startswith('__future__')][0]
        self.assertEqual(sub.identifier, 'mod')

    def testRelativeImport(self):
        node = self.mf.find_node('pkg.relative')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(node.identifier, 'pkg.relative')
        sub = [ n for n in self.mf.get_edges(node)[0] if not n.identifier.startswith('__future__') ][0]
        self.assertIsInstance(sub, modulegraph.Package)
        self.assertEqual(sub.identifier, 'pkg')

        node = self.mf.find_node('pkg.subpkg.relative')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(node.identifier, 'pkg.subpkg.relative')
        sub = [ n for n in self.mf.get_edges(node)[0] if not n.identifier.startswith('__future__') ][0]
        self.assertIsInstance(sub, modulegraph.Package)
        self.assertEqual(sub.identifier, 'pkg')

        node = self.mf.find_node('pkg.subpkg.mod2')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(node.identifier, 'pkg.subpkg.mod2')
        sub = [ n for n in self.mf.get_edges(node)[0] if not n.identifier.startswith('__future__') ][0]
        self.assertIsInstance(sub, modulegraph.SourceModule)
        self.assertEqual(sub.identifier, 'pkg.sub2.mod')

        node = self.mf.find_node('pkg.subpkg.relative2')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(node.identifier, 'pkg.subpkg.relative2')

        node = self.mf.find_node('pkg.relimport')
        self.assertIsInstance(node, modulegraph.SourceModule)

class TestRegressions1 (unittest.TestCase):
    if not hasattr(unittest.TestCase, 'assertIsInstance'):
        def assertIsInstance(self, value, types):
            if not isinstance(value, types):
                self.fail("%r is not an instance of %r", value, types)

    def setUp(self):
        root = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                'testpkg-regr1')
        self.mf = modulegraph.ModuleGraph(path=[ root ] + sys.path)
        self.mf.add_script(os.path.join(root, 'main_script.py'))

    def testRegr1(self):
        node = self.mf.find_node('pkg.a')
        self.assertIsInstance(node, modulegraph.SourceModule)
        node = self.mf.find_node('pkg.b')
        self.assertIsInstance(node, modulegraph.SourceModule)

    def testMissingPathEntry(self):
        root = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                'nosuchdirectory')
        try:
            mf = modulegraph.ModuleGraph(path=[ root ] + sys.path)
        except os.error:
            self.fail('modulegraph initialiser raises os.error')

class TestRegressions2 (unittest.TestCase):
    if not hasattr(unittest.TestCase, 'assertIsInstance'):
        def assertIsInstance(self, value, types):
            if not isinstance(value, types):
                self.fail("%r is not an instance of %r"%(value, types))

    def setUp(self):
        root = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                'testpkg-regr2')
        self.mf = modulegraph.ModuleGraph(path=[ root ] + sys.path)
        self.mf.add_script(os.path.join(root, 'main_script.py'))

    def testRegr1(self):
        node = self.mf.find_node('pkg.base')
        self.assertIsInstance(node, modulegraph.SourceModule)
        node = self.mf.find_node('pkg.pkg')
        self.assertIsInstance(node, modulegraph.SourceModule)

class TestRegressions3 (unittest.TestCase):
    # NOTE: in its original variant, this test was using the `distutils`
    # package as the test package; this has now been replaced with `json`.
    #
    # The reason is that with `setuptools`-provided distutils, modulegraph
    # fails to account for the meta-path loader used by setuptools to
    # override the distutils, and instead finds the stdlib `distutils`.
    # This failure is independent of the scenario of this test, which
    # involves `mypkg` creating a subpackage that essentially forwards
    # to a stdlib package.
    if not hasattr(unittest.TestCase, 'assertIsInstance'):
        def assertIsInstance(self, value, types):
            if not isinstance(value, types):
                self.fail("%r is not an instance of %r"%(value, types))

    def assertStartswith(self, value, test):
        if not value.startswith(test):
            self.fail("%r does not start with %r"%(value, test))

    def setUp(self):
        root = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                'testpkg-regr3')
        self.mf = modulegraph.ModuleGraph(path=[ root ] + sys.path)
        self.mf.add_script(os.path.join(root, 'script.py'))

    def testRegr1(self):
        node = self.mf.find_node('mypkg.json')
        self.assertIsInstance(node, modulegraph.Package)
        node = self.mf.find_node('mypkg.json.encoder')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertStartswith(node.filename, os.path.dirname(__file__))

        import json.encoder, json.decoder
        node = self.mf.find_node('json.encoder')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(os.path.dirname(node.filename),
                os.path.dirname(json.encoder.__file__))

        node = self.mf.find_node('json.decoder')
        self.assertIsInstance(node, modulegraph.SourceModule)
        self.assertEqual(os.path.dirname(node.filename),
                os.path.dirname(json.decoder.__file__))

class TestRegression4 (unittest.TestCase):
    if not hasattr(unittest.TestCase, 'assertIsInstance'):
        def assertIsInstance(self, value, types):
            if not isinstance(value, types):
                self.fail("%r is not an instance of %r"%(value, types))

    def setUp(self):
        root = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                'testpkg-regr4')
        self.mf = modulegraph.ModuleGraph(path=[ root ] + sys.path)
        self.mf.add_script(os.path.join(root, 'script.py'))

    def testRegr1(self):
        node = self.mf.find_node('pkg.core')
        self.assertIsInstance(node, modulegraph.Package)

        node = self.mf.find_node('pkg.core.callables')
        self.assertIsInstance(node, modulegraph.SourceModule)

        node = self.mf.find_node('pkg.core.listener')
        self.assertIsInstance(node, modulegraph.SourceModule)

        node = self.mf.find_node('pkg.core.listenerimpl')
        self.assertIsInstance(node, modulegraph.SourceModule)

class TestRelativeReferenceToToplevel (unittest.TestCase):
    if not hasattr(unittest.TestCase, 'assertIsInstance'):
        def assertIsInstance(self, value, types):
            if not isinstance(value, types):
                self.fail("%r is not an instance of %r"%(value, types))

    def test_relative_import_too_far(self):
        # pkg.mod tries to import "..sys" (outside of the package...)
        root = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                'testpkg-regr7')
        mf = modulegraph.ModuleGraph(path=[ root ] + sys.path)
        mf.add_script(os.path.join(root, 'script.py'))

        m = mf.find_node('')
        self.assertIs(m, None)

        m = mf.find_node('pkg.mod')
        self.assertIsInstance(m, modulegraph.SourceModule)

        imported = list(mf.get_edges(m)[0])
        self.assertEqual(len(imported), 5)

        im = imported[0]
        self.assertIsInstance(im, modulegraph.InvalidRelativeImport)
        self.assertEqual(im.relative_path, '..')
        self.assertEqual(im.from_name, 'sys')
        self.assertEqual(im.identifier, '..sys')

        im1 = imported[1]
        im2 = imported[2]
        if im1.identifier == '...xml':
            # Order of modules imported in a single 'from .. import a, b' list
            # is unspecified, ensure a fixed order for this test.
            im2, im1 = im1, im2

        self.assertIsInstance(im1, modulegraph.InvalidRelativeImport)
        self.assertEqual(im1.relative_path, '...')
        self.assertEqual(im1.from_name, 'os')
        self.assertEqual(im1.identifier, '...os')

        im = imported[2]
        self.assertIsInstance(im2, modulegraph.InvalidRelativeImport)
        self.assertEqual(im2.relative_path, '...')
        self.assertEqual(im2.from_name, 'xml')
        self.assertEqual(im2.identifier, '...xml')

        im = imported[3]
        self.assertIsInstance(im, modulegraph.InvalidRelativeImport)
        self.assertEqual(im.relative_path, '..foo')
        self.assertEqual(im.from_name, 'bar')
        self.assertEqual(im.identifier, '..foo.bar')

        im = imported[4]
        self.assertIs(im, mf.find_node('pkg'))

class TestInvalidAsyncFunction (unittest.TestCase):
    if not hasattr(unittest.TestCase, 'assertIsInstance'):
        def assertIsInstance(self, value, types):
            if not isinstance(value, types):
                self.fail("%r is not an instance of %r"%(value, types))

def test_extended_args_import():
    source = "".join(f"dummy_var{i} = {i}\n" for i in range(300)) + "import os\n"
    code = compile(source, "", "exec")
    node = modulegraph.Node("dummy_module")
    node._deferred_imports = []
    node.code = code
    graph = modulegraph.ModuleGraph()
    graph._scan_bytecode(node, code, True)
    assert node._deferred_imports[0][1][0] == "os"

if __name__ == "__main__":
    unittest.main()