import mimetypes

from scripts.geopackage.extensions.metadata.md_scope import MdScope

try:
    # python2
    from urlparse import urlparse
except:
    # python3
    from urllib.parse import urlparse

class Metadata(object):
    """
    Metadata object that represents an entry in the gpkg_metadata Table
    """

    def __init__(self,
                 md_standard_uri,
                 metadata="",
                 md_scope=MdScope.DATASET,
                 mime_type='text/xml'):
        """
        Constructor

        :param md_scope: Use MdScope Enum which has a case sensitive name of the data scope to which this metadata
        applies; see Metadata Scopes http://www.geopackage.org/spec121/#metadata_scopes
        :type md_scope: MdScope

        :param md_standard_uri: URI reference to the metadata structure definition authority
        :type md_standard_uri: str

        :param mime_type: MIME encoding of metadata
        :type mime_type: str

        :param metadata: Metadata text
        :type metadata: str
        """

        # check if the uri is a valid uri
        if not Metadata.__is_valid_uri(md_standard_uri):
            raise ValueError("Invalid URI: {uri}.".format(uri=md_standard_uri))

        # check to make sure the MdScope value is a valid value
        if md_scope.value not in [md_scope_enum.value for md_scope_enum in MdScope]:
            raise ValueError("Invalid md_scope value {md_scope}. Must be one of the following: {md_scopes}."
                             .format(md_scope=md_scope,
                                     md_scopes=[md_scope_enum.value for md_scope_enum in MdScope]))

        # check to see if the mime type is a known mime type
        if mime_type not in [value for ext, value in mimetypes.types_map.items()]:
            raise ValueError("Unknown mime type: {mime_type}. Must be one of the following: {types}."
                             .format(mime_type=mime_type,
                                     types=[value for ext, value in mimetypes.types_map.items()]))

        if metadata is None:
            raise ValueError("Metadata cannot be None.")

        self.md_scope = md_scope
        self.md_standard_uri = md_standard_uri
        self.mime_type = mime_type
        self.metadata = metadata

    @staticmethod
    def __is_valid_uri(uri):
        try:
            result = urlparse(uri)
            return all([result.scheme, result.netloc, result.path])
        except:
            return False


from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union

import attr

from ..models.characteristic import Characteristic
from ..models.operation_site import OperationSite
from ..models.parameter import Parameter
from ..types import UNSET, Unset
from ..util.serialization import is_not_none

T = TypeVar("T", bound="Operation")

@attr.s(auto_attribs=True)
class Operation:
    """An operation, with its sites and site-independent characteristics.

    Attributes:
        characteristics (List[Characteristic]): The list of site-independent characteristics of this operation.
        name (str): The name of the operation.
        parameters (List[Parameter]): The list of parameters. Each parameter must be uniquely named. May be empty.
        sites (List[OperationSite]): The list of sites at which this operation can be applied, together with its site-
            dependent characteristics.
        node_count (Union[Unset, int]): The number of nodes that this operation applies to. None if unspecified.
    """

    characteristics: List[Characteristic]
    name: str
    parameters: List[Parameter]
    sites: List[OperationSite]
    node_count: Union[Unset, int] = UNSET

    def to_dict(self, pick_by_predicate: Optional[Callable[[Any], bool]] = is_not_none) -> Dict[str, Any]:
        characteristics = []
        for characteristics_item_data in self.characteristics:
            characteristics_item = characteristics_item_data.to_dict()

            characteristics.append(characteristics_item)

        name = self.name
        parameters = []
        for parameters_item_data in self.parameters:
            parameters_item = parameters_item_data.to_dict()

            parameters.append(parameters_item)

        sites = []
        for sites_item_data in self.sites:
            sites_item = sites_item_data.to_dict()

            sites.append(sites_item)

        node_count = self.node_count

        field_dict: Dict[str, Any] = {}
        field_dict.update(
            {
                "characteristics": characteristics,
                "name": name,
                "parameters": parameters,
                "sites": sites,
            }
        )
        if node_count is not UNSET:
            field_dict["node_count"] = node_count

        field_dict = {k: v for k, v in field_dict.items() if v != UNSET}
        if pick_by_predicate is not None:
            field_dict = {k: v for k, v in field_dict.items() if pick_by_predicate(v)}

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        characteristics = []
        _characteristics = d.pop("characteristics")
        for characteristics_item_data in _characteristics:
            characteristics_item = Characteristic.from_dict(characteristics_item_data)

            characteristics.append(characteristics_item)

        name = d.pop("name")

        parameters = []
        _parameters = d.pop("parameters")
        for parameters_item_data in _parameters:
            parameters_item = Parameter.from_dict(parameters_item_data)

            parameters.append(parameters_item)

        sites = []
        _sites = d.pop("sites")
        for sites_item_data in _sites:
            sites_item = OperationSite.from_dict(sites_item_data)

            sites.append(sites_item)

        node_count = d.pop("node_count", UNSET)

        operation = cls(
            characteristics=characteristics,
            name=name,
            parameters=parameters,
            sites=sites,
            node_count=node_count,
        )

        return operation


#!/usr/bin/env python3
import sys
import os
import json
def createHelp(env,factoryFile,installDir):
    #jng: try to have scons handle all the creation but could not figure out how
    #     so handled dir creation manually
    try:
        os.makedirs(env['HELPER_BUILD_DIR'])
    except:
        # already exists
        pass
    try:
        #one could probably also use __import__ but needs to make sure the
        #the cwd is prepended to the sys.path otherwise if factoryFile = __init__.py
        #it will load the first one found
        import imp
        moduleList = env['ISCEPATH']
        package = "."
        for module in moduleList:
            if installDir.count(module):
                ls = installDir.replace(module,'').split("/")
                #remove empty element
                ls = [i for i in ls if i != '']
                package = ".".join(ls)
                #when module is the same as installDir package is empty
                if not package:
                    package = [i for i in installDir.split('/') if i != ''][-1]

        mod = imp.find_module(factoryFile.replace('.py',''))
        factModule = imp.load_module(factoryFile.replace('.py',''),mod[0],mod[1],mod[2])
        factoriesInfo = factModule.getFactoriesInfo()
        nameList = []
        for k,v in factoriesInfo.items():
            name = os.path.join(env['HELPER_BUILD_DIR'],k + '.hlp')
            nameList.append(name)
            v["package"] = package
            json.dump({k:v},open(name,'w'),indent=4)
    except:
        nameList = []
    return nameList,env['HELPER_DIR']


import csv
import datetime
import re
import os
import uuid

from google.cloud import vision
from google.oauth2.service_account import Credentials
import cv2
import matplotlib.pyplot as plt
import pandas as pd
import torch
from flask import Flask, render_template, request
from werkzeug.utils import secure_filename
import easyocr
creds = Credentials.from_service_account_file('credentials2.json')
client = vision.ImageAnnotatorClient(credentials=creds)

plt.switch_backend('agg')

app = Flask(__name__)
app.config['ALLOWED_EXTENSIONS'] = set(['png', 'jpg', 'jpeg'])
app.config['UPLOAD_FOLDER'] = 'static/uploads/'
reader = easyocr.Reader(['en'], gpu=False)

now = datetime.datetime.now()
timestamp = now.strftime('%Y-%m-%d %H:%M:%S')

def allowed_file(filename):
    return '.' in filename and \
        filename.rsplit('.', 1)[1].lower() in app.config['ALLOWED_EXTENSIONS']

@app.route("/")
def index():
    return render_template("index.html")

@app.route("/predict", methods=["GET", "POST"])
def predict():
    if request.method == "POST":
        image = request.files["image"]
        if image and allowed_file(image.filename):
            image.save(os.path.join(
                app.config['UPLOAD_FOLDER'], secure_filename(image.filename)))
            image_path = os.path.join(
                app.config['UPLOAD_FOLDER'], secure_filename(image.filename))
            model = torch.hub.load("ultralytics/yolov5",
                                   "custom",
                                   path="./best.pt",
                                   force_reload=True)
            result = model(image_path)
            print(result)
            bbox_raw = result.xyxy[0][0]
            bbox = []
            for bound in bbox_raw:
                bbox.append(int(bound.item()))
            bbox = bbox[:4]
            image = cv2.imread(image_path)
            cropped_image = image[bbox[1]:bbox[3], bbox[0]:bbox[2]]
            dynamic_filename = str(uuid.uuid4())
            cv2.imwrite("static/result/cropped_image_" +
                        dynamic_filename + ".jpg", cropped_image)
            cropped_image_path = "static/result/cropped_image_" + dynamic_filename + ".jpg"
            with open(cropped_image_path, 'rb') as f:
                image_bytes = f.read()
            image = vision.Image(content=image_bytes)
            response = client.text_detection(image=image)
            text = response.text_annotations[0].description
            # result = reader.readtext(image_bytes)
            # text = result[0][1]
            # integers = re.findall(r'\d+', text)
            # cleaned_text = ''.join(integers)
            row = {'timestamp': timestamp,
                   'uploaded_image_path': image_path,
                   'cropped_image_path': cropped_image_path,
                   'result_text': text}
            fields = ["timestamp", "uploaded_image_path",
                      "cropped_image_path", "result_text"]
            with open('ocr_results.csv', 'a') as f:
                writer = csv.DictWriter(f, fieldnames=fields)
                writer.writerow(row)
            return render_template("predict.html", result=cropped_image_path, water_meter=text)
        else:
            return render_template("predict.html", error="Silahkan upload gambar dengan format JPG")
    else:
        return render_template("predict.html")

@app.route("/history")
def history():
    df = pd.read_csv("ocr_results.csv")
    df = df.sort_values(by=['result_text'])
    x = list(range(len(df)))
    data = [int(val) for val in df['result_text'].iloc[:]]

    fig, ax = plt.subplots(figsize=(12, 6))
    ax.plot(x, data, marker='o', markersize=8,
            linestyle='-', linewidth=1, label='Result Text')
    ax.set_xlabel('Data Point')
    ax.set_ylabel('Result Text')
    ax.set_title('Result Text Trend')
    for i, val in enumerate(data):
        ax.text(x[i], data[i], str(val), ha='center', va='bottom')

    plot_path = 'static/result/plot.png'
    if not os.path.exists(os.path.dirname(plot_path)):
        os.makedirs(os.path.dirname(plot_path))
    plt.savefig(plot_path)

    return render_template("history.html", df=df, plot_path=plot_path)

@app.route("/trend")
def trend():
    plot_path = 'static/result/plot.png'
    return render_template("trend.html", plot_path=plot_path)

if __name__ == "__main__":
    app.run()


# -*- coding: utf-8 -*-
from odoo import models, fields, api
from datetime import timedelta,datetime,date
from odoo.exceptions import Warning, ValidationError, UserError
from datetime import datetime
from dateutil.relativedelta import relativedelta
from calendar import monthrange
import datetime as dt
import calendar
# import pandas as pd

class SaleOrderExt(models.Model):
	_inherit='sale.order'

	no_of_months = fields.Float(string="No of Months")
	sequence_number = fields.Char(string="Sequence No", copy = False)
	per_month_gross_salary = fields.Float(string="Per Month Gross Salary")
	template = fields.Many2one('costcard.template', string="Template")
	job_pos = fields.Many2one('hr.job', string="Job Position")
	version = fields.Char(string="Version No")
	first_approval = fields.Many2one('res.users',string = "First Approval", copy = False)
	second_approval = fields.Many2one('res.users',string = "Second Approval", copy = False)
	# interval = fields.Integer(string="Interval")
	contract_start_date = fields.Date(string="Contract Start Date")
	contract_end_date = fields.Date(string="Contract End Date")
	estimated_start_date = fields.Date(string="Estimated Start Date")
	date_invoice = fields.Date(string="Invoice Date")
	invoice_amount = fields.Float(string="Invoice Amount")
	percentage = fields.Float(string="Percentage %" ,digits=(4,4))
	invoice_id = fields.Many2one('account.move', string="Invoice")
	applicant = fields.Many2one('hr.applicant', string="Applicant")
	employee = fields.Many2one('hr.employee', string="Employee")
	contract = fields.Many2one('hr.contract', string="Contract")
	candidate_name = fields.Char(string="Candidate Name")
	customer_po_no = fields.Char(string="Customer PO Number")
	month_days_deduction = fields.Boolean(string="Month Days Deduction")
	monthly_deduction = fields.Boolean(string="Monthly Deduction")
	applicant_approve_check = fields.Boolean(string="Approved Applicant")
	budget = fields.Float(string="Budget", related="job_pos.budget")
	unlock_check = fields.Boolean(string="Un Lock Check")
	out_of_system_invoiced_amount = fields.Float(string="Out Of System Invoiced Amount")
	inv_attention = fields.Char("Invoice Attention")
	invoice_requester = fields.Many2one('res.partner',string="Invoice Requester")
	invoice_buyer = fields.Many2one('res.partner',string="Invoice Buyer")
	hiring_contact_client_dom = fields.Many2many('res.partner',compute = "GetContactDOM")

	order_line_2 = fields.One2many('sale.order.line', 'order_id', string='Order Lines', states={'cancel': [('readonly', True)], 'done': [('readonly', True)]}, copy=True, auto_join=True)

	so_type = fields.Selection([
		('cost_card','Cost Card'),
		('sale_order','Sale Order'),
		], string='SO Type',default = 'cost_card')

	costcard_type = fields.Selection([
		('estimate','Estimate'),
		('cost_card','Cost Card'),
		], string='Cost Card Type')
	work_days_type = fields.Selection([
		('twenty_two_days','22 Days'),
		('calender_days','Calender Days'),
		('actual_working_days','Actual Working Days'),
		('twenty_six_days','26 Days'),
		], string='Work Days Type', default="twenty_two_days", required = True)
	leave_type = fields.Selection([
		('one_day','One Day / Week'),
		('two_days','Two Days / Week'),
		], string='Leave Type', default="two_days")
	contract_state = fields.Selection([
		('draft','New'),
		('open','Running'),
		('close','Expired'),
		('cancel','Cancelled'),
		], string='Contract State', default="draft")

	def FirstApproval(self):
		self.first_approval = self.env.uid
		if self.first_approval and self.second_approval:
			self.state = "done"
			self.applicant.approve_btn()

	def SecondApproval(self):
		self.second_approval = self.env.uid
		if self.first_approval and self.second_approval:
			self.state = "done"
			self.applicant.approve_btn()

	def ActionCancel(self):
		self.state = "draft"
		self.unlock_check = True

	def GetContactDOM(self):
		for x in self:
			id_list = []
			for index in x.partner_id.child_ids:
				id_list.append(index.id)

		self.hiring_contact_client_dom = [(6, 0, id_list)]

	@api.depends('order_line.invoice_lines')
	def _get_invoiced(self):
		# The invoice_ids are obtained thanks to the invoice lines of the SO
		# lines, and we also search for possible refunds created directly from
		# existing invoices. This is necessary since such a refund is not
		# directly linked to the SO.

		if self.so_type == "cost_card":

			for order in self:
				invoices = self.env['account.move'].search([('sale_order_id','=',order.id)])
				# invoices = order.order_line.invoice_lines.move_id.filtered(lambda r: r.type in ('out_invoice', 'out_refund'))
				order.invoice_ids = invoices
				order.invoice_count = len(invoices)
		elif self.so_type == "sale_order":
			for order in self:
				invoices = order.order_line.invoice_lines.move_id.filtered(lambda r: r.type in ('out_invoice', 'out_refund'))
				order.invoice_ids = invoices
				order.invoice_count = len(invoices)
		else:
			for order in self:
				order.invoice_count = 0

	def approve_applicant(self):
		if self.applicant:
			recs = self.env['hr.recruitment.stage'].search([('name','=','In Approvals')])
			self.applicant.stage_id = recs.id
			# self.applicant.approve_btn()
			self.applicant_approve_check = True

	def get_handle_sequence(self):
		if self.template:
			for lines in self.order_line:
				costcard_line_rec = self.env['costcard.template.tree'].search([('service_name','=',lines.product_id.id),('tree_link','=',self.template.id)])
				lines.handle = costcard_line_rec.handle

	def UpdateSOName(self):
		cost_card_name = ""
		if self.costcard_type == "estimate":
			cost_card_name = "Estimate"
		if self.costcard_type == "cost_card":
			cost_card_name = "Cost Card"

		applicant_name = ""
		if self.contract:
			applicant_name = self.contract.employee_id.name
		elif self.job_pos:
			applicant_name = self.job_pos.name
		else:
			applicant_name = self.partner_id.name
		if not self.sequence_number:
			self.sequence_number = self.name
		self.name = self.sequence_number + "-" + cost_card_name + "-" + applicant_name

	# @api.onchange('contract_start_date','no_of_months')
	# def get_contract_end_date(self):
	#   if self.contract_start_date and self.no_of_months:
	#       self.contract_end_date = self.contract_start_date + (relativedelta(months = self.no_of_months))
	#   else:
	#       self.contract_end_date = self.contract_start_date

	# def create_invoice(self):
	#   self.create_journal_entry()
	#   print ("Create Journal Entry")

	# new way of creating invoice START
	def prepare_invoice(self,date_invoice,month_invoice,from_date,to_date):
		journal = self.env['account.move'].with_context(force_company=self.company_id.id, default_type='out_invoice')._get_default_journal()
		if not journal:
			raise UserError(_('Please define an accounting sales journal for the company %s (%s).') % (self.company_id.name, self.company_id.id))
		# if self.contract and self.contract_state =='open':
		invoice_vals = {
			# 'ref': self.client_order_ref or '',
			'ref': self.name,
			'type': 'out_invoice',
			'narration': self.note,
			'currency_id': self.pricelist_id.currency_id.id,
			'campaign_id': self.campaign_id.id,
			'medium_id': self.medium_id.id,
			'automated_invoice': True,
			'source_id': self.source_id.id,
			'invoice_user_id': self.user_id and self.user_id.id,
			'team_id': self.team_id.id,
			'partner_id': self.partner_invoice_id.id,
			'employee': self.contract.employee_id.id,
			'partner_shipping_id': self.partner_shipping_id.id,
			'fiscal_position_id': self.fiscal_position_id.id or self.partner_invoice_id.property_account_position_id.id,
			'invoice_origin': self.name,
			'invoice_date':month_invoice,
			'invoice_month':date_invoice,
			'from_date':from_date,
			'to_date':to_date,
			'invoice_payment_term_id': self.payment_term_id.id,
			'invoice_payment_ref': self.reference,
			'sale_order_id': self.id,
			'transaction_ids': [(6, 0, self.transaction_ids.ids)],
			'invoice_line_ids': [],
		}
		# else:
		#   raise ValidationError('Contract not available or not in Open state.')

		return invoice_vals

		# old method to create invoice
		# self.create_journal_entry()

	# Function to count do of days in a month
	def number_of_days_in_month(self, year, month):
		return monthrange(year, month)[1]

	def add_days_to_date(self, date):
		temp = str(date)
		days = int(temp[-2:])
		days += 1
		temp = temp[:-2]+str(days)

	def add_month_to_date(self, date):
		temp = str(date)
		month = int(temp[5:7])
		year = int(temp[0:4])
		day = int(temp[8:])
		if month == 12:
			day = 1
			month = 1
			year += 1
			temp = str(year)+"-0"+str(month)+"-0"+str(day)
		else:
			month += 1
			temp = temp[0:5]+str(month)+temp[7:]
		return temp

	def sub_days_to_date(self, date):
		temp = str(date)
		days = int(temp[-2:])
		days -= 1
		temp = temp[:-2]+str(days)

	def sub_month_to_date(self, date):
		temp = str(date)
		day = int(temp[8:])
		month = int(temp[5:7])
		year = int(temp[0:4])
		if month == 1:
			day = self.number_of_days_in_month(int(year), int(month))
			month = 12
			year -=1
			temp = str(year)+"-"+str(month)+"-"+str(day)
		else:
			month -= 1
			temp = temp[0:5]+str(month)+temp[7:]
		return temp

	############## Function to calculate leave balance total START ##############
	def calculate_leave_balance(self,date_invoice,leave_name):

		unique_holidays = []
		holiday_rec = self.env['custom.holiday.tree'].search([('tree_link.year','=',str(date_invoice.year))])
		for holiday_index in holiday_rec:
			if not holiday_index.day in unique_holidays:
				unique_holidays.append(holiday_index.date)

		employee_id = None
		if self.employee:
			employee_id = self.employee.id
		elif self.contract:
			employee_id = self.contract.employee_id.id
		leaves = self.env['hr.leave'].search([('employee_id','=',employee_id),('state','=','validate')])
		total_leaves = 0
		leave_name_days = 0
		leave_name_days = 0
		leave_days_list_specific = []
		leave_days_list = []
		for x in leaves:
			per_request_leaves = 0

			if date_invoice.replace(day=1) == x.request_date_from.replace(day=1) or date_invoice.replace(day=1) == x.request_date_to.replace(day=1):

				# getting list of days
				delta = x.request_date_to - x.request_date_from

				if delta.days == 0:
					delta_days = 1
				else:
					delta_days = delta.days
				for i in range(delta_days + 1):
					day = x.request_date_from + timedelta(days=i)
					if day.replace(day=1) == date_invoice.replace(day=1):
						if self.leave_type == "two_days":
							if day.weekday() != 4 and day.weekday() != 5:
								leave_days_list.append(day)
								if x.request_unit_half:
									total_leaves += 0.5
									if x.holiday_status_id.name == leave_name:
										leave_name_days += 0.5
										leave_days_list_specific.append(day)
								else:
									total_leaves += 1
									if x.holiday_status_id.name == leave_name:
										leave_name_days += 1
										leave_days_list_specific.append(day)
						if self.leave_type == "one_day":
							if day.weekday() != 4 :
								leave_days_list.append(day)
								if x.request_unit_half:
									total_leaves += 0.5
									if x.holiday_status_id.name == leave_name:
										leave_name_days += 0.5
										leave_days_list_specific.append(day)
								else:
									total_leaves += 1
									if x.holiday_status_id.name == leave_name:
										leave_name_days += 1
										leave_days_list_specific.append(day)

				for z in unique_holidays:
					if z in leave_days_list:
						leave_days_list.remove(z)

						total_leaves -= 1
					if z in leave_days_list_specific:
						leave_days_list_specific.remove(z)

						leave_name_days -= 1

		return [total_leaves,leave_name_days]

	def CalculateLeavesPayroll(self,date,leave_name):
		return self.calculate_leave_balance(date,leave_name)[1]

	############### Function to calculate leave balance total ENDS  #############

	############## Function to calculate Holidays total START ##############
	def calculate_holidays(self,interval_type,date_invoice):
		holidays = 0

		if interval_type == "start":
			holiday_rec = self.env['custom.holiday.tree'].search([('tree_link.year','=',str(date_invoice.year)),('date','>=',self.contract_start_date)])
		elif interval_type == "end":
			holiday_rec = self.env['custom.holiday.tree'].search([('tree_link.year','=',str(date_invoice.year)),('date','<=',self.contract_end_date)])
			print (holiday_rec)
		else:
			holiday_rec = self.env['custom.holiday.tree'].search([('tree_link.year','=',str(date_invoice.year))])
		for y in holiday_rec:
			if date_invoice.replace(day=1) == y.date.replace(day=1):
				if self.leave_type == "two_days":
					if y.date.weekday() !=4 and y.date.weekday() != 5 :

						holidays +=1
				if self.leave_type == "one_day":
					if y.date.weekday() !=4:
						holidays +=1
		return holidays
	############### Function to calculate Holidays total ENDS  #############

	################# Calculating salary according to days START #################
	def per_day_devisor(self,date_invoice):

		if self.work_days_type == 'twenty_two_days':
			return 22

		if self.work_days_type == 'twenty_six_days':
			return 26

		if self.work_days_type == "actual_working_days":
			total_days = self.number_of_days_in_month(date_invoice.year, date_invoice.month)
			weekends = self.calculate_weekends(date_invoice,"regular")
			# holidays = self.calculate_holidays()
			net_days = total_days - weekends

			return net_days

		if self.work_days_type == 'calender_days':

			return self.number_of_days_in_month(date_invoice.year, date_invoice.month)

	def calculate_working_days(self, contract_date, month_interval):
		working_days = 0
		relevant_date = contract_date

		last_day = calendar.monthrange(int(relevant_date.year),int(relevant_date.month))[1]
		last_date = relevant_date.replace(day=last_day)

		day = contract_date.replace(day=1)
		single_day = dt.timedelta(days=1)

		if month_interval == "start":
			start_date = relevant_date
			end_date = last_date
		if month_interval == "end":
			start_date = day
			end_date = relevant_date

		if self.work_days_type == 'twenty_two_days':
			while start_date <= end_date:
				if start_date.weekday() != 4 and start_date.weekday() != 5:
					if working_days < 22:
						working_days += 1
				start_date += single_day

		if self.work_days_type == 'calender_days':
			while start_date <= end_date:
				working_days += 1
				start_date += single_day

		if self.work_days_type == 'twenty_six_days':
			while start_date <= end_date:
				if start_date.weekday() != 4:
					if working_days < 26:
						working_days += 1
				start_date += single_day

		if self.work_days_type == 'actual_working_days':
			while start_date <= end_date:
				if self.leave_type == 'one_day':
					if start_date.weekday() != 4:
						working_days += 1
					start_date += single_day
				if self.leave_type == "two_days":
					if start_date.weekday() != 4 and start_date.weekday() != 5:
						working_days += 1
					start_date += single_day

		return working_days

	def calculate_weekends(self,contract_date, month_interval):

		relevant_date = contract_date

		last_day = calendar.monthrange(int(relevant_date.year),int(relevant_date.month))[1]
		last_date = relevant_date.replace(day=last_day)

		day = contract_date.replace(day=1)
		single_day = dt.timedelta(days=1)

		if month_interval == "start":
			start_date = relevant_date
			end_date = last_date
		if month_interval == "end":
			start_date = day
			end_date = relevant_date
		if month_interval == "regular":
			start_date = day
			end_date = last_date
		days_to_deduct = 0

		while start_date <= end_date:
			if self.leave_type == "two_days":
				if start_date.weekday() == 4 or start_date.weekday() == 5:
					days_to_deduct += 1
				start_date += single_day
			if self.leave_type == "one_day":
				if start_date.weekday() == 4:
					days_to_deduct += 1
				start_date += single_day

		return days_to_deduct
	################# Calculating salary according to days ENDS #################

	def recompute_func(self, line, code_dict,cumulative_total):
		costcard_template_rec = self.env['costcard.template.tree'].search([('service_name','=',line.product_id.id),('tree_link','=',self.template.id),('code','=',line.code)])

		global compute_result
		global compute_qty

		compute_result = 0
		compute_qty = 0

		salary = self.per_month_gross_salary
		no_months = self.no_of_months
		edari_service_percent = self.percentage
		# cumulative_total = 0
		edari_fee = 0

		if costcard_template_rec.computation_formula:
			expression = 'global compute_result;\n'+costcard_template_rec.computation_formula
		else:
			expression = 'global compute_result;\n'

		try:
			exec(expression)
			code_dict[line.code] = compute_result

			globals().update(code_dict)
		except Exception as e:
			raise ValidationError('Error..!\n'+str(e))

		return compute_result

	def FinalWorkingDays(self,date_invoice,starting_month,ending_month):
		t_date = date_invoice

		# divisor and working_days variable are used for finding invoice line amount
		divisor = self.per_day_devisor(date_invoice)
		working_days = divisor
		month_interval = "regular"
		if starting_month == True:
			t_date = self.contract_start_date
			# divisor = self.per_day_devisor(t_date)
			month_interval = "start"
			working_days = self.calculate_working_days(t_date, month_interval)
		if ending_month == True:
			month_interval = "end"
			t_date = self.contract_end_date
			# divisor = self.per_day_devisor(t_date)
			working_days = self.calculate_working_days(t_date, month_interval)

		no_of_holidays = 0
		no_of_holidays_wd = 0
		if self.work_days_type == "actual_working_days":
			no_of_holidays = self.calculate_holidays("Regular",date_invoice)
			no_of_holidays_wd = self.calculate_holidays(month_interval,date_invoice)
			divisor -= no_of_holidays

		working_days -= (self.calculate_leave_balance(date_invoice,None)[0] + no_of_holidays_wd)

		return [working_days,divisor]

	def create_invoice(self,date_invoice,month_invoice):
		if not self.contract_start_date:
			raise ValidationError("Please input Contract Start Date")
		if not self.contract_end_date:
			raise ValidationError("Please input Contract End Date")
		# for line in self.order_line:

		#####################################################################################
		edari_product = self.env['product.product'].search([('name','=','Edari Service Fee')],limit=1)

		# Untaxed amounts
		print ("Check1")
		move_lines_list = []
		credit_sum = 0
		starting_month = False
		ending_month = False
		lines_not_to_add = []
		if date_invoice.month == self.contract_start_date.month and date_invoice.year == self.contract_start_date.year:
			lines_not_to_add = ['end']
			starting_month = True
			invoice_date_from = self.contract_start_date
			invoice_date_to = monthrange(date_invoice.year,date_invoice.month)[1]
			invoice_date_to_date = dt.date(date_invoice.year, date_invoice.month, invoice_date_to)

		elif date_invoice.month == self.contract_end_date.month and date_invoice.year == self.contract_end_date.year:
			lines_not_to_add = ['upfront']
			ending_month = True
			invoice_date_from = self.contract_end_date.replace(day=1)
			invoice_date_to_date = self.contract_end_date

		else:
			lines_not_to_add = ['end','upfront']
			invoice_date_from = dt.date(date_invoice.year, date_invoice.month, 1)
			invoice_date_to = monthrange(date_invoice.year,date_invoice.month)[1]
			invoice_date_to_date = dt.date(date_invoice.year, date_invoice.month, invoice_date_to)

		invoice_vals_list = []
		invoice_vals = self.prepare_invoice(date_invoice,month_invoice,invoice_date_from,invoice_date_to_date)

		values = self.FinalWorkingDays(date_invoice,starting_month,ending_month)
		working_days = values[0]
		divisor = values[1]

		print (values)

		code_dict_new = {}
		total_invoice_amount = 0
		for line in self.order_line:
			if line.price_unit != 0:
				amount = 0

				if not line.payment_type in lines_not_to_add:
					print (line.code)

					# qty in months check
					if line.as_of_date:
						line_start_date = line.as_of_date.replace(day=1)
						line_end_date = line_start_date+(relativedelta(months = (int(line.product_uom_qty)), days=-1))
					else:
						line_end_date = self.contract_start_date.replace(day=1)+(relativedelta(months = (int(line.product_uom_qty) + int(line.offset)), days=-1))
						line_start_date = self.contract_start_date+(relativedelta(months = (int(line.offset))))

					if (date_invoice.replace(day=1) <= line_end_date.replace(day=1)) and (date_invoice.replace(day=1) >= line_start_date.replace(day=1)):

						if line.payment_type == 'interval':
							amount = line.price_unit
							total_invoice_amount += line.price_unit
						else:
							amount = line.price_subtotal
							total_invoice_amount += line.price_subtotal

						if line.based_on_wd:
							per_day = amount/divisor
							print (per_day)
							amount = per_day*working_days

						# Calculate leave balance
						balance = amount
						recompute_result = 0
						code_dict_new[line.code] = balance
						globals().update(code_dict_new)
						if line.recomputable:
							cumulative_total = 0
							for rec in self.order_line:
								if rec.code == line.code:
									break
								cumulative_total += code_dict_new[rec.code]

							recompute_result = self.recompute_func(line, code_dict_new,cumulative_total)
							balance = recompute_result

						invoice_vals['invoice_line_ids'].append(line.prepare_invoice_line(balance,line.product_id.name,line.product_id.id))
						credit_sum += balance
					else:
						balance = 0
						code_dict_new[line.code] = balance
						globals().update(code_dict_new)

				# setting those variables which are not falling in date limit as zero
				else:
					print (line.code)
					balance = 0
					code_dict_new[line.code] = balance
					globals().update(code_dict_new)
			else:
				balance = 0
				code_dict_new[line.code] = balance
				globals().update(code_dict_new)

		if not invoice_vals['invoice_line_ids']:
			raise UserError('There is no invoiceable line. If a product has a Delivered quantities invoicing policy, please make sure that a quantity has been delivered.')

		remaining_amount = 0
		if ending_month == True:

			all_invoices_amount = 0
			if self.out_of_system_invoiced_amount:
				all_invoices_amount += self.out_of_system_invoiced_amount
			all_invoices = self.env['account.move'].search([('sale_order_id','=',self.id)])
			for inv in all_invoices:
				all_invoices_amount += inv.amount_untaxed
			remaining_amount = self.amount_untaxed -(all_invoices_amount + credit_sum)
			if remaining_amount != 0:

				invoice_vals['invoice_line_ids'].append(line.prepare_invoice_line(remaining_amount,"Final Invoice Adjustment",None))

		invoice_vals_list.append(invoice_vals)

		total_invoice_amount += remaining_amount

		# 3) Create invoices.
		moves = self.env['account.move'].with_context(default_type='out_invoice').create(invoice_vals_list)

		moves.pro_rate_adjust = total_invoice_amount - moves.amount_untaxed
		moves.leave_taken = self.CalculateLeavesPayroll(date_invoice,"Annual Leave Days")
		moves.sick_days_taken = self.CalculateLeavesPayroll(date_invoice,"Sick Leave Days")

		########### update accrual entries ##################

		for acc in self.order_line:
			if acc.product_id.accruing_account_id:
				if acc.product_id.property_account_expense_id:
					debit_account = acc.product_id.property_account_income_id.id
				else:
					debit_account = acc.product_id.categ_id.property_account_income_categ_id.id
				for mv in moves.line_ids:
					if mv.name == acc.product_id.name:
						line_amount = mv.credit
						break

				if line_amount > 0:

					moves.write({
						'line_ids': [
								(0, 0, {
									'account_id':debit_account,
									'name':"Accrued Adjustment",
									'debit':line_amount,
									'credit':0,
									'partner_id':self.partner_id.id,
								}),

								(0, 0, {
									'account_id':acc.product_id.accruing_account_id.id,
									'name':"Accrued Adjustment",
									'debit':0,
									'credit':line_amount,
									'partner_id':self.contract.employee_id.partner_id.id,

								}),
							],
						})

		return moves

	def CalculateLeaveDays(self,record_id):
		salary = self.per_month_gross_salary
		no_months = self.no_of_months
		edari_service_percent = self.percentage
		cumulative_total = 0
		edari_fee = 0

		global compute_result

		if record_id.computation_formula:
			expression = 'global compute_result;\n'+record_id.computation_formula
		else:
			expression = 'global compute_result;\n'

		try:
			exec(expression)

		except Exception as e:
			raise ValidationError('Error..!\n'+str(e))

		return compute_result

	# @api.onchange('template')
	def get_order_lines(self):

		# computed_dict = {}
		for x in self.order_line:
			if not x.costcard_type == 'manual':
			#   if x.product_uom_qty > 1:
			#       computed_dict[x.product_id.id] = x.product_uom_qty
				x.unlink()

		if self.template:

			code_dict = {}

			salary = self.per_month_gross_salary
			no_months = self.no_of_months
			edari_service_percent = self.percentage
			cumulative_total = 0
			edari_fee = 0

			template_tree_recs = self.env['costcard.template.tree'].search([('tree_link','=',self.template.id)], order='handle')
			# for x in self.template.template_tree:
			for x in template_tree_recs:
				cumulative_check = True
				manual_amount_cumulative = 0
				for lines in self.order_line:
					if x.code == lines.code:
						if lines.costcard_type == "manual":
							cumulative_check = False
							manual_amount_cumulative = lines.price_unit
				if x.costcard_type == "calculation":
					cumulative_check = False

				# manual_amount_cumulative = 0
				# if x.costcard_type == "manual":
				# 	for lines in self.order_line:
				# 		if x.code == lines.code:
				# 			manual_amount_cumulative = lines.price_unit

				global compute_result
				global compute_qty

				compute_result = 0
				compute_qty = 0
				# if ' ' in x.computation_formula:
				# result = eval(x.computation_formula)
				if x.computation_formula:
					expression = 'global compute_result;\n'+x.computation_formula
				else:
					expression = 'global compute_result;\n'

				# Calculating qty
				if x.computation_qty:
					# print (compute_qty)
					qty_expression = 'global compute_qty;\n'+x.computation_qty
				else:
					qty_expression = 'global compute_qty;\n'
				# expression = x.computation_formula
				# expression.replace("result", "cost_card_compute_x1")
				# exec(x.computation_formula)

				try:
					exec(expression)
					exec(qty_expression)

				except Exception as e:
					raise ValidationError('Error..!\n'+str(e))

				# print (compute_result)

				qty = 0
				# if x.costcard_type in ['fixed','calculation']:
				# if x.costcard_type in ['manual']:
				# 	qty = 1

				# if x.costcard_type:
				# if x.costcard_type == 'manual':
				# else:
				#   # Changed to get date from compute date field
				qty = self.no_of_months

					# compute_result = 0
				if x.payment_type in ['upfront','end'] and x.costcard_type != 'manual':
					qty = 1

				if x.computation_qty:
					qty = compute_qty
					if compute_result and self.no_of_months and qty > 0:
						compute_result = compute_result * (self.no_of_months / qty)

				if cumulative_check == True:
					cumulative_total += compute_result

				cumulative_total += manual_amount_cumulative

				# order_lines_list.append({
				manual_check = True
				for index in self.order_line:
					if index.product_id.id == x.service_name.id:
						manual_check = False
						index.get_manual_price_unit()
						code_dict[x.code] = index.price_unit

				if manual_check:
					# if x.service_name.id in computed_dict:
					#   qty = computed_dict[x.service_name.id]

					self.order_line.create({
						'product_id':x.service_name.id,
						# 'sale_order_template_id':self.id,
						'order_id':self.id,
						# 'product_uom_qty':self.no_of_months,
						'product_uom_qty':qty,
						# 'product_uom_qty':compute_qty,
						'price_unit':compute_result,
						'based_on_wd':x.based_on_wd,
						'payment_type':x.payment_type,
						'code':x.code,
						'recomputable':x.recomputable,
						'handle':x.handle,
						'categ_id':x.service_name.categ_id.id,
						'name':x.code or "",
						'costcard_type':x.costcard_type,
						'chargable':x.chargable,
						})
					code_dict[x.code] = compute_result

				globals().update(code_dict)
				del compute_result
				del compute_qty

			# deleting global variables

			for x in code_dict.keys():
				del x

			for x in self.order_line:
				if x.costcard_type == 'calculation':
					x.unlink()

		else:
			self.order_line = None

	def CreateActivities(self):
		users = []
		first_approval = self.env.ref('so_extension.first_approval_cost_card_group')
		second_approval = self.env.ref('so_extension.second_approval_cost_card_group')
		for fir in first_approval.users:
			if fir.id not in [1,2]:
				users.append(fir.id)
		for sec in second_approval.users:
			if sec.id not in [1,2]:
				if sec.id not in users:
					users.append(sec.id)

		for us in users:
			self.env['mail.activity'].create({
					'res_id': self.id,
					'res_model_id': self.env['ir.model']._get('sale.order').id,
					'activity_type_id': self.env.ref('so_extension.activity_type').id,
					'summary': "Cost Card Number-" +self.name + " Needs your approval",
					'note': "Cost Card Number-" +self.name + " Needs your approval",
					'user_id': us,
				})

	def action_confirm(self):
		res = super(SaleOrderExt, self).action_confirm()
		if self.so_type == "cost_card":
			if not self.first_approval and not self.second_approval:
				self.CreateActivities()
			if self.first_approval and self.second_approval:
				self.state = "done"
		return res

	def create_edari_fee(self):
		return True

	@api.model
	def create(self, vals):
		new_record = super(SaleOrderExt, self).create(vals)

		if new_record.so_type == 'cost_card':
			self.UpdateLeaveType()

			if new_record.job_pos:
				print (new_record.job_pos.name)
				records = self.env['sale.order'].search([('job_pos','=',new_record.job_pos.id),('state','not in',['cancel','done'])])
				rec_len = len(records)
				if records:
					new_record.version = "V"+str(rec_len)
					print (new_record.version)
				else:
					new_record.version = "V1"
					print (new_record.version)
			new_record.get_handle_sequence()

			# new_record.get_contract_end_date()
			new_record.get_order_lines()

			# updating wage in employee
			if new_record.employee:
				new_record.employee.wage = new_record.per_month_gross_salary
		return new_record

	def write(self, vals):
		before=self.write_date

		rec = super(SaleOrderExt, self).write(vals)

		if 'per_month_gross_salary' in vals:
			self.applicant.salary_expected = vals['per_month_gross_salary']

			# updating wage in employee
			if self.employee:
				self.employee.wage = vals['per_month_gross_salary']

		after = self.write_date
		if before != after:
			if self.so_type == 'cost_card':
				self.get_order_lines()
				self.get_handle_sequence()

		if 'name' not in vals:
			if self.so_type == 'cost_card':
				self.UpdateSOName()

		if 'work_days_type' in vals:
			self.UpdateLeaveType()

		return rec

	def UpdateLeaveType(self):
		if self.work_days_type == "twenty_two_days":
			self.leave_type = "two_days"
		if self.work_days_type == "twenty_six_days":
			self.leave_type = "one_day"

	def GetDivisorPayroll(self,date_from):
		no_of_holidays = 0
		days = 0
		days = self.per_day_devisor(date_from)
		no_of_holidays = self.calculate_holidays("Regular",date_from)
		return days - no_of_holidays

	def GetWorkingDaysPayroll(self,date_from,date_to):
		result = 0
		starting_month = False
		ending_month = False
		if date_from <= self.contract_start_date  <= date_to:
			starting_month = True
		if date_from <= self.contract_end_date  <= date_to:
			ending_month = True

		values = self.FinalWorkingDays(date_from,starting_month,ending_month)
		return values[0]

	@api.onchange('no_of_months')
	def UpdateManualMonths(self):
		for lines in self.order_line:
			if lines.costcard_type == "manual":
				lines.product_uom_qty = self.no_of_months
				lines.get_manual_price_unit()

class SOLineExt(models.Model):
	_inherit='sale.order.line'
	_order = 'handle'

	code = fields.Char(string="Code")
	handle = fields.Char(string="Handle", readonly=True, copy=False)
	chargable = fields.Boolean(string="Chargable")
	manual_amount = fields.Float(string="Manual Amount")
	categ_id = fields.Many2one('product.category', string="Product Category")
	based_on_wd = fields.Boolean(string="Based on WD")
	recomputable = fields.Boolean(string="Recomputable")
	offset = fields.Float(string="Offset")
	as_of_date = fields.Date(string="As Of Month")

	def _check_line_unlink(self):
		return False
		"""
		Check wether a line can be deleted or not.

		Lines cannot be deleted if the order is confirmed; downpayment
		lines who have not yet been invoiced bypass that exception.
		:rtype: recordset sale.order.line
		:returns: set of lines that cannot be deleted
		"""
		# return self.filtered(lambda line: line.state in ('sale', 'done') and (line.invoice_lines or not line.is_downpayment))

	def prepare_invoice_line(self,amount,name,product_id):
		"""
		Prepare the dict of values to create the new invoice line for a sales order line.
		:param qty: float quantity to invoice
		"""
		self.ensure_one()

		return {
			'display_type': self.display_type,
			'sequence': self.sequence,
			'name': name,
			'product_id': product_id,
			'product_uom_id': self.product_uom.id,
			'quantity': 1,
			'discount': self.discount,
			# 'price_unit': self.price_unit,
			'price_unit': amount,
			'tax_ids': [(6, 0, self.tax_id.ids)],
			'analytic_account_id': self.order_id.analytic_account_id.id,
			'analytic_tag_ids': [(6, 0, self.analytic_tag_ids.ids)],
			'sale_line_ids': [(4, self.id)],
		}

	@api.onchange('manual_amount','product_uom_qty')
	def get_manual_price_unit(self):
		if self.costcard_type == 'manual' and self.order_id.no_of_months>0:
			if not self.product_uom_qty:
				self.product_uom_qty = self.order_id.no_of_months
			self.price_unit = self.manual_amount/self.product_uom_qty

	@api.onchange('product_uom_qty')
	def get_price_unit_on_uom_change(self):
		if self.costcard_type == 'manual' and self.order_id.no_of_months>0:
			self.price_unit = self.manual_amount/self.product_uom_qty

	@api.onchange('product_uom')
	def product_uom_change(self):
		if not self.product_uom or not self.product_id:
			self.price_unit = 0.0
			return
		if self.order_id.pricelist_id and self.order_id.partner_id:
			product = self.product_id.with_context(
				lang=self.order_id.partner_id.lang,
				partner=self.order_id.partner_id,
				quantity=self.product_uom_qty,
				date=self.order_id.date_order,
				pricelist=self.order_id.pricelist_id.id,
				uom=self.product_uom.id,
				fiscal_position=self.env.context.get('fiscal_position')
			)
			self.price_unit = self.env['account.tax']._fix_tax_included_price_company(self._get_display_price(product), product.taxes_id, self.tax_id, self.company_id)

	payment_type = fields.Selection([
		('upfront','Upfront'),
		('end','End'),
		('interval','Interval')
		], string='Payment Type')

	costcard_type = fields.Selection([
		('fixed','Fixed'),
		('manual','Manual'),
		('calculation','Calculation'),
		], string='Type')


#!/usr/bin/python
# -*- coding: utf-8 -*-

from abc import ABC
import logging
import os
import sys
import pandas as pd
from sklearn.preprocessing import MultiLabelBinarizer

# setup logger
logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

from .operator import IOperator

class CSV(IOperator, ABC):

    """ Instance Object for COCO annotation format """

    def __init__(self, dataset):
        super().__init__(dataset)
        self._dataset = dataset
        self.attrs = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']

    def extract(self, path: str):
        """
        all the annotations in the file convert into general dataframe object.
        :param path: string, relative / absolute path
        :return: generalize pandas.DataFrame type object.
        """
        if os.path.exists(path):
            ann_df = pd.read_csv(path)
            attr_df_list = list(ann_df.columns)
            if all(x in attr_df_list for x in self.attrs):
                new_ann_df = self.__dfUpdates(ann_df)
                self.__updateDataset(new_ann_df.loc[:, ["name", "width", "height", "image_id"]])
                self.__setAnn(new_ann_df)
            else:
                logger.error(f"\n ERROR : Entered annotation file does not contains all the required attributes. \n {self.attrs}")
                sys.exit()

        else:
            logger.error(f"\n ERROR : Entered directory {path}, does not exsist.")
            sys.exit()

    def archive(self, location, df):
        """ save csv annotation file in the given location

        :param location: .csv file saving location
        :param df: finalized DataFrame object from the self.translate()
        :return: None
        """
        if os.path.exists(os.path.dirname(location)):
                df.to_csv(location, index=False)
        else:
            logger.error("\n ERROR : There are no such parent directory to file save.")
            sys.exit(1)

    def translate(self, is_multilabel: bool=False):
        """ translate common schema into csv compatible format.

        :return: pd.DataFrame object with ["filename", "width", "height", "class", "xmin", "ymin", "xmax", "ymax"] columns.
        """
        csv_ann_df = self.annotations.copy()

        class_series = pd.Series(self.classes)
        csv_ann_df.loc[:,"class"] = csv_ann_df["class_id"].map(class_series)

        filename_series = pd.Series(self._dataset['name'].tolist(),
                                    index=self._dataset["image_id"].tolist())
        csv_ann_df.loc[:,"filename"] = csv_ann_df["image_id"].map(filename_series)

        width_series = pd.Series(self._dataset['width'].tolist(),
                                    index=self._dataset["image_id"].tolist())
        csv_ann_df.loc[:, "width"] = csv_ann_df["image_id"].map(width_series)

        height_series = pd.Series(self._dataset['height'].tolist(),
                                    index=self._dataset["image_id"].tolist())
        csv_ann_df.loc[:, "height"] = csv_ann_df["image_id"].map(height_series)

        if (pd.isnull(csv_ann_df["class"]).sum() + pd.isnull(csv_ann_df["filename"]).sum()) != 0:
            logger.error(f"\n ERROR : There are not enough data in past annotation file to create annotation file. {pd.isnull(csv_ann_df['class']).sum()}, {pd.isnull(csv_ann_df['filename']).sum()}")
            sys.exit(1)
        else:
            csv_ann_df.rename(columns={"x_min": "xmin", "y_min": "ymin", "x_max": "xmax", "y_max": "ymax"},
                              inplace=True)
            res_df =  csv_ann_df.copy().loc[:, ["filename", "width", "height", "class", "xmin", "ymin", "xmax", "ymax"]]

            if is_multilabel:
                res_df = self.to_multilabel(res_df)

            return res_df

    def __dfUpdates(self, full_df):
        """add id, image width & height columns to self.dataset

        :param full_df: read .csv file from annotation file.
        :return: refine DataFrame object with column of ["name" ,"obj_id", "image_id", "class_id", "x_min", "y_min", "x_max", "y_max"]
        """
        full_df = full_df.copy()

        uni_files = list(full_df.loc[:, "filename"].unique())
        ns = len(uni_files)
        file_id_col = pd.Series(range(1, ns + 1), index=uni_files)
        full_df["image_id"] = full_df["filename"].map(file_id_col)

        cats = list(full_df.loc[:, "class"].unique())
        nc = len(cats)
        cat_id_col = pd.Series(range(1, nc + 1), cats)
        full_df["class_id"] = full_df["class"].map(cat_id_col)

        self.__defineClasses(nc, cats)
        full_df.drop("class", inplace=True, axis=1)

        full_df["obj_id"] = pd.Series(range(1, full_df.shape[0] + 1))
        full_df.rename(columns={"filename": "name", "xmin": "x_min", "ymin": "y_min", "xmax": "x_max", "ymax": "y_max"},
                       inplace=True)

        return full_df

    def __defineClasses(self, n_ids, classes):
        """

        :param n_ids: number of unique classes
        :param classes: class list
        :return: dictionary object of type {id : class-name}
        """
        if n_ids == len(classes):
            ids = range(1, n_ids + 1)
            super(CSV, self).set_classes(dict(zip(ids, classes)))
        else:
            logger.error(f"\nlength of class names[{len(classes)}] and class ids[{n_ids}] are not equal.")
            sys.exit(1)

    def __setAnn(self, full_df):
        """

        :param full_df: refined DataFrame object.
        :return: set generalized annotation df object as self.annotations
        """
        full_df = full_df.copy()
        col_lis = ["obj_id", "image_id", "class_id", "x_min", "y_min", "x_max", "y_max"]
        if all(y in list(full_df.columns) for y in col_lis):
            ann_df = full_df.loc[:, col_lis]
            super(CSV, self).set_annotations(ann_df)
        else:
            logger.error(f"\nthere are missing of required columns in {full_df.columns}")
            sys.exit(1)

    def __updateDataset(self, image_df):
        """

        :param image_df: image attributes DataFrame
        :return: merge current self.__dataset with image_df.
        """
        partial_df = image_df.copy()
        partial_df.drop_duplicates(inplace=True)
        res_df = pd.merge(self._dataset, partial_df, on="name")
        super(CSV, self).set_dataset(res_df)

    def to_multilabel(self, df):
        if isinstance(df, str):
            df = pd.read_csv(df)

        grp_df = df.groupby('filename')['class'].apply(list).reset_index(name='labels')
        mlb = MultiLabelBinarizer(sparse_output=True)
        res_df = grp_df.join(pd.DataFrame.sparse.from_spmatrix(mlb.fit_transform(grp_df.pop('labels')),index=grp_df.index,columns=mlb.classes_))
        res_df.drop_duplicates(inplace=True)
        return res_df


from django.http import HttpResponse
from django.template import RequestContext
from django.shortcuts import get_object_or_404, render_to_response
from models import *
import settings

'''
'''
def tradeoff_analysis(request):
    if (request.POST):
        x_axis = request.POST.getlist('selected_x')[0]
        y_axis = request.POST.getlist('selected_y')[0]
        folder_id = getlist(request, 'folder_id')[0]
        smp_ids = getlist(request, 'smps[]')
        aoi_ids = getlist(request, 'aois[]')

        folder_obj = get_object_or_404(AnalysisFolder, pk=folder_id)
        #check permissions
        viewable, response = folder_obj.is_viewable(request.user)
        if not viewable:
            return response

        from tradeoff.tradeoff_analysis import display_tradeoff_analysis
        return display_tradeoff_analysis(request, folder_obj, x_axis, y_axis, smp_ids, aoi_ids)

'''
'''
def tradeoff_table(request):
    if (request.POST):
        folder_id = getlist(request, 'folder_id')[0]
        smp_ids = getlist(request, 'smps[]')
        aoi_ids = getlist(request, 'aois[]')
        folder_obj = get_object_or_404(AnalysisFolder, pk=folder_id)
        #check permissions
        viewable, response = folder_obj.is_viewable(request.user)
        if not viewable:
            return response

        from tradeoff.tradeoff_analysis import display_tradeoff_table
        return display_tradeoff_table(request, folder_obj, smp_ids, aoi_ids)

def getlist(request, param_string):
    string_list = request.POST.getlist(param_string)
    list = []
    for s in string_list:
        list.append(int(s))
    return list


# !/usr/local/bin/python3.6
# MIT licensed
# Copyright (c) 2018 White Russsian
# Github: <https://github.com/WhiteRusssian/PySpy>**********************
''' The primary function is main(), which takes a set of EVE Online
character names and gathers useful information from CCP's ESI API and
zKIllboard's API, to be stored in a temporary in-memory SQLite3 database.
'''
# **********************************************************************
import logging
import json
import threading
import queue
import time

import apis
import config
import db
import statusmsg
# cSpell Checker - Correct Words****************************************
# // cSpell:words affil, zkill, blops, qsize, numid, russsian, ccp's
# **********************************************************************
Logger = logging.getLogger(__name__)
# Example call: Logger.info("Something badhappened", exc_info=True) ****

def main(char_names):
    conn, cur = db.connect_db()
    chars_found = get_char_ids(conn, cur, char_names)
    if chars_found > 0:
        char_ids = cur.execute(
            "SELECT char_id FROM characters ORDER BY char_name"
            ).fetchall()
        q_main = queue.Queue()
        t = zKillStats(char_ids, q_main)
        t.start()
        get_char_affiliations(conn, cur)
        get_affil_names(conn, cur)
        t.join()
        zkill_stats = q_main.get()
        query_string = (
            '''UPDATE characters SET kills=?, blops_kills=?, hic_losses=?,
            week_kills=?
            WHERE char_id=?'''
            )
        db.write_many_to_db(conn, cur, query_string, zkill_stats)
        output = output_list(cur)
        conn.close()
        return output
    else:
        return

def get_char_ids(conn, cur, char_names):
    char_names = json.dumps(char_names[0:config.MAX_NAMES])  # apis max char is 1000
    statusmsg.push_status("Resolving character names to IDs...")
    try:
        characters = apis.post_req_ccp("universe/ids/", char_names)
        characters = characters['characters']
    except:
        return 0
    records = ()
    for r in characters:
        records = records + ((r["id"], r["name"]),)
    query_string = (
        '''INSERT INTO characters (char_id, char_name) VALUES (?, ?)'''
        )
    records_added = db.write_many_to_db(conn, cur, query_string, records)
    return records_added

def get_char_affiliations(conn, cur):
    char_ids = cur.execute("SELECT char_id FROM characters").fetchall()
    char_ids = json.dumps(tuple([r[0] for r in char_ids]))
    statusmsg.push_status("Retrieving character affiliation IDs...")
    try:
        affiliations = apis.post_req_ccp("characters/affiliation/", char_ids)
    except:
        Logger.info("Failed to obtain character affiliations.", exc_info=True)
        raise Exception

    records = ()
    for r in affiliations:
        corp_id = r["corporation_id"]
        alliance_id = r["alliance_id"] if "alliance_id" in r else 0
        faction_id = r["faction_id"] if "faction_id" in r else 0
        char_id = r["character_id"]
        records = records + ((corp_id, alliance_id, faction_id, char_id), )

    query_string = (
        '''UPDATE characters SET corp_id=?, alliance_id=?, faction_id=?
        WHERE char_id=?'''
        )
    records_added = db.write_many_to_db(conn, cur, query_string, records)

def get_affil_names(conn, cur):
    # use select distinct to get corp and alliance ids and reslove them
    alliance_ids = cur.execute(
        '''SELECT DISTINCT alliance_id FROM characters
        WHERE alliance_id IS NOT 0'''
        ).fetchall()
    corp_ids = cur.execute(
        "SELECT DISTINCT corp_id FROM characters WHERE corp_id IS NOT 0"
        ).fetchall()

    ids = alliance_ids + corp_ids
    ids = json.dumps(tuple([r[0] for r in ids]))

    statusmsg.push_status("Obtaining corporation and alliance names and zKillboard data...")
    try:
        names = apis.post_req_ccp("universe/names/", ids)
    except:
        Logger.info("Failed request corporation and alliance names.",
                    exc_info=True)
        raise Exception

    alliances, corporations = (), ()
    for r in names:
        if r["category"] == "alliance":
            alliances = alliances + ((r["id"], r["name"]),)
        elif r["category"] == "corporation":
            corporations = corporations + ((r["id"], r["name"]),)
    if alliances:
        query_string = ('''INSERT INTO alliances (id, name) VALUES (?, ?)''')
        db.write_many_to_db(conn, cur, query_string, alliances)
    if corporations:
        query_string = ('''INSERT INTO corporations (id, name) VALUES (?, ?)''')
        db.write_many_to_db(conn, cur, query_string, corporations)

class zKillStats(threading.Thread):

    def __init__(self, char_ids, q_main):
        super(zKillStats, self).__init__()
        self.daemon = True
        self._char_ids = char_ids
        self._q_main = q_main

    def run(self):
        count = 0
        max = config.ZKILL_CALLS
        threads = []
        q_sub = queue.Queue()
        for id in self._char_ids:
            t = apis.Query_zKill(id[0], q_sub)
            threads.append(t)
            t.start()
            count += 1
            time.sleep(config.ZKILL_DELAY)
            if count >= max:
                break
        for t in threads:
            t.join(5)
        zkill_stats = []
        while q_sub.qsize():
            # Run through each queue item and prepare response list.
            s = q_sub.get()
            kills = str(s[0])
            blops_kills = str(s[1])
            hic_losses = str(s[2])
            week_kills = str(s[3])
            id = str(s[4])
            zkill_stats.append([kills, blops_kills, hic_losses, week_kills, id])
        self._q_main.put(zkill_stats)
        return

def output_list(cur):
    query_string = (
        '''SELECT
        ch.char_id, ch.faction_id, ch.char_name, co.name, al.name, fa.name,
        ac.numid, ch.week_kills, ch.kills, ch.blops_kills, hic_losses
        FROM characters AS ch
        LEFT JOIN alliances AS al ON ch.alliance_id = al.id
        LEFT JOIN corporations AS co ON ch.corp_id = co.id
        LEFT JOIN factions AS fa ON ch.faction_id = fa.id
        LEFT JOIN (SELECT alliance_id, COUNT(alliance_id) AS numid FROM characters GROUP BY alliance_id) AS ac ON
        ch.alliance_id = ac.alliance_id
        ORDER BY ch.char_name'''
        )
    return cur.execute(query_string).fetchall()


import numpy as np
import pandas as pd
import dask as ds
import dask.dataframe
from performance.utils import TimeFunction
from multiprocessing.pool import Pool
from collections import deque

""" Number of elements for the simple data transformation """
DATA_SIZE = 10000

""" Size for the DataFrames to be used on the join operation """
CITY_COUNT = 120
PEOPLE_COUNT = 10000
NUM_PARTITIONS=10
NUM_CORES=8

@TimeFunction
def calculate_on_dask():
    df = pd.DataFrame(np.random.randn(DATA_SIZE, 1), columns=['values'])
    df = ds.dataframe.from_pandas(df, 10)

    df.apply(axis=1, func=lambda x: x * x)

    df.compute()

@TimeFunction
def calculate_on_pandas():
    df = pd.DataFrame(np.random.randn(DATA_SIZE, 1), columns=['values'])

    df.apply(axis=1, func=lambda x: x * x)

def generate_person_table():
    names = ['Harold', 'Nahuel', 'Kirsten', 'Katherina']
    return [(i % CITY_COUNT, names[i % 4] + str(i)) for i in range(PEOPLE_COUNT)]

def generate_city_table():
    names = ['Montevideo', 'Canelones', 'Maldonado', 'Soriano', 'Minas', 'Rivera', 'Salto', 'Paysandu']
    return [(i % CITY_COUNT, names[i % 8] + str(i)) for i in range(CITY_COUNT)]

def parallelize_dataframe(df, func):
    df_split = np.array_split(df, NUM_PARTITIONS)
    pool = Pool(NUM_CORES)
    try:
        df = pd.concat(pool.map(func, df_split))
    finally:
        pool.close()
        pool.join()
    return df

@TimeFunction
def pandas_join():
    people = pd.DataFrame(generate_person_table(), columns=['city_id', 'names'])
    cities = pd.DataFrame(generate_city_table(), columns=['city_id', 'city'])

    result = people.merge(cities, on='city_id', how='left')

def merge_helper(df):
    return df.merge(cities, on='city_id', how='left')

@TimeFunction
def pandas_multiprocessing():
    global cities
    people = pd.DataFrame(generate_person_table(), columns=['city_id', 'names'])
    cities = pd.DataFrame(generate_city_table(), columns=['city_id', 'city'])

    parallelize_dataframe(people, merge_helper)

    cities = None

@TimeFunction
def dask_join():
    people = pd.DataFrame(generate_person_table(), columns=['city_id', 'names'])
    cities = pd.DataFrame(generate_city_table(), columns=['city_id', 'city'])

    people_dask = ds.dataframe.from_pandas(people,20)
    cities_dask = ds.dataframe.from_pandas(cities,20)

    result_dask = people_dask.merge(cities_dask, how='left', on='city_id').compute()

@TimeFunction
def dask_filter():
    people = pd.DataFrame(generate_person_table(), columns=['city_id', 'names'])
    people_dask = ds.dataframe.from_pandas(people, 10)

    people_dask[people_dask['city_id'] == 1].compute()

@TimeFunction
def pandas_filter():
    people = pd.DataFrame(generate_person_table(), columns=['city_id', 'names'])

    people[people['city_id'] == 1]

if __name__ == '__main__':
    for i in range(10):
        calculate_on_dask()
        calculate_on_pandas()
        pandas_join()
        dask_join()
        pandas_multiprocessing()
        dask_filter()
        pandas_filter()

    pandas_join.print_stats()
    dask_join.print_stats()
    calculate_on_dask.print_stats()
    calculate_on_pandas.print_stats()
    pandas_multiprocessing.print_stats()
    dask_filter.print_stats()
    pandas_filter.print_stats()


import os
import numpy as np
import torch
from PIL import Image

class PennFudanDataset(torch.utils.data.Dataset):
    def __init__(self, root, transforms):
        self.root = root
        self.transforms = transforms
        # load all image files, sorting them to
        # ensure that they are aligned
        self.imgs = list(sorted(os.listdir(os.path.join(root, "PNGImages"))))
        self.masks = list(sorted(os.listdir(os.path.join(root, "PedMasks"))))

    def __getitem__(self, idx):
        # load images and masks
        img_path = os.path.join(self.root, "PNGImages", self.imgs[idx])
        mask_path = os.path.join(self.root, "PedMasks", self.masks[idx])
        img = Image.open(img_path).convert("RGB")
        # note that we haven't converted the mask to RGB,
        # because each color corresponds to a different instance
        # with 0 being background
        mask = Image.open(mask_path)
        # convert the PIL Image into a numpy array
        mask = np.array(mask)
        # instances are encoded as different colors
        obj_ids = np.unique(mask)
        # first id is the background, so remove it
        obj_ids = obj_ids[1:]

        # split the color-encoded mask into a set
        # of binary masks
        masks = mask == obj_ids[:, None, None]

        # get bounding box coordinates for each mask
        num_objs = len(obj_ids)
        boxes = []
        for i in range(num_objs):
            pos = np.where(masks[i])
            xmin = np.min(pos[1])
            xmax = np.max(pos[1])
            ymin = np.min(pos[0])
            ymax = np.max(pos[0])
            boxes.append([xmin, ymin, xmax, ymax])

        # convert everything into a torch.Tensor
        boxes = torch.as_tensor(boxes, dtype=torch.float32)
        # there is only one class
        labels = torch.ones((num_objs,), dtype=torch.int64)
        masks = torch.as_tensor(masks, dtype=torch.uint8)

        image_id = torch.tensor([idx])
        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
        # suppose all instances are not crowd
        iscrowd = torch.zeros((num_objs,), dtype=torch.int64)

        target = {}
        target["boxes"] = boxes
        target["labels"] = labels
        target["masks"] = masks
        target["image_id"] = image_id
        target["area"] = area
        target["iscrowd"] = iscrowd

        if self.transforms is not None:
            img, target = self.transforms(img, target)

        return img, target

    def __len__(self):
        return len(self.imgs)

import torchvision
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor

def get_instance_segmentation_model(num_classes):
    # load an instance segmentation model pre-trained on COCO
    model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True)

    # get the number of input features for the classifier
    in_features = model.roi_heads.box_predictor.cls_score.in_features
    # replace the pre-trained head with a new one
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)

    # now get the number of input features for the mask classifier
    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
    hidden_layer = 256
    # and replace the mask predictor with a new one
    model.roi_heads.mask_predictor = MaskRCNNPredictor(
        in_features_mask, hidden_layer, num_classes
    )

    return model

device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

# our dataset has two classes only - background and person
num_classes = 2

# get the model using our helper function
model = get_instance_segmentation_model(num_classes)
# move model to the right device
model.to(device)

# construct an optimizer
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)

from engine import train_one_epoch, evaluate
import utils
import transforms as T

class CocoDetection(torchvision.datasets.CocoDetection):
    def __init__(
        self, root, annFile, transform=None, target_transform=None, transforms=None
    ):
        super().__init__(root, annFile, transform, target_transform, transforms)

        ids = []
        for img in self.coco.imgs.values():
            anns = self.coco.imgToAnns[img['id']]
            if len(anns) > 1:
                ids.append(img['id'])
        self.ids = list(sorted(ids))

    def __getitem__(self, index):
        """
        Args:
            index (int): Index

        Returns:
            tuple: Tuple (image, target). target is the object returned by ``coco.loadAnns``.
        """
        coco = self.coco
        img_id = self.ids[index]
        ann_ids = coco.getAnnIds(imgIds=img_id)
        anns = coco.loadAnns(ann_ids)
        # print(img_id, anns)

        path = coco.loadImgs(img_id)[0]['file_name']

        img = Image.open(os.path.join(self.root, path)).convert('RGB')

        # split the color-encoded mask into a set
        # of binary masks
        if len(anns) > 1:
            masks: np.ndarray = np.stack([coco.annToMask(ann) for ann in anns])
        else:
            masks = coco.annToMask(anns[0])[np.newaxis, ...]

        # get bounding box coordinates for each mask
        boxes = []
        area = []
        labels = []
        iscrowd = []
        for ann in anns:
            bbox = ann['bbox']
            xmin = np.min(bbox[0])
            xmax = np.max(bbox[0] + bbox[2])
            ymin = np.min(bbox[1])
            ymax = np.max(bbox[1] + bbox[3])
            boxes.append([xmin, ymin, xmax, ymax])

            area.append(ann['area'])

            labels.append(ann['category_id'])

        # convert everything into a torch.Tensor
        boxes = torch.as_tensor(boxes, dtype=torch.float32)
        # there is only one class
        labels = torch.tensor(labels, dtype=torch.int64)
        masks = torch.as_tensor(masks, dtype=torch.uint8)

        image_id = torch.tensor([img_id])
        area = torch.tensor(area)
        # suppose all instances are not crowd
        iscrowd = torch.tensor(iscrowd, dtype=torch.int64)

        target = {}
        target["boxes"] = boxes
        target["labels"] = labels
        target["masks"] = masks
        target["image_id"] = image_id
        target["area"] = area
        target["iscrowd"] = iscrowd

        if self.transforms is not None:
            img, target = self.transforms(img, target)

        return img, target

    def __len__(self):
        return len(self.ids)

def get_transform(train):
    transforms = []
    # converts the image, a PIL image, into a PyTorch Tensor
    transforms.append(T.ToTensor())
    # if train:
    #     # during training, randomly flip the training images
    #     # and ground-truth for data augmentation
    #     transforms.append(T.RandomHorizontalFlip(0.5))
    return T.Compose(transforms)

# # use our dataset and defined transformations
# dataset = PennFudanDataset(
#     '/home/tankred/other_projects/PennFudanPed', get_transform(train=True)
# )
# dataset_test = PennFudanDataset(
#     '/home/tankred/other_projects/PennFudanPed', get_transform(train=False)
# )

dataset = CocoDetection(
    root='/home/ginjinn2/pollen_alnus_dataprep/pollen_alnus_full_coco/images',
    annFile='/home/ginjinn2/pollen_alnus_dataprep/pollen_alnus_full_coco/annotations.json',
    transforms=get_transform(train=True),
)

dataset_test = CocoDetection(
    root='/home/ginjinn2/pollen_alnus_dataprep/pollen_alnus_full_coco/images',
    annFile='/home/ginjinn2/pollen_alnus_dataprep/pollen_alnus_full_coco/annotations.json',
    transforms=get_transform(train=False),
)

# split the dataset in train and test set
torch.manual_seed(1)
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-10])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-10:])

# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(
    dataset, batch_size=1, shuffle=True, num_workers=1, collate_fn=utils.collate_fn
)

data_loader_test = torch.utils.data.DataLoader(
    dataset_test,
    batch_size=2,
    shuffle=False,
    num_workers=4,
    collate_fn=utils.collate_fn,
)

# train
from torch.optim.lr_scheduler import StepLR

lr_scheduler = StepLR(optimizer, step_size=3, gamma=0.1)

num_epochs = 5

for epoch in range(num_epochs):
    # train for one epoch, printing every 10 iterations
    train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq=10)
    # update the learning rate
    lr_scheduler.step()
    # evaluate on the test dataset
    evaluate(model, data_loader_test, device=device)


from django.urls import path

from .views import TeaListView, TeaCreateView, TeaDetailView

app_name='teas'

urlpatterns = [
    path('', TeaListView.as_view(), name='list'),
    path('create', TeaCreateView.as_view(), name='create'),
    path('<uuid:pk>', TeaDetailView.as_view(), name='detail')
]


import torch
import torch.nn as nn
from torchvision import models
from args import args
import coatnet
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.cnn = nn.Sequential(
            nn.Conv2d(3, 32, 7),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(2, 2, 0),

            nn.Conv2d(32, 64, 7),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2, 2, 0),

            nn.Conv2d(64, 128, 5),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.MaxPool2d(2, 2, 0),

            nn.Conv2d(128, 256, 5),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.MaxPool2d(2, 2, 0),

            nn.Conv2d(256, 512, 3),
            nn.BatchNorm2d(512),
            nn.ReLU(),
            nn.MaxPool2d(2, 2, 0),

            nn.Conv2d(512, 1024, 3),
            nn.BatchNorm2d(1024),
            nn.ReLU(),
            nn.MaxPool2d(2, 2, 0),
        )
        self.fc = nn.Sequential(
            nn.Linear(1024, 512),
            nn.BatchNorm1d(512),
            nn.ReLU(),
            nn.Dropout(),
            nn.Linear(512, 14),
        )

    def forward(self, x):
        out = self.cnn(x)
        out = out.view(out.size()[0], -1)
        return self.fc(out)

def get_models(model_name, pretrained):
    # pretrained model path
    model_path ="{}/{}/ckpt_best.ckpt".format(args["save_dir"], model_name)
    model = None
    if model_name == 'coatnet':
        model = coatnet.coatnet_hx()
        if pretrained:
            try:
                model.load_state_dict(torch.load(model_path))
                print(f"pretrained {model_name} model is loaded!")
            except:
                print(f"pretrained model doesn't exist! model path: {model_path}")
            if args['use_swa']:
                try:
                    swa_model = torch.optim.swa_utils.AveragedModel(model)
                    swa_model.load_state_dict(torch.load("{}/{}/swa.ckpt".format(args["save_dir"], model_name)))
                    print(f"loading SWA model!")
                except:
                    print("can't load SWA model")
        return model

    if model_name == "CNN":
        model = CNN()
        if pretrained:
            try:
                model.load_state_dict(torch.load(model_path))
                print(f"pretrained {model_name} model is loaded!")
            except:
                print(f"pretrained model doesn't exist! model path: {model_path}")
            if args['use_swa']:
                try:
                    swa_model = torch.optim.swa_utils.AveragedModel(model)
                    swa_model.load_state_dict(torch.load("{}/{}/swa.ckpt".format(args["save_dir"], model_name)))
                    print(f"loading SWA model!")
                except:
                    print("can't load SWA model")
        return model
    if model_name == "Res18":
        model = models.resnet18(pretrained=True)
    elif model_name == "Res34":
        model = models.resnet34(pretrained=True)
    elif model_name == "Res50":
        model = models.resnet50(pretrained=True)
    elif model_name == "wide_res":
        model = models.wide_resnet50_2(pretrained=True)

    if model != None:
        fc_inputs = model.fc.in_features

        model.fc = nn.Sequential(
            nn.Linear(fc_inputs, 512),
            nn.ReLU(),
            nn.Dropout(0.5),
            nn.Linear(512, 14),
        )
        if pretrained:
            try:
                model.load_state_dict(torch.load(model_path))
                print(f"pretrained {model_name} model is loaded!")
            except:
                print(f"pretrained model doesn't exist! model path: {model_path}")
            if args['use_swa']:
                try:
                    swa_model = torch.optim.swa_utils.AveragedModel(model)
                    swa_model.load_state_dict(torch.load("{}/{}/swa.ckpt".format(args["save_dir"], model_name)))
                    print(f"loading SWA model!")
                except:
                    print("can't load SWA model")
        return model

    if model_name == "densenet169":
        model = models.densenet169(pretrained=True)
    elif model_name == "densenet121":
        model = models.densenet121(pretrained=True)
    elif model_name == "densenet201":
        model = models.densenet201(pretrained=True)

    fc_inputs = model.classifier.in_features
    model.classifier = nn.Sequential(
        nn.Linear(fc_inputs, 512),
        nn.ReLU(),
        nn.Dropout(0.5),
        nn.Linear(512, 14),
    )
    if pretrained:
        try:
            model.load_state_dict(torch.load(model_path))
            print(f"pretrained {model_name} model is loaded!")
        except:
            print(f"pretrained model doesn't exist! model path: {model_path}")
        if args['use_swa']:
            try:
                swa_model = torch.optim.swa_utils.AveragedModel(model)
                swa_model.load_state_dict(torch.load("{}/{}/swa.ckpt".format(args["save_dir"], model_name)))
                print(f"loading SWA model!")
            except:
                print("can't load SWA model")
    return model


import webapp2
import json
from google.appengine.datastore.datastore_query import Cursor
from google.appengine.api import taskqueue
from RecipeParser import parseRecipeUrlToJson
from BlogParser import fetch_recipe_feed_entries, get_latest_recipes, find_image, add_feed, get_feeds, fetch_all_recipe_feed_entries, add_recipe_feed_set, fetch_recipe_feed_entries_new, get_latest_recipes_new

class RecipeParser(webapp2.RequestHandler):
    def get(self):
    	if self.request.get('url') == '':
    		self.response.write(json.dumps({"Error" : "Missing url parameter.  Example: http://recipe-api.appspot.com/?url=http://allrecipes.com/Recipe/Easy-Chicken-Pasta-Alfredo/"}))
    	else:
	        self.response.headers['Content-Type'] = 'application/json'
	        self.response.write(parseRecipeUrlToJson(self.request.get('url')))

class FetchRecipeFeedEntries(webapp2.RequestHandler):
	def get(self):
		url = self.request.get('url')
		if url == '':
			self.response.write(json.dumps({"Error" : "Missing 'url' parameter."}))
		else:
			self.response.headers['Content-Type'] = 'application/json'
			response = fetch_recipe_feed_entries_new(url)
			self.response.write(response)

class LatestRecipes(webapp2.RequestHandler):
	def get(self):
		self.response.headers['Content-Type'] = 'application/json'
		curs = Cursor(urlsafe=self.request.get('cursor'))
		response = get_latest_recipes_new(curs)
		self.response.write(response)

class Thumbnailer(webapp2.RequestHandler):
	def get(self):
		if self.request.get('url') == '':
			self.response.write(json.dumps({"Error" : "Missing 'url' parameter."}))
		else:
			self.response.headers['Content-Type'] = 'application/json'
			response = json.dumps({"image_url" : find_image(self.request.get('url'))})
			self.response.write(response)

class AddRecipeFeed(webapp2.RequestHandler):
	def get(self):
		url = self.request.get('url')
		if url == '':
			self.response.write(json.dumps({"Error" : "Missing 'url' parameter."}))
		else:
			self.response.headers['Content-Type'] = 'application/json'
			response = add_feed(url)
			self.response.write(response)

class AddRecipeFeedSet(webapp2.RequestHandler):
	def get(self):
		self.response.headers['Content-Type'] = 'application/json'
		response = add_recipe_feed_set()
		self.response.write(response)

class GetRecipeFeeds(webapp2.RequestHandler):
	def get(self):
		self.response.headers['Content-Type'] = 'application/json'
		response = get_feeds()
		self.response.write(response)

class FetchAllRecipeFeedEntries(webapp2.RequestHandler):
	def get(self):
		taskqueue.add(url='/recipe_feeds/create_fetch_all_task/', method='GET')
		self.response.headers['Content-Type'] = 'application/json'
		self.response.write(json.dumps({"response" : "started fetching all recipe feed entries"}))

class CreateFetchAllRecipeFeedEntriesTask(webapp2.RequestHandler):
	def get(self):
		self.response.headers['Content-Type'] = 'application/json'
		self.response.write(fetch_all_recipe_feed_entries())

application = webapp2.WSGIApplication([
    ('/recipe_parser/', RecipeParser),
    ('/recipe_feeds/add/', AddRecipeFeed),
    ('/recipe_feeds/add_set/', AddRecipeFeedSet),
    ('/recipe_feeds/', GetRecipeFeeds),
    ('/recipe_feeds/fetch_all/', FetchAllRecipeFeedEntries),
    ('/recipe_feeds/create_fetch_all_task/', CreateFetchAllRecipeFeedEntriesTask),
    ('/thumbnailer/', Thumbnailer),
    ('/recipe_feed/fetch/', FetchRecipeFeedEntries),
    ('/latest_recipes/', LatestRecipes),
    ('/', RecipeParser),
], debug=True)


from django.shortcuts import render
from django.views import generic

from .models import Actor, Director, Movie
#from .forms import PersonForm
from django.views.generic.edit import CreateView, UpdateView

#from django.views.generic.edit import FormView

def home(request):
    return render(request, 'home.html')

class ActorIndexView(generic.ListView):
    template_name = 'actor_list.html'
    context_object_name = 'actors'

    def get_queryset(self):
        """Return the last five published questions."""
        return Actor.objects.order_by('birthday')

class ActorDetailView(generic.DetailView):
    model = Actor
    template_name = 'actor_detail.html'

class ActorCreateView(CreateView):
    template_name = 'actor_form.html'
    model = Actor
    success_url = '../'
    fields = [
        'first_name',
        'last_name',
        'slug',
        'birthday',
    ]

class ActorUpdateView(UpdateView):
    template_name = 'actor_form.html'
    model = Actor
    fields = [
        'first_name',
        'last_name',
        'slug',
        'birthday',
    ]

class MovieIndexView(generic.ListView):
    template_name = 'movie_list.html'
    context_object_name = 'movies'

    def get_queryset(self):
        """Return the last five published questions."""
        return Movie.objects.order_by('year')

class MovieDetailView(generic.DetailView):
    model = Movie
    template_name = 'movie_detail.html'

class MovieCreateView(CreateView):
    template_name = 'movie_form.html'
    model = Movie
    success_url = '../'
    fields = [
        'name',
        'year',
        'rating',
        'director',
        'actors',
    ]

class MovieUpdateView(UpdateView):
    template_name = 'movie_form.html'
    model = Movie
    fields = [
        'name',
        'year',
        'rating',
        'director',
        'actors',
    ]

class DirectorIndexView(generic.ListView):
    template_name = 'director_list.html'
    context_object_name = 'directors'

    def get_queryset(self):
        """Return the last five published questions."""
        return Director.objects.order_by('birthday')

class DirectorDetailView(generic.DetailView):
    model = Director
    template_name = 'director_detail.html'

class DirectorCreateView(CreateView):
    template_name = 'director_form.html'
    model = Director
    success_url = '../'
    fields = [
        'first_name',
        'last_name',
        'slug',
        'birthday',
    ]

class DirectorUpdateView(UpdateView):
    template_name = 'director_form.html'
    model = Director
    fields = [
        'first_name',
        'last_name',
        'slug',
        'birthday',
    ]


import urllib.error
import urllib.parse
import urllib.request
import traceback
from collections import OrderedDict
from pprint import pprint
from datetime import datetime

from bs4 import BeautifulSoup
from twitter import Twitter, OAuth, api

class TweetUpdate(object):

    '''
    This class is used to tweet the latest update from a RSS feed, once that
    update has been handled by the MonitorFeedUpdate object.
    '''

    URLS_OF_IMAGES_NO_NEED_TO_UPLOAD_TO_TWITTER = (
        "https://a.fsdn.com/sd/twitter_icon_large.png",
    )

    def __init__(self, oauth_key, oauth_secret, consumer_key, consumer_secret):
        '''
        Initializes a Twitter object from the twitter module, using the given
        API credentials.
        '''

        # Twitter object
        self.twitter_api = Twitter(auth=OAuth(oauth_key, oauth_secret,
                                              consumer_key, consumer_secret))
        self.msg = OrderedDict((
            ('title', ''),
            ('url', ''),
            ('summary', ''),
            ('img_url', ''),
        ))

    def reset_msg(self):
        for key in self.msg:
            self.msg[key] = ''

    def delete_last_tweet(self):
        '''
        Deletes the last tweet in the timeline.
        This method is only called when an element in the feed is modified.
        '''

        last_tweet = self.twitter_api.statuses.home_timeline(count=1)[0]
        return self.twitter_api.statuses.destroy(id=last_tweet['id'])

    def get_entry_img_url(self, feed_entry):
        '''
        If feed entry has <img> then return the url of the image,
        else return None
        '''

        if hasattr(feed_entry, 'content'):
            entry_html = feed_entry.content[0].value
        elif hasattr(feed_entry, 'description'):
            entry_html = feed_entry.description
        else:
            return None

        soup = BeautifulSoup(entry_html, 'html.parser')
        img_tag = soup.find('img')
        try:
            img_url = img_tag['src']
        except KeyError:
            pprint(img_tag)
            pprint("This <img> tag has no <src>")
            return None
        except TypeError:   # entry_html doesn't have <img> tag
            return None
        else:
            img_url_splited = urllib.parse.urlsplit(img_url)
            img_url = img_url_splited._replace(
                netloc=urllib.parse.quote(img_url_splited.netloc),
                path=urllib.parse.quote(img_url_splited.path),
            ).geturl()

            if img_url in self.URLS_OF_IMAGES_NO_NEED_TO_UPLOAD_TO_TWITTER:
                return None
            return img_url

    def msg_to_string(self):
        '''
        Combine the elements in msg into a string.
        Add '\n' between each element.
        '''

        return '\n'.join(filter(bool, self.msg.values()))

    def msg_length(self):
        '''
        Get the length of the msg to be sent as a tweet.
        '''

        return len(self.msg_to_string())

    def get_msg_limit_length_and_urls(self, feed_entry):
        '''
        Calculating the limit length of current tweet
        and returns the url and img_url for cram_the_msg()
        '''

        TWEET_URL_LENGTH = 24
        TWEET_IMG_LENGTH = 25
        msg_limit_length = 280

        try:
            url = feed_entry['link']
        except:
            url = ''
        if url:
            msg_limit_length -= TWEET_URL_LENGTH

        try:
            img_url = self.get_entry_img_url(feed_entry)
        except:
            img_url = ''
        if img_url:
            msg_limit_length -= TWEET_IMG_LENGTH

        return msg_limit_length, url, img_url

    def cram_the_msg(self, feed_entry, msg_limit_length, url, img_url):
        '''
        Cram the contents in msg by calculating the limit length of a tweet.
        '''

        try:
            self.msg['title'] = feed_entry['title'].strip()
        except:
            pass
        else:
            if self.msg_length() > msg_limit_length:
                self.msg['title'] = self.msg['title'][:msg_limit_length]

        try:
            soup = BeautifulSoup(feed_entry['summary'], 'html.parser')
            self.msg['summary'] = ' '.join(
                map(str.strip, soup.get_text().split('\n'))
            ).strip()
        except:
            pass
        else:
            if self.msg_length() > msg_limit_length:
                print('msg({} chars) is longer than {}.'.format(
                    self.msg_length(), msg_limit_length
                ))
                print('summary before trimmed:\n{}'.format(
                    self.msg['summary']
                ))

                index_trimmed = msg_limit_length - self.msg_length()
                self.msg['summary'] = self.msg['summary'][:index_trimmed]

                print('summary after trimmed:\n{}'.format(self.msg['summary']))
                print('trimmed msg: {} chars.'.format(self.msg_length()))

        self.msg['url'] = url
        self.msg['img_url'] = img_url

    def tweet_with_media(self):
        '''
        Send a tweet with image.
        Try to retrieve the image first,
        then remove the img_url in msg and tweet with the image.
        If fail in uploading, add back the img_url into msg
        and tweet with img_url only.
        '''

        try:    # retrieve the image
            tempfile, headers = urllib.request.urlretrieve(self.msg['img_url'])
        except TypeError as e:
            pprint(e)
            pprint('img_url is None, tweet with media url.')
            raise
        except urllib.error.URLError as e:
            pprint(e)
            pprint('Error while urlretrieving media, tweet with media url.')
            raise
        except:
            traceback.print_exc()
            raise

        with open(tempfile, 'rb') as imgfile:
            img = imgfile.read()

        urllib.request.urlcleanup()
        img_url = self.msg.pop('img_url')
        params = {
            'status': self.msg_to_string(),
            'media[]': img,
        }
        try:
            return self.twitter_api.statuses.update_with_media(**params)
        except api.TwitterHTTPError:
            pprint('Cannot tweet with media, tweet with media url.')
            del img
            self.msg['img_url'] = img_url
            raise
        except:
            traceback.print_exc()

    def tweet_with_no_media(self):
        '''
        Send a pure text tweet.
        '''

        try:
            return self.twitter_api.statuses.update(
                status=self.msg_to_string()
            )
        except api.TwitterHTTPError as e:
            pprint(self.msg_length())
            pprint(self.msg)
            pprint("Cannot send this tweet.")
            pprint(e)
            print(datetime.now())
            print()
        except:
            traceback.print_exc()

    def tweet_latest_update(self, feed_entry):
        '''
        Tweets the latest update, logs when doing so.
        '''

        msg_limit_length, url, img_url = self.get_msg_limit_length_and_urls(
            feed_entry
        )
        self.cram_the_msg(feed_entry, msg_limit_length, url, img_url)

        try:
            self.tweet_with_media()
        except (TypeError, urllib.error.URLError, api.TwitterHTTPError):
            self.tweet_with_no_media()
        except:
            traceback.print_exc()

    def send_dm(self, user_screen_name=None, text=''):
        user_id = self.twitter_api.users.show(screen_name=user_screen_name)["id"]
        self.twitter_api.direct_messages.events.new(
            _json={
                "event": {
                    "type": "message_create",
                    "message_create": {
                        "target": {
                            "recipient_id": user_id
                        },
                        "message_data": {
                            "text": text
                        }
                    }
                }
            }
        )


# -*- coding: UTF-8 -*-
"""
Created on 2017年12月28日
@author: Leo
"""

# Python内置库
import sys
import getopt

# 项目内部库
from db_connect.mgo import Mgo
from db_connect.mysql import Mysql
from db_connect.db_handler import DBHandler
from utils.other_utils import Utils
from logger.LoggerHandler import Logger
from spiders.category_spider import CategorySpider
from spiders.data_spider import NationalDataWebsiteSpider

# 日志中心
logger = Logger(logger='starter.py').get_logger()

class Starter:
    def __init__(self, argv):
        # 命令行参数
        self._args = argv

        # 分类爬虫初始化
        self._category = None

        # 数据爬虫初始化
        self._data_spider = None

        # 工具类
        self._utils = Utils()

        # 初始化数据库连接池
        self.db_connect_pool = []

        # 加载数据库
        # # MongoDB
        self._mgo = Mgo().conn
        self._connect_info(conn_res=self._mgo, db_type="MongoDB")
        # # MySQL
        self._mysql = Mysql(json_config=True).conn
        self._connect_info(conn_res=self._mysql, db_type="MySQL")

    def _connect_info(self, conn_res, db_type: str):
        """
        检查数据库连接情况,并写入连接池
        :param conn_res: 连接结果
        :param db_type: 数据库类型名称
        :return: 无返回值
        """
        if conn_res is not None:
            logger.info("%s 数据库连接成功!" % db_type)
            self.db_connect_pool.append(conn_res)
            return True
        else:
            logger.info("%s 数据源没有启用,请检查配置文件或检查数据库连接情况!" % db_type)
            return False

    def command(self):
        """
        获取用户指令
        :return: 无返回值
        """
        try:
            opts, args = getopt.getopt(self._args, 'hc:i:', ["category=", "pid="])
        except getopt.GetoptError:
            logger.info('starter.py -c 获取目录信息的id')
            logger.info('starter.py -i <pid> [获取某个id下的年度数据]')
            sys.exit(2)
        # 判断恶意传入参数的行为
        if len(opts) >= 2:
            opts = opts[:1]
        # 获取参数
        for opt, args in opts:
            if opt == ('-h', '--help'):
                logger.info('starter.py -c 获取目录信息的id')
                logger.info('starter.py -i <pid> [获取某个id下的年度数据]')
                sys.exit()
            elif opt in ('-c', '--category'):
                self.category_start(data_type=args)

            elif opt in ('-i', '--pid'):
                logger.info("Choose a data_type: (hgnd is only choice support now, so we set it default!)")
                logger.info("I will expand other choices in next version.")
                self.data_start(pid=args, data_type="hgnd")

    def category_start(self, data_type: str):
        """
        分类爬虫的启动方法

        注: 暂时只是输出. 存储到数据库的版本在下个版本进行开发
        :return: 无返回值
        """
        if self._utils.check_data_type(data_type=data_type):
            logger.info("开始分类爬虫...")
            self._category = CategorySpider(data_type=data_type)
            try:
                self._category.req_data()
                self._category.parse()
                result = self._category.node_list
            except Exception as err:
                logger.error(err)
            else:
                logger.info("分类总条数: %d 条" % len(result))
                if len(self.db_connect_pool) != 0:
                    logger.info("爬取结束,数据开始入库...")
                    db = DBHandler(
                        db_connect_pool=self.db_connect_pool,
                        data_list=result,
                        mgo_col_name="t_category",
                        mysql_query="INSERT INTO t_category(category_name, category_id, category_parent, category_type)"
                        " VALUES (%s, %s, %s, %s)"
                    )
                    db.insert_data()
                else:
                    self._utils.export_category_to_file(data=result, file_name=data_type)

        else:
            logger.error("Data type is error!")
            sys.exit(1)

    def data_start(self, pid: str, data_type: str):
        """
        数据爬虫的启动方法
        :param pid: 数据的ID
        :param data_type: 数据类型(目前只支持hgnd)
        :return:
        """
        if self._utils.check_data_type(data_type=data_type):
            logger.info("开始数据爬虫!...")
            self._data_spider = NationalDataWebsiteSpider(pid=pid,
                                                          data_type=data_type)
            try:
                result = self._data_spider.start_spider()
            except Exception as err:
                logger.error(err)
            else:
                if len(self.db_connect_pool) != 0:
                    logger.info("爬取结束开始入库...")
                    db = DBHandler(
                        db_connect_pool=self.db_connect_pool,
                        data_list=result,
                        mgo_col_name="t_year_data",
                        mysql_query="INSERT INTO t_year_data(data_info, data_year, data_name, data_unit) "
                                    "VALUES (%s, %s, %s, %s)"
                    )
                    db.insert_data()
                else:
                    self._utils.export_data_to_file(data=result[1], file_name=pid)
        else:
            logger.error("Data type or pid is error!")
            sys.exit(1)

if __name__ == '__main__':
    s = Starter(argv=sys.argv[1:])
    s.command()


from sklearn import linear_model
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

df_car = pd.DataFrame({'type':['A','B','C','D','E','F','G'],
'hp':[130,250,190,300,210,220,170],
'ef':[16.3,10.2,11.1,7.1,12.1,13.2,14.2]})

x = df_car['hp'].to_numpy()
x = x[:,np.newaxis]
y = df_car['ef'].to_numpy()

regr = linear_model.LinearRegression()
regr.fit(x,y)

df_car.plot(kind='scatter',x='hp',y='ef')
y_pred = regr.predict([[0],[300]])

plt.plot([0,300],y_pred)

print("계수: ",regr.coef_)
print("절편: ",regr.intercept_)
print('예측 점수 ')
new_car = round(regr.predict([[270]])[0],2)
print("270 마력 자동차의 예상 연비 : ",new_car,"km/l")
print("예측 점수 :",regr.score(x,y))