from io import BytesIO
from PIL import Image, ImageDraw, ImageFont
from tqdm import tqdm
from typing import List

import imageio
import os
import pandas as pd
import requests

PATH_FILE = os.path.dirname(os.path.abspath(__file__))
ARIAL_PATH = os.path.join(PATH_FILE, "arial.ttf")

def get_detections(oid: str) -> [dict, None]:
    url = f"https://api.alerce.online/ztf/v1/objects/{oid}/lightcurve"
    querystring = {"survey_id": "ztf"}
    payload = ""
    response = requests.request("GET", url, data=payload, params=querystring)
    if response.status_code == 200:
        data = response.json()
        detections = data["detections"]
        return detections
    return None

def get_stamp(oid: str, candid: str, source="science"):
    url = "https://avro.alerce.online/get_stamp"
    querystring = {"oid": oid, "candid": candid, "type": source, "format": "png"}
    payload = ""
    response = requests.request("GET", url, data=payload, params=querystring)
    if response.status_code == 200:
        img = Image.open(BytesIO(response.content))
        return img
    return None

def make_gif(oid: str, path: str = "./") -> List:
    detections = get_detections(oid)
    detections = pd.DataFrame(detections)
    detections.sort_values(by="mjd", inplace=True, ascending=True)
    t_0 = detections["mjd"].values[0]
    stamps = []

    progress_bar = tqdm(detections.iterrows(), total=len(detections))
    for i, d in progress_bar:
        progress_bar.set_description(f"Retrieving {oid}: {d['candid']}")
        stamp = get_stamp(oid, d["candid"])
        if stamp is not None:
            draw = ImageDraw.Draw(stamp)
            font = ImageFont.truetype(ARIAL_PATH, 20)
            mjd = d["mjd"]
            delta = mjd - t_0
            mjd = f"{mjd:0.5f} Δ{delta:0.3f}"
            draw.text((10, 10), mjd, (255, 255, 255), font=font)
            stamps.append(stamp)
    output_path = os.path.join(path, f"{oid}.gif")
    imageio.mimsave(output_path, stamps, duration=0.5)
    return stamps


import web

#405 error when incorrect action is used on a resource
class NotSupportedError(web.HTTPError):
	def __init__(self):
		status = '405 Method Not Alloweds'
		headers = {'Content-Type': 'text/html'}
		data = 'That method is not valid for this action.'
		web.HTTPError.__init__(self, status, headers, data)

#400 error when invalid data is sent to a resource
class BadRequestError(web.HTTPError):
	def __init__(self):
		status = '400'
		headers = {'Content-Type': 'text/html'}
		data = 'Invalid data supplied in request.'
		web.HTTPError.__init__(self, status, headers, data)


#Python Imports
import math
from math import degrees, radians, pi

#Blender Imports
import bpy
from mathutils import Vector

#Addon Imports
from Addon_utils import odcutils

#Popup message box function :

def ShowMessageBox(message="", title="INFO", icon="INFO"):
    def draw(self, context):
        self.layout.label(text=message)

    bpy.context.window_manager.popup_menu(draw, title=title, icon=icon)

color = {
    "No color selected": [1.0, 0.9, 0.8, 1],
    "Green": [0, 1.0, 0.0, 1],
    "Blue": [0.0, 0.02, 0.19, 1],
    "Violet": [0.16, 0.0, 0.19, 1],
    "Pink": [1.0, 0.3, 1.0, 1],
}

class OPENDENTAL_OT_survey_model(bpy.types.Operator):
    """Calculates silhouette of object which surveys convexities AND concavities from the current view axis"""

    bl_idname = "opendental.view_silhouette_survey"
    bl_label = "Survey Model From View"
    bl_options = {"REGISTER", "UNDO"}

    world = bpy.props.BoolProperty(
        default=True,
        name="Use world coordinate for calculation...almost always should be true.",
    )
    smooth = bpy.props.BoolProperty(
        default=True,
        name="Smooth the outline.  Slightly less acuurate in some situations but more accurate in others.  Default True for best results",
    )

    @classmethod
    def poll(cls, context):
        # restoration exists and is in scene
        C0 = context.space_data.type == "VIEW_3D"
        C1 = context.object != None
        if C1:
            C2 = context.object.type == "MESH"
        else:
            C2 = False
        return C0 and C1 and C2

    def execute(self, context):

        colorprop = context.scene.UNDERCUTS_view_props.colorprop

        context.scene.UNDERCUTS_view_props.survey_quaternion = context.space_data.region_3d.view_rotation

        bpy.types.Scene.pre_surveyed = True

        if bpy.context.selected_objects == []:

            message = " Please select the Model to survey !"
            ShowMessageBox(message=message, icon="COLORSET_02_VEC")

            return {"CANCELLED"}

        elif colorprop == "No color selected" :

            message = " Please select a color for the survey zone !"
            ShowMessageBox(message=message, icon="COLORSET_01_VEC")

            return {"CANCELLED"}

        else:

            # ...........................Prepare scene settings : ..............................................

            # bpy.ops.view3d.snap_cursor_to_center()
            bpy.context.scene.transform_orientation_slots[0].type = "GLOBAL"
            bpy.context.scene.tool_settings.transform_pivot_point = "ACTIVE_ELEMENT"
            bpy.context.scene.tool_settings.use_snap = False
            bpy.context.tool_settings.mesh_select_mode = (False, False, True)
            bpy.ops.object.mode_set(mode = 'OBJECT')

            # Get active Object :..........................................................

            Model = bpy.context.view_layer.objects.active
            Model_name = Model.name

            if not f"_survey({colorprop})" in Model_name :

                # Remove old survey model and silhouette :
                for obj in bpy.data.objects :

                    if obj.name.endswith((f"_survey({colorprop})", f"_survey({colorprop})_silhouette"))  :

                        bpy.ops.object.hide_view_clear()
                        bpy.ops.object.select_all(action="DESELECT")
                        obj.select_set(True)
                        bpy.context.view_layer.objects.active = obj

                        bpy.ops.object.delete(use_global=False, confirm=False)

                # duplicate Model :

                bpy.ops.object.select_all(action="DESELECT")
                Model.select_set(True)
                bpy.context.view_layer.objects.active = Model
                bpy.ops.object.duplicate_move()
                Model_Survey = bpy.context.view_layer.objects.active
                mesh_Survey = Model_Survey.data
                bpy.ops.object.select_all(action="DESELECT")
                Model_Survey.select_set(True)

                # Rename Model_Survey :

                if "_solid_base" in Model_name :
                    Model_Survey.name = Model_name.replace("_solid_base", "_")

                Model_Survey.name += f"survey({colorprop})"
                mesh_Survey.name = f"{Model_Survey.name}_mesh"

            else :

                Model_Survey = context.active_object
                mesh_Survey = Model_Survey.data

                # Remove old silhouette object :
                for obj in bpy.data.objects :

                    if obj.name.endswith(f"_survey({colorprop})_silhouette")  :

                        bpy.ops.object.hide_view_clear()
                        bpy.ops.object.select_all(action="DESELECT")
                        obj.select_set(True)
                        bpy.context.view_layer.objects.active = obj

                        bpy.ops.object.delete(use_global=False, confirm=False)

            # Model_Survey add material :

            bpy.ops.object.select_all(action="DESELECT")
            Model_Survey.select_set(True)
            bpy.context.view_layer.objects.active = Model_Survey

            for _ in range(1,len(Model_Survey.material_slots)) :

                bpy.ops.object.material_slot_remove()

            mat_list = []

            for mat in bpy.data.materials:

                mat_list.append(mat.name)

            #if  Model.material_slot

            if not "my_Neutral" in mat_list:

                Model_mat = bpy.data.materials.new("my_Neutral")
                Model_mat.diffuse_color = [0.8, 0.8, 0.8, 1]

            else:

                Model_mat = bpy.data.materials["my_Neutral"]

            mesh_Survey.materials.append(Model_mat)

            survey_matname = f"survey_materiel({colorprop})"
            survey_matcolor = color[colorprop]

            if not survey_matname in mat_list:

                Survey_mat = bpy.data.materials.new(survey_matname)
                Survey_mat.diffuse_color = survey_matcolor
                Survey_mat.roughness = 0.4

            else:

                Survey_mat = bpy.data.materials[survey_matname]

            mesh_Survey.materials.append(Survey_mat)
            Model_Survey.active_material_index = 1

            # #############################____Surveying____###############################

            global survey_faces_index_list

            survey_faces_index_list = []

            bpy.ops.object.select_all(action="DESELECT")
            Model_Survey.select_set(True)
            bpy.context.view_layer.objects.active = Model_Survey

            ob = Model_Survey
            world_view = context.space_data.region_3d.view_rotation @ Vector((0,0,1))
            local_view = ob.matrix_world.inverted().to_quaternion() @ world_view

            bpy.context.tool_settings.mesh_select_mode = (False, False, True)

            for f in ob.data.polygons:

                if f.normal.dot(local_view) < -0.000001:
                    survey_faces_index_list.append(f.index)

            # 4_# select survey faces :

            # ....Deselect everything first
            bpy.ops.object.mode_set(mode="EDIT")
            bpy.ops.mesh.select_all(action="DESELECT")

            # ....it seems we can only select faces during object mode
            bpy.ops.object.mode_set(mode="OBJECT")

            for i in survey_faces_index_list:
                face = ob.data.polygons[i]
                face.select = True

            # Remove old vertex groups :

            ob.vertex_groups.clear()

            # Add vertex group :

            bpy.ops.object.mode_set(mode = 'EDIT')
            bpy.context.tool_settings.mesh_select_mode = (True, False, False)
            Model_Survey.vertex_groups.new(name=f"my_survey_vgroup({colorprop})")
            bpy.ops.object.vertex_group_assign()
            bpy.ops.object.material_slot_assign()
            bpy.ops.object.mode_set(mode = 'OBJECT')

            bpy.ops.object.select_all(action="DESELECT")
            Model_Survey.select_set(True)

            ob = Model_Survey
            view = context.space_data.region_3d.view_rotation @ Vector((0, 0, 1))
            odcutils.silouette_brute_force(
                context, ob, view, self.world, self.smooth #, debug=dbg
            )
            silhouette = bpy.context.view_layer.objects.active
            bpy.ops.object.mode_set(mode = 'OBJECT')
            bpy.ops.object.select_all(action="DESELECT")
            silhouette.select_set(True)
            Model_Survey.select_set(True)
            bpy.context.view_layer.objects.active = Model_Survey
            bpy.ops.object.hide_view_set(unselected=True)
            silhouette.select_set(False)

        return {"FINISHED"}

class OPENDENTAL_OT_blockout_model(bpy.types.Operator):
    """Calculates silhouette of object which surveys convexities AND concavities from the current view axis"""

    bl_idname = "opendental.blockout_model"
    bl_label = "Blockout Model From View"
    bl_options = {"REGISTER", "UNDO"}
    """
    world = bpy.props.BoolProperty(
        default=True,
        name="Use world coordinate for calculation...almost always should be true.",
    )
    smooth = bpy.props.BoolProperty(
        default=True,
        name="Smooth the outline.  Slightly less acuurate in some situations but more accurate in others.  Default True for best results",
    )

    @classmethod
    def poll(cls, context):
        # restoration exists and is in scene
        C0 = context.space_data.type == "VIEW_3D"
        C1 = context.object != None
        if C1:
            C2 = context.object.type == "MESH"
        else:
            C2 = False
        return C0 and C1 and C2
    """
    def execute(self, context):

        Modelsprop = bpy.context.scene.UNDERCUTS_props.Modelsprop
        if "Preview" in Modelsprop:
            bmesh_fns.remove_undercuts(context, ob, view, self.world, self.smooth)
        elif "Solid" in Modelsprop:
            bpy.ops.opendental.view_blockout_undercuts_solid()
        return {"FINISHED"}

class OPENDENTAL_OT_blockout_model_solid(bpy.types.Operator): #produces watertight blockout mesh when supplied watertight mesh
    bl_idname = 'opendental.view_blockout_undercuts_solid'
    bl_label = "Blockout Model From Z-axis"
    bl_options = {'REGISTER','UNDO'}

    def execute(self, context):

        extrude_z = -10

        if bpy.context.selected_objects == []:

            message = " Please select the Model to Blockout !"
            ShowMessageBox(message=message, icon="COLORSET_02_VEC")

            return {"CANCELLED"}

        else:

            if context.object.type != "MESH" :

                message = " Please select a valid Model (mesh object) !"
                ShowMessageBox(message=message, icon="COLORSET_02_VEC")

                return {"CANCELLED"}

            else :

                # ...........................Prepare scene settings : ..............................................

                # bpy.ops.view3d.snap_cursor_to_center()
                bpy.context.scene.transform_orientation_slots[0].type = "GLOBAL"
                bpy.context.scene.tool_settings.transform_pivot_point = "ACTIVE_ELEMENT"
                bpy.context.scene.tool_settings.use_snap = False

                # Get active Object :..........................................................

                ob = bpy.context.view_layer.objects.active

                ###  PATRICKS TEST ###############################
                ##################################################
                if bpy.types.Scene.pre_surveyed == True:
                    world_view = context.scene.UNDERCUTS_view_props.survey_quaternion @ Vector((0,0,1))
                else:
                    world_view = context.space_data.region_3d.view_rotation @ Vector((0,0,1))

                local_view = ob.matrix_world.inverted().to_quaternion() @ world_view

                bpy.context.tool_settings.mesh_select_mode = (False, False, True)
                for v in ob.data.vertices:
                    v.select = False
                for ed in ob.data.edges:
                    ed.select = False

                for f in ob.data.polygons:
                    if f.normal.dot(local_view) < -0.000001:
                        f.select = True
                    else:
                        f.select = False

                bpy.ops.object.mode_set(mode = 'EDIT')

                bpy.context.scene.transform_orientation_slots[0].type = "LOCAL"
                extrude_vec = extrude_z * local_view
                bpy.ops.mesh.extrude_region_move()
                bpy.ops.transform.translate(
                    value=(extrude_vec[0], extrude_vec[1], extrude_vec[2]), constraint_axis=(False, False, False)
                )
                bpy.context.tool_settings.mesh_select_mode = (True, False, False)
                bpy.ops.object.mode_set(mode = 'OBJECT')
                bpy.ops.opendental.remesh_model("INVOKE_DEFAULT")

                bpy.types.Scene.pre_surveyed = False

                # Rename Model_blocked :

                colorprop = context.scene.UNDERCUTS_view_props.colorprop

                if "_solid_base" in ob.name :
                    ob_name = ob.name
                    ob.name = ob_name.replace("_solid_base", "_")

                if f"_survey({colorprop})" in ob.name :
                    ob_name = ob.name
                    ob.name = ob_name.replace(f"_survey({colorprop})", "_")

                ob.name += "blocked"
                ob.data.name = f"{ob.name}_mesh"

        return {'FINISHED'}
        ### END PATRICK"S TEST   #########################
        ###################################################

def register():
    bpy.utils.register_class(OPENDENTAL_OT_survey_model)
    bpy.utils.register_class(OPENDENTAL_OT_blockout_model)
    bpy.utils.register_class(OPENDENTAL_OT_blockout_model_solid)

def unregister():
    bpy.utils.unregister_class(OPENDENTAL_OT_survey_model)
    bpy.utils.unregister_class(OPENDENTAL_OT_blockout_model)
    bpy.utils.unregister_class(OPENDENTAL_OT_blockout_model_solid)


import glob
import match
import pandas as pd
import os
import csv
import fnmatch
from datetime import date
import subprocess
import re

mp_path = 'Mediaplan/result/*'
#files = os.listdir(mp_path)
mpfiles = []
mp_frames =[]

vendornetworkmap =  pd.read_csv('Mediaplan/vendor-network-id-mapping.csv')
reg = vendornetworkmap['Match'].values.tolist()
val = vendornetworkmap['Value'].values.tolist()
#print(vendornetworkmap)

#mediafacts = pd.DataFrame(columns = ["Campaign Name", "Mediaplan", "Campaign Mp Label", 'Datasource'])
mediaplans = pd.DataFrame(columns = ['Vendor', 'Network / Site', 'Campaign Mp Label', 'Mediaplan','Campaign Name', 'VendorNetworkID', 'Datasource'])
#df = pd.read_excel('Mediaplan/result/2021_ktbo_kube_Adverity_Mediaplan_Abril-Caricam.csv', sheet_name = 'MEDIA PLAN', skiprows = 10)
#print(df)

def taxonomytrim(campaign, datasource):
    campaigntrimmed = "undefined"
    if datasource == "Facebook Ads Insights":
        campaigntrimmed = "_". join(campaign.split("_")[0:12])
    elif datasource == "Google Adwords":
         campaigntrimmed= "_". join(campaign.split("_")[0:12])
    elif datasource == "Other Channels":
         campaigntrimmed= "_". join(campaign.split("_")[0:11])
    elif datasource == "Sizmek Sas":
         campaigntrimmed= "_". join(campaign.split("_")[0:10])
    elif datasource == "Twitter":
         campaigntrimmed= "_". join(campaign.split("_")[0:12])
    else:
        print("something went wrong")
        print(datasource)
    return(campaigntrimmed)

for filename in glob.iglob(mp_path, recursive=True):
    if os.path.isfile(filename): # filter dirs
        print(filename)
        mpfiles.append(filename)
        df = pd.read_excel(filename, usecols = ['Campaign Initiative', 'Filename', 'Vendor', 'Network / Site'], sheet_name = 'MEDIA PLAN', skiprows = 10)
        df.columns = ['Vendor', 'Network / Site','Campaign Mp Label', 'Mediaplan' ]
        df['Campaign upper'] = df['Campaign Mp Label'].str.upper().str.replace(" ", "")
        df['VendorNetworkID'] = df['Vendor'].str.upper().str.cat(df['Network / Site'].str.upper(), sep = "_")
        mapping_id = list(range(11))
        for i in mapping_id:
            df.loc[df['VendorNetworkID'].str.match(reg[i]), 'Datasource'] = val[i]
        df.loc[df['Datasource'].isnull(), 'Datasource']='Other Channels'
        print(df['Datasource'].unique())
        df['Campaign Name'] = df.apply(lambda row : taxonomytrim(row['Campaign upper'], row['Datasource']), axis = 1)
        mediaplans = pd.concat([mediaplans, df])

mediaplans[['Vendor', 'Network / Site', 'VendorNetworkID', 'Datasource', 'Mediaplan','Campaign Mp Label','Campaign upper', 'Campaign Name']].to_csv("mpcampaigns.csv", index=False)

#Mediafacts creation using a pd concat between a export from the explorer view and the other channels file
mediafactsexport = pd.read_csv('Campaigns/campaigns_unified.csv', usecols = ['Datasource', 'Campaign Name'])
mediafactsother = pd.read_excel('OtherCampaigns/pautas_locales_acumulado_other.xlsx', usecols = ['Campaign'])
mediafactsother['Datasource'] = 'Other Channels'
mediafactsother['Datastream'] = 'OtherCampaigns'
mediafactsother['Campaign upper'] = mediafactsother['Campaign'].str.upper().str.replace(" ", "")
mediafactsother['Campaign Name'] = mediafactsother.apply(lambda row : taxonomytrim(row['Campaign upper'], row['Datasource']), axis = 1)

mediafacts =  pd.concat([mediafactsother, mediafactsexport])
#mediaplans =  pd.read_csv('Mediaplan/mp_processed/Adverity-export_Plan.csv')

print('Mediaplan')
print(mediaplans)

print('Mediafacts')
print(mediafacts)

mr = match.mp_match(mediaplans, mediafacts)
print(mr)

subprocess.call('aws s3 cp matching_campaigns.csv s3://testingmidktbo/matched.csv >> aws_upd.log', shell=True)
subprocess.call('aws s3 cp unmatching_campaigns.csv s3://testingmidktbo/unmatched.csv >> aws_upd.log', shell=True)

today = date.today()
print(today)

#file = open('unmatched.csv', 'w+', newline ='')
#write = csv.writer(file)
#write.writerows(mr)


from django.shortcuts import render
from rest_framework.decorators import api_view
import json
from django.conf import settings
import redis
import csv, io, zipfile
import requests
from django.http import HttpResponse
# from vue_app.models import Post
from datetime import datetime
from bs4 import BeautifulSoup
# from django.core.cache import cache

# Create your views here.

# def test_vue(request):
#     return render(request, 'vue_app/test.html',data)

# Connect to our Redis instance
r = redis.StrictRedis(host=settings.REDIS_HOST,
                                  port=settings.REDIS_PORT, db=0)

def home(request):
    # data = get_stock_by_name("")
    getBhavCopy()
    search_term = ''
    if 'search' in request.GET:
        search_term = request.GET['search']
    print(search_term)
    data = get_stock_by_name(search_term)
    return render(request, 'vue_app/test.html',{'data': data})

def get_stock_by_name(name):
        stock_gen = r.scan_iter('*name:{}*'.format(name.lower()))
        return get_stocks(stock_gen)

def get_stocks(iterator):
    stocks = []
    for item in iterator:
        stock = dict()
        stock_prop_values = r.hmget(item, columns)
        for ind, prop in enumerate(stock_prop_values):
            stock[columns[ind]] = prop.decode("utf-8")
        stocks.append(stock)
    return stocks

columns = ['code', 'name', 'open', 'high', 'low', 'close']

def create_stock(data):
    pipe = r.pipeline()
    # print("lkhkj",data['name'])
    primary_key = 'code:{}::name:{}'.format(data['code'], data['name'].lower())
    # print("before",data)
    pipe.hmset(primary_key, data)
    # pipe.zadd('top_ten', {primary_key: data['code']})  # use data['turnover'] if needed
    pipe.execute()

# def get_top_ten():
#     top_ten = r.zrange('top_ten', 0, 9)
#     return get_stocks(top_ten)

FIELD_CSV_MAP = {
    'code': 'SC_CODE',
    'name': 'SC_NAME',
    'open': 'OPEN',
    'high': 'HIGH',
    'low': 'LOW',
    'close': 'CLOSE'
}

BASE_URL = "https://www.bseindia.com/"
HOMEPAGE_PATH = "/markets/MarketInfo/BhavCopy.aspx"
DOWNLOAD_LINK_ELEMENT_ID = "ContentPlaceHolder1_btnhylZip"
def get_download_page(page_path):  # /markets/MarketInfo/BhavCopy.aspx
        response = requests.get(BASE_URL+page_path)
        page = response.content
        return page

def get_download_url(page):  # /download/BhavCopy/Equity/EQ250719_CSV.ZIP
    parsed_page = BeautifulSoup(page, 'html.parser')
    download_path = parsed_page.find(id=DOWNLOAD_LINK_ELEMENT_ID).get('href')
    return download_path

# @staticmethod
def get_zip(file_url):
    try:
        res = requests.get(file_url)
        zipped_file = ZipFile(BytesIO(res.content))
        return zipped_file
    except:  # TODO : Handle individual exceptions, create a decorator
        raise Exception("Some error in downloading processing zip file")

# @staticmethod
def extract_csv_from_zip(zipped_file):
    csv_filename = zipped_file.infolist()[0].filename
    op_file_name = zipped_file.extract(csv_filename)
    zipped_file.close()
    return op_file_name

def process_csv_to_db(csv_file):
    with open(csv_file) as stocks:
        csv_reader = csv.reader(stocks, delimiter=",")
        field_index = dict()
        line_count = 0
        for row in csv_reader:
            if line_count == 0:
                for field in FIELD_CSV_MAP.keys():
                    field_index[field] = row.index(FIELD_CSV_MAP[field])
            else:
                data = dict()
                for field, index in field_index.items():
                    field_value = row[index]
                    if isinstance(field_value, str):
                        field_value = field_value.strip()
                    data[field] = field_value
                # print(data)
                create_stock(data)
            line_count += 1

# home_page = get_download_page(HOMEPAGE_PATH)
# print(home_page)
# zip_dload_url = get_download_url(home_page)
# zipfile_obj = get_zip(zip_dload_url)
# extracted_csv_filename = extract_csv_from_zip(zipfile_obj)
# process_csv_to_db(extracted_csv_filename)

BSE_URL = 'http://www.bseindia.com/markets/equity/EQReports/BhavCopyDebt.aspx?expandable=3'
def getBhavCopy():
    # bse_page = requests.get(BSE_URL)
    # soup = BeautifulSoup(bse_page.content,'lxml')

    # iframe_url = soup.find('iframe').attrs.get('src')
    # frame_page = requests.get(BSE_URL[:BSE_URL.rfind('/')+1] + iframe_url)
    # print(frame_page)
    # soup = BeautifulSoup(frame_page.content, 'lxml')
    # zip_file_url = soup.find(id='btnhylZip').attrs.get('href')
    r = requests.get('http://www.bseindia.com/markets/equity/EQReports/BhavCopyDebt.aspx?expandable=3')
    print("kjhgjmhh")
    soup = BeautifulSoup(r.text, 'lxml')
    stories = []

    for a in soup.find_all('a', attrs={'class': 'btnhylZip'}):
        stories.append([a.text, a['href']])
        print("here",stories[0])
        zip_file = requests.get(stories)
        print("got",zip_file)
        z = zipfile.ZipFile(io.BytesIO(zip_file.content))
        z.extractall('zips')
        return str('zips' + '/' + z.namelist()[0])


from __future__ import annotations

from fastapi import HTTPException, status

from fideslib.oauth.scopes import SCOPES

class AuthenticationError(HTTPException):
    """To be raised when attempting to fetch an access token using
    invalid credentials.
    """

    def __init__(self, detail: str) -> None:
        super().__init__(
            status_code=status.HTTP_401_UNAUTHORIZED,
            detail=detail,
        )

class AuthorizationError(HTTPException):
    """Throws an HTTP 403"""

    def __init__(self, detail: str) -> None:
        """Override the regular HTTPException throwing only a 403"""
        super().__init__(status_code=status.HTTP_403_FORBIDDEN, detail=detail)

class ClientWriteFailedError(HTTPException):
    """To be raised when a client cannot be created."""

    def __init__(self) -> None:
        super().__init__(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Failed to create client",
        )

class ClientNotFoundError(HTTPException):
    """To be raised when attempting to fetch a client that does not exist."""

    def __init__(self, client_id: str) -> None:
        super().__init__(
            status_code=status.HTTP_404_NOT_FOUND,
            detail={
                "error": "Client does not exist",
                "id": client_id,
            },
        )

class ExpiredTokenError(HTTPException):
    """To be raised when a provided token is expired."""

    def __init__(self) -> None:
        super().__init__(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="OAuth token expired",
        )

class InvalidAuthorizationSchemeError(HTTPException):
    """To be raised when attempting to authenticate with an unexpected
    Authorization header value.
    """

    def __init__(self) -> None:
        super().__init__(
            status_code=status.HTTP_401_UNAUTHORIZED,
            detail="Failed to authenticate",
            headers={"WWW-Authenticate": "Bearer"},
        )

class InvalidScopeError(HTTPException):
    """To be raised when a provided scope does not exist."""

    def __init__(self, invalid_scopes: list[str]) -> None:
        SCOPES.sort()

        super().__init__(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail={
                "error": "Invalid scope provided",
                "invalid_scopes": invalid_scopes,
                "valid_scopes": SCOPES,
            },
        )

class KeyOrNameAlreadyExists(Exception):
    """A resource already exists with this key or name."""

class KeyValidationError(Exception):
    """The resource you're trying to create has a key specified but not
    a name specified.
    """

class MissingConfig(Exception):
    """Custom exception for when no valid configuration file is provided."""


#!/usr/bin/env python3

import os
import sys
import pathlib
import io
import re
import requests
import json
import yaml
import argparse
from dotenv import load_dotenv
from datetime import date

VERBOSE=False

# ---------------------------------------------------------------------------------------------------------------------

def verbose(msg):
  if VERBOSE:
    print(msg)

# ---------------------------------------------------------------------------------------------------------------------

def get_compatibility_code(conversionCode):
  """
  Load the CompatibilityCode and adapt it.
  """

  # copatibility code laden

  with open(selfpath+"./.clobber/CompatibilityCode.c", "r") as f:
    compatibilityCode=f.read()

  # gucken, welche keys es im compatibility code gibt

  keys = {}

  for line in compatibilityCode.split("\n"):
    match=re.match(r'^\s*\/\*\|([^\|]*)\|\*\/(.*)$',line)
    if match != None:
      key=match.group(1).strip()
      if key != '' and not key in keys:
        keys[key]=False

  # gucken, welche keys im conversion code vorkommen

  for key in keys:
    if re.search(r'[^\w]'+key+r'[^\w]',conversionCode) != None:
      keys[key]=True

  # zeilen rausfiltern, deren key nicht verwendet wird

  abridgedCompatibilityCode=""

  for line in compatibilityCode.split("\n"):
    match=re.match(r'^\s*\/\*\|([^\|]*)\|\*\/(.*)$',line)
    if match==None:
      abridgedCompatibilityCode = abridgedCompatibilityCode+line+"\n"
    else:
      key  = match.group(1).strip()
      code = match.group(2)
      if key=='' or keys[key]:
        abridgedCompatibilityCode = abridgedCompatibilityCode+code+"\n"

  # ueberfluessegen whitespace eliminieren

  abridgedCompatibilityCode=re.sub(r'\t','  ',abridgedCompatibilityCode)
  abridgedCompatibilityCode=re.sub(r' +\n','\n',abridgedCompatibilityCode)
  abridgedCompatibilityCode=re.sub(r'\n +\n','\n\n',abridgedCompatibilityCode)
  abridgedCompatibilityCode=re.sub(r'\n{3,}','\n\n',abridgedCompatibilityCode)

  # und damit ist der compatibility code entsprechend angepasst

  return abridgedCompatibilityCode

# ---------------------------------------------------------------------------------------------------------------------

def get_basefilename(path,id):
  """
  Get the name preceding the '.<ID>.c' suffix.
  """

  files=[entry for entry in os.scandir(path) if entry.is_file() and entry.name.endswith(f".{id}.c")]

  if len(files) < 1:
    raise Exception(f"no such file '{path}*.{id}.c'")

  if len(files) > 1:
    raise Exception(f"there are multiple files matching '{path}*.{id}.c'")

  name = re.sub(r'^(.+)\.'+id+'\.c$',r'\1',files[0].name)

  # print(f"fuse conversion '{path}{name}.{id}.c'")

  return name

# ---------------------------------------------------------------------------------------------------------------------

def write_markdown_file(info,target_file,force=False):

  fuse_name     = info['fuse']['name']
  fuse_author   = info['fuse']['author']
  shader_author = info['shader']['author']
  shader_id     = info['shader']['id']
  shader_name   = info['shader']['name']
  description   = info['shader'].get('description','')

  if os.path.isfile(target_file) and not force:
    print("file '"+target_file+"' already exists - use force option to overwrite")
    return

  dlbtn = f"[![Download Installer](https://img.shields.io/static/v1?label=Download&message={fuse_name}-Installer.lua&color=blue)]({fuse_name}-Installer.lua \"Installer\")"
  with open(target_file, 'w') as f:
    f.write( "" \
      f"# Here insert a description of the shader! And maybe a gif\n\n" \
      f"# Description of the Shader in Shadertoy: \n{description}\n\n" \
      f"[![Thumbnail]({fuse_name}_screenshoot.png)]({fuse_name}.fuse)\n\n"
      )

    verbose("wrote file '"+target_file+"'")

# ---------------------------------------------------------------------------------------------------------------------

def write_sfi_file(info,target_file,force=False):

  fuse_name     = info['fuse']['name']
  fuse_author   = info['fuse']['author']
  shader_author = info['shader']['author']
  shader_id     = info['shader']['id']
  shader_name   = info['shader']['name']
  description   = info['shader'].get('description','')
  creationdate  = date.today().strftime("%Y-%m-%d")

  if os.path.isfile(target_file) and not force:
    print("file '"+target_file+"' already exists - use force option to overwrite")
    return

  with open(target_file, 'w') as f:
    f.write( "" \
      f"info = {{\n\n   -- https://www.shadertoy.com/view/{shader_id}\n\n   Shadertoy = {{\n      Name = \"{shader_name}\",\n      Author = \"{shader_author}\",\n      ID = \"{shader_id}\",\n   }},\n\n" \
      f"   Fuse = {{\n      Author = \"{fuse_author}\",\n      Date = \"{creationdate}\",\n   }},\n\n" \
      f"   Compatibility = {{\n      macOS_Metal = false,\n      macOS_OpenCL = false,\n      Windows_CUDA = false,\n      Windows_OpenCL = false,\n   }},\n}}"
      )

    verbose("wrote file '"+target_file+"'")

# ---------------------------------------------------------------------------------------------------------------------

def patch_kernel(conversionCode,kernelName,has_param,part=""):

  conversionKernelName=kernelName+"Fuse"+part

  verbose("Patch Kernel '"+conversionKernelName+"'")

  m = re.search(r'__KERNEL__\s+void\s+'+conversionKernelName+r'\s*\(([^\)]*)\)\s*\{',conversionCode)

  if not m:
    if part=="":
      raise Exception("no image kernel found!")
    return conversionCode

  kernel_parameters=m.group(1)

  fragColor_name = None # float4
  fragCoord_name = None # float2

  # ja, okay, jetzt wird's echt bescheuert ...

  i=-1 # iteration to allow fragColor and Coord only as the first two parameters

  kernel_parameters=kernel_parameters.strip()
  verbose("kernel parameters='"+kernel_parameters+"'")

  for p in kernel_parameters.split(','):

    i+=1

    t = None
    n = None
    m = re.search(r'^\s*([A-Za-z_][A-Za-z0-9_]*)\s+([A-Za-z_][A-Za-z_0-9]*)\s*$', p) #match "type name"
    if m:
        t = m.group(1)
        n = m.group(2)
    else:
      m = re.search(r'^\s*([A-Za-z_][A-Za-z0-9_]*)\s*\*\s*([A-Za-z_][A-Za-z_0-9]*)\s*$', p) # match "type * name" (yes, type*name matches, so what)
      if m:
        t = m.group(1)+'*'
        n = m.group(2)
      else:
        m= re.search(r'^\s*([A-Za-z_][A-Za-z0-9_]*)\s+([A-Za-z_][A-Za-z_0-9]*)\s*\[\s*\]\s*$', p) # match "type name [ ]"
        if m:
          t = m.group(1)+'*'
          n = m.group(2)

    if not m:
      raise Exception("parameter='"+p+"' has bad format")

    verbose("parameter"+str(i)+": type='"+t+"', name='"+n+"'")
    if has_param.get(n,None)==None:
      if i==0 and t=='float4' and fragColor_name==None:
        fragColor_name=n # first and only the first parameter can be the color (yes, stupid idea to call that color iDate)
      elif i<2 and t=='float2' and fragCoord_name==None:
        fragCoord_name=n # first or second parameter can be the coord
      else:
        raise Exception("unknown parameter '"+t+" "+n+"'")
    else:
      if t != has_param[n]['Type']:
        raise Exception("bad type '"+t+"' for '"+n+"' ('"+has_param[n]['Type']+"' expected)")

      if has_param[n]['Present']:
        print("multiple occurences of '"+t+" "+n+"'")
      else:
        has_param[n]['Present']=True

  # even 'gaps' in the channels (unused channels) must be counted
  # so if we have iChannel3 and only iChannel3, then the number
  # of channels is 4 (iChannel0 to iChannel3).

  NUM_INPUT_CHANNELS=0
  kernel_parameters = "__CONSTANTREF__ Params*  params, "
  for i in range(16):
    if has_param["iChannel"+str(i)]['Present']:
      kernel_parameters += " __TEXTURE2D__ iChannel"+str(i)+", "
      NUM_INPUT_CHANNELS=i+1

  kernel_parameters += "__TEXTURE2D_WRITE__ destinationTexture"

  if NUM_INPUT_CHANNELS==0:
    has_param['iChannelResolution']['Code'] = '/* no iChannels!?! */\n'
    has_param['iChannelTime']['Code']       = '/* no iChannels!?! */\n'
  else:

    # iChannelResolution[]
    # iChannelTime[]
    has_param['iChannelResolution']['Code']='  float2 iChannelResolution['+ str(NUM_INPUT_CHANNELS) +'];\n'
    has_param['iChannelTime']['Code']='  float iChannelTime['+ str(NUM_INPUT_CHANNELS) +'];\n'
    for i in range(NUM_INPUT_CHANNELS):
      if has_param["iChannel"+str(i)]['Present']:
        has_param['iChannelResolution']['Code'] += "  iChannelResolution["+str(i)+"] = to_float2(params->iChannelResolution["+str(i)+"][0], params->iChannelResolution["+str(i)+"][1]);\n"
        has_param['iChannelTime']['Code']       += "  iChannelTime["+str(i)+"] = params->iChannelTime["+str(i)+"];\n"
      else:
        has_param['iChannelResolution']['Code'] += "  iChannelResolution["+str(i)+"] = to_float2(0.0f,0.0f);\n"
        has_param['iChannelTime']['Code']       += "  iChannelTime["+str(i)+"] = 0.0f;\n"

  variable_declarations = ""

  for n,p in has_param.items():
    if p['Present'] and p['Code']!='':
      variable_declarations += p['Code']

  if fragColor_name != None:
    variable_declarations+="  float4 "+fragColor_name+"   = to_float4_s(0.0f);\n"

  if fragCoord_name != None:
    variable_declarations+="  float2 "+fragCoord_name+"   = to_float2(fusion_x,fusion_y);\n"

  m=r'__KERNEL__\s+void\s+'+conversionKernelName+r'\s*\([^\)]*\)\s*\{'

  r=  "__KERNEL__ void "+conversionKernelName+"("+kernel_parameters+")\n" \
      "{\n" \
      "   DEFINE_KERNEL_ITERATORS_XY(fusion_x, fusion_y);\n\n" \
      "   if (fusion_x >= params->width || fusion_y >= params->height)\n" \
      "     return;\n\n"+variable_declarations+"\n\n<<CODEFROMCONTROLS>>  // --------\n\n"

  conversionCode  = re.sub( m,r,conversionCode)

  conversionCode  = re.sub(
      r'(\s+)SetFragmentShaderComputedColor\s*\(\s*',
      r'\1_tex2DVec4Write(destinationTexture, fusion_x, fusion_y, ',
      conversionCode)

  return conversionCode

# ---------------------------------------------------------------------------------------------------------------------

def check_controls(conversionCode):

  controls = re.findall(r'\s+CONNECT_([A-Z]+)(\d+)\s*\(\s*([A-Za-z_][\w]*)\s*,\s*([^\)]+)\)\s*;',conversionCode)

  fullCodeCreate=''
  fullCodeProcess=''
  fullCodeKernel=''
  fullCodeParams=''

  for ctrl in controls:

    ctrlType    = ctrl[0]
    ctrlIdx     = ctrl[1]
    ctrlVar     = ctrl[2]
    ctrlParams  = ctrl[3] # ctrl[3].split(",")

    # print(f"create {ctrlType} for {ctrlVar} with parameters '{ctrlParams}'")

    if ctrlType == 'SLIDER' or ctrlType == 'SCREW':

      m = re.match(r'^\s*(-?\d+\.\d+)f\s*,\s*(-?\d+\.\d+)f\s*,\s*(-?\d+\.\d+)f\s*$',ctrlParams)

      if m == None:
        raise Exception (f"failed to match '{ctrlParams}' as {ctrlType} parameters")

      min, max, default = m.group(1), m.group(2), m.group(3)

      codeParams  = f"float  {ctrlVar};"
      codeKernel  = f"float  {ctrlVar} = params->{ctrlVar};"
      codeProcess = f"params.{ctrlVar} = In{ctrlVar}Slider:GetValue(req).Value"
      if ctrlType == 'SLIDER':
          codeCreate  = f"""
              In{ctrlVar}Slider = self:AddInput("{ctrlVar}", "{ctrlVar}", {{
                  LINKID_DataType    = "Number",
                  INPID_InputControl = "SliderControl",
                  INP_MinScale       = {min},
                  INP_MaxScale       = {max},
                  INP_Default        = {default},
              }})"""
      else:
          codeCreate  = f"""
              In{ctrlVar}Slider = self:AddInput("{ctrlVar}", "{ctrlVar}", {{
                  LINKID_DataType    = "Number",
                  INPID_InputControl = "ScrewControl",
                  INP_MinScale       = {min},
                  INP_MaxScale       = {max},
                  INP_Default        = {default},
              }})"""

    elif ctrlType == 'INTSLIDER':

      m = re.match(r'^\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*$',ctrlParams)

      if m == None:
        raise Exception (f"failed to match '{ctrlParams}' as {ctrlType} parameters")

      min, max, default = m.group(1), m.group(2), m.group(3)

      codeParams  = f"int    {ctrlVar};"
      codeKernel  = f"int    {ctrlVar} = params->{ctrlVar};"
      codeProcess = f"params.{ctrlVar} = In{ctrlVar}Slider:GetValue(req).Value"
      codeCreate  = f"""
          In{ctrlVar}Slider = self:AddInput("{ctrlVar}", "{ctrlVar}", {{
              LINKID_DataType    = "Number",
              INPID_InputControl = "SliderControl",
              INP_MinScale       = {min},
              INP_MaxScale       = {max},
              INP_Default        = {default},
              INP_Integer        = true,
          }})"""

    elif ctrlType == 'POINT':

      m = re.match(r'^\s*(-?\d+\.\d+)f\s*,\s*(-?\d+\.\d+)f\s*$',ctrlParams)

      if m == None:
        raise Exception (f"failed to match '{ctrlParams}' as {ctrlType} parameters")

      x, y = m.group(1), m.group(2)

      codeParams  = f"float  {ctrlVar}[2];"
      codeKernel  = f"float2 {ctrlVar} = to_float2(params->{ctrlVar}[0], params->{ctrlVar}[1]);"
      codeProcess = f"params.{ctrlVar} = {{In{ctrlVar}Point:GetValue(req).X,In{ctrlVar}Point:GetValue(req).Y}}"
      codeCreate  = f"""
          In{ctrlVar}Point = self:AddInput("{ctrlVar}", "{ctrlVar}", {{
              LINKID_DataType    = "Point",
              INPID_InputControl = "OffsetControl",
              INPID_PreviewControl  = "CrosshairControl",
              INP_DefaultX          = {x},
              INP_DefaultY          = {y},
          }})"""

    elif ctrlType == 'CHECKBOX':

      m = re.match(r'^\s*(1|true|0|false)\s*$',ctrlParams)

      if m == None:
        raise Exception (f"failed to match '{ctrlParams}' as {ctrlType} parameters")

      default = m.group(1)
      if default=="1" or default=="true":
        default=1
      else:
        default=0

      codeParams  = f"bool   {ctrlVar};"
      codeKernel  = f"bool   {ctrlVar} = params->{ctrlVar};"
      codeProcess = f"params.{ctrlVar} = In{ctrlVar}Checkbox:GetValue(req).Value"
      codeCreate  = f"""
          In{ctrlVar}Checkbox = self:AddInput("{ctrlVar}", "{ctrlVar}", {{
            LINKID_DataType     = "Number",
            INPID_InputControl  = "CheckboxControl",
            INP_Integer         = true,
            INP_Default         = {default},
          }})"""

    elif ctrlType == 'COLOR':

      m = re.match(r'^\s*(-?\d+\.\d+)f\s*,\s*(-?\d+\.\d+)f\s*,\s*(-?\d+\.\d+)f\s*,\s*(-?\d+\.\d+)f\s*$',ctrlParams)

      if m == None:
        raise Exception (f"failed to match '{ctrlParams}' as {ctrlType} parameters")

      r, g, b, a = m.group(1), m.group(2), m.group(3), m.group(4)

      codeParams  = f"float  {ctrlVar}[4];"
      codeKernel  = f"float4 {ctrlVar} = to_float4(params->{ctrlVar}[0], params->{ctrlVar}[1], params->{ctrlVar}[2], params->{ctrlVar}[3]);"
      codeProcess = f"params.{ctrlVar} = {{\n    In{ctrlVar}ColorR:GetValue(req).Value,\n    In{ctrlVar}ColorG:GetValue(req).Value,\n    In{ctrlVar}ColorB:GetValue(req).Value,In{ctrlVar}ColorA:GetValue(req).Value\n  }}"
      codeCreate  = f"""
          self:BeginControlNest("{ctrlVar}", "{ctrlVar}", true, {{}})

            ctrl_grp_cnt = (ctrl_grp_cnt==nil) and 1 or (ctrl_grp_cnt+1)

            attrs = {{
              ICS_Name = "{ctrlVar}",
              LINKID_DataType = "Number",
              INPID_InputControl = "ColorControl",
              INP_MinScale = 0.0,
              INP_MaxScale = 1.0,
              IC_ControlGroup = ctrl_grp_cnt,
            }}

            In{ctrlVar}ColorR = self:AddInput("Red",   "{ctrlVar}Red",   {{ INP_Default  = {r}, IC_ControlID = 0, attrs}})
            In{ctrlVar}ColorG = self:AddInput("Green", "{ctrlVar}Green", {{ INP_Default  = {g}, IC_ControlID = 1, attrs}})
            In{ctrlVar}ColorB = self:AddInput("Blue",  "{ctrlVar}Blue",  {{ INP_Default  = {b}, IC_ControlID = 2, attrs}})
            In{ctrlVar}ColorA = self:AddInput("Alpha", "{ctrlVar}Alpha", {{ INP_Default  = {a}, IC_ControlID = 3, attrs}})

          self:EndControlNest()"""

    elif ctrlType == 'BUTTON':
      #print(f"create {ctrlType} for {ctrlVar} with parameters '{ctrlParams}'")

      m=ctrlParams.split(",")

      if m == None:
        raise Exception (f"failed to match '{ctrlParams}' as {ctrlType} parameters")

      #print("m",m, m[0])

      if (m[0] == "0"):
        _type = ""
      if (m[0] == "1"):
        _type = f""" MBTNC_Type         = "Toggle", """

      buttons = ""
      m.pop(0)
      for i in m:
        buttons += f""" {{ MBTNC_AddButton  = "{i}", }}, """

      codeParams  = f"float  {ctrlVar};"
      codeKernel  = f"float  {ctrlVar} = params->{ctrlVar};"
      codeProcess = f"params.{ctrlVar} = In{ctrlVar}Button:GetValue(req).Value"
      codeCreate  = f"""
          In{ctrlVar}Button = self:AddInput("{ctrlVar}", "{ctrlVar}", {{
              LINKID_DataType    = "Number",
              INPID_InputControl = "MultiButtonControl",
              MBTNC_ForceButtons = true,
        {_type}
              MBTNC_ShowName     = false,
        {buttons}
		      MBTNC_StretchToFit = true,
              IC_NoLabel         = true,
		      INP_Default        = 0,
              IC_Visible         = true,
          }})"""

    else:
      raise Exception(f"unknow resp. not yet implemented contol {ctrlType} for {ctrlVar}")

    codeCreate  = re.sub(r'\n        ',  '\n', codeCreate)
    codeCreate  = re.sub(r'^\s+',        '  ', codeCreate)
    codeParams  = "  "+codeParams
    codeKernel  = "  "+codeKernel
    codeProcess = "  "+codeProcess

    fullCodeCreate  += codeCreate  + "\n\n"
    fullCodeParams  += codeParams  + "\n"
    fullCodeKernel  += codeKernel  + "\n"
    fullCodeProcess += codeProcess + "\n"

  return fullCodeCreate, "\n"+fullCodeProcess, fullCodeKernel+"\n",  fullCodeParams

# ---------------------------------------------------------------------------------------------------------------------

def fuse_it(id,force=False):

  source_path=selfpath+'./Conversions/'+folder+'/'
  target_path=selfpath+'./Converted/'

  conversion_file = get_basefilename(source_path,id)

  verbose(f"process '{conversion_file}'")

  # read .yaml file

  info=None
  with open(f"{source_path}{conversion_file}.{id}.yaml", "r") as f:
    info=yaml.safe_load(f)

  fuse_name     = info['fuse']['name']  # FuReg name ... this is what's displayed in the tool selection menu
  fuse_kernel   = conversion_file       # name of the fragment shader kernel function
  fuse_id       = info['fuse'].get('id',conversion_file) # FuReg ID (must follow identifier syntax)
  fuse_file     = info['fuse'].get('file',fuse_id)  # filename to save the fuse as

  if info['shader']['id'] != id:
    raise Exception("inconsistency in your source and yaml file")

  # write .md file

  write_markdown_file(info, target_file = target_path+fuse_file+".md", force = force )

  # write .sfi file

  write_sfi_file(info, target_file = target_path+fuse_file+".sfi", force = force )

  # read conversion code

  with open(f"{source_path}{conversion_file}.{id}.c", "r") as f:
    conversionCode=f.read()

  conversionCode = re.sub(r'\t','  ',conversionCode)
  conversionCode = re.sub(r' +\n','\n',conversionCode)

  # create compatibility code for this conversion

  compatibilityCode = get_compatibility_code(conversionCode)

  # load the data from yaml file

  with open(selfpath+".clobber/fuse-parameters.yaml", "r") as f:
    src=f.read()

  has_param=yaml.load(src, Loader=yaml.FullLoader)

  # iChannel<0..15> with 15 as the maximum number of input channels supported for the time being

  for i in range(16):
    has_param['iChannel'+str(i)]={ 'Type':'sampler2D', 'Code': '' }

  # do some code indention

  for k,v in has_param.items():
    for f in ['Code','Proc','Create','Param']:
      if v.get(f,None)==None:
        v[f]=''
      elif v[f]!='':
        v[f]='  '+v[f].replace('\n','\n  ')+'\n'

  # patch all kernel signatures

  channelsSeen    = {}
  channelsSeenNum = 0

  for i in range(16):
    channelsSeen['iChannel'+str(i)]=False

  for part in ['__Buffer_A','__Buffer_B','__Buffer_C','__Buffer_D','']:

    for k,v in has_param.items():
      v['Present'] = False

    conversionCode = patch_kernel(conversionCode,fuse_kernel,has_param,part)

    for i in range(16):
      channel='iChannel'+str(i)
      if has_param[channel]['Present']:
        channelsSeen[channel]=True
        if i+1>channelsSeenNum:
          channelsSeenNum=i+1

  ichan_create    = ''
  ichan_process1  = ''
  ichan_process2  = ''
  ichan_process2_time  = ''
  ichan_process2_res   = ''
  ichan_process3  = ''
  j=0 # how to handle gaps when it comes to LINK_Main?!?
  for i in range(channelsSeenNum):
    channel='iChannel'+str(i)
    if channelsSeen[channel]:
      j+=1
      ichan_create   += f'  InChannel{i} = self:AddInput( "iChannel{i}",  "iChannel{i}",  {{ LINKID_DataType = "Image", LINK_Main = {j}, INP_Required = false  }})\n'
      ichan_process1 += f'  iChannel{i} = InChannel{i}:GetValue(req)\n\n  if iChannel{i}==nil then\n    iChannel{i} = Image(imgattrs)\n    iChannel{i}:Fill(black)\n  end\n\n'
      ichan_process2_time += f'  params.iChannelTime[{i}] = 0\n'
      ichan_process2_res  += f'  params.iChannelResolution[{i}][0] = iChannel{i}.DataWindow:Width()    -- or maybe: iChannel{i}.Width\n  params.iChannelResolution[{i}][1] = iChannel{i}.DataWindow:Height()   -- or maybe: iChannel{i}.Height\n'
      ichan_process3 += f'  node:AddInput("iChannel{i}",iChannel{i}) -- TODO: add a better channel name\n'
    else:
      ichan_process2_time += f'  params.iChannelTime[{i}] = 0\n'
      ichan_process2_res  += f'  params.iChannelResolution[{i}][0] = 0\n  params.iChannelResolution[{i}][1] = 0\n'

  # if channelsSeenNum==0:
  #   ichan_process2 = f'  params.iChannelTime[0] = 0\n  params.iChannelResolution[0][0] = 0\n  params.iChannelResolution[0][1] = 0\n'
  # else:
  #   ichan_process2 = ichan_process2_time + "\n" + ichan_process2_res

  fuse_param    = ''
  fuse_create   = ''
  fuse_process  = ''

  for n,p in has_param.items():
    if p['Present'] and p['Code']!='':
      fuse_param   +=  p['Param']
      fuse_create  +=  p['Create']
      fuse_process +=  p['Proc']

  with open(selfpath+".clobber/fuse-template.lua", "r") as f:
    fuse=f.read()

  # now we need to equip the fuse with all the controls

  create, process, kernel, params = check_controls(conversionCode)

  # remove all controls from source

  conversionCode = re.sub(r' +CONNECT_[A-Z]+\d+\s*\(\s*[A-Za-z_][\w]*\s*,\s*[^\)]+\)\s*; *\n', '', conversionCode)

  fuse_param   += params
  fuse_create  += create
  fuse_process += process

  #print(create)
  #print(params)
  #print(kernel)
  #print(process)

  conversionCode = re.sub(r'<<CODEFROMCONTROLS>>',kernel,conversionCode)

  # load some author information from yaml file

  fuse_author_url   = ''
  fuse_author_logo  = ''

  with open(selfpath+".clobber/fuse-authors.yaml", "r") as f:
    tmp=f.read()

  tmp=yaml.load(tmp, Loader=yaml.FullLoader)

  for author in tmp:
    if author['Name']==info['fuse']['author']:
      fuse_author_url   = author['URL']
      fuse_author_logo  = author['Logo']

  fuse=re.sub(r'<<<SHADER_ID>>>',info['shader']['id'],fuse)
  fuse=re.sub(r'<<<SHADER_AUTHOR>>>',info['shader']['author'],fuse)
  fuse=re.sub(r'<<<SHADER_NAME>>>',info['shader']['name'],fuse)
  fuse=re.sub(r'<<<FUSE_NAME>>>',info['fuse']['name'],fuse)
  fuse=re.sub(r'<<<FUSE_AUTHOR>>>',info['fuse']['author'],fuse)
  fuse=re.sub(r'<<<FUSE_CREATION_DATE>>>',date.today().strftime("%B %d, %Y"),fuse)
  fuse=re.sub(r'<<<FUSE_AUTHOR_URL>>>',fuse_author_url,fuse)
  fuse=re.sub(r'<<<FUSE_AUTHOR_LOGO>>>',fuse_author_logo,fuse)

  fuse=re.sub(r'<<<CODE_COMPATIBILITY>>>',compatibilityCode,fuse)
  fuse=re.sub(r'<<<CODE_KERNEL>>>',conversionCode,fuse)
  fuse=re.sub(r'<<<CODE_PARAMETERS>>>',"\n"+fuse_param,fuse)
  fuse=re.sub(r'<<<CODE_CREATE>>>',fuse_create,fuse)
  fuse=re.sub(r'<<<CODE_PROCESS>>> *\n',fuse_process,fuse)

  fuse=re.sub(r'<<<ICHANNELS_CREATE>>>',ichan_create,fuse)
  fuse=re.sub(r'<<<ICHANNELS_PROCESS1>>> *\n',ichan_process1,fuse)
  fuse=re.sub(r'<<<ICHANNELS_PROCESS2>>> *\n',ichan_process2,fuse)
  fuse=re.sub(r'<<<ICHANNELS_PROCESS3>>> *\n',ichan_process3,fuse)

  # nur zum mal gucken wo wir stehen:
  with open(target_path+fuse_file+".fuse", 'w') as f:
    f.write(fuse)

print("\n#################### Fuse Script ###################")
#print("Aufruf :",sys.argv[0]) #ok

if (sys.argv[0] != "fuse.py" and sys.argv[0] != "./fuse.py" ):
 selfpath = os.path.dirname(sys.argv[0])+"\\"

 print("#SELFPATH:",selfpath)
 print(sys.argv, len(sys.argv))
 #print("##Argv2##",id,param,txt)

 print("Folder: ",folder)

 CONVERSIONS_PATH = selfpath+"\Conversions\\"+folder+"\\"

 #VERBOSE   = verbose
 ID = id
 FORCE = pforce
else:
 selfpath = ""
 folder   = ""

 parser = argparse.ArgumentParser(description='Encapsulate conversion in a Fuse.')
 parser.add_argument('-f','--force',action='store_true',help='overwrite fuse and/or markdown file even if it already exists')
 parser.add_argument('-v','--verbose',action='store_true',help="tell what's happening")
 parser.add_argument('-i','--id', help="shadertoy id as it can be found in the conversion's filename right before the'.c' suffix", required=True)
 args = parser.parse_args()

 VERBOSE=args.verbose
 FORCE  =args.force
 ID     =args.id

load_dotenv(selfpath+".env")

#fuse_it(args.id,force=args.force)
fuse_it(ID,force=FORCE)

# try:
#   fuse_it(args.id)
# except Exception as e:
#   print("outch: "+str(e))


import tkinter
from tkinter import messagebox
import datetime
import json
import socket
from _thread import *

e = datetime.datetime.now()
# This code is to hide the main tkinter window
root = tkinter.Tk()
root.withdraw()

ServerSideSocket = socket.socket()
# Change host to server machine's ipv4 address when allowing connection to ESP32 or other external devices
host = '127.0.0.1'
port = 2004
ThreadCount = 0

microwave_list = []

def multi_threaded_client(connection):
    connection.send(str.encode('Server is working:'))
    while True:
        data = connection.recv(2048)
        if not data:
            break
    connection.close()

try:
    ServerSideSocket.bind((host, port))
except socket.error as e:
    print(str(e))
print('Socket is listening..')

'''
 If there's an error on the next line for Windows, try:
 > open CMD
 > netstat -ano | findstr :2004
 > tskill <PID from result, rightmost column>
'''
ServerSideSocket.listen(5)

while True:
    Client, address = ServerSideSocket.accept()
    print('Connected to: ' + address[0] + ':' + str(address[1]))
    start_new_thread(multi_threaded_client, (Client, ))
    ThreadCount += 1
    print('Thread Number: ' + str(ThreadCount))
    client_data = Client.recv(2048)
    print(client_data)
    info = json.loads(client_data.decode("utf-8").replace("'", '"'))
    #read json file

    for val in info['MotionData']:
        if info['MotionData'][val] == "True":
            microwave_list.append(val + ": True")
            # messagebox.showerror("Alert", "Date of Alert: %s/%s/%s" % (e.day, e.month, e.year) + "\nTime of Alert: %s:%s:%s" % (e.hour, e.minute, e.second) + "\nMotion detected at: ")
    print(microwave_list)
    microwave_list.clear()

ServerSideSocket.close()


from django.test import TestCase

from .models import Bares,Tapas

class TestBar(TestCase):

	def test_crear_bar(self):
		bar = Bares(nombre="bar", direccion="direccion",visitas=1)
		bar.save()
		self.assertEqual(bar.nombre, "bar")
		print("Bar creado correctamente")


from __future__ import absolute_import

import os
import logging

import h5py
import numpy as np

import luchador
import luchador.util
from luchador import nn

_LG = logging.getLogger('luchador')

_BE = luchador.get_nn_backend()
_CONV = luchador.get_nn_conv_format()

def _parse_command_line_args():
    from argparse import ArgumentParser as AP
    ap = AP(
        description='Feed batch data to layer and save the output to file'
    )
    ap.add_argument(
        'config',
        help='File contains layer and run config.'
    )
    ap.add_argument(
        '--output',
        help='Output data file.'
    )
    ap.add_argument(
        '--debug',
        help='Enable debug log', action='store_true'
    )
    return ap.parse_args()

def _run_forward_prop(layer, input_value, parameter_file, iteration=1):
    sess = nn.Session()
    if parameter_file:
        _LG.info('Loading parameter values from %s', parameter_file)
        sess.load_from_file(parameter_file, strict=False)

    _LG.info('Running forward path for %s times', iteration)
    for _ in range(iteration):
        ret = sess.run(
            outputs=layer.output, updates=layer.get_update_operations(),
            inputs={layer.input: input_value.astype(layer.input.dtype)},
            name='test',
        )
    _LG.info('Run forward path. Output shape: %s', ret.shape)
    return ret

def _transpose_needed(input_ndim):
    return input_ndim == 4 and _BE == 'tensorflow' and _CONV == 'NHWC'

def _load_input_value(filepath):
    _LG.info('Loading input value from %s', filepath)
    file_ = h5py.File(filepath, 'r')
    value = np.asarray(file_['input'])
    file_.close()

    _LG.info('  Shape %s', value.shape)
    _LG.info('  Dtype %s', value.dtype)
    return value

def _save_output(filepath, data):
    directory = os.path.dirname(filepath)
    if not os.path.exists(directory):
        os.makedirs(directory)

    _LG.info('Saving output value to %s', filepath)
    _LG.info('  Shape %s', data.shape)
    _LG.info('  Dtype %s', data.dtype)
    file_ = h5py.File(filepath, 'w')
    file_.create_dataset('output', data=data)
    file_.close()

def _load_config(config_path):
    cfg = luchador.util.load_config(config_path)

    cfg_dir = os.path.dirname(config_path)
    input_file = os.path.join(cfg_dir, cfg['input'])
    param_file = (
        os.path.join(cfg_dir, cfg['parameter'])
        if 'parameter' in cfg else None)
    return cfg, input_file, param_file

def _init_logger(debug):
    from luchador.util import initialize_logger
    message_format = (
        '%(asctime)s: %(levelname)5s: %(funcName)10s: %(message)s'
        if debug else '%(asctime)s: %(levelname)5s: %(message)s'
    )
    level = logging.DEBUG if debug else logging.INFO
    initialize_logger(
        name='luchador', message_format=message_format, level=level)

def _make_model(config_path, input_shape):
    config = luchador.nn.get_model_config(config_path, input_shape=input_shape)
    model_def = config['model']
    return nn.make_model(model_def)

def _main():
    args = _parse_command_line_args()
    _init_logger(args.debug)

    cfg, input_file, param_file = _load_config(args.config)
    input_value = _load_input_value(input_file)

    if _transpose_needed(input_value.ndim):
        # All the test data is created floowing the Theano format, which
        # is NCHW for input data. So when running this test in Tensorflow
        # backend, we reorder the input data to NHWC
        _LG.info('  * Converting input value from NCHW -> NHWC')
        input_value = input_value.transpose((0, 2, 3, 1))

    model = _make_model(args.config, input_shape=input_value.shape)

    output = _run_forward_prop(
        layer=model,
        input_value=input_value,
        parameter_file=param_file,
        **cfg.get('run', {})
    )

    if _transpose_needed(input_value.ndim):
        # So as to make the output comarison easy, we revert the oreder
        # from NHWC to NCHW.
        output_ = output.transpose((0, 3, 1, 2))
        _LG.info(
            '  * Rearranging output shape from %s to %s',
            output.shape, output_.shape)
        output = output_

    if args.output:
        _save_output(args.output, output)

if __name__ == '__main__':
    _main()


from RedditWebCrawler_Class import RedditCrawler
import string
import time
from collections import Counter

###
# this class will create/update a stock list
# this stock list will be composed of stocks discussed on specific subreddits
# in this case the "wallstreetbets" subreddit. It works taking as input webscraped reddit post titles,
# and finding names of stocks discussed therein. The names of stocks will be analyzed
# by way of word frequency, and any stock being mentioned more than num times will be added
# to the stock list
###

class StockListGenerator:
    ### file path is used for read/writing of stockList
    def __init__(self, filePath, posts):
        self.filePath = filePath
        self.titleString = ''
        self.posts = posts
        self.wordFreqArr = []

    ### receives array of titles (posts) and concantenates all titles
    ### into one big string, in order to parse and analyze word count
    ### each element in array has [post title, self text, and post id] - we're only using post title
    def prepareString(self):
        str = ''
        for post in self.posts:
            str = str +  ' ' + post[0] + ' '
        filter = string.punctuation + "0123456789"
        cleanStr = ''
        for char in str:
            if char not in filter:
                cleanStr = cleanStr + char
        self.titleString = cleanStr
        # print(self.titleString)
        return self.titleString

    ### takes a string of words, and firstly only uses words that are all caps
    ### this is to filter for potential stocks as generally they are written in all caps
    ### will return a 2-d array containing the words and their frequency
    ### uses self.titleString as string, must call prepare string first to populate titleString
    def wordFreqAllCaps(self):
        wordsFreq = []
        arr = Counter(self.titleString.split()).most_common()
        for word, count in arr:
            if (word.isupper()):
                # print(word)
                wordsFreq.append([word,count])
        self.wordFreqArr = wordsFreq
        # print(wordsFreq)
        return wordsFreq

    ### will open file for writing at given filePath, and create it if it doesn't exist
    ### appends the provided string parameter to end of file
    def writeToTxt(self, str):
        # Open a file with access mode 'a'
        file_object = open(self.filePath, 'a')
        writeStr = '\n' + str
        file_object.write(writeStr)
        # Close the file
        file_object.flush()
        file_object.close()

    ### will attempt to open file at given filePath
    ### for reading and will strip each line of surrounding whitespace
    def readTxt(self):
        try:
            readFile = open(self.filePath, 'r')

            str = readFile.readlines()

            readFile.close()
            for x in range(len(str)):
                str[x] = str[x].strip()
            return str
        except:
            return['']

    def theStockList(self, ocurrenceNum):
        redditCrawler = RedditCrawler()
        self.prepareString()
        self.wordFreqAllCaps()

        # print(self.wordFreqArr)
        print('Updating Stock List...')
        for entry in self.wordFreqArr:
            if (entry[1] >= ocurrenceNum): # if this specifc word is mentioned equal to or more than occurrenceNum times
                trackedStocks = self.readTxt()  # loads tracked stock list
                # whitelist = set('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ') # this was used to filter out punctuation/numbers but this is handled elsewhere
                # filteredStr = ''.join(filter(whitelist.__contains__, entry[0]))
                if (entry[0] not in trackedStocks): # checks if specific stock is already in stockList
                    time.sleep(7.6)
                    assetPrice = redditCrawler.realTimePrice(entry[0])
                    if (assetPrice != -1): #if the word is a real stock, it adds it
                        self.writeToTxt(entry[0])

        return self.wordFreqArr


import glob
import numpy as np
import argparse
import os
from skimage import io, color
from ast import literal_eval
from matplotlib import pyplot as plt
import json
import shutil
from utils.xview_synthetic_util import process_wv_coco_for_yolo_patches_for_aircraft_split as pwv

IMG_FORMAT0 = '.jpg'
IMG_FORMAT = '.png'
TXT_FORMAT = '.txt'

def merge_clean_origin_syn_bkg_image_files(file_path, cities, streets, tile_size=608, resolution=0.3, white_thresh=0.5):
    '''
    merge all the origin synthetic data into one folder
    then remove rgb images those contain more than white_thresh*100% white pixels
    :param file_path:
    :param cities:
    :param streets:
    :param tile_size:
    :param resolution:
    :return:
    '''
    step = tile_size * resolution
    image_folder_name = '{}_{}_{}_images_step{}'

    new_img_folder = '{}_all_images_step{}'.format(syn_args.syn_display_type, step)
    new_lbl_folder = '{}_all_annos_step{}'.format(syn_args.syn_display_type, step)
    des_img_path = os.path.join(file_path, new_img_folder)
    des_lbl_path = os.path.join(file_path, new_lbl_folder)
    if not os.path.exists(des_img_path):
        os.mkdir(des_img_path)
    else:
        shutil.rmtree(des_img_path)
        os.mkdir(des_img_path)
    if not os.path.exists(des_lbl_path):
        os.mkdir(des_lbl_path)
    else:
        shutil.rmtree(des_lbl_path)
        os.mkdir(des_lbl_path)

    for i in range(len(cities)):
        image_path = os.path.join(file_path,
                                  image_folder_name.format(syn_args.syn_display_type, cities[i], streets[i], step))
        image_files = np.sort(glob.glob(os.path.join(image_path, '*{}'.format(IMG_FORMAT))))
        for img in image_files:
            shutil.copy(img, des_img_path)

    all_images = np.sort(glob.glob(os.path.join(des_img_path, '*{}'.format(IMG_FORMAT))))
    for ix, f in enumerate(all_images):
        img = io.imread(f)
        img = color.rgb2gray(img)
        white_num = np.sum(img == 1)
        white_ratio = white_num / img.shape[0] / img.shape[1]

        if white_ratio > white_thresh:
            os.remove(f)
        else:
            shutil.copy(f, os.path.join(syn_args.syn_images_save_dir, os.path.basename(f)))

            txt_name = os.path.basename(f).replace(IMG_FORMAT, TXT_FORMAT)
            f_txt = open(os.path.join(des_lbl_path, txt_name), 'w')
            f_txt.close()
            shutil.copy(os.path.join(des_lbl_path, txt_name), os.path.join(syn_args.syn_annos_save_dir, txt_name))

def get_part_syn_args():
    parser = argparse.ArgumentParser()

    parser.add_argument("--syn_bkg_img_anno_dir", type=str, help="images and annotations of synthetic airplanes",
                        default='/media/lab/Yang/data/synthetic_data/syn_background/')

    parser.add_argument("--syn_bkg_txt_dir", type=str, help="txt labels of synthetic airplanes",
                        default='/media/lab/Yang/data/synthetic_data/syn_background_txt_xcycwh/{}/')

    parser.add_argument("--syn_bkg_gt_bbox_dir", type=str, help="gt images with bbox of synthetic airplanes",
                        default='/media/lab/Yang/data/synthetic_data/syn_background_gt_bbox/{}/')

    parser.add_argument("--syn_images_save_dir", type=str, help="rgb images of synthetic airplanes",
                        default='/media/lab/Yang/data/xView_YOLO/images/{}_{}/')
    parser.add_argument("--syn_annos_save_dir", type=str, help="gt of synthetic airplanes",
                        default='/media/lab/Yang/data/xView_YOLO/labels/{}/{}_{}_cls_xcycwh/')
    parser.add_argument("--syn_txt_save_dir", type=str, help="gt related files of synthetic airplanes",
                        default='/media/lab/Yang/data/xView_YOLO/labels/{}/{}_{}_cls/')

    parser.add_argument("--data_txt_dir", type=str, help="to save txt files",
                        default='/media/lab/Yang/data/xView_YOLO/labels/{}/{}_cls/')

    parser.add_argument("--data_xview_dir", type=str, help="to save data files",
                        default='/media/lab/Yang/code/yolov3/data_xview/{}_cls/')

    parser.add_argument("--syn_data_list_dir", type=str, help="to syn data list files",
                        default='/media/lab/Yang/code/yolov3/data_xview/{}_{}_cls/')

    parser.add_argument("--results_dir", type=str, help="to save category files",
                        default='/media/lab/Yang/code/yolov3/result_output/{}_cls/{}_{}/')

    parser.add_argument("--cat_sample_dir", type=str, help="to save figures",
                        default='/media/lab/Yang/data/xView_YOLO/cat_samples/{}/{}_cls/')

    parser.add_argument("--val_percent", type=float, default=0.20,
                        help="Percent to split into validation (ie .25 = val set is 25% total)")
    parser.add_argument("--font3", type=str, help="legend font",
                        default="{'family': 'serif', 'weight': 'normal', 'size': 10}")
    parser.add_argument("-ft2", "--font2", type=str, help="legend font",
                        default="{'family': 'serif', 'weight': 'normal', 'size': 13}")

    parser.add_argument("--class_num", type=int, default=1, help="Number of Total Categories")  # 60  6
    parser.add_argument("--tile_size", type=int, default=608, help="image size")  # 300 416

    parser.add_argument("--syn_display_type", type=str, default='syn_background',
                        help="syn_background")  # ######*********************change
    parser.add_argument("--syn_ratio", type=float, default=0.75,
                        help="ratio of synthetic data: 0.25, 0.5, 0.75, 1.0  0")  # ######*********************change

    parser.add_argument("--min_region", type=int, default=100, help="the smallest #pixels (area) to form an object")
    parser.add_argument("--link_r", type=int, default=15,
                        help="the #pixels between two connected components to be grouped")

    parser.add_argument("--resolution", type=float, default=0.3, help="resolution of synthetic data")

    parser.add_argument("--cities", type=str,
                        default="['barcelona', 'berlin', 'francisco', 'hexagon', 'radial', 'siena', 'spiral']",
                        help="the synthetic data of cities")
    parser.add_argument("--streets", type=str, default="[200, 200, 200, 200, 200, 250, 130]",
                        help="the  #streets of synthetic  cities ")
    syn_args = parser.parse_args()
    syn_args.cities = literal_eval(syn_args.cities)
    syn_args.streets = literal_eval(syn_args.streets)
    syn_args.data_xview_dir = syn_args.data_xview_dir.format(syn_args.class_num)
    syn_args.data_txt_dir = syn_args.data_txt_dir.format(syn_args.tile_size, syn_args.class_num)
    syn_args.cat_sample_dir = syn_args.cat_sample_dir.format(syn_args.tile_size, syn_args.class_num)
    if not os.path.exists(syn_args.data_xview_dir):
        os.makedirs(syn_args.data_xview_dir)
    if not os.path.exists(syn_args.data_txt_dir):
        os.makedirs(syn_args.data_txt_dir)
    if not os.path.exists(syn_args.cat_sample_dir):
        os.makedirs(syn_args.cat_sample_dir)

    syn_args.syn_images_save_dir = syn_args.syn_images_save_dir.format(syn_args.tile_size, syn_args.syn_display_type)
    syn_args.syn_annos_save_dir = syn_args.syn_annos_save_dir.format(syn_args.tile_size, syn_args.syn_display_type, syn_args.class_num)
    syn_args.syn_txt_save_dir = syn_args.syn_txt_save_dir.format(syn_args.tile_size, syn_args.syn_display_type, syn_args.class_num)
    if not os.path.exists(syn_args.syn_images_save_dir):
        os.makedirs(syn_args.syn_images_save_dir)
    if not os.path.exists(syn_args.syn_txt_save_dir):
        os.makedirs(syn_args.syn_txt_save_dir)
    if not os.path.exists(syn_args.syn_annos_save_dir):
        os.makedirs(syn_args.syn_annos_save_dir)

    return syn_args

if __name__ == '__main__':
    syn_args = get_part_syn_args()

    # white_thresh = 0.5
    # merge_clean_origin_syn_bkg_image_files(syn_args.syn_bkg_img_anno_dir, syn_args.cities, syn_args.streets,
    #                                         syn_args.tile_size, syn_args.resolution, white_thresh)

    '''
    xview with syn_background
    split train:val randomly split chips
    default split
    '''
    # comments = '_px6whr4_ng0'
    # seed = [3, 5, 9]
    # data_name = 'xview'
    # for s in seed:
    #     comments = '_px6whr4_ng0_seed{}'.format(s)
    #     pwv.split_trn_val_with_chips(data_name, comments, s)

    '''
    combine xview & syn_background dataset [0.1, 0.2, 0.3]
    '''
    # seed = [3, 5, 9]
    # syn_ratio = [0.1, 0.2, 0.3]
    # dt = 'syn_background'
    # for s in seed:
    #     comments = '_px6whr4_ng0_seed{}'.format(s)
    #     for sr in syn_ratio:
    #         pwv.combine_xview_syn_by_ratio(dt, sr, comments, seed=s)

    ''''
    create xview_syn_background_*_px6whr4_ng0_seed{}.data
    xview_syn_background_*_px6whr4_ng0.data
    '''
    # comments = '_px6whr4_ng0_seed{}'
    # seed = [3, 5, 9]
    # syn_ratio = [0.1, 0.2, 0.3, 0]
    # for s in seed:
    #     for sr in syn_ratio:
    #         pwv.create_xview_syn_data('syn_background', sr, comments, trn_comments=True, seed=s)


import logging
import os
from contextlib import closing

import barman
from barman.cloud import CloudInterface, configure_logging
from barman.utils import force_str
from barman.xlog import hash_dir, is_any_xlog_file, is_backup_file

try:
    import argparse
except ImportError:
    raise SystemExit("Missing required python module: argparse")

def main(args=None):
    """
    The main script entry point

    :param list[str] args: the raw arguments list. When not provided
        it defaults to sys.args[1:]
    """
    config = parse_arguments(args)
    configure_logging(config)

    # Validate the WAL file name before downloading it
    if not is_any_xlog_file(config.wal_name):
        logging.error('%s is an invalid name for a WAL file' % config.wal_name)
        raise SystemExit(1)

    try:
        cloud_interface = CloudInterface(
            url=config.source_url,
            encryption=config.encryption,
            profile_name=config.profile,
            endpoint_url=config.endpoint_url)

        with closing(cloud_interface):
            downloader = S3WalDownloader(
                cloud_interface=cloud_interface,
                server_name=config.server_name)

            if not cloud_interface.test_connectivity():
                raise SystemExit(1)
            # If test is requested, just exit after connectivity test
            elif config.test:
                raise SystemExit(0)

            if not cloud_interface.bucket_exists:
                logging.error("Bucket %s does not exist",
                              cloud_interface.bucket_name)
                raise SystemExit(1)

            downloader.download_wal(config.wal_name, config.wal_dest)

    except Exception as exc:
        logging.error("Barman cloud WAL restore exception: %s",
                      force_str(exc))
        logging.debug('Exception details:', exc_info=exc)
        raise SystemExit(1)

def parse_arguments(args=None):
    """
    Parse command line arguments

    :return: The options parsed
    """

    parser = argparse.ArgumentParser(
        description='This script can be used as a `restore_command` '
                    'to download WAL files previously archived with '
                    'barman-cloud-wal-archive command. '
                    'Currently only AWS S3 is supported.',
        add_help=False
    )

    parser.add_argument(
        'source_url',
        help='URL of the cloud source, such as a bucket in AWS S3.'
             ' For example: `s3://bucket/path/to/folder`.'
    )
    parser.add_argument(
        'server_name',
        help='the name of the server as configured in Barman.'
    )
    parser.add_argument(
        "wal_name",
        help="The value of the '%%f' keyword "
             "(according to 'restore_command').",
    )
    parser.add_argument(
        "wal_dest",
        help="The value of the '%%p' keyword "
             "(according to 'restore_command').",
    )
    parser.add_argument(
        '-V', '--version',
        action='version', version='%%(prog)s %s' % barman.__version__
    )
    parser.add_argument(
        '--help',
        action='help',
        help='show this help message and exit')
    verbosity = parser.add_mutually_exclusive_group()
    verbosity.add_argument(
        '-v', '--verbose',
        action='count',
        default=0,
        help='increase output verbosity (e.g., -vv is more than -v)')
    verbosity.add_argument(
        '-q', '--quiet',
        action='count',
        default=0,
        help='decrease output verbosity (e.g., -qq is less than -q)')
    parser.add_argument(
        '-P', '--profile',
        help='profile name (e.g. INI section in AWS credentials file)',
    )
    parser.add_argument(
        "-e", "--encryption",
        help="Enable server-side encryption for the transfer. "
             "Allowed values: 'AES256', 'aws:kms'",
        choices=['AES256', 'aws:kms'],
        metavar="ENCRYPTION",
    )
    parser.add_argument(
        "-t", "--test",
        help="Test cloud connectivity and exit",
        action="store_true",
        default=False
    )
    parser.add_argument(
        "--endpoint-url",
        help="Override default S3 endpoint URL with the given one",
    )
    return parser.parse_args(args=args)

class S3WalDownloader(object):
    """
    S3 download client
    """

    # Allowed compression algorithms
    ALLOWED_COMPRESSIONS = {'.gz': 'gzip', '.bz2': 'bzip2'}

    def __init__(self, cloud_interface,
                 server_name):
        """
        Object responsible for handling interactions with S3

        :param CloudInterface cloud_interface: The interface to use to
          upload the backup
        :param str server_name: The name of the server as configured in Barman
        """

        self.cloud_interface = cloud_interface
        self.server_name = server_name

    def download_wal(self, wal_name, wal_dest):
        """
        Download a WAL file from S3

        :param str wal_name: Name of the WAL file
        :param str wal_dest: Full path of the destination WAL file
        """

        # Correctly format the source path on s3
        source_dir = os.path.join(
            self.cloud_interface.path,
            self.server_name,
            'wals',
            hash_dir(wal_name)
        )
        # Add a path separator if needed
        if not source_dir.endswith(os.path.sep):
            source_dir += os.path.sep

        wal_path = os.path.join(source_dir, wal_name)

        remote_name = None
        # Automatically detect compression based on the file extension
        compression = None
        for item in self.cloud_interface.list_bucket(source_dir):
            # perfect match (uncompressed file)
            if item == wal_path:
                remote_name = item
            # look for compressed files or .partial files
            elif item.startswith(wal_path):
                # Detect compression
                basename = item
                for e, c in self.ALLOWED_COMPRESSIONS.items():
                    if item[-len(e):] == e:
                        # Strip extension
                        basename = basename[:-len(e)]
                        compression = c
                        break

                # Check basename is a known xlog file (.partial?)
                if not is_any_xlog_file(basename):
                    logging.warning("Unknown WAL file: %s", item)
                    continue
                # Exclude backup informative files (not needed in recovery)
                elif is_backup_file(basename):
                    logging.info("Skipping backup file: %s", item)
                    continue

                # Found candidate
                remote_name = item
                logging.info("Found WAL %s for server %s as %s",
                             wal_name, self.server_name, remote_name)
                break

        if not remote_name:
            logging.info("WAL file %s for server %s does not exists",
                         wal_name, self.server_name)
            raise SystemExit(1)

        # Download the file
        logging.debug("Downloading %s to %s (%s)",
                      remote_name, wal_dest,
                      "decompressing " + compression if compression
                      else "no compression")
        self.cloud_interface.download_file(remote_name, wal_dest, compression)

if __name__ == '__main__':
    main()


'''
 Authors : Bianchi Alexandre and Moulin Vincent
 Class : INF3dlma
 HE-Arc 2019-2020
 PedestriArc : Detection of pedestrian crossing

 Papers:    https://www.researchgate.net/publication/320675020_Crosswalk_navigation_for_people_with_visual_impairments_on_a_wearable_device
			https://ieeexplore.ieee.org/document/7873114
			https://pdfs.semanticscholar.org/19d9/becce00a500bdd1cad5a19f9f16175347096.pdf
'''

#==========================#
#----------import----------#
#==========================#

import cv2 as cv
import numpy as np
from matplotlib import pyplot as plt
import random as rng
from sklearn import linear_model, datasets

#==========================#
#---------functions--------#
#==========================#

def imgThreshold(img):
	pass

def process(img):
	pass

def lineCalc(vx, vy, x0, y0):
	scale = 10
	x1 = x0+scale*vx
	y1 = y0+scale*vy
	m = (y1-y0)/(x1-x0)
	b = y1-m*x1
	return m, b

# the angle at the vanishing point

def angle(pt1, pt2):
	x1, y1 = pt1
	x2, y2 = pt2
	inner_product = x1*x2 + y1*y2
	len1 = math.hypot(x1, y1)
	len2 = math.hypot(x2, y2)
	print(len1)
	print(len2)
	a = math.acos(inner_product/(len1*len2))
	return a*180/math.pi

# vanishing point - cramer's rule

def lineIntersect(m1, b1, m2, b2):
	# a1*x+b1*y=c1
	# a2*x+b2*y=c2
	# convert to cramer's system
	a_1 = -m1
	b_1 = 1
	c_1 = b1

	a_2 = -m2
	b_2 = 1
	c_2 = b2

	d = a_1*b_2 - a_2*b_1  # determinant
	dx = c_1*b_2 - c_2*b_1
	dy = a_1*c_2 - a_2*c_1

	intersectionX = dx/d
	intersectionY = dy/d
	return intersectionX, intersectionY

def is_contour_bad(c, HEIGHT, WIDTH):
	size_max = 0.2 * HEIGHT * WIDTH
	size_min = 0.5 * 10 ** -3 * HEIGHT * WIDTH

	if len(c) > 5:
		ellipse = cv.fitEllipse(c)
		area = ellipse[1][0] * ellipse[1][1]
		print(ellipse)
		if ellipse[2] < 20:
			return True
		if area > size_max or area < size_min:
			return True
	else:
		return True

# Main
# Used for debugging the modules
if __name__ == "__main__":

	radius = 500 #px

	bxLeft = []
	byLeft = []
	bxbyLeftArray = []
	bxbyRightArray = []
	bxRight = []
	byRight = []
	boundedLeft = []
	boundedRight = []

	img_color = cv.imread('tests/cam_20200119_103520.jpg', cv.IMREAD_UNCHANGED)
	img = cv.imread('tests/cam_20200119_103520.jpg', 0)
	img_blur = cv.medianBlur(img, 5)

	HEIGHT, WIDTH = img_blur.shape

	# The block size parameters is base on the size of the given dataset
	# A neural network based on smaller resolution could be useful for the future (cf: first papers)
	th2 = cv.adaptiveThreshold(
		img_blur, 255, cv.ADAPTIVE_THRESH_MEAN_C, cv.THRESH_BINARY, 257, 2)
	th3 = cv.adaptiveThreshold(
		img_blur, 255, cv.ADAPTIVE_THRESH_MEAN_C, cv.THRESH_BINARY, 257, 2)

	# Finding the potential candidates
	CANDIDATES = []

	size_max = 0.2 * HEIGHT * WIDTH
	size_min = 0.5 * 10 ** -3 * HEIGHT * WIDTH
	print(f'sizeMax: {size_max}, sizeMin: {size_min}')

	# 3. find contours and  draw the green lines on the white strips
	contours, hierarchy = cv.findContours(
		th3, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_NONE)

	mask = np.ones(th3.shape[:2], dtype="uint8") * 255

	print(f'ContoursNB: {len(contours)}')

	# loop over the contours
	for c in contours:
		if is_contour_bad(c, HEIGHT, WIDTH):
			cv.drawContours(mask, [c], -1, 0, -1)

	cv.imshow("MASK", mask)
	mask = cv.bitwise_and(th3, th3, mask=mask)

	titles = ['Original Image', 'Adaptive Mean Thresholding',
			  'Adaptive Gaussian Thresholding', 'After Mask']
	images = [img_color, th2, th3, mask]

	for i in range(4):
		plt.subplot(2, 2, i+1), plt.imshow(images[i], 'gray')
		plt.title(titles[i])
		plt.xticks([]), plt.yticks([])
	plt.show()

	contours, hierarchy = cv.findContours(
		mask, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)

	cv.drawContours(img_color, contours, -1, (255, 0, 0))

	for i in contours:
		if len(i) > 5:
			ellipse = cv.fitEllipse(i)
			cv.ellipse(img_color, ellipse, (0,255,0))
			minRect = cv.minAreaRect(i)
			box = cv.boxPoints(minRect)
			box = np.intp(box)
			cv.drawContours(img_color, [box], -1, (0,0,255))

			bx_1, by_1, bx_2, by_2 = box[0][0], box[0][1], box[2][0], box[2][1]
			#cv.rectangle(img_color,(bx,by),(bx+bw,by+bh),(0,255,0),2)

			cv.line(img_color, (box[0][0], box[0][1]), (box[2][0], box[2][1]),
					(0, 255, 0), 2)  # draw the a contour line
			bxRight.append(bx_2)  # right line
			byRight.append(by_2)  # right line
			bxLeft.append(bx_1)  # left line
			byLeft.append(by_1)  # left line
			bxbyLeftArray.append([bx_1, by_1])  # x,y for the left line
			bxbyRightArray.append([bx_2, by_2])  # x,y for the left line
			cv.circle(img_color, (int(bx_1), int(by_1)), 5,
					(0, 250, 250), 2)  # circles -> left line
			cv.circle(img_color, (int(bx_2), int(by_2)), 5,
					(250, 0, 0), 2)  # circles -> right line

	cv.imshow("TEST", img_color)
	cv.waitKey()

	# calculate median average for each line
	medianR = np.median(bxbyRightArray, axis=0)
	medianL = np.median(bxbyLeftArray, axis=0)

	bxbyLeftArray = np.asarray(bxbyLeftArray)
	bxbyRightArray = np.asarray(bxbyRightArray)

	# 4. are the points bounded within the median circle?
	for i in bxbyLeftArray:
		if (((medianL[0] - i[0])**2 + (medianL[1] - i[1])**2) < radius**2) == True:
			boundedLeft.append(i)

	boundedLeft = np.asarray(boundedLeft)

	for i in bxbyRightArray:
		if (((medianR[0] - i[0])**2 + (medianR[1] - i[1])**2) < radius**2) == True:
			boundedRight.append(i)

	boundedRight = np.asarray(boundedRight)

	# 5. RANSAC Algorithm

	# select the points enclosed within the circle (from the last part)
	bxLeft = np.asarray(boundedLeft[:, 0])
	byLeft = np.asarray(boundedLeft[:, 1])
	bxRight = np.asarray(boundedRight[:, 0])
	byRight = np.asarray(boundedRight[:, 1])

	# transpose x of the right and the left line
	bxLeftT = np.array([bxLeft]).reshape(-1, 1)
	bxRightT = np.array([bxRight]).reshape(-1, 1)

	# run ransac for LEFT
	model_ransac = linear_model.RANSACRegressor()
	ransacX = model_ransac.fit(bxLeftT, byLeft)
	inlier_maskL = model_ransac.inlier_mask_  # right mask

	# run ransac for RIGHT
	ransacY = model_ransac.fit(bxRightT, byRight)
	inlier_maskR = model_ransac.inlier_mask_  # left mask

	# draw RANSAC selected circles
	for i, element in enumerate(boundedRight[inlier_maskR]):
	   # print(i,element[0])
		# circles -> right line
		cv.circle(img_color, (element[0], element[1]), 10, (250, 100, 100), 2)

	for i, element in enumerate(boundedLeft[inlier_maskL]):
	   # print(i,element[0])
		# circles -> right line
		cv.circle(img_color, (element[0], element[1]), 10, (100, 100, 250), 2)

	# 6. Calcuate the intersection point of the bounding lines
	# unit vector + a point on each line
	vx, vy, x0, y0 = cv.fitLine(boundedLeft[inlier_maskL],cv.DIST_L2,0,0.01,0.01)
	vx_R, vy_R, x0_R, y0_R = cv.fitLine(boundedRight[inlier_maskR],cv.DIST_L2,0,0.01,0.01)

	# get m*x+b
	m_L,b_L=lineCalc(vx, vy, x0, y0)
	m_R,b_R=lineCalc(vx_R, vy_R, x0_R, y0_R)

	# calculate intersention
	intersectionX,intersectionY = lineIntersect(m_R,b_R,m_L,b_L)

	# 7. draw the bounding lines and the intersection point
	m = radius*10
	if (intersectionY < HEIGHT/2 ):
		cv.circle(img_color,(int(intersectionX),int(intersectionY)),10,(0,0,255),15)
		cv.line(img_color,(x0-m*vx, y0-m*vy), (x0+m*vx, y0+m*vy),(255,0,0),3)
		cv.line(img_color,(x0_R-m*vx_R, y0_R-m*vy_R), (x0_R+m*vx_R, y0_R+m*vy_R),(255,0,0),3)

	# cv.drawContours(img, contours, -1, (255, 255, 0))

	plt.show()

	titles = ['Original Image', 'Adaptive Mean Thresholding',
			  'Adaptive Gaussian Thresholding']
	images = [img_color, th2, th3]

	for i in range(3):
		plt.subplot(2, 2, i+1), plt.imshow(images[i], 'gray')
		plt.title(titles[i])
		plt.xticks([]), plt.yticks([])
	plt.show()