"""Administrative command-line interface."""

from __future__ import annotations

import json
import sys
from pathlib import Path

import click
import structlog
import uvicorn
from cryptography.fernet import Fernet
from fastapi.openapi.utils import get_openapi
from safir.asyncio import run_with_asyncio
from safir.click import display_help
from safir.database import create_database_engine, initialize_database
from safir.slack.blockkit import SlackMessage
from sqlalchemy import text

from .dependencies.config import config_dependency
from .factory import Factory
from .keypair import RSAKeyPair
from .main import create_app
from .models.token import Token
from .schema import Base

__all__ = [
    "audit",
    "delete_all_data",
    "generate_key",
    "generate_token",
    "help",
    "init",
    "main",
    "maintenance",
    "openapi_schema",
    "run",
]

@click.group(context_settings={"help_option_names": ["-h", "--help"]})
@click.version_option(message="%(version)s")
def main() -> None:
    """Administrative command-line interface for gafaelfawr."""

@main.command()
@click.argument("topic", default=None, required=False, nargs=1)
@click.pass_context
def help(ctx: click.Context, topic: str | None) -> None:
    """Show help for any command."""
    display_help(main, ctx, topic)

@main.command()
@click.option(
    "--fix", default=False, is_flag=True, help="Fix issues found, if possible"
)
@click.option(
    "--config-path",
    envvar="GAFAELFAWR_CONFIG_PATH",
    type=click.Path(path_type=Path),
    default=None,
    help="Application configuration file.",
)
@run_with_asyncio
async def audit(*, fix: bool, config_path: Path | None) -> None:
    """Run a consistency check on Gafaelfawr's data stores.

    Any problems found will be reported to Slack.
    """
    if config_path:
        config_dependency.set_config_path(config_path)
    config = await config_dependency()
    logger = structlog.get_logger("gafaelfawr")
    logger.debug("Starting audit")
    engine = create_database_engine(
        config.database_url, config.database_password
    )
    async with Factory.standalone(config, engine) as factory:
        slack = factory.create_slack_client()
        if not slack:
            msg = "Slack alerting required for audit but not configured"
            raise click.UsageError(msg)
        token_service = factory.create_token_service()
        async with factory.session.begin():
            alerts = await token_service.audit(fix=fix)
        if alerts:
            message = (
                "Gafaelfawr data inconsistencies found:\n• "
                + "\n• ".join(alerts)
            )
            await slack.post(SlackMessage(message=message))
    await engine.dispose()
    logger.debug("Finished audit")

@main.command()
@click.option(
    "--config-path",
    envvar="GAFAELFAWR_CONFIG_PATH",
    type=click.Path(path_type=Path),
    default=None,
    help="Application configuration file.",
)
@run_with_asyncio
async def delete_all_data(*, config_path: Path | None) -> None:
    """Delete all data from Redis and the database.

    Intended for destructive upgrades, such as when switching from one
    upstream authentication provider to another when all of the usernames will
    change.  This does not delete or reset UID and GID assignments from
    Firestore.
    """
    if config_path:
        config_dependency.set_config_path(config_path)
    config = await config_dependency()
    logger = structlog.get_logger("gafaelfawr")
    logger.debug("Starting to delete all data")
    engine = create_database_engine(
        config.database_url, config.database_password
    )
    tables = (t.name for t in Base.metadata.sorted_tables)
    async with Factory.standalone(config, engine) as factory:
        admin_service = factory.create_admin_service()
        async with factory.session.begin():
            stmt = text(f'TRUNCATE TABLE {", ".join(tables)}')
            logger.info("Truncating all tables")
            await factory.session.execute(stmt)
            await admin_service.add_initial_admins(config.initial_admins)
        token_service = factory.create_token_service()
        logger.info("Deleting all tokens from Redis")
        await token_service.delete_all_tokens()
        if config.oidc_server:
            oidc_service = factory.create_oidc_service()
            logger.info("Deleting all OpenID Connect codes from Redis")
            await oidc_service.delete_all_codes()
    await engine.dispose()
    logger.debug("Finished deleting all data")

@main.command()
def generate_key() -> None:
    """Generate a new RSA key pair and print the private key."""
    keypair = RSAKeyPair.generate()
    sys.stdout.write(keypair.private_key_as_pem().decode())

@main.command()
def generate_session_secret() -> None:
    """Generate a new Gafaelfawr session secret."""
    sys.stdout.write(Fernet.generate_key().decode() + "\n")

@main.command()
def generate_token() -> None:
    """Generate an encoded token (such as the bootstrap token)."""
    sys.stdout.write(str(Token()) + "\n")

@main.command()
@click.option(
    "--config-path",
    envvar="GAFAELFAWR_CONFIG_PATH",
    type=click.Path(path_type=Path),
    default=None,
    help="Application configuration file.",
)
@run_with_asyncio
async def init(*, config_path: Path | None) -> None:
    """Initialize the database storage."""
    if config_path:
        config_dependency.set_config_path(config_path)
    config = await config_dependency()
    logger = structlog.get_logger("gafaelfawr")
    logger.debug("Initializing database")
    engine = create_database_engine(
        config.database_url, config.database_password
    )
    await initialize_database(engine, logger, schema=Base.metadata)
    async with Factory.standalone(config, engine) as factory:
        admin_service = factory.create_admin_service()
        logger.debug("Adding initial administrators")
        async with factory.session.begin():
            await admin_service.add_initial_admins(config.initial_admins)
        if config.firestore:
            firestore = factory.create_firestore_storage()
            logger.debug("Initializing Firestore")
            await firestore.initialize()
    await engine.dispose()
    logger.debug("Finished initializing data stores")

@main.command()
@click.option(
    "--config-path",
    envvar="GAFAELFAWR_CONFIG_PATH",
    type=click.Path(path_type=Path),
    default=None,
    help="Application configuration file.",
)
@run_with_asyncio
async def maintenance(*, config_path: Path | None) -> None:
    """Perform background maintenance."""
    if config_path:
        config_dependency.set_config_path(config_path)
    config = await config_dependency()
    logger = structlog.get_logger("gafaelfawr")
    logger.debug("Starting background maintenance")
    engine = create_database_engine(
        config.database_url, config.database_password
    )
    async with Factory.standalone(config, engine, check_db=True) as factory:
        token_service = factory.create_token_service()
        async with factory.session.begin():
            logger.info("Marking expired tokens in database")
            await token_service.expire_tokens()
            logger.info("Truncating token history")
            await token_service.truncate_history()
    await engine.dispose()
    logger.debug("Finished background maintenance")

@main.command()
@click.option(
    "--add-back-link",
    default=False,
    is_flag=True,
    help="Add back link (used when generating application documentation)",
)
@click.option(
    "--output",
    default=None,
    type=click.Path(path_type=Path),
    help="Output path (output to stdout if not given).",
)
def openapi_schema(*, add_back_link: bool, output: Path | None) -> None:
    """Generate the OpenAPI schema."""
    app = create_app(load_config=False)
    description = app.description
    if add_back_link:
        description += "\n\n[Return to Gafaelfawr documentation](api.html)."
    schema = get_openapi(
        title=app.title,
        description=description,
        version=app.version,
        routes=app.routes,
    )
    if output:
        output.parent.mkdir(exist_ok=True)
        with output.open("w") as f:
            json.dump(schema, f)
    else:
        json.dump(schema, sys.stdout)

@main.command()
@click.option(
    "--port", default=8080, type=int, help="Port to run the application on."
)
def run(*, port: int) -> None:
    """Run the application (for testing only)."""
    uvicorn.run(
        "gafaelfawr.main:create_app",
        factory=True,
        port=port,
        reload=True,
        reload_dirs=["src"],
    )


from keras.models import Model
from keras.layers import Input, Conv1D, BatchNormalization, MaxPool1D, Conv2D, Deconv2D, Concatenate, Lambda, MaxPooling2D, ZeroPadding2D, Reshape, Activation, Conv2DTranspose
from keras import backend as K

from keras_layers.keras_layer_AnchorBoxes import AnchorBoxes
from keras_layers.keras_layer_L2Normalization import L2Normalization
from keras_layers.keras_layer_DecodeDetections import DecodeDetections
from keras_layers.keras_layer_DecodeDetectionsFast import DecodeDetectionsFast

D = 9
N = 100
C = 64

def make_pillar_feature_net():
    x = Input(shape=(N, D), name='input')
    conv_1 = Conv1D(filters=C, kernel_size=1, strides=1, activation='relu', name='conv_1')(x)
    bn_1 = BatchNormalization(name='bn_1')(conv_1)
    pool_1 = MaxPool1D(pool_size=100, strides=1, name='output')(bn_1)

    return Model(inputs=x, outputs=pool_1)

def make_backbone(width, height, feature_C):
    x = Input(shape=(height, width, feature_C), name='input')

    y = rpn_conv_block(x, feature_C, 3, name='rpn_conv1')
    y_deconv1 = Conv2DTranspose(2*feature_C, 3, strides=1, padding='same', name='rpn_deconv1')(y)
    y = rpn_conv_block(y, 2*feature_C, 5, name='rpn_conv2')
    y_deconv2 = Conv2DTranspose(2*feature_C, 2, strides=2, padding='same', name='rpn_deconv2')(y)
    y = rpn_conv_block(y, 4*feature_C, 5, name='rpn_conv3')
    y = Conv2DTranspose(2*feature_C, 4, strides=4, padding='same', name='rpn_deconv3')(y)
    y = Concatenate()([y, y_deconv2, y_deconv1])

    print (K.int_shape(y))

    return Model(inputs=x, outputs=ssd_300(y))

def _rpn_conv(x, cout, k, s, p, name, i=None):
    if i is not None:
        name += '_blk' + str(i)
    x = ZeroPadding2D(p, name=name + '_pad')(x)
    x = Conv2D(cout, k, strides=s, name=name + '_conv')(x)
    x = BatchNormalization(name=name + '_bn')(x)
    x = Activation('relu', name=name + '_relu')(x)
    return x

def rpn_conv_block(x, cout, q, name='rpn_conv'):
    x = _rpn_conv(x, cout, 3, 2, 1, name)
    for i in range(q):
        x = _rpn_conv(x, cout, 3, 1, 1, name, i + 1)
    return x

def ssd_300(x):
    n_classes = 2
    two_boxes_for_ar1 = True
    aspect_ratios_per_layer = [[1.0, 2.0, 0.5],
                               [1.0, 2.0, 0.5, 3.0, 1.0/3.0],
                                [1.0, 2.0, 0.5, 3.0, 1.0/3.0],
                                [1.0, 2.0, 0.5, 3.0, 1.0/3.0],
                                [1.0, 2.0, 0.5],
                                [1.0, 2.0, 0.5]]

    n_boxes = []
    for ar in aspect_ratios_per_layer:
        if (1 in ar) & two_boxes_for_ar1:
            n_boxes.append(len(ar) + 1)  # +1 for the second box for aspect ratio 1
        else:
            n_boxes.append(len(ar))

    conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_1')(x)
    conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_2')(conv1_1)
    pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1')(conv1_2)

    conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_1')(pool1)
    conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_2')(conv2_1)
    pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2')(conv2_2)

    conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_1')(pool2)
    conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_2')(conv3_1)
    conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_3')(conv3_2)
    pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3')(conv3_3)

    conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_1')(pool3)
    conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_2')(conv4_1)
    conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_3')(conv4_2)
    pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4')(conv4_3)

    conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_1')(pool4)
    conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_2')(conv5_1)
    conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_3')(conv5_2)
    pool5 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same', name='pool5')(conv5_3)

    fc6 = Conv2D(1024, (3, 3), dilation_rate=(6, 6), activation='relu', padding='same', name='fc6')(pool5)

    fc7 = Conv2D(1024, (1, 1), activation='relu', padding='same', name='fc7')(fc6)

    conv6_1 = Conv2D(256, (1, 1), activation='relu', padding='same', name='conv6_1')(fc7)
    conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv6_padding')(conv6_1)
    conv6_2 = Conv2D(512, (3, 3), strides=(2, 2), activation='relu', padding='valid', name='conv6_2')(conv6_1)

    conv7_1 = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv7_1')(conv6_2)
    conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv7_padding')(conv7_1)
    conv7_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', name='conv7_2')(conv7_1)

    conv8_1 = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv8_1')(conv7_2)
    conv8_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', name='conv8_2')(conv8_1)

    conv9_1 = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv9_1')(conv8_2)
    conv9_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', name='conv9_2')(conv9_1)

    # Feed conv4_3 into the L2 normalization layer
    conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3)

    conv4_3_norm_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', name='conv4_3_norm_mbox_conf')(conv4_3_norm)
    fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', name='fc7_mbox_conf')(fc7)
    conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', name='conv6_2_mbox_conf')(conv6_2)
    conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', name='conv7_2_mbox_conf')(conv7_2)
    conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', name='conv8_2_mbox_conf')(conv8_2)
    conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', name='conv9_2_mbox_conf')(conv9_2)
    # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4`
    # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)`
    conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', name='conv4_3_norm_mbox_loc')(conv4_3_norm)
    fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', name='fc7_mbox_loc')(fc7)
    conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', name='conv6_2_mbox_loc')(conv6_2)
    conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', name='conv7_2_mbox_loc')(conv7_2)
    conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', name='conv8_2_mbox_loc')(conv8_2)
    conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', name='conv9_2_mbox_loc')(conv9_2)

    conv4_3_norm_mbox_conf_reshape = Reshape((-1, n_classes), name='conv4_3_norm_mbox_conf_reshape')(
        conv4_3_norm_mbox_conf)
    fc7_mbox_conf_reshape = Reshape((-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf)
    conv6_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf)
    conv7_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf)
    conv8_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf)
    conv9_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf)
    # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)`
    # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
    conv4_3_norm_mbox_loc_reshape = Reshape((-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc)
    fc7_mbox_loc_reshape = Reshape((-1, 4), name='fc7_mbox_loc_reshape')(fc7_mbox_loc)
    conv6_2_mbox_loc_reshape = Reshape((-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc)
    conv7_2_mbox_loc_reshape = Reshape((-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc)
    conv8_2_mbox_loc_reshape = Reshape((-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc)
    conv9_2_mbox_loc_reshape = Reshape((-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc)
    # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)`

    mbox_conf = Concatenate(axis=1, name='mbox_conf')([conv4_3_norm_mbox_conf_reshape,
                                                       fc7_mbox_conf_reshape,
                                                       conv6_2_mbox_conf_reshape,
                                                       conv7_2_mbox_conf_reshape,
                                                       conv8_2_mbox_conf_reshape,
                                                       conv9_2_mbox_conf_reshape])

    mbox_loc = Concatenate(axis=1, name='mbox_loc')([conv4_3_norm_mbox_loc_reshape,
                                                     fc7_mbox_loc_reshape,
                                                     conv6_2_mbox_loc_reshape,
                                                     conv7_2_mbox_loc_reshape,
                                                     conv8_2_mbox_loc_reshape,
                                                     conv9_2_mbox_loc_reshape])

    mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf)
    predictions = Concatenate(axis=2, name='predictions')([mbox_conf_softmax, mbox_loc])
    return predictions

if __name__ == '__main__':

    # pillar_feature_net_model = make_pillar_feature_net()
    # pillar_feature_net_model.summary()

    backbone_model = make_backbone(440, 800, 64)
    backbone_model.summary()


from django.urls import path

from . import views

urlpatterns = [
  path('signup/', views.signup, name='signup'),
  path('login/', views.do_login, name='login'),
  path('logout/', views.do_logout, name='logout'),
  path('profile/<int:primary_key>/', views.profile, name='profile'),
  path('rate/<int:primary_key>/', views.rate_user, name='rating'),
]


# The selenium module
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

import requests, time, re
import ExcelOperations as EO
from bs4 import BeautifulSoup
import DownloadImage as downloader

already_found_url = []

def get_html(url):
    error_count = 0
    error = True
    page_obj = None
    while error and error_count < 3:
        try:
            r = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=30)
            r.encoding = "utf-8"
            page = r.text
            page_obj = BeautifulSoup(page, "lxml")
            error = False
        except:
            error = True
            error_count = error_count + 1
            print("%s URL not accessible " % (url))

    return page_obj

def get_alphabets_unit(value):
    only_alpha = ""
    for char in value:
        if char.isalpha():
            only_alpha += char
    return only_alpha

def convert_to_standard_unit(value):
    non_decimal = re.compile(r'[^\d.-]+')
    unit = get_alphabets_unit(value).strip()
    try:
        if unit.lower() == "kg":
            return value
        elif unit.lower() == "cm":
            return value
        elif unit.lower() == "g":
            value = non_decimal.sub('', value).strip()
            if "." in value:
                value = str(float(value) / 1000) + "kg"
            else:
                value = str(int(value) / 1000) + "kg"
        elif unit.lower() == "mm":
            value = non_decimal.sub('', value).strip()
            if "." in value:
                value = str(float(value) / 10) + "cm"
            else:
                value = str(int(value) / 10) + "cm"
        elif unit.lower() == "m":
            value = non_decimal.sub('', value).strip()
            if "." in value:
                value = str(float(value) * 100) + "cm"
            else:
                value = str(int(value) * 100) + "cm"
    except Exception as e:
        pass
    return value

def increase_price_15_percent(price):
    price = price.replace("£", "").strip()
    price = float(price)
    price = price + (price * 0.15)
    return "£" + str(price)

def scrape_product(driver, main_cat, cat_name, cat_link, sub_cat_name, sub_cat_link, sub_sub_cat_name, sub_sub_cat_link, product_title, product_link, start_url):
    try:
        driver.get(product_link)
        time.sleep(1)
        page_obj = BeautifulSoup(driver.page_source, "lxml")
        if page_obj is not None:
            product_price = page_obj.find("div", attrs={"data-test-id": "product-primary-price"}).text.strip().replace("\n", "").replace(",", "").replace('"', '').strip()
            product_price = increase_price_15_percent(product_price)
            product_details_section = page_obj.find("section", attrs={"class": "_2cf5ecfb _042bbf7f"})
            product_details_section = product_details_section.find("div", attrs={"class": "fef30cae _5e7ce7a9 _461d0ef9 d4281212"})
            #product_details_div = product_details_section.find("div", attrs={"data-test-id": "ProductDescText"})
            product_info_section = product_details_section.findAll("div", attrs={"class": "_7be6e5a0 _66dabd6a e0657c31"})
            product_details_div = product_info_section[0]
            spec_table = product_details_section.find("table", attrs={"class": "f16ac490 eddf1b8e"}).find("tbody").findAll("tr")
            spec_table_html = str(product_info_section[-1])
            special_fields = {}
            for data in spec_table:
                heading = data.find("th").text.replace("\n", "").replace(",", "").replace('"', '').replace("|", "").replace("#", "").strip()
                val = data.find("td").text.replace("\n", "").replace(",", "").replace('"', '').replace("|", "").replace("#", "").strip()
                value = convert_to_standard_unit(val)
                if val != value:
                    spec_table_html = spec_table_html.replace(val, value)
                if "height" in heading.lower():
                    special_fields["height"] = value
                elif "length" in heading.lower():
                    special_fields["length"] = value
                elif "weight" in heading.lower():
                    special_fields["weight"] = value
                elif "width" in heading.lower():
                    special_fields["width"] = value
                elif "thickness" in heading.lower():
                    special_fields["thickness"] = value
                elif "diameter" in heading.lower():
                    special_fields["diameter"] = value
                elif "depth" in heading.lower():
                    special_fields["depth"] = value
                elif "mesh size" in heading.lower():
                    special_fields["mesh size"] = value
                elif "product code" in heading.lower():
                    special_fields["code"] = value

            spec_table_html = spec_table_html.replace(",", "")

            try:
                #images_div = page_obj.findAll("div", attrs={"class": "slick-list"})[-1]
                images_div = page_obj.findAll("div", attrs={"class": "slick-list"})[0]
                images_list = images_div.findAll("img")
                list_images = []
                for img in images_list:
                    list_images.append(img.attrs["src"])
                list_images = list(set(list_images))
                while len(list_images) < 5:
                    list_images.append("")
            except Exception as e:
                list_images = ["", "", "", "", ""]

            list_images_names = []
            for x in range(0, 5):
                if list_images[x] is not "":
                    file_name = special_fields["code"] + "_img_" + str(x)
                    file_name = downloader.download_image(list_images[x], file_name)
                    list_images_names.append(file_name)
                else:
                    list_images_names.append("")

            try:
                try:
                    product_details_div.find("button").decompose()
                except:
                    pass
                product_info = str(product_details_div).replace(",", "")
                product_feat = ""
            except:
                product_info = ""
                product_feat = ""
            print("=> Scraped product = " + product_title)
            EO.write_excel_file(main_cat, cat_name, sub_cat_name, sub_sub_cat_name, product_title, product_price, list_images_names, spec_table_html, special_fields, product_info)
    except Exception as e:
        print(e)

def scrape_products_list(CHROME_PATH, start_url, base_url, main_cat, cat_name, cat_link, sub_cat_name, sub_cat_link, sub_sub_cat_name, sub_sub_cat_link, products_page_link):
    link_to_go = products_page_link
    if link_to_go in already_found_url:
        print("Already scraped => " + link_to_go)
    else:
        WINDOW_SIZE = "1920,1080"
        CHROME_DRIVER_PATH = "ChromDriver/chromedriver"
        driver = None
        chrome_options = Options()
        # chrome_options.add_argument("--headless")
        chrome_options.add_argument("--window-size=%s" % WINDOW_SIZE)
        chrome_options.binary_location = CHROME_PATH
        chrome_options.add_argument('disable-infobars')
        chrome_options.add_argument("disable-notifications")
        driver = webdriver.Chrome(executable_path=CHROME_DRIVER_PATH, options=chrome_options)
        try:
            driver.get(link_to_go)
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(2)
            execute = True
            counter = 0
            loading = True
            try:
                load_more = driver.find_element_by_xpath('//*[@id="content"]/div/div[1]/div/div/div[2]/div[3]/div/div[3]/div[2]/main/div/a[2]')
            except:
                loading = False
                load_more = None
            if loading:
                while execute or counter < 3:
                    try:
                        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                        load_more.click()
                        time.sleep(1)
                        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                        time.sleep(3)
                        counter = 0
                        execute = True
                    except:
                        execute = False
                        counter = counter + 1
            page_obj = BeautifulSoup(driver.page_source, 'lxml')
            products_div = page_obj.findAll("ul", attrs={"class": "_40158784 _6b5bb6a7 _190cafcd"})[0]
            products_list = products_div.findAll("li", attrs={"class": "b9bdc658"})
            for product in products_list:
                try:
                    a_tag = product.find("a", attrs={"data-test-id": "product-panel-main-section"})
                    product_title = a_tag.find("p", attrs={"data-test-id": "productTitle"}).text.replace("\n", "").replace("\r", "").replace(",", "").replace('"', '').strip()
                    product_link = base_url + a_tag.attrs["href"]
                    scrape_product(driver, main_cat, cat_name, cat_link, sub_cat_name, sub_cat_link, sub_sub_cat_name, sub_sub_cat_link, product_title, product_link, start_url)
                except Exception as e:
                    pass
        except Exception as e:
            print(e)
        finally:
            driver.close()
            already_found_url.append(link_to_go)

def scrape_sub_categories(main_cat, cat_name, cat_link, sub_cat_name, sub_cat_link, CHROME_PATH, start_url, base_url):
    page_obj = get_html(sub_cat_link)
    if page_obj is not None:
        try:
            #sub_sub_categories = page_obj.findAll("div", attrs={"class": "sc-1ulaoyw-0 fqZatT"})[0]
            #sub_sub_categories = sub_sub_categories.find("div", attrs={"data-test-id": "grid-sections"})
            #sub_sub_categories_list = sub_sub_categories.findAll("a", attrs={"class": "sc-cnb8ll-0 hHvddy"})

            sub_sub_categories_list = page_obj.findAll("ul", attrs={"id": "side-navigation-menu-1"})[0].findAll("li")

            for sub_sub_cat in sub_sub_categories_list:
                try:
                    a_tag = sub_sub_cat.find("a")
                    sub_sub_cat_name = a_tag.text.replace("\n", "").replace("\r", "").replace(",", "").replace('"', '').strip()
                    #sub_sub_cat_name = sub_sub_cat.find("div", attrs={"data-test-id": "category-tile-content"}).find("h3").text.replace("\n", "").replace("\r", "").replace(",", "").replace('"', '').strip()
                    #sub_sub_cat_link = base_url + sub_sub_cat.attrs["href"]
                    sub_sub_cat_link = base_url + a_tag.attrs["href"]
                    print("Sub Sub Category = " + sub_sub_cat_name)
                    scrape_products_list(CHROME_PATH, start_url, base_url, main_cat, cat_name, cat_link, sub_cat_name, sub_cat_link, sub_sub_cat_name, sub_sub_cat_link, sub_sub_cat_link)
                except Exception as e:
                    pass
        except:
            print("Sub Sub Category = ")
            scrape_products_list(CHROME_PATH, start_url, base_url, main_cat, cat_name, cat_link, sub_cat_name, sub_cat_link, "", "", sub_cat_link)

def scrape_categories(main_cat, cat_name, cat_url, CHROME_PATH, start_url, base_url):
    page_obj = get_html(cat_url)
    if page_obj is not None:
        try:
            #sub_categories = page_obj.findAll("div", attrs={"class": "sc-1ulaoyw-0 fqZatT"})[0]
            #sub_categories = sub_categories.find("div", attrs={"data-test-id": "grid-sections"})
            #sub_categories_list = sub_categories.findAll("a", attrs={"class": "sc-cnb8ll-0 hHvddy"})

            ####
            sub_categories_list = page_obj.findAll("ul", attrs={"id": "side-navigation-menu-1"})[0].findAll("li")
            for sub_cat in sub_categories_list:
                try:
                    a_tag = sub_cat.find("a")
                    # sub_cat_name = sub_cat.find("div", attrs={"data-test-id": "category-tile-content"}).find("h3").text.replace("\n", "").replace("\r", "").replace(",", "").replace('"', '').strip()
                    sub_cat_name = a_tag.text.replace("\n", "").replace("\r", "").replace(",", "").replace('"',
                                                                                '').strip()
                    # sub_cat_link = base_url + sub_cat.attrs["href"]
                    sub_cat_link = base_url + a_tag.attrs["href"]
                    print("Sub Category = " + sub_cat_name)
                    scrape_sub_categories(main_cat, cat_name, cat_url, sub_cat_name, sub_cat_link, CHROME_PATH,
                                          start_url, base_url)
                except Exception as e:
                    pass
        except:
            print("Sub Sub Category = ")
            scrape_products_list(CHROME_PATH, start_url, base_url, main_cat, cat_name, cat_url, "",
                                 "", "", "", cat_url)


from django.core.management.base import BaseCommand, CommandError
import logging
logger = logging.getLogger('labsmanager')

from django.db.models import Q
from fund.models import AmountHistory

class Command(BaseCommand):
    help = 'remove duplicate in AmountHistory'

    # def add_arguments(self, parser):
    #     # Named (optional) arguments
    #     parser.add_argument(
    #         '--force',
    #         action='store_true',
    #         help='Force Update clean',
    #     )

    def handle(self, *args, **options):
        logger.debug("COMMAND remove_duplicate CALLED")
        self.stdout.write("COMMAND remove_duplicate", ending='\n')
        unik=AmountHistory.objects.values('content_type','object_id', 'value_date', 'amount').distinct()
        logger.debug(f" {AmountHistory.objects.count()} object found and {unik.count()} unic entry found")

        for ah in unik:
            sel=AmountHistory.objects.filter(content_type=ah['content_type'], object_id=ah['object_id'], value_date=ah['value_date']).values_list('pk', flat=True)[1:]
            if sel.count()>0:
                AmountHistory.objects.filter(pk__in=sel).delete()
            #AmountHistory.objects.filter(pk__in=Email.objects.filter(email=email).values_list('id', flat=True)[1:]).delete()


import flask
from flask import request, url_for, redirect
from flask.ext.jwt import current_user

from segue.core import db, config
from segue.factory import Factory
from segue.decorators import jsoned, jwt_only
from segue.responses import Response

from jwt import Signer
from models import Account, ResetPassword
from services import AccountService
from errors import InvalidLogin, EmailAlreadyInUse, NotAuthorized
from responses import AccountResponse, EmployeesListResponse

class AccountController(object):
    def __init__(self, service=None):
        self.service = service or AccountService()
        self.current_user = current_user

    @jwt_only
    @jsoned
    def get_one(self, account_id):
        result = self.service.get_one(account_id, by=self.current_user) or flask.abort(404)
        return Response(result, AccountResponse).create(), 200

    @jwt_only
    @jsoned
    def modify(self, account_id):
        data = request.get_json()
        result = self.service.modify(account_id, data, by=self.current_user) or flask.abort(404)
        return result, 200

    @jsoned
    def schema(self, name):
        #TODO: REMOVE
        return {}

    @jsoned
    def create(self):
        data = request.get_json()
        return self.service.create(data), 201

    @jsoned
    def login(self):
        data = request.get_json()
        return self.service.login(**data), 200

    @jwt_only
    @jsoned
    def set_certificate_name(self, account_id):
        new_name = request.get_json().get('name',None)
        if not new_name: flask.abort(400)
        return self.service.set_certificate_name(account_id, new_name, by=self.current_user), 200

    @jwt_only
    @jsoned
    def list_employees(self, account_id):
        #TODO: FIX
        from segue.purchase.promocode.models import PromoCodePayment, PromoCode
        from segue.purchase.models import Purchase

        if current_user.id != account_id:
            return [], 404

        result = db.session.query(PromoCodePayment)\
            .join(Purchase)\
            .join(Account)\
            .filter(PromoCodePayment.promocode_id.in_(
                db.session.query(PromoCode.id).filter(PromoCode.creator_id == account_id))).all()

        if result:
            return Response(result, EmployeesListResponse).create(), 200
        return [], 200

    def list_proposals(self, account_id):
        query_string = "?owner_id={}".format(account_id)
        return redirect(url_for('proposals.list') + query_string)

    def list_purchases(self, account_id):
        query_string = "?customer_id={}".format(account_id)
        return redirect(url_for('purchases.list') + query_string)

    def list_promocodes(self, account_id):
        query_string = "?creator_id={}".format(account_id)
        return redirect(url_for('promocodes.list') + query_string)

    def get_caravan(self, account_id):
        query_string = "?owner_id={}".format(account_id)
        return redirect(url_for('caravans.get_one') + query_string)

    def ask_reset(self):
        data = request.get_json()
        self.service.ask_reset(data.get('email'))
        return '', 200

    def get_reset(self, account_id, hash_code):
        reset = self.service.get_reset(account_id, hash_code)
        path = '/#/account/{}/reset/{}'.format(account_id, hash_code)
        return flask.redirect(config.FRONTEND_URL + path)

    @jsoned
    def perform_reset(self, account_id, hash_code):
        data = request.get_json()
        result = self.service.perform_reset(account_id, hash_code, data['password'])
        return dict(email=result.account.email), 200


import asyncio
import nest_asyncio
nest_asyncio.apply()
import ccxt.async_support as ccxta
import time
import numpy as np
import pandas as pd

async def async_client(exchange_id, run_time: int, symbol: str):
    orderbook = None
    exchange = getattr(ccxta, exchange_id)()
    time_1 = time.time()
    time_f = 0
    ob = []
    while time_f <= run_time:
        try:
            await exchange.load_markets()
            market = exchange.market(symbol)
            orderbook = await exchange.fetch_order_book(market["symbol"])
            ohlc = await exchange.fetch_ohlcv(symbol=market['symbol'], timeframe='1m')
            datetime = exchange.iso8601(exchange.milliseconds())
            # Unpack values
            bid = orderbook['bids'][0][0] if len (orderbook['bids']) > 0 else None # Get the highest bid
            ask = orderbook['asks'][0][0] if len (orderbook['asks']) > 0 else None # Get the lowest ask

            bid_volume = sum([bid[1] for bid in orderbook['bids']]) if len (orderbook['bids']) > 0 else None # Get the total bid volume
            ask_volume = sum([ask[1] for ask in orderbook['asks']]) if len (orderbook['asks']) > 0 else None # Get the total ask volume
            total_volume = bid_volume + ask_volume

            mid_price = (bid+ask)/2
            spread = np.round(ask - bid, 4)
            # Final data format for the results
            ob.append(
                {
                    "exchange": exchange_id,
                    "datetime": datetime,
                    "level": len(orderbook['asks']),
                    "bid_volume": bid_volume,
                    "ask_volume": ask_volume,
                    "total_volume": total_volume,
                    "mid_price": mid_price,
                    "spread": spread,
                    "close_price": ohlc[0][4],
                    "Symbol": symbol, #placeholder

                    #"orderbook": {
                     #   "ask_size": ask_size.tolist(),
                      #  "ask": ask_price.tolist(),
                       # "bid": bid_price.tolist(),
                        #"bid_size": bid_size.tolist(),
                        #"spread": spread.tolist(),
                    #},
                }
            )
            # End time
            time_2 = time.time()
            time_f = round(time_2 - time_1, 4)
        except Exception as e:
            time_2 = time.time()
            time_f = round(time_2 - time_1, 4)
            print(type(e).__name__, str(e))
    await exchange.close()
    return ob

async def multi_orderbooks(exchanges, run_time: int, symbol: str):
    input_coroutines = [
        async_client(exchange, run_time, symbol) for exchange in exchanges
    ]
    orderbooks = await asyncio.gather(*input_coroutines, return_exceptions=True)
    return orderbooks

#if __name__ == "__main__":
#    exchanges = ["kucoin", "bittrex", "bitfinex", "poloniex", "huobipro"]
#    run_time = 10  # seconds
#    symbol = "ETH/BTC"
#    data = asyncio.run(multi_orderbooks(exchanges, run_time=run_time, symbol=symbol))
#    data = [item for sublist in data for item in sublist]
#    data = pd.DataFrame(data)
#
#    print(data.head())

#async def orderbooks_df(exchanges:list, run_time:int, symbol:str):
##     exchanges = ["kucoin", "bittrex", "bitfinex", "poloniex", "huobipro"]
##     run_time = 10  # seconds
##     symbol = "ETH/BTC"
#
#     data = asyncio.run(multi_orderbooks(exchanges, run_time=run_time, symbol=symbol))
#     data = [item for sublist in data for item in sublist]
#     data = pd.DataFrame(data)
#     data.set_index('exchange', inplace=True)
#
#     return(data)

async def orderbooks_df(exchanges:list, symbol:str):
    exchanges = ["kucoin", "bittrex", "bitfinex"]
    run_time = 10  # seconds
    #symbol = "BTC/USDT"

    #symbol = "ETH/EUR"

    data = asyncio.run(multi_orderbooks(exchanges, run_time=run_time, symbol=symbol))
    data = [item for sublist in data for item in sublist]
    data = pd.DataFrame(data)

    return(data)


from bs4 import BeautifulSoup
import requests
import pprint
from scrapers.base_scraper import BaseScraper

class CNAScraper(BaseScraper):
	@classmethod
	def get_text(cls, url):
	    res = requests.get(url, allow_redirects=False)
	    soup = BeautifulSoup(res.text, 'html.parser')
	    print("Prasing article..")
	    print(res.status_code)
	    return ''.join([_.get_text() for _ in soup.find_all('div', class_='c-rte--article')])

from sys import argv
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--url', type=str, help='url to scrape')

if __name__ == '__main__':
	args = parser.parse_args(argv[1:])
	url = args.url
	print(CNAScraper.get_text(url))


#!/usr/bin/python3
"""Script that fetches https://alx-intranet.hbtn.io/status"""
import requests
import sys

if __name__ == "__main__":
    res_msg = requests.get(sys.argv[1])
    response = res_msg.status_code
    if response >= 400:
        print(f"Error code: {response}")
    else:
        print(res_msg.text)


import pygame_toolbox.graphics as ptg
import pygame_toolbox.graphics.widgets as ptgw
import pygame, sys, random

def close():
    pygame.quit()
    sys.exit()

class Screen_1(ptg.Menu):
    def __init__(self):
        self.header = ['Screen 1']
        self.buttons = [['Screen 2',lambda:2],['Quit',close]]
        ptg.Menu.__init__(self,(800,600),(150,69,69),self.header,self.buttons)

        self.widgetlist += [ptgw.Checkbox("option a",(200,200),(17,17),True,self.image,'c')]

class Screen_2(ptg.Menu):
    def __init__(self,widget_info):
        header = ['Screen 2','%s was %s' % (widget_info[0][0],bool(widget_info[0][1]))]
        buttons = [['Screen 1',lambda:1],['Quit',close]]
        ptg.Menu.__init__(self,(800,600),(150,69,150),header,buttons)

class Main(object):
    def __init__(self):
        self.progress = 1
        self.clock = pygame.time.Clock()

    def update(self,screen):
        while True:
            if self.progress == 1:
                self.progress = Screen_1().update(screen,self.clock)
                self.widget_info = self.progress[1]
                self.progress = self.progress[0]
            elif self.progress == 2:
                self.progress = Screen_2(self.widget_info).update(screen,self.clock)

if __name__ == '__main__':
    pygame.init()
    screen = pygame.display.set_mode((800,600))
    Main().update(screen)

General Python/widgets develop/widgets checkbox.py


"""
Plot adaptive experimental results
"""
import pickle
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
import numpy as np
import pandas as pd
import os
import glob
import math
import scipy.interpolate as interp

font = {'family' : 'DejaVu Sans',
#        'weight' : 'bold',
        'size'   : 20}
plt.rc('font', **font)
colors=['#6aa2fc', '#fc8181', '#a5ff9e', '#3639ff', '#ff3636', '#13ba00', '#ff62f3']

input_size = {'mimic3': 178, 'gina': 242, 'activity': 140, 'phishing': 10, 'sylva': 54}
num_samples = {'mimic3': 14681, 'gina': 3468*0.8, 'activity': 7352, 'phishing': 11055*0.8, 'sylva': 14395*0.8}
embedding_size = 16
post_training = 100

def all_seeds(prefix, selector, epochs, dataset, num_clients, batch_size, lambdc, lambds, emb_sizes, red):
    """
    Averages the results of all seeds
    """
    files = glob.glob(f'{prefix}_featureSGD_L21_BS{batch_size}_NC{num_clients}_epochs{epochs}_selector{selector}_dataset{dataset}_lambdc{lambdc}_lambds{lambds}_red{red}_seed*.pkl')
    files_base = glob.glob(f'/hdd/home/feature_results/{prefix}_feature_L21_BS{batch_size}_NC{num_clients}_epochs0_selectorbaseline_dataset{dataset}_lambdc0.0_lambds0.0_red0.0_seed*.pkl')
    pickles = []
    pickles_bas = []
    min_len = 10000
    for f,f_bas in zip(files, files_base):
        # Get accuracy pickles
        seed = f.split('_')[-1][4:-4]
        pkl = pickle.load(open(f, 'rb'))
        pkl_bas = pickle.load(open(f_bas, 'rb'))
        pkl = np.array(np.array(pkl)[:,0,1], dtype=np.float32)
        pkl_bas = np.array(np.array(pkl_bas)[:,0,1], dtype=np.float32)*100
        pickles_bas.append(pkl_bas)
        if selector == 'sparse':
            pkl = np.concatenate((pkl[0:epochs+1], pkl[150+epochs:]))
        elif selector == 'emb_sparse':
            pkl = np.concatenate((pkl[0:epochs+1], pkl[300+epochs:]))
        pickles.append(pkl*100)

        if selector == 'emb_sparse':
            # Get percentage of embedding components kept during post feature selection
            filename = f'out_feature_emb_sparse_{dataset}_lambdc{lambdc}_lambds{lambds}_red{red}_seed{seed}.txt'
            out = open(filename, 'r')
            substring = 'embedding components'
            lines = out.readlines()
            embs_removed = [line for line in lines if substring in line]
            out.close()
            emb_sizes.append((64-float(embs_removed[0].split(' ')[1]))/64)

    target = np.max(pickles_bas)
    pickles = np.array(pickles)

    if selector == 'centralized':
        # Get percentage of spurious features removed for all iterations
        files_imp = glob.glob(f'importances_all_featureSGD_L21_BS{batch_size}_NC{num_clients}_epochs{epochs}_selector{selector}_dataset{dataset}_lambdc{lambdc}_lambds{lambds}_red{red}_seed*.pkl')
        red_pers = []
        for i, f in enumerate(files_imp):
            importances = pickle.load(open(f, 'rb'))[:-1]
            red_per = []
            for importance in importances:
                correct = 0
                total = 0
                for client in range(num_clients):
                    start = input_size[dataset]
                    end = int(input_size[dataset]*(1+red))
                    red_new = np.arange(start,end)
                    total += len(red_new)
                    correct += len(list(set(red_new).intersection(importance[client])))
                red_per.append(correct/total)
            while len(red_per) < len(pickles[i]):
                red_per.append(red_per[-1])
            red_pers.append(red_per)
    else:
        # Get percentage of spurious features removed total
        files = glob.glob(f'importances_featureSGD_L21_BS{batch_size}_NC{num_clients}_epochs{epochs}_selector{selector}_dataset{dataset}_lambdc{lambdc}_lambds{lambds}_red{red}_seed*.pkl')
        red_pers = []
        for i, f in enumerate(files):
            importance = pickle.load(open(f, 'rb'))
            importance = tf.concat(importance, 0)
            zero = np.where(importance <= 0)[0]
            start = 0
            end = 0
            redundant_features = []
            for client in range(num_clients):
                start += input_size[dataset]
                end += int(input_size[dataset]*(1+red))
                redundancy = np.arange(start,end)
                redundant_features = np.concatenate((redundant_features, redundancy))
                start = end
            correct = list(set(redundant_features).intersection(zero))
            red_pers.append(len(correct)/len(redundant_features))

    # Get size of embeddings
    if selector == 'emb_sparse':
        embedding_per = np.average(emb_sizes)
    else:
        embedding_per = 1.0

    # Calculate the bits to reach targets
    bits_list = []
    for i in range(len(pickles)):
        if i >= len(red_pers):
            continue
        min_bits_per_epoch = min(emb_sizes)*embedding_size*num_samples[dataset]*num_clients*32
        bits_per_epoch_pre = embedding_size*num_samples[dataset]*num_clients*32
        bits_per_epoch_post = embedding_per*embedding_size*num_samples[dataset]*num_clients*32

        # Cost during pre-training
        final_epoch = 0
        bits_so_far = 0
        if epochs > 0:
            while final_epoch <= epochs:
                bits_so_far += bits_per_epoch_pre
                final_epoch += 1

        # Cost after pre-training
        cur_red = red_pers[i]
        while final_epoch < len(pickles[i]):
            if selector == 'centralized':
                cur_red = red_pers[i][final_epoch]
            if pickles[i][final_epoch] > target*0.9 and cur_red > 0.8:
                break
            bits_so_far += bits_per_epoch_post
            final_epoch += 1
        if final_epoch < len(pickles[i]) or (pickles[i][-1] > target*0.9 and cur_red > 0.8):
            bits_list.append(bits_so_far)

    # Average and print results
    avg_bits = np.average(bits_list)
    std_bits = np.std(bits_list)
    print(f'{avg_bits/(8*(2**20)):.2f} $\pm$ {std_bits/(8*(2**20)):.2f}')

    return emb_sizes

types = ['accs_test']
datasets = ['mimic3', 'gina', 'sylva', 'activity', 'phishing']
batches = [1000,100,1000,1000,1000]
num_clients = [4,4,4,4,3]

centralized_lambds = [0.25, 0.1, 1.0, 0.1, 0.25]
sparse_lambds =      [40.0, 0.025, 0.1, 0.1, 0.1]
emb_sparse_lambdsc = [40.0, 0.025, 0.1, 0.1, 0.1]
emb_sparse_lambdss = [0.1, 0.005, 0.01, 0.25, 0.005]
epochs_emb = [1,1,1,2,1]
epochs_sparse = [1,1,1,3,1]

for t in types:
    for dataset, epoch_emb, epoch_sparse, batch, clients, cent_lambd, sparse_lambd, emb_lambdc, emb_lambds in zip(datasets, epochs_emb, epochs_sparse, batches, num_clients, centralized_lambds, sparse_lambds, emb_sparse_lambdsc, emb_sparse_lambdss):
        # Print communication cost to reach target accuracy and spurious features removed
        emb_sizes = all_seeds(t,"emb_sparse", epoch_emb, dataset, clients, batch, emb_lambdc, emb_lambds, [], 0.5)
        _ = all_seeds(t,"centralized", 0, dataset, clients, batch, cent_lambd, 0.0, emb_sizes, 0.5)
        _ = all_seeds(t,"sparse", epoch_sparse, dataset, clients, batch, sparse_lambd, 0.0, emb_sizes, 0.5)

        print('\midrule')
        print()


from mathutils import Vector
from array import array
import bpy
from .Branch import Branch
from .Leaf import Leaf, l
from bpy.props import BoolProperty
from math import sqrt
import math
from functools import partial
from random import randint, random, uniform
from copy import copy, deepcopy
#from .scaoperator import bpy

#https://blenderartists.org/t/detecting-if-a-point-is-inside-a-mesh-2-5-api/485866/4
def point_inside(point,obj):
    axes = [ Vector((1,0,0)), Vector((0,1,0)), Vector((0,0,1)) ]
    outside = False
    mat = obj.matrix_world.copy()
    mat.invert()
    for axis in axes:
        orig = mat*point
        count = 0
        while True:
            a,location,normal,index = obj.ray_cast(orig,orig+axis*1.84467e+19)
            if index == -1: break
            count += 1
            orig = location + axis*0.00001
        if count%2 == 0:
            outside = True
            break
    return not outside

def closest(pos, count, n, x, y, z):
    d2 = 1e30
    for i in range(n):
        if count[i] > 1:
            continue
        dx, dy, dz = x-pos[i*3], y-pos[i*3+1], z-pos[i*3+2]
        d = dx*dx + dy*dy + dz*dz
        if d < d2:
            d2 = d
            ci = i
            v = dx, dy, dz
    return d2, ci, v

def getDistance(p0, p1):
        return sqrt((p0[0] - p1[0])**2 + (p0[1] - p1[1])**2 + (p0[2] - p1[2])**2)

# importiert von https://blender.stackexchange.com/questions/5898/how-can-i-create-a-cylinder-linking-two-points-with-python

class Tree:
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'TOOLS'
    bl_label = 'Tools Tab Label'
    bl_context = 'objectmode'
    bl_category = 'SC'

    def __init__(self):
        # Vektor operationen

        def div(x,y):
            return(x[0]/y, x[1]/y, x[2]/y)

        def sub(x,y):
            return(x[0]-y[0], x[1]-y[1], x[2]-y[2])

        def normieren(x,y):
            länge = sqrt(x[0]**2 + x[1]**2 + x[2]**2)
            return((x[0]/länge + x[1]/länge+ x[2]/länge)*y)

        def length_squared(u):
            return sum([a ** 2 for a in u])

        def add(u, v):
            return [a + b for a, b in zip(u, v)]

        def norm(u):
            return setlength(u, 0.5)

        def scale_by_scalar(u, scalar):
            return [a * scalar for a in u]

        def length(u):
            return math.sqrt(length_squared(u))

        def setlength(u, l):
            return scale_by_scalar(u, l / length(u))

        branches = []
        leaves = []
        verts = ()
        closeEnough = 0 # False

        #leaves generieren
        for i in range(0,bpy.context.scene.leaf):
            leaves.append(Leaf((uniform(-bpy.context.scene.xyvalue,bpy.context.scene.xyvalue),uniform(-bpy.context.scene.xyvalue,bpy.context.scene.xyvalue),uniform(bpy.context.scene.zvaluebot,bpy.context.scene.zvaluetop)),0))
            #bpy.ops.mesh.primitive_uv_sphere_add(size=0.1, location=leaves[i].pos)
            i =+ 1

        #Prüfen für Objekt
        if bpy.context.scene.Growth_limitation == True:
            for i in reversed(range(len(leaves))):
                myobj = bpy.context.object
                #mypoint = mathutils.Vector(leaves[i].pos)
                if (point_inside(Vector(leaves[i].pos),myobj)) == False:
                    del leaves[i]

        #Stamm machen
        root = Branch(None,(0,0,0),(0,0,0.5))
        branches.append(root)
        current = root

        max_dist = bpy.context.scene.maxdist#3
        min_dist = bpy.context.scene.mindist#1
        while closeEnough == 0: # False
            for i in range (len(leaves)):
                g = 0
                g = getDistance(current.pos, leaves[i].pos)
                #print(g)
                if (g < max_dist):
                    #print('schlaufe 1')
                    closeEnough = 1
                if (closeEnough == 0):
                    #print('         schlaufe 2          ')
                    nextPos = tuple(add(branches[i].pos,branches[i].dir))
                    branch = Branch(i,nextPos,branches[i].dir)
                    current = branch
                    branches.append(current)

        #Der eigentliche Algorithmus
        def grow(self):
            self.record = 100000
            for i in range(len(leaves)):
                leaf = leaves[i]
                closestBranch = None
                self.closestDir = None

                #Den nächsten Ast bestimmen
                for j in range(len(branches)):
                    #print(j)
                    branch = branches[j]
                    dir = sub(leaf.pos ,branch.pos)
                    d = length(dir)
                    #print(d)
                    if (d < min_dist):
                        leaf.reached()
                        break
                    #elif(d > max_dist):
                    elif (closestBranch == None or d < self.record):
                        closestBranch = branch
                        self.record = d

                #Dessen Richtung bestimmen
                if (closestBranch is not None):
                    newDir = tuple(sub(leaf.pos, closestBranch.pos))
                    newDirnormed = norm(newDir)
                    #print(newDir)
                    closestBranch.dir = tuple(add(closestBranch.dir, newDirnormed))
                    closestBranch.count += 1

            #alle Blätter entfernen die erreicht wurden
            for i in reversed(range(len(leaves))):
                if (leaves[i].erreicht == 1):
                    del(leaves[i])

            branchcopy = deepcopy(branches)
            #print(branches[0].dir)
            for i in reversed(range(len(branches))):
                branch = branches[i]
                #print(branch.count)
                if (branch.count > 0):
                    branch.dir = tuple(div(branch.dir,branch.count))
                    newPos = tuple(add(branch.pos,branch.dir))
                    newB = Branch(i, newPos, branch.dir)
                    branches.append(newB)
                    branch.reset()

        for i in range(70):
            grow(self)

        # this assumes all vertex positions are unique (= do not overlap)
        # create a mapping from vertex position to vertex index
        verts = {b.pos:i for i,b in enumerate(branches)}
        # create vertex pairs as edge by locating vert index based on coordinates
        edges = [ (verts[b.pos],verts[branches[b.parent].pos])for b in branches if b.parent is not None]
        # replace vertex mapping by simple list of coordinates
        verts = [b.pos for b in branches]

        mesh = bpy.data.meshes.new("tree_mesh")
        mesh.from_pydata(verts, edges, faces=[])
        mesh.update()

        obj = bpy.data.objects.new("Tree", mesh)

        scene = bpy.context.scene
        scene.objects.link(obj)
        #remove doubles muss man noch machen

tree= Tree()


from numpy import *
import matplotlib.pyplot as plt
from pandas import *
from math import *
from numpy.linalg import inv

def findRMSE(n,err):
	return ((1/n)*dot(err.transpose(),err))**0.5

def gradientDescent(x,y,theta,iterations,learning_rate):
	rmse=zeros(iterations)
	n=len(x)
	y=y.reshape(y.shape[0],1)
	for i in range(iterations):
		pred=dot(x,theta)
		err=pred-y
		rmse[i]=findRMSE(n,err)
		theta=theta-(learning_rate*(1/n)*dot(x.transpose(),err))
	return theta,rmse

def gradientDescentWithL1(x,y,theta,iterations,learning_rate,alpha):
	rmse=zeros(iterations)
	n=len(x)
	y=y.reshape(y.shape[0],1)
	for i in range(iterations):
		pred=dot(x,theta)
		err=pred-y
		rmse[i]=findRMSE(n,err)
		theta=theta-(learning_rate*((1/n)*dot(x.transpose(),err)))-((learning_rate*alpha*sign(theta))/n)
	return theta,rmse

def gradientDescentWithL2(x,y,theta,iterations,learning_rate,alpha):
	rmse=zeros(iterations)
	n=len(x)
	y=y.reshape(y.shape[0],1)
	for i in range(iterations):
		pred=dot(x,theta)
		err=pred-y
		rmse[i]=findRMSE(n,err)
		theta=theta*(1-(alpha/n))-((learning_rate/n)*(dot(x.transpose(),err)))
	return theta,rmse

def run():
	data=read_csv('data.csv')
	y=data.iloc[:,1].values
	y.reshape(y.shape[0],1)
	learning_rate=0.0001
	alpha=5
	iterations=100000
	x=data.iloc[:,0:1]
	xCopy=copy(x)
	a=ones((x.shape[0],1),dtype='double')
	x=append(a,x,axis=1)
	theta=zeros((len(x[0]),1))
	theta,cost=gradientDescent(x,y,theta,iterations,learning_rate)
	print("Gradient descent without regularisation:")
	print("Slope: ",theta[1][0])
	print("Intercept: ",theta[0][0])

	y1=zeros((len(xCopy),1))
	count=0
	for i in range(len(xCopy)):
		res=theta[1]*xCopy[i]+theta[0]
		y1[i]=res
	err=y1-y
	print("RMSE: ",findRMSE(len(y1),err)[0][0])
	print()
	plt.figure("Without regularisation")
	plt.title("Without regularisation")
	plt.scatter(xCopy,y)
	plt.plot(xCopy,y1,'r')
	plt.xlabel("Brain Weight")
	plt.ylabel("Body Weight")
	plt.grid()

	#L1
	theta2=zeros((len(x[0]),1))
	theta2,cost2=gradientDescentWithL1(x,y,theta2,iterations,learning_rate,alpha)
	print("Gradient descent with L1 regularisation:")
	print("Slope: ",theta2[1][0])
	print("Intercept: ",theta2[0][0])
	y3=zeros((len(xCopy),1))
	for i in range(len(xCopy)):
		res=theta2[1]*xCopy[i]+theta2[0]
		y3[i]=res
	err=y3-y
	print("RMSE: ",findRMSE(len(y3),err)[0][0])
	print()
	plt.figure("L1 regularisation")
	plt.title("L1 regularisation")
	plt.scatter(xCopy,y)
	plt.plot(xCopy,y3,'r')
	plt.xlabel("Brain Weight")
	plt.ylabel("Body Weight")
	plt.grid()

	#L2
	theta1=zeros((len(x[0]),1))
	theta1,cost1=gradientDescentWithL2(x,y,theta1,iterations,learning_rate,alpha)
	print("Gradient descent with L2 regularisation:")
	print("Slope: ",theta1[1][0])
	print("Intercept: ",theta1[0][0])
	y2=zeros((len(xCopy),1))
	for i in range(len(xCopy)):
		res=theta1[1]*xCopy[i]+theta1[0]
		y2[i]=res
	err=y2-y
	print("RMSE: ",findRMSE(len(y2),err)[0][0])
	print()
	plt.figure("L2 regularisation")
	plt.title("L2 regularisation")
	plt.scatter(xCopy,y)
	plt.plot(xCopy,y2,'r')
	plt.xlabel("Brain Weight")
	plt.ylabel("Body Weight")
	plt.grid()

	plt.show()

if __name__=='__main__':
	run()


from django.shortcuts import render
from .forms import StudentForm
from .models import Studentmodel
from django.http import JsonResponse
# Create your views here.

def home(request):
    # if request.method == 'POST':
    form = StudentForm(request.POST)
    stud = Studentmodel.objects.all()
    return render (request,'enroll/home.html',{'form':form,'stud':stud})

def save_data(request):
    if request.method == 'POST':
        form = StudentForm(request.POST)
        if form.is_valid():
            sid = request.POST.get('stuid')
            name = request.POST['name']
            email = request.POST['email']
            desc = request.POST['desc']
            if(sid == ''):
                usr = Studentmodel(name=name , email = email , desc= desc)
            else:
                usr = Studentmodel(id =sid,name=name , email = email , desc= desc)
            usr.save()
            stud = Studentmodel.objects.values()
            # print(stud)
            student_data = list(stud)
            print(student_data)
            return JsonResponse({'status':'save','student_data':student_data})
        else:
            return JsonResponse({'status':0})

def delete_data(request):
    if request.method == 'POST':
        id = request.POST.get('sid')
        print(id)
        pi = Studentmodel.objects.get(pk=id)
        pi.delete()
        return JsonResponse({'status':1})
    else:
        return JsonResponse({'status':0})

def edit_data(request):
    if request.method == 'POST':
        id = request.POST.get('sid')
        print(id)
        student = Studentmodel.objects.get(pk=id)
        student_data = {'id':student.id,'name':student.name,'email':student.email,'desc':student.desc}
        return JsonResponse(student_data)


from flask import Flask, jsonify, request
from flask_cors import CORS
from llama_cpp import Llama
import re

llm = Llama(model_path="./vicuna-7b-1.1.ggmlv3.q4_K_S.bin")
# llm = Llama(model_path="./ggjtv1-model-q4_0.bin")
# llm = Llama(model_path="./Wizard-Vicuna-13B-Uncensored.ggmlv3.q4_K_S.bin")

app = Flask(__name__)
CORS(app)

def extract_last_answer(dialogs):
    answers = re.findall(r"A:\s*([^Q]+)", dialogs)
    if answers:
        last_answer = answers[-1].strip()
        return last_answer
    else:
        return None

@app.route('/', methods=['POST'])
def chatbot():
    if request.method == "POST":
        user_input = request.json.get('userObject').get('userInput')
        # print(f"User Input: {user_input}")

        prompt = "Q: You are now a medical doctor, and I am a patient. I will describe my conditions and symptoms and you will give me medical suggestions. You may ask me to do further medical testings if you are not sure about my symptoms. Ok? A: Sure. Q: "
        human_input = prompt + user_input["message"] + " A:"
        # print(human_input)
        output = llm(human_input, max_tokens=128, stop=["Q:"], echo=True) #"\n"
        print(output)
        response = output["choices"][0]["text"]

        return jsonify({"response": extract_last_answer(response)})

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=80)


import os
import sys
import json
import warnings
warnings.filterwarnings("ignore")

import math
import time
import random
import argparse
import numpy as np
from copy import deepcopy
from tqdm import tqdm

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import models, transforms, datasets

from models import get_backbone_class
from util.methods import get_semisup_method_class
from util.dataset import ImageFolderSemiSup, x_u_split, MergeDataset
from util.transform import get_semisup_transform_class
from util.misc import *

DATASET_CONFIG = {'cars': (196, 8144), 'flowers': (102, 2040), 'pets': (37, 3680), 'aircraft': (100, 6667), 'cub': (200, 5990),
                  'dogs': (120, 12000), 'mit67': (67, 5360), 'stanford40': (40, 4000), 'dtd': (47, 3760), 'celeba': (307, 4263),
                  'food11': (11, 13296), 'imagenet100': (100, 130000), 'imagenet': (1000, 1281167)}

def parse_args():
    parser = argparse.ArgumentParser('argument for open-set semi-supervised learning')

    parser.add_argument('--seed', type=int, default=0)
    parser.add_argument('--print_freq', type=int, default=10)
    parser.add_argument('--save_freq', type=int, default=10)
    parser.add_argument('--save_dir', type=str, default='./save')
    parser.add_argument('--tag', type=str, default='',
                        help='tag for experiment name')

    # optimization
    parser.add_argument('--optimizer', type=str, default='sgd')
    parser.add_argument('--learning_rate', type=float, default=0.03)
    parser.add_argument('--lr_decay_epochs', type=str, default='60,80')
    parser.add_argument('--lr_decay_rate', type=float, default=0.1)
    parser.add_argument('--weight_decay', type=float, default=1e-4)
    parser.add_argument('--momentum', type=float, default=0.9)
    parser.add_argument('--cosine', action='store_true',
                        help='using cosine annealing')
    parser.add_argument('--warm', action='store_true',
                        help='warm-up for large batch training')
    parser.add_argument('--warm_epochs', type=int, default=0,
                        help='warmup epochs')
    parser.add_argument('--total_step', default=2**14, type=int,
                        help='number of total steps to run, SimCore (2**14) and scratch (2**16)')
    parser.add_argument('--eval_step', default=512, type=int,
                        help='number of eval steps to run')

    # dataset
    parser.add_argument('--dataset', type=str, default='cars')
    parser.add_argument('--data_folder', type=str, default='./data/')
    parser.add_argument('--num_workers', type=int, default=16)
    parser.add_argument('--img_size', type=int, default=224)
    parser.add_argument('--batch_size', type=int, default=64,
                        help='only Self-Training with SimCore ckpt used 256')
    parser.add_argument('--label_ratio', type=float, default=0.5,
                        help='ratio for the labeled samples')
    parser.add_argument('--mu', type=float, default=1,
                        help='coefficient of unlabeled batch size, SelfTrain (1) and OpenMatch (2)')
    # for Self-Training
    parser.add_argument('--data_folder2', type=str, default='./data/',
                        help='data folder root for WebFG-496 dataset')
    parser.add_argument('--teacher_batch_size', type=int, default=64,
                        help='batch size for teacher_labeled_loader, SimCore (256) and scratch (64)')

    # model & method
    parser.add_argument('--model', type=str, default='resnet50')
    parser.add_argument('--method', type=str, default=None)
    parser.add_argument('--pretrained', action='store_true')
    parser.add_argument('--pretrained_ckpt', type=str, default=None,
                        help='path to pre-trained model')
    parser.add_argument('--teacher_pretrained_ckpt', type=str, default=None,
                        help='path to pre-trained teacher model')
    parser.add_argument('--use_ema', action='store_true',
                        help='whether to use exponential moving average of encoder model')
    parser.add_argument('--ema_decay', default=0.999, type=float,
                        help='EMA decay rate')
    parser.add_argument('--clip', default=0, type=float,
                        help='clip gradient value')

    parser.add_argument('--semisup_method', default='self_training', type=str,
                        help='semi-supervised learning algorithm')

    ### Self-Training ###
    parser.add_argument('--teacher_epochs', default=100, type=int,
                        help='epochs to train teacher network, SimCore (100) and scratch (500)')
    parser.add_argument('--T', default=1, type=float,
                        help='pseudo label temperature')
    parser.add_argument('--lambda_u', default=0.5, type=float,
                        help='coefficient of unlabeled loss')

    ### OpenMatch ###
    parser.add_argument('--start_fix', default=5, type=int,
                        help='epoch to start fixmatch training, SimCore (5) and scratch (20)')
    parser.add_argument('--lambda_oem', default=0.1, type=float,
                        help='coefficient of OEM loss')
    parser.add_argument('--lambda_socr', default=0.5, type=float,
                        help='coefficient of SOCR loss')
    # parser.add_argument('--T', default=1, type=float,
    #                     help='pseudo label temperature')
    parser.add_argument('--threshold', default=0.0, type=float,
                        help='pseudo label threshold')

    args = parser.parse_args()

    args.epochs = math.ceil(args.total_step / args.eval_step)
    iterations = args.lr_decay_epochs.split(',')
    args.lr_decay_epochs = list([])
    for it in iterations:
        args.lr_decay_epochs.append(int(it))
    args.wd_scheduler = False

    args.model_name = '{}_{}'.format(args.dataset, args.model)

    args.model_name += '_{}_semisup'.format(args.semisup_method)

    if not args.pretrained:
        assert args.pretrained_ckpt is None
        args.model_name += '_from_scratch'

    if args.tag:
        args.model_name += '_{}'.format(args.tag)

    args.save_folder = os.path.join(args.save_dir, args.model_name)
    if not os.path.isdir(args.save_folder):
        os.makedirs(args.save_folder)

    if args.warm:
        args.warmup_from = args.learning_rate * 0.1**3
        args.warm_epochs = args.warm_epochs
        if args.cosine:
            eta_min = args.learning_rate * (args.lr_decay_rate ** 3)
            args.warmup_to = eta_min + (args.learning_rate - eta_min) * (
                    1 + math.cos(math.pi * args.warm_epochs / args.epochs)) / 2
        else:
            args.warmup_to = args.learning_rate

    if args.dataset in DATASET_CONFIG:
        args.n_cls, args.n_data = DATASET_CONFIG[args.dataset]
    else:
        raise NotImplementedError

    args.num_labeled = int(args.n_data * args.label_ratio)
    assert args.num_labeled >= args.batch_size

    return args

def set_loader(args):
    # construct data loader
    if args.dataset in DATASET_CONFIG:
        mean = (0.485, 0.456, 0.406)
        std = (0.229, 0.224, 0.225)
    else:
        raise NotImplementedError
    normalize = transforms.Normalize(mean=mean, std=std)

    train_transform = transforms.Compose([
        transforms.RandomResizedCrop(size=args.img_size, scale=(0.2, 1.)),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        normalize,
    ])
    val_transform = transforms.Compose([transforms.Resize(256),
                                        transforms.CenterCrop(args.img_size),
                                        transforms.ToTensor(),
                                        normalize])

    # split labeled and unlabeled dataset
    train_labeled_idxs, train_unlabeled_idxs = x_u_split(args)
    TRANSFORM = get_semisup_transform_class(args.semisup_method)

    if args.dataset == 'imagenet':
        traindir = os.path.join(args.data_folder, 'train') # under ~~/Data/CLS-LOC
        valdir = os.path.join(args.data_folder, 'val')
    else: # for fine-grained dataset
        if args.dataset == 'aircraft' or args.dataset == 'cars':
            traindir = os.path.join(args.data_folder, args.dataset, 'train')
            valdir = os.path.join(args.data_folder, args.dataset, 'test')
        elif args.dataset == 'celeba':
            traindir = os.path.join(args.data_folder, 'celeba_maskhq', 'train')
            valdir = os.path.join(args.data_folder, 'celeba_maskhq', 'test')
        else:
            traindir = os.path.join(args.data_folder, args.dataset, 'train')
            valdir = os.path.join(args.data_folder, args.dataset, 'test')

    if args.semisup_method == 'openmatch':
        train_transform = TRANSFORM(args.img_size, mean, std)

    train_labeled_dataset = ImageFolderSemiSup(root=traindir,
                                                transform=train_transform,
                                                index=train_labeled_idxs)
    train_unlabeled_dataset = ImageFolderSemiSup(root=traindir,
                                                 transform=TRANSFORM(args.img_size, mean, std),
                                                 index=train_unlabeled_idxs)

    traindir = os.path.join(args.data_folder2, 'train')
    webfg_ul_dataset = datasets.ImageFolder(root=traindir, transform=TRANSFORM(args.img_size, mean, std))
    train_unlabeled_dataset = MergeDataset(train_unlabeled_dataset, webfg_ul_dataset)

    val_dataset = datasets.ImageFolder(root=valdir,
                                       transform=val_transform)

    if args.semisup_method == 'self_training':
        train_labeled_loader = [torch.utils.data.DataLoader(train_labeled_dataset, batch_size=args.batch_size, shuffle=True,
                                                            num_workers=8, pin_memory=True, drop_last=True),
                                torch.utils.data.DataLoader(train_labeled_dataset, batch_size=args.teacher_batch_size, shuffle=True,
                                                            num_workers=8, pin_memory=True, drop_last=True)]
    elif args.semisup_method == 'openmatch':
        train_labeled_loader = torch.utils.data.DataLoader(
                train_labeled_dataset, batch_size=args.batch_size, shuffle=True,
                num_workers=4, pin_memory=True, drop_last=True)

    train_unlabeled_loader = torch.utils.data.DataLoader(
            train_unlabeled_dataset, batch_size=int(args.batch_size*args.mu), shuffle=True,
            num_workers=4, pin_memory=True, drop_last=True)
    val_loader = torch.utils.data.DataLoader(
            val_dataset, batch_size=args.batch_size, shuffle=False,
            num_workers=4, pin_memory=True)

    return train_labeled_loader, train_unlabeled_loader, val_loader, args

def set_model(args):
    model = get_backbone_class(args.model)()
    feat_dim = model.final_feat_dim
    classifier = nn.Linear(feat_dim, args.n_cls)  # reset fc layer
    criterion = nn.CrossEntropyLoss()
    semisup_criterion = get_semisup_method_class(args.semisup_method)(args)

    if args.pretrained_ckpt is not None:
        ckpt = torch.load(args.pretrained_ckpt, map_location='cpu')
        state_dict = ckpt['model']

        # HOTFIX: always dataparallel during pretraining
        new_state_dict = {}
        for k, v in state_dict.items():
            if "module." in k:
                k = k.replace("module.", "")
            if "backbone." in k:
                k = k.replace("backbone.", "")
            new_state_dict[k] = v
        state_dict = new_state_dict
        model.load_state_dict(state_dict, strict=False)
        print('pretrained model loaded from: {}'.format(args.pretrained_ckpt))

    if torch.cuda.device_count() > 1:
        model = torch.nn.DataParallel(model)
    else:
        raise NotImplementedError

    model.cuda()
    ema_model = ModelEMA(model, args.ema_decay) if args.use_ema else None

    classifier.cuda()
    criterion = criterion.cuda()
    semisup_criterion = semisup_criterion.cuda()
    optim_params = list(model.parameters()) + list(classifier.parameters())

    if args.semisup_method == 'openmatch':
        classifier_open = nn.Linear(feat_dim, args.n_cls*2, bias=False).cuda()
        optim_params += list(classifier_open.parameters())
        classifier = (classifier, classifier_open)

    if args.optimizer == 'sgd':
        optimizer = optim.SGD(optim_params,
                            lr=args.learning_rate,
                            momentum=args.momentum,
                            weight_decay=args.weight_decay,
                            nesterov=True)
    elif args.optimizer == 'adam':
        optimizer = optim.Adam(optim_params,
                               lr=args.learning_rate,
                               weight_decay=args.weight_decay)

    return model, ema_model, classifier, criterion, semisup_criterion, optimizer

def train_teacher(train_loader, model, classifier, criterion, optimizer, epoch, args):
    model.train()
    classifier.train()

    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()

    end = time.time()
    for idx, (images, labels) in enumerate(train_loader):
        data_time.update(time.time() - end)

        images = images.cuda(non_blocking=True)
        labels = labels.cuda(non_blocking=True)
        bsz = labels.shape[0]

        # warmup_learning_rate(args, epoch, idx, len(train_loader), optimizer)

        if not args.e2e:
            with torch.no_grad():
                features = model(images)
            output = classifier(features.detach())
        else:
            output = classifier(model(images))
        loss = criterion(output, labels)

        losses.update(loss.item(), bsz)
        [acc1, acc5], _ = accuracy(output, labels, topk=(1, 5))
        top1.update(acc1[0], bsz)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # print info
        if (idx + 1) % args.print_freq == 0:
            print('Train: [{0}][{1}/{2}]\t'
                  'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'loss {loss.val:.3f} ({loss.avg:.3f})\t'
                  'Acc@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                   epoch, idx + 1, len(train_loader), batch_time=batch_time,
                   data_time=data_time, loss=losses, top1=top1))
            sys.stdout.flush()

    return losses.avg, top1.avg

def train_selftraining(train_labeled_loader, train_unlabeled_loader, model, ema_model, classifier, semisup_criterion, optimizer, scaler, epoch, args, teacher_model=None):
    model.train()
    classifier.train()

    batch_time, data_time = AverageMeter(), AverageMeter()
    l_losses, u_losses = AverageMeter(), AverageMeter()
    top1 = AverageMeter()

    labeled_iter = iter(train_labeled_loader)
    unlabeled_iter = iter(train_unlabeled_loader)

    end = time.time()
    for idx in range(args.eval_step):
        try:
            labeled_data = next(labeled_iter)
        except:
            labeled_iter = iter(train_labeled_loader)
            labeled_data = next(labeled_iter)

        try:
            unlabeled_data = next(unlabeled_iter)
        except:
            unlabeled_iter = iter(train_unlabeled_loader)
            unlabeled_data = next(unlabeled_iter)

        data_time.update(time.time() - end)

        warmup_learning_rate(args, epoch, idx, args.eval_step, optimizer)

        # semi-supervised learning methods
        epoch_info = [epoch, idx]
        with torch.cuda.amp.autocast():
            if teacher_model is None:
                loss, Lx, Lu, logits_x, targets_x = semisup_criterion(labeled_data, unlabeled_data, model, classifier, epoch_info)
            else:
                loss, Lx, Lu, logits_x, targets_x = semisup_criterion(labeled_data, unlabeled_data, model, classifier, epoch_info, teacher_model)

        bsz = targets_x.shape[0]
        l_losses.update(Lx.item(), bsz)
        u_losses.update(Lu.item(), bsz)
        [acc1, acc5], _ = accuracy(logits_x, targets_x, topk=(1, 5))
        top1.update(acc1[0], bsz)

        optimizer.zero_grad()
        scaler.scale(loss).backward()
        if args.clip > 0:
            torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
            torch.nn.utils.clip_grad_norm_(classifier.parameters(), args.clip)

        scaler.step(optimizer)
        scaler.update()
        if args.use_ema:
            ema_model.update(model)

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # print info
        if (idx + 1) % args.print_freq == 0:
            print('Train: [{0}][{1}/{2}]\t'
                  'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'l_loss {l_loss.val:.3f} ({l_loss.avg:.3f})\t'
                  'u_loss {u_loss.val:.3f} ({u_loss.avg:.3f})\t'
                  'Acc@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                   epoch, idx + 1, args.eval_step, batch_time=batch_time, data_time=data_time,
                   l_loss=l_losses, u_loss=u_losses, top1=top1))
            sys.stdout.flush()

    return l_losses.avg, top1.avg

def exclude_dataset(args, dataset, model, classifier, classifier_open, exclude_known=False):
    end = time.time()
    # dataset.init_index()
    test_loader = torch.utils.data.DataLoader(
        dataset,
        batch_size=args.batch_size,
        num_workers=4,
        drop_last=True,
        shuffle=False)

    model.eval()
    classifier.eval()
    classifier_open.eval()
    with torch.no_grad():
        for batch_idx, ((_, _, inputs), targets) in enumerate(tqdm(test_loader)):
            inputs = inputs.cuda(non_blocking=True)
            feat = model(inputs)
            outputs, outputs_open = classifier(feat), classifier_open(feat)
            outputs = F.softmax(outputs, 1)
            out_open = F.softmax(outputs_open.view(outputs_open.size(0), 2, -1), 1)
            tmp_range = torch.range(0, out_open.size(0) - 1).long().cuda()
            pred_close = outputs.data.max(1)[1]
            unk_score = out_open[tmp_range, 0, pred_close]
            known_ind = unk_score < 0.5
            if batch_idx == 0:
                known_all = known_ind
            else:
                known_all = torch.cat([known_all, known_ind], 0)
    known_all = known_all.data.cpu().numpy()
    if exclude_known:
        ind_selected = np.where(known_all == 0)[0]
    else:
        ind_selected = np.where(known_all != 0)[0]
    print("selected ratio %s"%( (len(ind_selected)/ len(known_all))))
    model.train()
    classifier.train()
    classifier_open.train()
    return ind_selected

def train_openmatch(train_labeled_loader, train_unlabeled_loader, model, ema_model, classifier, semisup_criterion, optimizer, scaler, epoch, args, teacher_model=None):
    model.train()
    classifier, classifier_open = classifier
    classifier.train()
    classifier_open.train()

    batch_time, data_time = AverageMeter(), AverageMeter()
    losses, losses_x = AverageMeter(), AverageMeter()
    losses_o, losses_oem, losses_socr, losses_fix = AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter()
    top1 = AverageMeter()

    labeled_iter = iter(train_labeled_loader)
    unlabeled_all_iter = iter(train_unlabeled_loader)

    unlabeled_dataset = deepcopy(train_unlabeled_loader.dataset)

    if epoch >= args.start_fix:
        indices_inlier = exclude_dataset(args, unlabeled_dataset, model, classifier, classifier_open)
    else:
        indices_inlier = range(len(unlabeled_dataset))

    unlabeled_sampler = torch.utils.data.SubsetRandomSampler(indices_inlier, generator=None)
    unlabeled_trainloader = torch.utils.data.DataLoader(
                            unlabeled_dataset, batch_size=int(args.batch_size*args.mu), sampler=unlabeled_sampler,
                            num_workers=4, pin_memory=True, drop_last=True)
    unlabeled_iter = iter(unlabeled_trainloader)

    end = time.time()
    for idx in range(args.eval_step):
        try:
            labeled_data = next(labeled_iter)
        except:
            labeled_iter = iter(train_labeled_loader)
            labeled_data = next(labeled_iter)

        try:
            unlabeled_data = next(unlabeled_iter)
        except:
            unlabeled_iter = iter(unlabeled_trainloader)
            unlabeled_data = next(unlabeled_iter)

        try:
            unlabeled_data_all = next(unlabeled_all_iter)
        except:
            unlabeled_all_iter = iter(train_unlabeled_loader)
            unlabeled_data_all = next(unlabeled_all_iter)

        data_time.update(time.time() - end)

        warmup_learning_rate(args, epoch, idx, args.eval_step, optimizer)

        # semi-supervised learning methods
        epoch_info = [epoch, idx]
        with torch.cuda.amp.autocast():
            loss, Lx, Lo, L_oem, L_socr, L_fix, logits_x, targets_x = semisup_criterion(labeled_data, unlabeled_data, unlabeled_data_all, model, classifier, classifier_open, epoch_info)

        bsz = targets_x.shape[0]
        losses.update(loss.item(), bsz)
        losses_x.update(Lx.item(), bsz)
        losses_o.update(Lo.item(), bsz)
        losses_oem.update(L_oem.item(), bsz)
        losses_socr.update(L_socr.item(), bsz)
        losses_fix.update(L_fix.item(), bsz)

        [acc1, acc5], _ = accuracy(logits_x, targets_x, topk=(1, 5))
        top1.update(acc1[0], bsz)

        optimizer.zero_grad()
        scaler.scale(loss).backward()
        if args.clip > 0:
            torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
            torch.nn.utils.clip_grad_norm_(classifier.parameters(), args.clip)

        scaler.step(optimizer)
        scaler.update()
        if args.use_ema:
            ema_model.update(model)

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # print info
        if (idx + 1) % args.print_freq == 0:
            print('Train: [{0}][{1}/{2}]\t'
                  'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'loss {losses.val:.3f} ({losses.avg:.3f})\t'
                  'loss_x {losses_x.val:.3f} ({losses_x.avg:.3f})\t'
                  'loss_o {losses_o.val:.3f} ({losses_o.avg:.3f})\t'
                  'loss_oem {losses_oem.val:.3f} ({losses_oem.avg:.3f})\t'
                  'loss_socr {losses_socr.val:.3f} ({losses_socr.avg:.3f})\t'
                  'loss_fix {losses_fix.val:.3f} ({losses_fix.avg:.3f})\t'
                  'Acc@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                   epoch, idx + 1, args.eval_step, batch_time=batch_time, data_time=data_time,
                   losses=losses, losses_x=losses_x, losses_o=losses_o, losses_oem=losses_oem,
                   losses_socr=losses_socr, losses_fix=losses_fix, top1=top1))
            sys.stdout.flush()

    return losses_x.avg, top1.avg

def validate(val_loader, model, classifier, criterion, args, best_acc, best_model):
    model.eval()
    if isinstance(classifier, tuple):
        classifier = classifier[0]
    classifier.eval()

    batch_time = AverageMeter()
    losses = AverageMeter()
    top1, top5 = AverageMeter(), AverageMeter()
    if args.dataset in ['aircraft', 'pets', 'flowers', 'mit67']: # mean per-class accuracy
        top1, top5 = AverageClassMeter(args.n_cls), AverageClassMeter(args.n_cls)

    with torch.no_grad():
        end = time.time()
        for idx, (images, labels) in enumerate(val_loader):
            images = images.cuda(non_blocking=True)
            labels = labels.cuda(non_blocking=True)
            bsz = labels.shape[0]

            output = classifier(model(images))
            loss = criterion(output, labels)

            losses.update(loss.item(), bsz)
            top1, top5 = update_metric(output, labels, top1, top5, args)
            batch_time.update(time.time() - end)
            end = time.time()
            if (idx + 1) % args.print_freq == 0:
                print('Test: [{0}/{1}]\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                      'Acc@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                       idx + 1, len(val_loader), batch_time=batch_time,
                       loss=losses, top1=top1))

    print(' * Acc@1 {top1.avg:.2f}, Acc@5 {top5.avg:.2f}'.format(top1=top1, top5=top5))
    best_acc, bool = get_best_acc(top1.avg, top5.avg, best_acc)
    if bool:
        best_model = [deepcopy(model.state_dict()), deepcopy(classifier.state_dict())]

    return best_acc, best_model

def main():
    args = parse_args()
    with open(os.path.join(args.save_folder, 'train_args.json'), 'w') as f:
        json.dump(vars(args), f, indent=4)

    # fix seed
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    best_acc, best_model = [0, 0, 0], None
    train_labeled_loader, train_unlabeled_loader, val_loader, args = set_loader(args)
    model, ema_model, classifier, criterion, semisup_criterion, optimizer = set_model(args)
    if args.semisup_method == 'self_training':
        model2, _, classifier2, _, _, optimizer2 = set_model(args)
    scaler = torch.cuda.amp.GradScaler()

    teacher_model = None
    if args.semisup_method == 'self_training':
        # for Self-Training, train a teacher network with labeled dataset
        train_labeled_loader, teacher_labeled_loader = train_labeled_loader
        if args.teacher_pretrained_ckpt is None:
            args.e2e = True
            for t_epoch in range(1, args.teacher_epochs+1):
                adjust_lr_wd(args, optimizer2, t_epoch)
                loss, acc = train_teacher(teacher_labeled_loader, model2, classifier2, criterion, optimizer2, t_epoch, args)

                if t_epoch % 5 == 0:
                    best_acc, best_model = validate(val_loader, model2, classifier2, criterion, args, best_acc, best_model)

            save_file = os.path.join(args.save_folder, 'teacher_best.pth')
            model2.load_state_dict(best_model[0])
            classifier2.load_state_dict(best_model[1])
            save_model(model2, optimizer2, args, t_epoch, save_file, classifier=classifier2)

            _, _ = validate(val_loader, model2, classifier2, criterion, args, best_acc, best_model)

            teacher_model = [model2, classifier2]
            best_acc, best_model = [0, 0, 0], None
        else:
            print('teacher pretrained model loaded from: {}'.format(args.teacher_pretrained_ckpt))
            ckpt = torch.load(args.teacher_pretrained_ckpt, map_location='cpu')
            model2.load_state_dict(ckpt['model'])
            classifier2.load_state_dict(ckpt['classifier'])
            teacher_model = [model2, classifier2]

            _, _ = validate(val_loader, model2, classifier2, criterion, args, best_acc, best_model)

        model2.eval()
        classifier2.eval()

    for epoch in range(1, args.epochs+1):
        adjust_lr_wd(args, optimizer, epoch)

        # train for one epoch
        time1 = time.time()
        if args.semisup_method == 'self_training':
            loss, acc = train_selftraining(train_labeled_loader, train_unlabeled_loader, model, ema_model, classifier, semisup_criterion, optimizer, scaler, epoch, args, teacher_model)
        elif args.semisup_method == 'openmatch':
            loss, acc = train_openmatch(train_labeled_loader, train_unlabeled_loader, model, ema_model, classifier, semisup_criterion, optimizer, scaler, epoch, args)

        time2 = time.time()
        print('Train epoch {}, total time {:.2f}, accuracy:{:.2f}'.format(
            epoch, time2-time1, acc))
        best_acc[2] = acc.item()

        # eval for one epoch
        if args.use_ema:
            best_acc, best_model = validate(val_loader, ema_model.model, classifier, criterion, args, best_acc, best_model)
        else:
            best_acc, best_model = validate(val_loader, model, classifier, criterion, args, best_acc, best_model)

        if epoch % args.save_freq == 0:
            save_file = os.path.join(args.save_folder, 'epoch_{}.pth'.format(epoch))
            if args.use_ema:
                save_model(ema_model.model, optimizer, args, epoch, save_file)
            else:
                save_model(model, optimizer, args, epoch, save_file)

    save_file = os.path.join(args.save_folder, 'best.pth')
    model.load_state_dict(best_model[0])
    save_model(model, optimizer, args, epoch, save_file)
    update_json('%s' % args.model_name, best_acc, path=os.path.join(args.save_dir, 'results.json'))

if __name__ == '__main__':
    main()


""" Portlet for content
"""
from zope.component import getMultiAdapter
from zope.publisher.browser import BrowserPage
from allen.utils.cache import servercache
from zope.traversing.browser import absoluteURL

class ContentPortlet(BrowserPage):
    """ Display content as a 5 items portlet.
    """
    def news(self, max_items=5):
        """ Return news items
        """
        view = getMultiAdapter((self.context, self.request), name=u'index.html')
        index = 0
        for newsitem in view.items:
            index += 1
            if index > max_items:
                raise StopIteration
            yield newsitem

    def thumbnail(self, newsitem):
        """ Return icon
        """
        index = getMultiAdapter((newsitem, self.request), name=u'index.html')
        thumb = getattr(index, 'thumbnail', None)
        thumb = thumb and thumb(from_parent=True)
        if thumb:
            return thumb
        return ''

    @property
    def etag(self):
        """ Used by servercache decorator
        """
        etag = getMultiAdapter((self.context, self.request), name=u'index.etag')
        etag = etag(content_portlet=absoluteURL(self.context, self.request))
        return etag

    def relative_id(self, newsitem):
        view = getMultiAdapter((newsitem, self.request), name=u'index.html')
        return getattr(view, 'relative_id', '')

    @servercache
    def __call__(self, **kwargs):
        return self.index()