"""
CTD evaluation.

Note: the CTD training set has some oddities

E.g. scopolamine induces drug overdose

Biocreativ results here:

https://biocreative.bioinformatics.udel.edu/media/store/files/2015/BC5CDR_overview.final.pdf

A total of 18 teams successfully submitted CID results in 46 runs. As
shown in Table 3 (only the best run of each team is included),
the Fscore ranges from 32.01% to 57.03% (team 288) with an average of
43.37%. All teams outperformed the baseline results by the simple abstract-level
co-occurrence method (16.43% in precision, 76.45% in recall and 27.05% in F-score).

"""
import gzip
import logging
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
from random import shuffle
from typing import Any, Dict, Iterable, List, Set, Tuple

import yaml
from bioc import biocxml
from oaklib import BasicOntologyInterface, get_implementation_from_shorthand
from pydantic.v1 import BaseModel

from ontogpt.engines.knowledge_engine import chunk_text
from ontogpt.engines.spires_engine import SPIRESEngine
from ontogpt.evaluation.evaluation_engine import SimilarityScore, SPIRESEvaluationEngine
from ontogpt.templates.core import Publication, Triple
from ontogpt.templates.ctd import (
    ChemicalToDiseaseDocument,
    ChemicalToDiseaseRelationship,
    TextWithTriples,
)

THIS_DIR = Path(__file__).parent
DATABASE_DIR = Path(__file__).parent / "database"
TEST_SET_BIOC = DATABASE_DIR / "CDR_TestSet.BioC.xml.gz"
TRAIN_SET_BIOC = DATABASE_DIR / "CDR_TrainSet.BioC.xml.gz"

RMAP = {"CID": "induces"}

logger = logging.getLogger(__name__)

def negated(Triple) -> bool:
    return Triple.qualifier and Triple.qualifier.lower() == "not"

class PredictionRE(BaseModel):
    """A prediction for a relationship extraction task."""

    test_object: TextWithTriples = None
    """source of truth to evaluate against"""

    true_positives: List[Tuple] = None
    num_true_positives: int = None
    false_positives: List[Tuple] = None
    num_false_positives: int = None
    false_negatives: List[Tuple] = None
    num_false_negatives: int = None
    scores: Dict[str, SimilarityScore] = None
    predicted_object: TextWithTriples = None
    named_entities: List[Any] = None

    def calculate_scores(self, labelers: List[BasicOntologyInterface] = None):
        self.scores = {}

        def label(x):
            if labelers:
                for labeler in labelers:
                    lbl = labeler.label(x)
                    if lbl:
                        return f"{x} {lbl}"
            return x

        def all_objects(dm: TextWithTriples):
            return list(
                set(link.subject for link in dm.triples if not negated(link))
                | set(link.object for link in dm.triples if not negated(link))
            )

        def pairs(dm: TextWithTriples) -> Set:
            return set(
                (label(link.subject), label(link.object))
                for link in dm.triples
                if not negated(link)
            )

        self.scores["similarity"] = SimilarityScore.from_set(
            all_objects(self.test_object),
            all_objects(self.predicted_object),
            labelers=labelers,
        )
        pred_pairs = pairs(self.predicted_object)
        test_pairs = pairs(self.test_object)
        self.true_positives = list(pred_pairs.intersection(test_pairs))
        self.false_positives = list(pred_pairs.difference(test_pairs))
        self.false_negatives = list(test_pairs.difference(pred_pairs))
        self.num_false_negatives = len(self.false_negatives)
        self.num_false_positives = len(self.false_positives)
        self.num_true_positives = len(self.true_positives)

class EvaluationObjectSetRE(BaseModel):
    """A result of predicting relationextractions."""

    precision: float = None
    recall: float = None
    f1: float = None

    training: List[TextWithTriples] = None
    predictions: List[PredictionRE] = None
    test: List[TextWithTriples] = None

@dataclass
class EvalCTD(SPIRESEvaluationEngine):
    # ontology: OboGraphInterface = None
    subject_prefix = "MESH"
    object_prefix = "MESH"

    def __post_init__(self):
        self.extractor = SPIRESEngine("ctd.ChemicalToDiseaseDocument")
        # synonyms are derived entirely from training set
        self.extractor.load_dictionary(DATABASE_DIR / "synonyms.yaml")

    def load_test_cases(self) -> Iterable[ChemicalToDiseaseDocument]:
        return self.load_cases(TEST_SET_BIOC)

    def load_training_cases(self) -> Iterable[ChemicalToDiseaseDocument]:
        return self.load_cases(TRAIN_SET_BIOC)

    def load_cases(self, path: Path) -> Iterable[ChemicalToDiseaseDocument]:
        logger.info(f"Loading {path}")
        with gzip.open(str(path), "rb") as f:
            collection = biocxml.load(f)
            triples_by_text = defaultdict(list)
            for document in collection.documents:
                doc = {}
                for p in document.passages:
                    doc[p.infons["type"]] = p.text
                title = doc["title"]
                abstract = doc["abstract"]
                # text = f"Title: {title} Abstract: {abstract}"
                for r in document.relations:
                    i = r.infons
                    t = Triple(
                        subject=f"{self.subject_prefix}:{i['Chemical']}",
                        predicate=RMAP[i["relation"]],
                        object=f"{self.object_prefix}:{i['Disease']}",
                    )
                    triples_by_text[(title, abstract)].append(t)
        for (title, abstract), triples in triples_by_text.items():
            pub = Publication(title=title, abstract=abstract)
            logger.debug(f"Triples: {len(triples)} for Title: {title} Abstract: {abstract}")
            yield ChemicalToDiseaseDocument(publication=pub, triples=triples)

    def create_training_set(self, num=100):
        ke = self.extractor
        docs = list(self.load_test_cases())
        shuffle(docs)
        for doc in docs[0:num]:
            text = doc.text
            prompt = ke.get_completion_prompt(None, text)
            completion = ke.serialize_object(m)
            yield dict(prompt=prompt, completion=completion)

    def eval(self) -> EvaluationObjectSetRE:
        """Evaluate the ability to extract relations."""
        labeler = get_implementation_from_shorthand("sqlite:obo:mesh")
        num_test = self.num_tests
        ke = self.extractor
        docs = list(self.load_test_cases())
        shuffle(docs)
        eos = EvaluationObjectSetRE(
            test=docs[:num_test],
            training=[],
            predictions=[],
        )
        n = 1
        for doc in eos.test:
            logger.info(f"Iteration {n} of {num_test}")
            n += 1
            logger.info(doc)
            text = f"Title: {doc.publication.title} Abstract: {doc.publication.abstract}"
            predicted_obj = None
            named_entities = []
            for chunked_text in chunk_text(text):
                extraction = ke.extract_from_text(chunked_text)
                logger.info(
                    f"{len(extraction.extracted_object.triples)}\
                        triples from window: {chunked_text}"
                )
                if not predicted_obj:
                    predicted_obj = extraction.extracted_object
                else:
                    predicted_obj.triples.extend(extraction.extracted_object.triples)
                    logger.info(f"{len(predicted_obj.triples)} total triples, after concatenation")
                    logger.debug(f"concatenated triples: {predicted_obj.triples}")
                named_entities.extend(extraction.named_entities)

            # title_extraction = ke.extract_from_text(doc.publication.title)
            # logger.info(f"{len(title_extraction.extracted_object.triples)}\
            # triples from: Title {doc.publication.title}")
            # abstract_extraction = ke.extract_from_text(doc.publication.abstract)
            # logger.info(f"{len(abstract_extraction.extracted_object.triples)}\
            # triples from: Abstract {doc.publication.abstract}")
            # ke.merge_resultsets([results, results2])
            # predicted_obj = title_extraction.extracted_object
            # predicted_obj.triples.extend(abstract_extraction.extracted_object.triples)
            # logger.info(f"{len(predicted_obj.triples)} total triples, after concatenation")
            # logger.debug(f"concatenated triples: {predicted_obj.triples}")

            def included(t: ChemicalToDiseaseRelationship):
                return (
                    t
                    and t.subject
                    and t.object
                    and t.subject.startswith("MESH:")
                    and t.object.startswith("MESH:")
                    and t.predicate.lower() == "induces"
                )

            predicted_obj.triples = [t for t in predicted_obj.triples if included(t)]
            logger.info(
                f"{len(predicted_obj.triples)} filtered triples (CID only, between MESH only)"
            )
            pred = PredictionRE(predicted_object=predicted_obj, test_object=doc)
            pred.named_entities = named_entities
            logger.info("PRED")
            logger.info(yaml.dump(pred.dict()))
            logger.info("Calc scores")
            pred.calculate_scores(labelers=[labeler])
            logger.info(yaml.dump(pred.dict()))
            eos.predictions.append(pred)
        self.calc_stats(eos)
        return eos

    def calc_stats(self, eos: EvaluationObjectSetRE):
        num_true_positives = sum(p.num_true_positives for p in eos.predictions)
        num_false_positives = sum(p.num_false_positives for p in eos.predictions)
        num_false_negatives = sum(p.num_false_negatives for p in eos.predictions)
        if num_true_positives + num_false_positives == 0:
            logger.warning("No true positives or false positives")
            return
        eos.precision = num_true_positives / (num_true_positives + num_false_positives)
        eos.recall = num_true_positives / (num_true_positives + num_false_negatives)
        if eos.precision + eos.recall == 0:
            logger.warning("No precision or recall")
            return
        eos.f1 = 2 * (eos.precision * eos.recall) / (eos.precision + eos.recall)


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-i686/egg/cspace/hgsite/genshi.py
# Compiled at: 2013-06-09 10:57:05
"""Genshi rendering
Copyright (C) 2012 Remy Blank
"""
from __future__ import absolute_import
import os.path, weakref
try:
    from genshi import QName, Stream
    from genshi.core import Attrs, END, START, TEXT
    from genshi.template import Context, MarkupTemplate, NewTextTemplate, TemplateLoader
    from genshi.template.eval import Suite
    has_genshi = True
except ImportError:
    TemplateLoader = object
    has_genshi = False

try:
    from pygments.lexers import get_lexer_by_name
    from pygments.formatters.html import HtmlFormatter, _get_ttype_class
    from pygments.styles import get_style_by_name
    has_pygments = True
except ImportError:
    has_pygments = False

from .site import cache_must_revalidate, renderer
from .util import guess_mimetype, hex_node

class Loader(TemplateLoader):
    """A template loader propagating the revision to included templates"""

    def load(self, filename, relative_to=None, cls=None, encoding=None):
        if relative_to is not None and '@' not in filename:
            parts = relative_to.rsplit('@', 1)
            if len(parts) == 2:
                filename += '@' + parts[1]
        return super(Loader, self).load(filename, relative_to, cls, encoding)

_loaders = weakref.WeakKeyDictionary()

def get_template_loader(repo):
    """Return a template loader for the given repository."""
    try:
        return _loaders[repo]
    except KeyError:
        pass

    def load_template(filename):
        path, rev = filename.rsplit('@', 1)
        rev = bytes(rev) if rev else None
        fctx = repo[rev][path]

        def uptodate():
            return rev is not None

        return (filename, filename, fctx.data(), uptodate)

    loader = _loaders[repo] = Loader(load_template, auto_reload=True, max_cache_size=100, default_encoding='utf-8', variable_lookup='strict')
    return loader

def execute(path, context):
    """Execute a script in the given context."""
    with open(path, 'r') as (f):
        source = f.read()
    Suite(source, path).execute(context)

def make_context(req, path, **kwargs):
    """Create a rendering context."""
    context = Context(req=req, **kwargs)
    context_module = req.web.config('hgsite', 'context.genshi', '')
    if context_module:
        execute(os.path.join(req.repo.path, context_module), context)
    return context

translate_nop = ('').join(chr(i) for i in range(256))
invalid_control_chars = ('').join(chr(i) for i in range(32) if i not in (9, 10, 13))

@renderer('genshi:html', 'genshi:xhtml', 'genshi:xml', register=has_genshi)
def render_markup_template(req, path, name, config, **kwargs):
    """Render a Genshi markup template."""
    method = name.split(':', 1)[(-1)]
    mimetype = guess_mimetype(config.get('mimetype'), path, 'text/xml')
    encoding = config.get('encoding', 'utf-8')
    loader = get_template_loader(req.repo)
    template = loader.load('%s@%s' % (path, hex_node(req.ctx)), cls=MarkupTemplate)
    context = make_context(req, path, highlight=highlight, **kwargs)
    stream = template.generate(context)
    text = stream.render(method, encoding=encoding)
    text = text.translate(translate_nop, invalid_control_chars)
    return req.send(text, mimetype=mimetype, encoding=encoding, headers=cache_must_revalidate)

@renderer('genshi:text', register=has_genshi)
def render_text_template(req, path, name, config, **kwargs):
    """Render a Genshi text template."""
    method = name.split(':', 1)[(-1)]
    mimetype = guess_mimetype(config.get('mimetype'), path, 'text/plain')
    encoding = config.get('encoding', 'utf-8')
    loader = get_template_loader(req.repo)
    template = loader.load('%s@%s' % (path, hex_node(req.ctx)), cls=NewTextTemplate)
    stream = template.generate(make_context(req, path, **kwargs))
    text = stream.render(method, encoding=encoding)
    return req.send(text, mimetype=mimetype, encoding=encoding, headers=cache_must_revalidate)

@renderer('pygments:css', register=has_pygments, file_must_exist=False)
def render_pygments_css(req, path, name, config, **kwargs):
    """Render a (virtual) CSS file with Pygments styles."""
    style_cls = get_style_by_name(config.get('style', 'trac'))
    container = config.get('container', '')
    formatter = HtmlFormatter(style=style_cls, nobackground=True)
    content = formatter.get_style_defs(c.strip() for c in container.split(','))
    return req.send(content, mimetype='text/css', encoding='utf-8', headers=cache_must_revalidate)

def _group_by_class(tokens):
    """Group Pygments lexer tokens with the same CSS class."""
    cls = None
    text = []
    for ttype, value in tokens:
        c = _get_ttype_class(ttype)
        if c == cls:
            text.append(value)
            continue
        if value:
            yield (
             cls, ('').join(text))
            text = [value]
            cls = c

    if text:
        yield (
         cls, ('').join(text))
    return

def generate_highlight_markup(tokens):
    """Generate a Genshi stream for Pygments markup."""
    pos = (None, -1, -1)
    span = QName('span')
    class_ = QName('class')
    for cls, text_chunk in _group_by_class(tokens):
        if cls:
            yield (
             START, (span, Attrs([(class_, cls)])), pos)
            yield (TEXT, text_chunk, pos)
            yield (END, span, pos)
        else:
            yield (
             TEXT, text_chunk, pos)

    return

def generate_text(text):
    """Generate a Genshi stream with a single text node."""
    yield (TEXT, text, (None, -1, -1))
    return

def highlight(text, lexer, tabsize=0):
    """Syntax-highlight the given text with the given lexer, and return a
    Genshi stream of the result."""
    if has_pygments:
        try:
            lexer = get_lexer_by_name(lexer, stripnl=False, ensurenl=False, tabsize=tabsize)
            tokens = lexer.get_tokens(text)
            return Stream(generate_highlight_markup(tokens))
        except Exception:
            pass

    return Stream(generate_text(text))


import time
import struct
import os
from hyper import comms, udp
from lib import bno055, quat
import spidev
from RPi import GPIO

class Input:
    def __init__(self):
        # input backends
        self.comm_port = comms.CommPort()
        self.udp_port = udp.UDPClient()
        self.start_time = int(time.time() * 1000)

        self.voltmeter = AnalogSensor(19, self.comm_port)
        self.ammeter_highcurrent = AnalogSensor(18, self.comm_port)
        self.ammeter_lowcurrent = AnalogSensor(-1, self.comm_port)
        self.prox1 = DigitalSensor(0, self.comm_port)
        self.prox2 = DigitalSensor(1, self.comm_port)
        self.prox3 = DigitalSensor(2, self.comm_port)
        self.prox4 = DigitalSensor(3, self.comm_port)
        self.retro1 = DigitalSensor(4, self.comm_port)
        self.retro2 = DigitalSensor(5, self.comm_port)
        self.lf_lev_pot = AnalogSensor(3, self.comm_port)
        self.rf_lev_pot = AnalogSensor(2, self.comm_port)
        self.lb_lev_pot = AnalogSensor(1, self.comm_port)
        self.rb_lev_pot = AnalogSensor(0, self.comm_port)
        self.lf_lat_pot = AnalogSensor(9, self.comm_port)
        self.rf_lat_pot = AnalogSensor(11, self.comm_port)
        self.lb_lat_pot = AnalogSensor(16, self.comm_port)
        self.rb_lat_pot = AnalogSensor(17, self.comm_port)
        self.l_ebr_pot = AnalogSensor(10, self.comm_port)
        self.r_ebr_pot = AnalogSensor(9, self.comm_port)

        self.team_id = 1
        self.status = 2
        self.acceleration = 3  # cm/s^2
        self.position = 4  # cm
        self.velocity = 5  # cm/s
        self.battery_voltage = 6  # millivolts
        self.battery_current = 7  # milliamps
        self.battery_temperature = 8  # tenths of C
        self.pod_temperature = 9  # tenths of C
        self.stripe_count = 10

        # sensor inputs
        self.duration = 0

        self.emag_activated = False
        self.emag_timer = -1
        self.user_command = None

        self.voltage = 0
        self.amperage_highcurrent = 0
        self.amperage_lowcurrent = 0
        self.lf_lev_extension = 0
        self.rf_lev_extension = 0
        self.lb_lev_extension = 0
        self.rb_lev_extension = 0
        self.lf_lat_extension = 0
        self.rf_lat_extension = 0
        self.lb_lat_extension = 0
        self.rb_lat_extension = 0
        self.l_ebr_extension = 0
        self.r_ebr_extension = 0

        # outputs
        self.lev_extended = False
        self.lat_extended = False

    def updateFTA(self):
        self.lev_extended = False
        self.lat_extended = False
        pass

    def updateFTB(self):
        pass

    def updateFTC(self):
        pass

    def update(self):

        # SENSOR DATA
        self.voltage = self.voltmeter.get()
        self.amperage_highcurrent = self.ammeter_highcurrent.get()
        self.amperage_lowcurrent = self.ammeter_lowcurrent.get()
        self.lf_lev_extension = self.lf_lev_pot.get()
        self.rf_lev_extension = self.rf_lev_pot.get()
        self.lb_lev_extension = self.lb_lev_pot.get()
        self.rb_lev_extension = self.rb_lev_pot.get()
        self.lf_lat_extension = self.lf_lat_pot.get()
        self.rf_lat_extension = self.rf_lat_pot.get()
        self.lb_lat_extension = self.lb_lat_pot.get()
        self.rb_lat_extension = self.rb_lat_pot.get()
        self.l_ebr_extension = self.l_ebr_pot.get()
        self.r_ebr_extension = self.r_ebr_pot.get()

        # processing of data to be sent to GUI and SpaceX TODO: add other variable
        self.battery_voltage = self.voltage * 1000
        self.battery_current = self.amperage_highcurrent * 1000

        # CONTROL VARIABLES
        self.duration = int(time.time() * 1000) - self.start_time

        self.udp_port.update()
        self.emag_activated = self.udp_port.ebrake
        self.emag_timer = self.udp_port.timer
        self.user_command = self.udp_port.user_command

        self.updateFTA()
        # self.updateFTB()
        # self.updateFTC()
        # return

    def get_packed_data(self):
        """
        :return: The 34 byte telemetry frame of required data as specified by the SpaceX documentation.
        """
        input_data = [self.team_id, self.status, self.acceleration, self.position, self.velocity, self.battery_voltage,
                      self.battery_current, self.battery_temperature, self.pod_temperature, self.stripe_count]
        packer = struct.Struct('! B B i i i i i i i I')
        packed_data = packer.pack(*input_data)
        return packed_data

    def isReady(self):
        # if input is ready to be updated
        # check comms
        if not self.comm_port.readAnalog(31) == 512:
            return False

        # calibrate sensors

        return True

class DigitalSensor:
    def __init__(self, in_id, comm_port):
        self.in_id = in_id
        self.comm_port = comm_port

    def get(self):
        if self.in_id == -1:
            return -1
        return self.comm_port.readDigital(self.in_id)

class AnalogSensor:
    def __init__(self, in_id, comm_port):
        self.in_id = in_id
        self.comm_port = comm_port

    def get(self):
        if self.in_id == -1:
            return -1
        return self.comm_port.readAnalog(self.in_id)

class Voltmeter:
    def __init__(self, in_id, comm_port):
        self.in_id = in_id
        self.comm_port = comm_port

    def get(self):
        if self.in_id == -1:
            self.value = -1
            return
        val = self.comm_port.readAnalog(self.in_id)
        vout = (val * 5.0) / 1024.0
        vin = vout / (7500 / (30000 + 7500))
        vin = vin * .175
        return vin

class ThermoSensor:
    THERMO_PIN = 17

    def __init__(self, serial):
        self.serial = serial
        GPIO.setup(self.THERMO_PIN, GPIO.OUT, initial=GPIO.LOW)

        os.system('modprobe w1-gpio')
        os.system('modprobe w1-therm')
        self.temp_sensor = "sys/bus/w1/devices/28-" + self.serial + "/w1_slave"

        self.value = None

    def temp_raw(self):
        f = open(self.temp_sensor, 'r')
        lines = f.readlines()
        f.close()
        return lines

    def data_string(self):
        return "spi sensor, degrees celsius: " + str(self.value)

class DigitalOutput:
    def __init__(self, out_id, comm_port):
        self.out_id = out_id
        self.comm_port = comm_port

    def set(self, val):
        if self.out_id == -1:
            return
        self.comm_port.writeDigial(self.out_id, val)

class Actuator:
    def __init__(self, out_id, comm_port):
        self.out_id = out_id
        self.comm_port = comm_port

    def set(self, val):
        if self.out_id == -1:
            return
        self.comm_port.writeDigitalActuator(self.out_id, val)


#!/usr/bin/python
# -*- coding: utf-8 -*-

import utl
import argparse
import json
from DockerAPI import SwarmMaster
from DockerAPI import SwarmWorker
from DockerAPI import BaseDocker

base = BaseDocker()
master = SwarmMaster()
worker = SwarmWorker()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--action', choices=['initSwarm', 'newService', 'joinSwarm', 'rmService', 'leaveSwarm',
                                             'inspectTask', 'inspectTasks', 'listNodes', 'getNodeID', 'inspectTaskName'],
                        type=str, help='DynamicDockerSwarm action')
    parser.add_argument('--service', required=False, type=str, help='Service definition')
    parser.add_argument('--remote_addr', required=False, type=str, default=None, help='Remote address')
    parser.add_argument('--join_token', required=False, type=str, default=None, help='Docker Swarm join token.')
    parser.add_argument('--task_name', required=False, type=str, help='Specific task name')

    args = parser.parse_args()
    action = args.action
    serviceInfo = args.service
    remote_addr = args.remote_addr
    join_token = args.join_token

    if action == 'initSwarm':
        master.init_swarm(advertise_addr=utl.get_local_address())
        master.create_network(name='DynamicSwarmNetwork')
        master.get_join_token()
    elif action == 'joinSwarm':
        if not remote_addr or not join_token:
            print('Remote address and join_token must be specified together.')
        worker.join(remote_addr, join_token)
    elif action == 'newService':
        serviceInfo = serviceInfo.strip('\'')
        serviceInfo = json.loads(serviceInfo)
        master.create_service(serviceInfo)
    elif action == 'rmService':
        serviceName = serviceInfo
        master.rm_service(serviceName)
    elif action == 'leaveSwarm':
        base.leave()
    elif action == 'inspectTask':
        task = args.task_name
        master.inspect_task(task)
    elif action == 'inspectTasks':
        sv_name = serviceInfo
        master.inspect_tasks(sv_name)
    elif action == 'listNodes':
        master.list_nodes()
    elif action == 'getNodeID':
        base.getNodeID()
    elif action == 'inspectTaskName':
        sv_name = serviceInfo
        master.inspect_task_name(sv_name)


from math import sin

import matplotlib.pyplot as plt
import numpy as np

def SinC(t):
    ''' sine cardinal '''
    if t == 0:
        return 1
    else:
        return sin(t) / t

SinC = np.vectorize(SinC)

t = np.arange(-50, 50, 0.01)
plt.plot(t, SinC(t))
plt.grid(True)
plt.show()

Level 2/29 Work in progress/SignalProcessing/01_sinc.py


import os
from collections import Counter, defaultdict
import json
import sys

def find_all_filenames(dirname):
    all_filenames_dict = defaultdict(list)
    for directory, dirs, filenames in os.walk(dirname):
        for filename in filenames:
            path = os.path.join(directory, filename)
            size_of_file = os.path.getsize(path)
            all_filenames_dict[(filename, size_of_file)].append(path)
    return all_filenames_dict

def get_similar_filenames(all_filenames):
    duplicate_filenames = {}
    for file_info, path in all_filenames.items():
        if len(path) > 1:
            duplicate_filenames.update({file_info[0]: path})
    return json.dumps(duplicate_filenames, indent=2, ensure_ascii=False)

def print_similar_files(kind_of_files, duplicate_filenames):
    print(kind_of_files, duplicate_filenames)

if __name__ == "__main__":
    if len(sys.argv) > 1:
        dirname = sys.argv[1]
    else:
        exit("Не выбран каталог")
    if os.path.isfile(dirname):
        exit("Ошибка! Передан файл, а не каталог")
    all_filenames = find_all_filenames(dirname)
    duplicate_filenames = get_similar_filenames(all_filenames)
    print_similar_files("Найденные дубликаты:", duplicate_filenames)


# -*- coding: utf-8 -*-
# Store the version here so:
# 1) we don't load dependencies by storing it in __init__.py
# 2) we can import it in setup.py for the same reason
# 3) we can import it into the jira module
import datetime
import os
import subprocess

VERSION = (1, 0, 6, 'final', 0)

def get_version(version, filename=None):
    assert len(version) == 5
    assert version[3] in ('a', 'beta', 'rc', 'final')
    main = '.'.join(map(str, version[:3]))
    sub = ''
    # everything build based on develop branch is a .dev build
    # develop is the integration branch for git flow adopters
    if (version[3] == 'a' and version[4] == 0) or get_git_branch() == 'develop':
        git_changeset = get_git_changeset(filename)
        if git_changeset:
            sub = '.dev%s' % git_changeset
    if version[3] != 'final' and not sub:
        sub = '%s%s' % tuple(version[3:])
    return main + sub

def sh(command, cwd=None):
    return subprocess.Popen(command,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            shell=True,
                            cwd=cwd,
                            universal_newlines=True).communicate()[0]

def get_git_changeset(filename=None):
    """Returns a numeric identifier of the latest git changeset.

    The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
    This value isn't guaranteed to be unique, but collisions are very unlikely,
    so it's sufficient for generating the development version numbers.
    """
    dirname = os.path.dirname(filename or __file__)
    git_show = sh('git show --pretty=format:%ct --quiet HEAD',
                  cwd=dirname)
    timestamp = git_show.partition('\n')[0]
    try:
        timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))
    except ValueError:
        return None
    return timestamp.strftime('%Y%m%d%H%M%S')

def get_git_branch():
    # Jenkins may not do a full checkout so we use the branch reported via env variables.
    if 'BRANCH_NAME' in os.environ:
        branch = os.environ['BRANCH_NAME']
    elif 'GIT_BRANCH' in os.environ:
        branch = '/'.join(os.environ['GIT_BRANCH'].split('/')[1:])
    else:
        branch = sh("git branch | sed -n '/\* /s///p'").rstrip()
    return branch

FORMAT = '%n'.join(['%H', '%aN', '%ae', '%cN', '%ce', '%s'])

def gitrepo(root=None):
    if not root:
        cwd = root = os.getcwd()
    else:
        cwd = os.getcwd()
        if cwd != root:
            os.chdir(root)
    gitlog = sh('git --no-pager log -1 --pretty="format:%s"' % FORMAT,
                cwd=root).split('\n', 5)
    branch = sh('git rev-parse --abbrev-ref HEAD', cwd=root).strip()
    remotes = [x.split() for x in
               filter(lambda x: x.endswith('(fetch)'),
                      sh('git remote -v', cwd=root).strip().splitlines())]
    if cwd != root:
        os.chdir(cwd)
    return {
        "head": {
            "id": gitlog[0],
            "author_name": gitlog[1],
            "author_email": gitlog[2],
            "committer_name": gitlog[3],
            "committer_email": gitlog[4],
            "message": gitlog[5].strip(),
        },
        "branch": branch,
        "remotes": [{'name': remote[0], 'url': remote[1]}
                    for remote in remotes]
    }

__version__ = get_version(VERSION)


#!/usr/bin/python
import grequests
from bs4 import BeautifulSoup
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlite_decl import Base, RawDataUrl, RawDataArticle
import zip
import os

dataPath = '../data'
databasePath = '%s/datacollection.db' % (dataPath)
engine = create_engine('sqlite:///%s' % databasePath)
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
session = DBSession()

def store_url(url, content):
    global session
    raw_data_url = session.query(RawDataUrl).filter(RawDataUrl.url == url).first
    if raw_data_url is None:
        raw_data_url = RawDataUrl(id = RawDataUrl.getMaxId(session) + 1, url=url)
        session.add(raw_data_url)
        session.commit()

    zipped = zip.zipped(content, 'article.html')
    raw_data_article = RawDataArticle(id = RawDataArticle.getMaxId(session) + 1, content = zipped, rawDataUrl = None)
    session.add(raw_data_article)
    session.commit()

def get_urls_from_response(r):
    if r is None:
        return []
    print(r.url)
    soup = BeautifulSoup(r.text, 'html.parser')
    store_url(r.url, str(soup))
    urls = [link.get('href') for link in soup.find_all('a')]
    soup.decompose()
    return urls

def print_url(args):
    print (args['url'])

def exception_handler(r,e):
        print('failed: ',r.url,'\n')
        #changing the url just for doing sth
        r.url = 'http://httpbin.org/status/200'
        res = r.send().response
        return res

def chunks(l, n):
    n = max(1, n)
    return (l[i:i+n] for i in xrange(0, len(l), n))

def recursive_urls(urls):
    """
    Given a list of starting urls, recursively finds all descendant urls
    recursively
    """
    global base_url
    if len(urls) == 0:
        return
    print (urls)
    rs = [grequests.get(url) for url in urls]
    responses = grequests.map(rs, size=100)
    url_lists = [get_urls_from_response(response) for response in responses]
    urls = sum(url_lists, [])  # flatten list of lists into a list
    urls = [url for url in urls if url is not None and url.startswith(base_url)]
    urls = list(set(urls)) # remove double entries
    for chunked_urls in chunks(urls, 100):
    	recursive_urls(chunked_urls)

def iterative_loader():
    global loadedUrls, identifiedUrls, base_url
    urls = identifiedUrls[:100]
    identifiedUrls = identifiedUrls[100:]
    rs = [grequests.get(url) for url in urls]
    responses = grequests.map(rs, size=100)
    loadedUrls = list(sum([loadedUrls, urls], []))
    url_lists = [get_urls_from_response(response) for response in responses]
    responded_urls = list(set(sum(url_lists, [])))
    responded_urls = [url for url in responded_urls if url is not None and url.startswith(base_url)]
    responded_urls = [url for url in responded_urls if not url in loadedUrls]
    identifiedUrls = list(sum([identifiedUrls, responded_urls],[]))

    print ("----------------------------------------------------")
    print ("%s identified Urls, %s loaded urls" % (len(identifiedUrls), len(loadedUrls)))
    print ("----------------------------------------------------")

base_url = 'https://www.krone.at'
loadedUrls = []
identifiedUrls = [base_url]

iterative_loader()
while (len(identifiedUrls) > 0):
    iterative_loader()

#recursive_urls([base_url])


"""Returns tidy3d simulation from gdsfactory Component."""
import warnings
from typing import Dict, Optional

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pydantic
import tidy3d as td

import gdsfactory as gf
from gdsfactory.component import Component
from gdsfactory.components.extension import move_polar_rad_copy
from gdsfactory.config import logger
from gdsfactory.routing.sort_ports import sort_ports_x, sort_ports_y
from gdsfactory.simulation.gtidy3d.materials import (
    MATERIAL_NAME_TO_TIDY3D_INDEX,
    MATERIAL_NAME_TO_TIDY3D_NAME,
    get_index,
    get_medium,
)
from gdsfactory.tech import LAYER_STACK, LayerStack
from gdsfactory.types import ComponentOrFactory, Float2

@pydantic.validate_arguments
def get_simulation(
    component: ComponentOrFactory,
    port_extension: Optional[float] = 4.0,
    layer_stack: LayerStack = LAYER_STACK,
    thickness_pml: float = 1.0,
    xmargin: float = 0,
    ymargin: float = 0,
    xmargin_left: float = 0,
    xmargin_right: float = 0,
    ymargin_top: float = 0,
    ymargin_bot: float = 0,
    zmargin: float = 1.0,
    clad_material: str = "sio2",
    port_source_name: str = "o1",
    port_margin: float = 0.5,
    port_source_offset: float = 0.1,
    distance_source_to_monitors: float = 0.2,
    resolution: float = 50,
    wavelength_start: float = 1.50,
    wavelength_stop: float = 1.60,
    wavelength_points: int = 50,
    plot_modes: bool = False,
    num_modes: int = 2,
    run_time_ps: float = 10.0,
    dispersive: bool = False,
    material_name_to_tidy3d_index: Dict[str, float] = MATERIAL_NAME_TO_TIDY3D_INDEX,
    material_name_to_tidy3d_name: Dict[str, str] = MATERIAL_NAME_TO_TIDY3D_NAME,
    is_3d: bool = True,
    with_all_monitors: bool = False,
) -> td.Simulation:
    r"""Returns Simulation object from gdsfactory.component

    based on GDS example
    https://simulation.cloud/docs/html/examples/ParameterScan.html

    .. code::

         top view
              ________________________________
             |                               |
             | xmargin_left                  | port_extension
             |<------>          port_margin ||<-->
          ___|___________          _________||___
             |           \        /          |
             |            \      /           |
             |             ======            |
             |            /      \           |
          ___|___________/        \__________|___
             |   |                 <-------->|
             |   |ymargin_bot   xmargin_right|
             |   |                           |
             |___|___________________________|

        side view
              ________________________________
             |                     |         |
             |                     |         |
             |                   zmargin_top |
             |xmargin_left         |         |
             |<---> _____         _|___      |
             |     |     |       |     |     |
             |     |     |       |     |     |
             |     |_____|       |_____|     |
             |       |                       |
             |       |                       |
             |       |zmargin_bot            |
             |       |                       |
             |_______|_______________________|

    Args:
        component: gdsfactory Component.
        port_extension: extend ports beyond the PML.
        layer_stack: contains layer numbers (int, int) to thickness, zmin.
        thickness_pml: PML thickness (um).
        xmargin: left/right distance from component to PML.
        xmargin_left: left distance from component to PML.
        xmargin_right: right distance from component to PML.
        ymargin: left/right distance from component to PML.
        ymargin_top: top distance from component to PML.
        ymargin_bot: bottom distance from component to PML.
        zmargin: thickness for cladding above and below core.
        clad_material: material for cladding.
        port_source_name: input port name.
        port_margin: margin on each side of the port.
        distance_source_to_monitors: in (um) source goes before monitors.
        port_source_offset: mode solver workaround.
            positive moves source forward, negative moves source backward.
        resolution: in pixels/um (20: for coarse, 120: for fine)
        wavelength_start: in (um).
        wavelength_stop: in (um).
        wavelength_points: number of wavelengths.
        plot_modes: plot source modes.
        num_modes: number of modes to plot.
        run_time_ps: make sure it's sufficient for the fields to decay.
            defaults to 10ps and counts on automatic shutoff to stop earlier if needed.
        dispersive: False uses constant refractive index materials.
            True adds wavelength depending materials.
            Dispersive materials require more computation.
        material_name_to_tidy3d_index: not dispersive materials have a constant index.
        material_name_to_tidy3d_name: dispersive materials have a wavelength
            dependent index. Maps layer_stack names with tidy3d material database names.
        is_3d: if False, does not consider Z dimension for faster simulations.
        with_all_monitors: if True, includes field monitors which increase results file size.

    .. code::

        import matplotlib.pyplot as plt
        import gdsfactory as gf
        import gdsfactory.simulation.tidy3d as gt

        c = gf.components.bend_circular()
        sim = gt.get_simulation(c)
        gt.plot_simulation(sim)

    """
    component = component() if callable(component) else component
    assert isinstance(component, Component)

    layer_to_thickness = layer_stack.get_layer_to_thickness()
    layer_to_material = layer_stack.get_layer_to_material()
    layer_to_zmin = layer_stack.get_layer_to_zmin()
    # layer_to_sidewall_angle = layer_stack.get_layer_to_sidewall_angle()

    if dispersive:
        material_name_to_tidy3d = material_name_to_tidy3d_name
    else:
        material_name_to_tidy3d = material_name_to_tidy3d_index

    assert isinstance(
        component, Component
    ), f"component needs to be a gf.Component, got Type {type(component)}"
    if port_source_name not in component.ports:
        warnings.warn(
            f"port_source_name={port_source_name} not in {component.ports.keys()}"
        )
        port_source = component.get_ports_list(port_type="optical")[0]
        port_source_name = port_source.name
        warnings.warn(f"Selecting port_source_name={port_source_name} instead.")

    component_padding = gf.add_padding_container(
        component,
        default=0,
        top=ymargin or ymargin_top,
        bottom=ymargin or ymargin_bot,
        left=xmargin or xmargin_left,
        right=xmargin or xmargin_right,
    )
    component_extended = (
        gf.components.extension.extend_ports(
            component=component_padding, length=port_extension, centered=True
        )
        if port_extension
        else component_padding
    )

    gf.show(component_extended)
    component_extended = component_extended.flatten()

    component_ref = component_padding.ref()
    component_ref.x = 0
    component_ref.y = 0

    clad_material_name_or_index = material_name_to_tidy3d[clad_material]
    clad = td.Structure(
        geometry=td.Box(
            size=(td.inf, td.inf, td.inf),
            center=(0, 0, 0),
        ),
        medium=get_medium(name_or_index=clad_material_name_or_index),
    )
    structures = [clad]

    layers_thickness = [
        layer_to_thickness[layer]
        for layer in component.get_layers()
        if layer in layer_to_thickness
    ]

    if len(layer_to_thickness) < 1:
        raise ValueError(f"{component.get_layers()} not in {layer_to_thickness.keys()}")

    t_core = max(layers_thickness)
    cell_thickness = (
        thickness_pml + t_core + thickness_pml + 2 * zmargin
        if is_3d
        else 1 / resolution
    )

    sim_size = [
        component_ref.xsize + 2 * thickness_pml,
        component_ref.ysize + 2 * thickness_pml,
        cell_thickness,
    ]

    for layer in component.layers:
        if layer in layer_to_thickness and layer in layer_to_material:
            thickness = layer_to_thickness[layer]
            zmin = layer_to_zmin[layer] if is_3d else -td.inf
            zmax = zmin + thickness if is_3d else td.inf

            if (
                layer in layer_to_material
                and layer_to_material[layer] in material_name_to_tidy3d
            ):
                name_or_index = material_name_to_tidy3d[layer_to_material[layer]]
                medium = get_medium(name_or_index=name_or_index)
                index = get_index(name_or_index=name_or_index)
                logger.debug(
                    f"Add {layer}, {name_or_index!r}, index = {index:.3f}, "
                    f"thickness = {thickness}, zmin = {zmin}, zmax = {zmax}"
                )

                polygons = td.PolySlab.from_gds(
                    gds_cell=component_extended,
                    gds_layer=layer[0],
                    gds_dtype=layer[1],
                    axis=2,
                    slab_bounds=(zmin, zmax),
                )

                for polygon in polygons:
                    geometry = td.Structure(
                        geometry=polygon,
                        medium=medium,
                    )
                    structures.append(geometry)
            elif layer not in layer_to_material:
                logger.debug(f"Layer {layer} not in {layer_to_material.keys()}")
            elif layer_to_material[layer] not in material_name_to_tidy3d:
                materials = list(material_name_to_tidy3d.keys())
                logger.debug(f"material {layer_to_material[layer]} not in {materials}")

    # Add source
    port = component_ref.ports[port_source_name]
    angle = port.orientation
    width = port.width + 2 * port_margin
    size_x = width * abs(np.sin(angle * np.pi / 180))
    size_y = width * abs(np.cos(angle * np.pi / 180))
    size_x = 0 if size_x < 0.001 else size_x
    size_y = 0 if size_y < 0.001 else size_y
    size_z = cell_thickness - 2 * zmargin if is_3d else td.inf

    source_size = [size_x, size_y, size_z]
    source_center = port.center.tolist() + [0]  # (x, y, z=0)

    xy_shifted = move_polar_rad_copy(
        np.array(port.center), angle=angle * np.pi / 180, length=port_source_offset
    )
    source_center_offset = xy_shifted.tolist() + [0]  # (x, y, z=0)

    wavelengths = np.linspace(wavelength_start, wavelength_stop, wavelength_points)
    freqs = td.constants.C_0 / wavelengths
    freq0 = td.constants.C_0 / np.mean(wavelengths)
    fwidth = freq0 / 10

    msource = td.ModeSource(
        size=source_size,
        center=source_center,
        source_time=td.GaussianPulse(freq0=freq0, fwidth=fwidth),
        direction="+",
    )

    # Add port monitors
    monitors = {}
    ports = sort_ports_x(sort_ports_y(component_ref.get_ports_list()))
    for port in ports:
        port_name = port.name
        angle = port.orientation
        width = port.width + 2 * port_margin
        size_x = width * abs(np.sin(angle * np.pi / 180))
        size_y = width * abs(np.cos(angle * np.pi / 180))
        size_x = 0 if size_x < 0.001 else size_x
        size_y = 0 if size_y < 0.001 else size_y
        size = (size_x, size_y, size_z)

        # if monitor has a source move monitor inwards
        length = -distance_source_to_monitors if port_name == port_source_name else 0
        xy_shifted = move_polar_rad_copy(
            np.array(port.center), angle=angle * np.pi / 180, length=length
        )
        center = xy_shifted.tolist() + [0]  # (x, y, z=0)

        monitors[port_name] = td.ModeMonitor(
            center=center,
            size=size,
            freqs=freqs,
            mode_spec=td.ModeSpec(num_modes=1),
            name=port.name,
        )

    zcenter = (zmax + zmin) / 2 if is_3d else 0
    domain_monitor = td.FieldMonitor(
        center=[0, 0, zcenter],
        size=[sim_size[0], sim_size[1], 0] if is_3d else [td.inf, td.inf, 0],
        freqs=[freq0],
        name="field",
    )
    monitors = list(monitors.values())
    monitors += [domain_monitor] if with_all_monitors else []

    sim = td.Simulation(
        size=sim_size,
        grid_size=3 * [1 / resolution],
        structures=structures,
        sources=[msource],
        monitors=monitors,
        run_time=20 * run_time_ps / fwidth,
        pml_layers=3 * [td.PML()] if is_3d else [td.PML(), td.PML(), None],
    )

    if plot_modes:
        src_plane = td.Box(center=source_center_offset, size=source_size)
        ms = td.plugins.ModeSolver(simulation=sim, plane=src_plane, freq=freq0)
        mode_spec = td.ModeSpec(num_modes=num_modes)
        modes = ms.solve(mode_spec=mode_spec)

        print(
            "Effective index of computed modes: ",
            ", ".join([f"{mode.n_eff:1.4f}" for mode in modes]),
        )

        if is_3d:
            fig, axs = plt.subplots(num_modes, 2, figsize=(12, 12))
        else:
            fig, axs = plt.subplots(num_modes, 3, figsize=(12, 12))

        for mode_ind in range(num_modes):
            if is_3d:
                abs(modes[mode_ind].field_data.Ey).plot(
                    x="y", y="z", cmap="magma", ax=axs[mode_ind, 0]
                )
                abs(modes[mode_ind].field_data.Ez).plot(
                    x="y", y="z", cmap="magma", ax=axs[mode_ind, 1]
                )
            else:
                abs(modes[mode_ind].field_data.Ex).plot(ax=axs[mode_ind, 0])
                abs(modes[mode_ind].field_data.Ey).plot(ax=axs[mode_ind, 1])
                abs(modes[mode_ind].field_data.Ez).plot(ax=axs[mode_ind, 2])

                axs[mode_ind, 0].set_title(f"|Ex|: mode_index={mode_ind}")
                axs[mode_ind, 1].set_title(f"|Ey|: mode_index={mode_ind}")
                axs[mode_ind, 2].set_title(f"|Ez|: mode_index={mode_ind}")

        if is_3d:
            axs[mode_ind, 0].set_aspect("equal")
            axs[mode_ind, 1].set_aspect("equal")
        plt.show()
    return sim

def plot_simulation_yz(
    sim: td.Simulation,
    z: float = 0.0,
    y: float = 0.0,
    wavelength: Optional[float] = 1.55,
    figsize: Float2 = (11, 4),
):
    """Returns Simulation visual representation.
    returns two views for 3D component and one view for 2D

    Args:
        sim: simulation object
        z: (um)
        y: (um)
        wavelength: (um) for epsilon plot if None plot structures.
        figsize: figure size
    """
    fig = plt.figure(figsize=figsize)
    if sim.size[2] > 0.1 and sim.size[1] > 0.1:
        gs = mpl.gridspec.GridSpec(1, 2, figure=fig, width_ratios=[1, 1.4])
        ax1 = fig.add_subplot(gs[0, 0])
        ax2 = fig.add_subplot(gs[0, 1])
        if wavelength:
            freq = td.constants.C_0 / wavelength
            sim.plot_eps(z=z, ax=ax1, freq=freq)
            sim.plot_eps(y=y, ax=ax2, freq=freq)
        else:
            sim.plot(z=z, ax=ax1)
            sim.plot(y=y, ax=ax2)
    elif sim.size[2] > 0.1:  # 2D grating sim_size_y = 0
        gs = mpl.gridspec.GridSpec(1, 1, figure=fig, width_ratios=[1])
        ax1 = fig.add_subplot(gs[0, 0])
        if wavelength:
            freq = td.constants.C_0 / wavelength
            sim.plot_eps(y=y, ax=ax1, freq=freq)
        else:
            sim.plot(y=y, ax=ax1)

    else:  # 2D planar component size_z = 0
        gs = mpl.gridspec.GridSpec(1, 1, figure=fig, width_ratios=[1])
        ax1 = fig.add_subplot(gs[0, 0])
        if wavelength:
            freq = td.constants.C_0 / wavelength
            sim.plot_eps(z=z, ax=ax1, freq=freq)
        else:
            sim.plot(z=z, ax=ax1)

    plt.show()
    return fig

def plot_simulation_xz(
    sim: td.Simulation,
    x: float = 0.0,
    z: float = 0.0,
    wavelength: Optional[float] = 1.55,
    figsize: Float2 = (11, 4),
):
    """Returns figure with two axis of the Simulation.

    Args:
        sim: simulation object
        x: (um)
        z: (um)
        wavelength: (um) for epsilon plot if None plot structures.
        figsize: figure size
    """
    fig = plt.figure(figsize=figsize)
    gs = mpl.gridspec.GridSpec(1, 2, figure=fig, width_ratios=[1, 1.4])
    ax1 = fig.add_subplot(gs[0, 0])
    ax2 = fig.add_subplot(gs[0, 1])
    if wavelength:
        freq = td.constants.C_0 / wavelength
        sim.plot_eps(z=z, ax=ax1, freq=freq)
        sim.plot_eps(x=x, ax=ax2, freq=freq)
    else:
        sim.plot(z=z, ax=ax1)
        sim.plot(x=x, ax=ax2)
    plt.show()
    return fig

plot_simulation = plot_simulation_yz

if __name__ == "__main__":
    # c = gf.components.mmi1x2()
    # c = gf.components.bend_circular(radius=2)
    # c = gf.components.crossing()
    # c = gf.c.straight_rib()

    c = gf.c.straight(length=3)
    sim = get_simulation(c, plot_modes=False, is_3d=False)

    plot_simulation(sim)

    # filepath = pathlib.Path(__file__).parent / "extra" / "wg2d.json"
    # filepath.write_text(sim.json())

    # sim.plotly(z=0)
    # plot_simulation_yz(sim, wavelength=1.55)
    # fig = plt.figure(figsize=(11, 4))
    # gs = mpl.gridspec.GridSpec(1, 2, figure=fig, width_ratios=[1, 1.4])
    # ax1 = fig.add_subplot(gs[0, 0])
    # ax2 = fig.add_subplot(gs[0, 1])
    # sim.plot(z=0.0, ax=ax1)
    # sim.plot(x=0.0, ax=ax2)
    # plt.show()


import logging
import random
import re
import utils.globals as GG

log = logging.getLogger(__name__)

class SearchException(Exception):
    pass

def discord_trim(string):
    result = []
    trimLen = 0
    lastLen = 0
    while trimLen <= len(string):
        trimLen += 1999
        result.append(string[lastLen:trimLen])
        lastLen += 1999
    return result

async def get_next_case_num():
    reportNum = await GG.MDB['properties'].find_one({'key': 'caseId'})
    num = reportNum['amount'] + 1
    reportNum['amount'] += 1
    await GG.MDB['properties'].replace_one({"key": 'caseId'}, reportNum)
    return num

def make_ordinal(n):
    '''
    Convert an integer into its ordinal representation::

        make_ordinal(0)   => '0th'
        make_ordinal(3)   => '3rd'
        make_ordinal(122) => '122nd'
        make_ordinal(213) => '213th'
    '''
    n = int(n)
    suffix = ['th', 'st', 'nd', 'rd', 'th'][min(n % 10, 4)]
    if 11 <= (n % 100) <= 13:
        suffix = 'th'
    return str(n) + suffix

def gen_error_message():
    subject = random.choice(['A kobold', 'The green dragon', 'The Frost Mage', 'DiscordCrawler', 'The wizard',
                             'An iron golem', 'Your mom', 'This bot', 'You', 'Me', 'The president',
                             'The Queen', 'Xanathar', 'Volo', 'This world'])
    verb = random.choice(['must be', 'should be', 'has been', 'will be', 'is being', 'was being'])
    thing_to_do = random.choice(['stopped', 'killed', 'talked to', 'found', 'destroyed', 'fought'])
    return f"{subject} {verb} {thing_to_do}"

def a_or_an(string, upper=False):
    if string.startswith('^') or string.endswith('^'):
        return string.strip('^')
    if re.match('[AEIOUaeiou].*', string):
        return 'an {0}'.format(string) if not upper else f'An {string}'
    return 'a {0}'.format(string) if not upper else f'A {string}'

def camel_to_title(string):
    return re.sub(r'((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))', r' \1', string).title()


#!/usr/bin/env python2
""" Created on Wed Aug 29 16:05:01 2018 @author: jklhj """

import argparse
import cv2
import os
import shutil

parser = argparse.ArgumentParser(
        description='For Caffe ssd image detection (use python2), '
                    'and return the coordinates of the bounding boxes.')

parser.add_argument('--root_caffe_ssd',
                    default='',
                    help='Caffe ssd directory (python2).')

parser.add_argument('--set_gpu',
                    help='set to use GPU.',
                    default=False,
                    action='store_true')

parser.add_argument('--gpu_index',
                    help='the index of GPU.',
                    default=None)

parser.add_argument('--deploy_model',
                    default='',
                    help='the prototxt file of deployment.')

parser.add_argument('--pretrain_model',
                    default='',
                    help='the pretrain model (*.caffemodel) of deployment.')

parser.add_argument('--conf_threshold',
                    default=0.5,
                    help='the confidence threshold in ssd detection, default=0.5 .')

parser.add_argument('--image_file',
                    default='',
                    help='the image file to detect.')

parser.add_argument('--show_img',
                    help='set to show the image with bounding boxes.',
                    default=False,
                    action='store_true')

args = parser.parse_args()

if args.set_gpu and not args.gpu_index: print('Error: Need to set the index of GPU: '
                                                  '--gpu_index')

if not args.root_caffe_ssd: print('Error: Need to set the caffe ssd root: '
                                  '--root_caffe_ssd')
if not args.deploy_model: print('Error: Need to set deploy prototxt file: '
                                '--deploy_model')
if not args.pretrain_model: print('Error: Need to set pretrained model: '
                                  '--pretrain_model')
if not args.image_file: print('Error: Need to set image file to detect: '
                              '--image_file')

img = cv2.imread(args.image_file)
height, width, channels = img.shape

import sys
sys.path.insert(0, args.root_caffe_ssd + '/python/')

import caffe
from caffe.proto import caffe_pb2
from google.protobuf import text_format

if args.set_gpu:
    caffe.set_device(int(args.gpu_index))
    caffe.set_mode_gpu()

def get_modify_output_layer_param(conf_threshold, deploy_model):
    net = caffe_pb2.NetParameter()

    with open(deploy_model, 'r') as f:
        text_format.Merge(f.read(), net)

    layerName = [ l.name for l in net.layer ]
    idx = layerName.index('detection_out')

    output_layer = net.layer[idx]

    # modify output parameter
    output_layer.detection_output_param.confidence_threshold = float(conf_threshold)

    # get output parameter
    LabelMapFile = output_layer.detection_output_param.save_output_param.label_map_file

    shutil.copyfile(deploy_model, deploy_model + '_orig')
    os.remove(deploy_model)

    with open(deploy_model, 'w') as f:
        f.write(str(net))

    return LabelMapFile, conf_threshold

def caffe_ssd_img_detection(deploy_model,
                            pretrain_model,
                            image_file):

    net = caffe.Net(deploy_model,
                    pretrain_model,
                    caffe.TEST)

    transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
    transformer.set_transpose('data', (2, 0, 1))
#    transformer.set_mean('data', np.array([104,117,123])) # mean pixel

    ## the reference model operates on images in [0,255] range instead of [0,1]
    transformer.set_raw_scale('data', 255)
    ## the reference model has channels in BGR order instead of RGB
    transformer.set_channel_swap('data', (2,1,0))

    image = caffe.io.load_image(image_file)

    transformed_image = transformer.preprocess('data', image)

    net.blobs['data'].data[0,...] = transformed_image

    detections = net.forward()['detection_out']

    det_label = detections[0,0,:,1]
    det_conf  = detections[0,0,:,2]
    det_xmin  = detections[0,0,:,3]
    det_ymin  = detections[0,0,:,4]
    det_xmax  = detections[0,0,:,5]
    det_ymax  = detections[0,0,:,6]

    return det_label, det_conf, det_xmin, det_ymin, det_xmax, det_ymax

def show_image(img, xmin, xmax, ymin, ymax, label_names):
    for i in range(len(xmin)):
        cv2.rectangle( img, (xmin[i], ymin[i]), (xmax[i], ymax[i]), (0, 0, 255), 3 )
        font = cv2.FONT_HERSHEY_SIMPLEX
        cv2.putText( img, str(label_names[i]),
                     (xmin[i], ymin[i]), font, 0.7, (255, 255, 255), 1 )
        cv2.putText( img, '{:.2f}'.format(conf[i]),
                     (xmin[i], ymin[i]-20), font, 0.7, (255, 255, 255), 1 )

    cv2.imshow('Image', img)
    cv2.waitKey(0)

if __name__ == '__main__':

    labelmap_file, conf_thres = \
      get_modify_output_layer_param(args.conf_threshold,
                                    args.deploy_model)

    label, conf, xmin, ymin, xmax, ymax = \
      caffe_ssd_img_detection(args.deploy_model,
                              args.pretrain_model,
                              args.image_file)
    label = list(label)
    conf = list(conf)
    xmin = map( int, list(xmin*width) )
    xmax = map( int, list(xmax*width) )
    ymin = map( int, list(ymin*height) )
    ymax = map( int, list(ymax*height) )

    # get label names
    label_names = []
    with open(labelmap_file, 'r') as f:
        labelmap = caffe_pb2.LabelMap()
        text_format.Merge(str(f.read()), labelmap)
        for i in label:
            label_names.append( str(labelmap.item[int(i)].display_name) )

    print('conf threshold: ', conf_thres)
    print('label: ', list(label))
    print('label name:', label_names)
    print('conf: ', list(conf))
    print('xmin: ', map( int, xmin ))
    print('xmax: ', map( int, xmax ))
    print('ymin: ', map( int, ymin ))
    print('ymax: ', map( int, ymax ))
    print('image height, width, channels: ', height, width, channels)

    if args.show_img:
        show_image(img, xmin, xmax, ymin, ymax, label_names)


import os
from . import bpyhelper
from . import read_owmat
from . import owm_types
import bpy

def cleanUnusedMaterials(materials):
    if materials is None:
        return
    m = {}
    for name in materials[1]:
        mat = materials[1][name]
        if mat.users == 0:
            print('[import_owmat]: removed material: {}'.format(mat.name))
            bpy.data.materials.remove(mat)
        else:
            m[name] = mat
    bpyhelper.scene_update()
    t = {}
    for name in materials[0]:
        tex = materials[0][name]
        if tex.users == 0:
            bpy.data.textures.remove(tex)
        else:
            t[name] = tex
    bpyhelper.scene_update()
    return (t, m)

def mutate_texture_path(file, new_ext):
    return os.path.splitext(file)[0] + new_ext

def load_textures(texture, root, t):
    ''' Loads an overwatch texture.

    Priority (high to low): TIFF, TGA, DDS (doesn't work properly)
    '''
    realpath = bpyhelper.normpath(texture)
    if not os.path.isabs(realpath):
        realpath = bpyhelper.normpath('%s/%s' % (root, realpath))

    tga_file = mutate_texture_path(realpath, '.tga')
    if os.path.exists(tga_file):
        realpath = tga_file

    tif_file = mutate_texture_path(realpath, '.tif')
    if os.path.exists(tif_file):
        realpath = tif_file

    fn = os.path.splitext(os.path.basename(realpath))[0]

    try:
        tex = None
        if fn in t:
            tex = t[fn]
        else:
            img = None
            for eimg in bpy.data.images:
                if eimg.name == fn or eimg.filepath == realpath:
                    img = eimg
            if img is None:
                img = bpy.data.images.load(realpath, check_existing=True)
                img.name = fn
            tex = None
            for etex in bpy.data.textures:
                if etex.name == fn:
                    tex = etex
            if tex == None:
                tex = bpy.data.textures.new(fn, type='IMAGE')
                tex.image = img
            t[fn] = tex
        return tex
    except Exception as e:
        print('[import_owmat]: error loading texture: {}'.format(bpyhelper.format_exc(e)))
    return None

def read(filename, prefix = ''):
    root, file = os.path.split(filename)
    data = read_owmat.read(filename)
    if not data:
        return None

    t = {}
    m = {}

    for i in range(len(data.materials)):
        m[data.materials[i].key] = process_material(data.materials[i], prefix, root, t)

    return (t, m)

def process_material(material, prefix, root, t):
    if bpy.context.scene.render.engine == 'CYCLES':
        return process_material_Cycles(material, prefix, root, t)
    else:
        return process_material_BI(material, prefix, root, t)

def process_material_Cycles(material, prefix, root, t):
    mat = bpy.data.materials.new('%s%016X' % (prefix, material.key))

    # print('Processing material: ' + mat.name)
    mat.use_nodes = True

    tile = 300
    nodes = mat.node_tree.nodes
    links = mat.node_tree.links

    # Remove default nodes
    for node in [node for node in nodes if node.type != 'OUTPUT_MATERIAL']:
        nodes.remove(node)

    # Get material output node
    material_output = None
    try:
        material_output = [node for node in nodes if node.type == 'OUTPUT_MATERIAL'][0]
    except:
        material_output = nodes.new('ShaderNodeOutputMaterial')
    material_output.location = (tile, 0)

    # Create Overwatch NodeGroup Instance
    nodeOverwatch = nodes.new('ShaderNodeGroup')
    if material.shader != 0:
        print('[import_owmat]: {} uses shader {}'.format(mat.name, material.shader))
        nodeOverwatch.label = 'OWM Shader %d' % (material.shader)

    if str(material.shader) in owm_types.TextureTypes['NodeGroups'] and owm_types.TextureTypes['NodeGroups'][str(material.shader)] in bpy.data.node_groups:
        nodeOverwatch.node_tree = bpy.data.node_groups[owm_types.TextureTypes['NodeGroups'][str(material.shader)]]
    else:
        # print('[import_owmat]: could not find node group for shader %s, using default' % (material.shader))
        if owm_types.TextureTypes['NodeGroups']['Default'] in bpy.data.node_groups:
            nodeOverwatch.node_tree = bpy.data.node_groups[owm_types.TextureTypes['NodeGroups']['Default']]
        # else:
        #     print('[import_owmat]: could not find fallback shader %s' % (owm_types.TextureTypes['NodeGroups']['Default']))
    nodeOverwatch.location = (0, 0)
    nodeOverwatch.width = 250
    if nodeOverwatch.node_tree is not None:
        links.new(nodeOverwatch.outputs[0], material_output.inputs[0])

    tt = owm_types.TextureTypesById
    tm = owm_types.TextureTypes
    scratchSocket = {}
    for i, texData in enumerate(material.textures):
        nodeTex = nodes.new('ShaderNodeTexImage')
        nodeTex.location = (-tile, -tile*(i))
        nodeTex.width = 250
        nodeTex.color_space = 'NONE'

        tex = load_textures(texData[0], root, t)
        if tex is None:
            print('[import_owmat]: failed to load texture: {}'.format(texData[0]))
            continue
        nodeTex.image = tex.image

        if len(texData) == 2:
            continue

        if nodeOverwatch.node_tree is None:
            continue

        typ = texData[2]
        nodeTex['owm.material.typeid'] = str(typ)
        nodeTex.label = str(typ)
        if typ in tt:
            bfTyp = tm['Mapping'][tt[typ]]
            print('[import_owmat]: {} is {}'.format(os.path.basename(texData[0]), tt[typ]))
            nodeTex.label = str(tt[typ])
            nodeTex.name = str(tt[typ])
            for colorSocketPoint in bfTyp[0]:
                nodeSocketName = colorSocketPoint
                if colorSocketPoint in tm['Alias']:
                    nodeSocketName = tm['Alias'][colorSocketPoint]
                if colorSocketPoint not in scratchSocket:
                    if nodeSocketName in nodeOverwatch.inputs:
                        links.new(nodeTex.outputs['Color'], nodeOverwatch.inputs[nodeSocketName])
                        scratchSocket[colorSocketPoint] = nodeTex.outputs['Color']
                    else:
                        print('[import_owmat] could not find node %s on shader group' % (nodeSocketName))
            for alphaSocketPoint in bfTyp[1]:
                nodeSocketName = alphaSocketPoint
                if alphaSocketPoint in tm['Alias']:
                    nodeSocketName = tm['Alias'][alphaSocketPoint]
                if alphaSocketPoint not in scratchSocket:
                    if nodeSocketName in nodeOverwatch.inputs:
                        links.new(nodeTex.outputs['Alpha'], nodeOverwatch.inputs[nodeSocketName])
                        scratchSocket[alphaSocketPoint] = nodeTex.outputs['Alpha']
                    else:
                        print('[import_owmat] could not find node %s on shader group' % (nodeSocketName))

    if nodeOverwatch.node_tree is None:
        return mat

    for envPoint, basePoint in tm['Env'].items():
        if envPoint in scratchSocket or basePoint not in scratchSocket: continue
        nodeSocketName = envPoint
        if envPoint in tm['Alias']:
            nodeSocketName = tm['Alias'][envPoint]
        if nodeSocketName in nodeOverwatch.inputs:
            links.new(scratchSocket[basePoint], nodeOverwatch.inputs[nodeSocketName])
            scratchSocket[envPoint] = scratchSocket[basePoint]

    for activeNodePoint in tm['Active']:
        if activeNodePoint in scratchSocket:
            nodes.active = scratchSocket[activeNodePoint].node
            break

    for colorNodePoint in tm['Color']:
        if colorNodePoint in scratchSocket:
            scratchSocket[colorNodePoint].node.color_space = 'COLOR'

    return mat

def process_material_BI(material, prefix, root, t):
    mat = bpy.data.materials.new('%s%016X' % (prefix, material.key))
    mat.diffuse_intensity = 1.0

    mat.texture_slots.add() # for some reason the first texture duplicates itself

    tt = owm_types.TextureTypesById

    ti = 1

    for texturetype in material.textures:
        typ = texturetype[2]
        texture = texturetype[0]

        tex = load_textures(texture, root, t)

        if tex is None:
            print('[import_owmat]: failed to load texture: {}'.format(texturetype[0]))
            continue

        ti += 1

        mattex = mat.texture_slots.add()

        mattex.use_map_color_diffuse = True
        mattex.texture = tex
        mattex.texture_coords = 'UV'

        if typ in tt:
            if not tex.name.endswith(')'): tex.name = '%s (%s)' % (tt[typ], tex.name)
            if tt[typ].startswith('Normal'):
                tex.use_alpha = False
                tex.use_normal_map = True
                mattex.use_map_color_diffuse = False
                mattex.use_map_normal = True
                mattex.normal_factor = 1
            elif tt[typ].startswith('Diffuse'):
                mattex.diffuse_factor = 1
            else:
                mattex.use = False
        else:
            if not tex.name.endswith(')'): tex.name = '%s (%s)' % (str(typ), tex.name)
            mattex.use = False
    return mat


from flask import Blueprint, request, url_for, redirect, render_template, session
from models.colebrook.colebrook import Colebrook

colebrook_blueprint = Blueprint('colebrook', __name__)

@colebrook_blueprint.route('/calcular', methods = ['POST'])
def calcular():
    if request.method == "POST":
        rugosidade = request.form['rugosidade-absoluta']
        diametro = request.form['diametro-interno']
        reynolds = request.form['numero-reynolds']

        if rugosidade and diametro and reynolds:
            session['rugosidade'] = rugosidade
            session['diametro'] = diametro
            session['reynolds'] = reynolds

            colebrook = Colebrook(rugosidade, diametro, reynolds)
            return render_template('colebrook.html', fator_de_friccao=format(colebrook.calcula_fator_de_friccao(), '.6f'))
        else:
            return "Erro!"

    return render_template('colebrook.html')


import urllib.request, urllib.parse, urllib.error
# http://www.py4e.com/code3/bs4.zip
# and unzip it in the same directory as this file

from urllib.request import urlopen
import re
from bs4 import BeautifulSoup
import ssl
import sqlite3

conn = sqlite3.connect('wiki3.sqlite')
cur = conn.cursor()

cur.executescript('''

CREATE TABLE IF NOT EXISTS dict (
    word   TEXT UNIQUE PRIMARY KEY
);

''')

fhand=''
comm = 0

list_link = cur.execute(''' SELECT link FROM data where flag = ?''',(1,))

print(list_link)

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

#url = input('Enter - ')
#html = urlopen(url, context=ctx).read()

# html.parser is the HTML parser included in the standard Python 3 library.
# information on other HTML parsers is here:
# http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser
#soup = BeautifulSoup(html, "html.parser")

arr_junk =['http:','https:','/','<','>','=','1','2','3','4','5','6','7','8','9','0','\'','\"','}','{',']','[','(',')',':','-','+','!','~','|','\\','*','?',';','_','.','#','$','@','%','^','&','`']

def ext_wrd():
	for line in fhand:
		big_junk=line.decode().strip().split(' ')
		for junk in big_junk:
			flag=1
			for needle in arr_junk:
				if needle in junk:
					flag=0
			if ',' in junk:
				com_pos = junk.find(',')	# comma postion
				ext_wrd = junk[:com_pos]	# to extract word
			else:
				ext_wrd = junk
			if flag==1:
				#commit_Var = commit_Var + 1
				if ext_wrd != '':
					#print(ext_wrd)
					ex_wrd_l = ext_wrd.lower()
					print(ex_wrd_l)
					cur.execute('''INSERT OR IGNORE INTO dict (word)
                                        	VALUES ( ? )''', ( ex_wrd_l, ) )
					cur.execute('''UPDATE data SET flag = 2 WHERE link = ?''',(tlink1,))

for tlink in list_link:
	tlink1 = ''.join(tlink)
	print(tlink1)
	try:
		fhand = urllib.request.urlopen(tlink1)
	except:
		print("Sorry Link cannot be opened!",tlink1)
		cur.execute('''UPDATE data SET flag = 2 WHERE link = ?''',(tlink1,))
		continue
	print("Extracting words in the link .... : ",tlink1)
	ext_wrd()
	#comm = comm + 1
	#if comm % 4 == 0:
		#conn.commit()
	#print("???????????")
conn.commit()

#print("Var comm = ",comm)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import os
import subprocess
import sys
import shutil
import xml.dom.minidom

def format_file(xml_name):
    try:
        xml_actual = xml.dom.minidom.parse(xml_name)
    except Exception as e:
        print('could not read %s (%s)' % (xml_name, e))
        return 1

    def sort_key(node):
        if node.nodeType != xml.dom.Node.ELEMENT_NODE:
            return ''
        name = filter(lambda n: n.nodeName == 'name',
                      node.childNodes)[0].childNodes[0].nodeValue
        return name
    # class nodes order in instrumenter output is unstable,
    # so we sort classes here
    xml_actual.documentElement.childNodes[:] = \
        sorted(
            filter(lambda n: n.nodeType == xml.dom.Node.ELEMENT_NODE,
                   xml_actual.documentElement.childNodes),
            key=sort_key)

    actual = xml_actual.toprettyxml(indent='  ')

    try:
        open(xml_name, 'w').write(actual)
    except Exception as e:
        print('could not write %s (%s)' % (xml_name, e))
        return 1
    return 0

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'xml_path',
        help='The path to the instrumentation output.')
    args = parser.parse_args()

    err_code = format_file(args.xml_path)
    return err_code

if __name__ == '__main__':
    sys.exit(main())


# 实现voxel滤波，并加载数据集中的文件进行验证

import open3d as o3d
import os
import numpy as np
from pyntcloud import PyntCloud
import pandas as pd
import math
import random

# 功能：对点云进行voxel滤波
# 输入：
#     point_cloud：输入点云
#     leaf_size: voxel尺寸
def voxel_filter(point_cloud, leaf_size):
    filtered_points = []

    # 1. compute boundary:
    x_max = point_cloud.loc[:, 'x'].max()
    x_min = point_cloud.loc[:, 'x'].min()
    y_max = point_cloud.loc[:, 'y'].max()
    y_min = point_cloud.loc[:, 'y'].min()
    z_max = point_cloud.loc[:, 'z'].max()
    z_min = point_cloud.loc[:, 'z'].min()

    # 2. compute dim of voxel grid:
    d_x = (x_max-x_min)/leaf_size
    d_y = (y_max-y_min)/leaf_size

    # 3. compute index for each point:
    for index, row in point_cloud.iterrows():
        # floor, because voxel index begin from 0
        point_cloud.loc[index, 'h_x'] = h_x = math.floor((row['x']-x_min)/leaf_size)
        point_cloud.loc[index, 'h_y'] = h_y = math.floor((row['y']-y_min)/leaf_size)
        point_cloud.loc[index, 'h_z'] = h_z = math.floor((row['z']-z_min)/leaf_size)
        point_cloud.loc[index, 'h'] = math.floor(h_x + h_y*d_x + h_z*d_x*d_y)  # also floor

    # 4. sort the points with the voxel index:
    points_sorted = point_cloud.sort_values(by='h').loc[:, ['x', 'y', 'z', 'h']]
    points_sorted.index = range(len(points_sorted))  # after sorted, index have changed

    # 5. select points:

    # solution 1: centroid:
    count = points_sorted.loc[0, 'h']
    x_, y_, z_ = points_sorted.loc[0, ['x', 'y', 'z']]
    num = 1
    for index, row in points_sorted.iterrows():
        if row['h'] == count and index != 0:
            x_ += row['x']
            y_ += row['y']
            z_ += row['z']
            num += 1
            continue
        elif index != 0:
            filtered_points.append([x_ / num, y_ / num, z_ / num])
            count = row['h']
            x_, y_, z_ = row['x'], row['y'], row['z']
            num = 1
    # add last point:
    filtered_points.append([x_ / num, y_ / num, z_ / num])

    # solution 2: random:
    # start = end = 0
    # count = points_sorted.loc[0, 'h']
    # for index, row in points_sorted.iterrows():
    #     if row['h'] == count:
    #         end = index
    #         continue
    #     idx = random.randint(start, end)
    #     filtered_points.append(points_sorted.loc[idx, ['x', 'y', 'z']])
    #     start = end = index
    #     count = row['h']
    # idx = random.randint(start, end)
    # filtered_points.append(points_sorted.loc[idx, ['x', 'y', 'z']])

    # 把点云格式改成array，并对外返回
    filtered_points = np.array(filtered_points, dtype=np.float64)
    return filtered_points

# 功能：对点云进行voxel approximated滤波
# 输入：
#     point_cloud：输入点云
#     leaf_size: voxel尺寸
#     contain_size: 容器个数
def voxel_filter_approximated(point_cloud, leaf_size, contain_size):
    filtered_points = []

    # 1. compute boundary:
    x_max = point_cloud.loc[:, 'x'].max()
    x_min = point_cloud.loc[:, 'x'].min()
    y_max = point_cloud.loc[:, 'y'].max()
    y_min = point_cloud.loc[:, 'y'].min()
    z_max = point_cloud.loc[:, 'z'].max()
    z_min = point_cloud.loc[:, 'z'].min()

    # 2. compute dim of voxel grid: length should be a int
    d_x = math.ceil((x_max-x_min)/leaf_size)
    d_y = math.ceil((y_max-y_min)/leaf_size)

    # 3. compute index for each point:
    for index, row in point_cloud.iterrows():
        # floor, because voxel index begin from 0
        point_cloud.loc[index, 'h_x'] = h_x = math.floor((row['x']-x_min)/leaf_size)
        point_cloud.loc[index, 'h_y'] = h_y = math.floor((row['y']-y_min)/leaf_size)
        point_cloud.loc[index, 'h_z'] = h_z = math.floor((row['z']-z_min)/leaf_size)
        # compute index also floor:
        point_cloud.loc[index, 'h'] = math.floor(h_x + h_y*d_x + h_z*d_x*d_y)

    # 4. do hash map and solve hash conflict:
    # conflict happened when hash_idx and h are both same!
    containers = [[] for i in range(contain_size)]
    for index, row in point_cloud.iterrows():
        voxel_idx = row['h']
        hash_idx = int(row['h'] % contain_size)  # index should be int!

        # if conflict happened, select one them empty:
        if containers[hash_idx] and voxel_idx != point_cloud.loc[containers[hash_idx][0], 'h']:
            # select one point: centroid(or random)
            if len(containers[hash_idx]) > 1:
                point = [point_cloud.loc[containers[hash_idx], 'x'].mean(0),
                         point_cloud.loc[containers[hash_idx], 'y'].mean(0),
                         point_cloud.loc[containers[hash_idx], 'z'].mean(0)]
            else:
                point = [point_cloud.loc[containers[hash_idx], 'x'],
                         point_cloud.loc[containers[hash_idx], 'y'],
                         point_cloud.loc[containers[hash_idx], 'z']]
            filtered_points.append(point)
            # empty this container:
            containers[hash_idx] = []
        containers[hash_idx].append(index)

    # final output all points in containers:
    while containers:
        points_idx = containers.pop()
        point = [point_cloud.loc[points_idx, 'x'].mean(0),
                 point_cloud.loc[points_idx, 'y'].mean(0),
                 point_cloud.loc[points_idx, 'z'].mean(0)]
        filtered_points.append(point)

    # 把点云格式改成array，并对外返回
    filtered_points = np.array(filtered_points, dtype=np.float64)
    print("before filter: ", point_cloud.shape[0])
    print("after filter: ", len(filtered_points))
    print(filtered_points[0:5])
    return filtered_points

def main():
    classes = ["airplane", "bathtub", "bed", "bench", "bookshelf", "bottle", "bowl", "car", "chair", "cone",
               "cup", "curtain", "desk", "door", "dresser", "flower_pot", "glass_box", "guitar", "keyboard", "lamp",
               "laptop", "mantel", "monitor", "night_stand", "person", "piano", "plant", "radio", "range_hood", "sink",
               "sofa", "stairs", "stool", "table", "tent", "toilet", "tv_stand", "vase", "wardrobe", "xbox"]

    dir = "./modelnet40_normal_resampled/"

    for name in classes:

        filename = dir + name + "/" + name + "_0001.txt"
        points = np.genfromtxt(filename, delimiter=",")
        # points = np.genfromtxt("./modelnet40_normal_resampled/airplane/airplane_0001.txt", delimiter=",")
        points = pd.DataFrame(points[:, 0:3])
        points.columns = ['x', 'y', 'z']
        point_cloud_pynt = PyntCloud(points)

        # 转成open3d能识别的格式
        point_cloud_o3d = point_cloud_pynt.to_instance("open3d", mesh=False)
        # o3d.visualization.draw_geometries([point_cloud_o3d]) # 显示原始点云

        # 调用voxel滤波函数，实现滤波
        # filtered_cloud = voxel_filter(point_cloud_pynt.points, 0.1)  # 100 cause d_x,d_y too samll

        # 调用voxel approximated 滤波函数，实现滤波
        contain_size = 200
        filtered_cloud = voxel_filter_approximated(point_cloud_pynt.points, 0.1, contain_size)
        point_cloud_o3d.points = o3d.utility.Vector3dVector(filtered_cloud)
        # 显示滤波后的点云
        o3d.visualization.draw_geometries([point_cloud_o3d])

if __name__ == '__main__':
    main()


from game import Game
import config as c
import sys
from button import Button
from text_object import TextObject
import colors
import pygame
import sqlite3

class Menu(Game):
    def __init__(self):
        super().__init__('Breakout', c.screen_width, c.screen_height, c.background_image, c.frame_rate)
        self.menu_buttons = []
        self.nick = [i for i in open("name_user.txt", "r")][0]
        self.user_text = TextObject(c.screen_width - 100, 30, lambda: self.nick, colors.WHITE, c.font_name, c.font_size)
        self.user_text.draw(self.surface)
        self.objects.append(self.user_text)
        self.user_rec = TextObject(20, 30, lambda: str(0), colors.WHITE, c.font_name, c.font_size)
        self.user_rec.draw(self.surface)
        self.objects.append(self.user_rec)
        self.input = pygame.Rect(c.screen_width - 120, 30, 100, 30)
        self.input_nick = ""
        self.active = False

    c = 0
    def create_menu(self, c_menu):
        def on_play(button):
            c_menu.game_over = False
            if self.c == 0:
                c_menu.create_menu()
                self.c += 1
            c_menu.run()

        def on_quit(button):
            self.game_over = True
            self.is_game_running = False

        start_btns = (('PLAY', on_play), ('QUIT', on_quit))

        def handle_input_box(type, pos):
            if type == pygame.MOUSEBUTTONDOWN:
                if self.input.collidepoint(pos):
                    self.active = not self.active
                else:
                    self.active = False
                    self.input_nick = ""
                    self.objects.remove(self.user_text)
                    self.user_text = TextObject(c.screen_width - 100, 30, lambda: self.nick, colors.WHITE, c.font_name, c.font_size)
                    self.user_text.draw(self.surface)
                    self.objects.append(self.user_text)

        self.mouse_handlers.append(handle_input_box)

        for i, (text, click_handler) in enumerate(start_btns):
            b = Button(c.menu_offset_x,
                       c.menu_offset_y + (c.menu_button_h + 5) * i,
                       c.menu_button_w,
                       c.menu_button_h,
                       text,
                       click_handler,
                       padding=5)

            self.objects.append(b)
            self.menu_buttons.append(b)
            self.mouse_handlers.append(b.handle_mouse_event)

    def handle_events(self):
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()
            elif event.type == pygame.KEYDOWN:
                if self.active:
                    if event.key == pygame.K_BACKSPACE:
                        self.input_nick = self.input_nick[:-1]
                    elif event.key == 13:
                        self.nick = self.input_nick
                        txt = open("name_user.txt", "w")
                        txt.write(self.nick)
                        txt.close()
                        with sqlite3.connect("achivements") as con:
                            cur = con.cursor()
                            res = cur.execute("""select name from users""").fetchall()
                            res = [i[0] for i in res]
                            if self.nick not in res:
                                cur.execute("""insert into users (name) values("{}")""".format(self.nick))
                                cur.execute("""insert into scores (id_user, max_score) values(
                                            (select id_user from users where name = "{}"), 0)""".format(self.nick))
                    else:
                        self.input_nick += event.unicode
                    self.objects.remove(self.user_text)
                    self.user_text = TextObject(c.screen_width - 100, 30, lambda: self.input_nick, colors.WHITE, c.font_name, c.font_size)
                    self.user_text.draw(self.surface)
                    self.objects.append(self.user_text)
            elif event.type in (pygame.MOUSEBUTTONDOWN, pygame.MOUSEBUTTONUP, pygame.MOUSEMOTION):
                for handler in self.mouse_handlers:
                    handler(event.type, event.pos)

    def run(self):
        while not self.game_over:
            self.surface.blit(self.background_image, (0, 0))
            self.handle_events()
            self.update()
            self.draw()
            if self.active:
                pygame.draw.rect(self.surface, colors.WHITE, self.input, 2)
            else:
                pygame.draw.rect(self.surface, colors.BLACK, self.input, 2)

            pygame.display.update()
            self.clock.tick(self.frame_rate)

    def update(self):
        super().update()
        with sqlite3.connect("achivements") as con:
            cur = con.cursor()
            res = cur.execute("""select max_score from scores
                              where id_user = (select id_user from users
                              where name = "{}")""".format(self.nick)).fetchall()
            self.objects.remove(self.user_rec)
            self.user_rec = TextObject(20, 30, lambda: str(res[0][0]), colors.WHITE, c.font_name, c.font_size)
            self.user_rec.draw(self.surface)
            self.objects.append(self.user_rec)


import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from scipy.ndimage.interpolation import zoom

from tf_keras_vis.gradcam import Gradcam
from tf_keras_vis.utils import (is_mixed_precision, listify, standardize, zoom_factor)

class Scorecam(Gradcam):
    def __call__(self,
                 score,
                 seed_input,
                 penultimate_layer=-1,
                 seek_penultimate_conv_layer=True,
                 activation_modifier=lambda cam: K.relu(cam),
                 expand_cam=True,
                 batch_size=32,
                 max_N=None,
                 training=False,
                 standardize_cam=True):
        """Generate score-weighted class activation maps (CAM)
            by using gradient-free visualization method.

            For details on Score-CAM, see the paper:
            [Score-CAM: Score-Weighted Visual Explanations for Convolutional Neural Networks ]
            (https://arxiv.org/pdf/1910.01279.pdf).

        # Arguments
            score: A score function. If the model has multiple outputs, you can use a different
                score on each output by passing a list of scores.
            seed_input: An N-dim Numpy array. If the model has multiple inputs,
                you have to pass a list of N-dim Numpy arrays.
            penultimate_layer: A number of integer or a tf.keras.layers.Layer object.
            seek_penultimate_conv_layer: True to seek the penultimate layter that is a subtype of
                `keras.layers.convolutional.Conv` class.
                If False, the penultimate layer is that was elected by penultimate_layer index.
            activation_modifier: A function to modify activations.
            expand_cam: True to expand cam to same as input image size.
                ![Note] Even if the model has multiple inputs, this function return only one cam
                value (That's, when `expand_cam` is True, multiple cam images are generated from
                a model that has multiple inputs).
            batch_size: Integer or None. Number of samples per batch.
                If unspecified, batch_size will default to 32.
            max_N: Integer or None. Setting None or under Zero is that we do NOT recommend,
                because it takes huge time. If not None and over Zero of Integer,
                run as Faster-ScoreCAM.
                Set larger number, need more time to visualize CAM but to be able to get
                clearer attention images.
                (see for details: https://github.com/tabayashi0117/Score-CAM#faster-score-cam)
            training: A bool whether the model's trainig-mode turn on or off.
            standardize_cam: A bool. If True(default), cam will be standardized.
        # Returns
            The heatmap image or a list of their images that indicate the `seed_input` regions
                whose change would most contribute  the score value,
        # Raises
            ValueError: In case of invalid arguments for `score`, or `penultimate_layer`.
        """

        # Preparing
        scores = self._get_scores_for_multiple_outputs(score)
        seed_inputs = self._get_seed_inputs_for_multiple_inputs(seed_input)
        penultimate_output_tensor = self._find_penultimate_output(penultimate_layer,
                                                                  seek_penultimate_conv_layer)
        # Processing score-cam
        penultimate_output = tf.keras.Model(inputs=self.model.inputs,
                                            outputs=penultimate_output_tensor)(seed_inputs,
                                                                               training=training)
        if is_mixed_precision(self.model):
            penultimate_output = tf.cast(penultimate_output, self.model.variable_dtype)

        # For efficiently visualizing, extract maps that has a large variance.
        # This excellent idea is devised by tabayashi0117.
        # (see for details: https://github.com/tabayashi0117/Score-CAM#faster-score-cam)
        if max_N is not None and max_N > -1:
            activation_map_std = tf.math.reduce_std(penultimate_output,
                                                    axis=tuple(
                                                        range(penultimate_output.ndim)[1:-1]),
                                                    keepdims=True)
            _, top_k_indices = tf.math.top_k(activation_map_std, max_N)
            top_k_indices, _ = tf.unique(tf.reshape(top_k_indices, (-1, )))
            penultimate_output = tf.gather(penultimate_output, top_k_indices, axis=-1)
        channels = penultimate_output.shape[-1]

        # Upsampling activation-maps
        input_shapes = [seed_input.shape for seed_input in seed_inputs]
        factors = (zoom_factor(penultimate_output.shape[:-1], input_shape[:-1])
                   for input_shape in input_shapes)
        upsampled_activation_maps = [zoom(penultimate_output, factor + (1, )) for factor in factors]
        map_shapes = [activation_map.shape for activation_map in upsampled_activation_maps]

        # Normalizing activation-maps
        min_activation_maps = (np.min(activation_map,
                                      axis=tuple(range(activation_map.ndim)[1:-1]),
                                      keepdims=True)
                               for activation_map in upsampled_activation_maps)
        max_activation_maps = (np.max(activation_map,
                                      axis=tuple(range(activation_map.ndim)[1:-1]),
                                      keepdims=True)
                               for activation_map in upsampled_activation_maps)
        normalized_activation_maps = (
            (activation_map - min_activation_map) /
            (max_activation_map - min_activation_map + K.epsilon())
            for activation_map, min_activation_map, max_activation_map in zip(
                upsampled_activation_maps, min_activation_maps, max_activation_maps))

        # Masking inputs
        input_tile_axes = ((map_shape[-1], ) + tuple(np.ones(len(input_shape), np.int))
                           for input_shape, map_shape in zip(input_shapes, map_shapes))
        mask_templates = (np.tile(seed_input, axes)
                          for seed_input, axes in zip(seed_inputs, input_tile_axes))
        map_transpose_axes = ((len(map_shape) - 1, ) + tuple(range(len(map_shape))[:-1])
                              for map_shape in map_shapes)
        masks = (np.transpose(activation_map,
                              transpose_axis) for activation_map, transpose_axis in zip(
                                  normalized_activation_maps, map_transpose_axes))
        map_tile_axes = (tuple(np.ones(len(map_shape), np.int)) + (input_shape[-1], )
                         for input_shape, map_shape in zip(input_shapes, map_shapes))
        masks = (np.tile(np.expand_dims(activation_map, axis=-1), tile_axis)
                 for activation_map, tile_axis in zip(masks, map_tile_axes))
        masked_seed_inputs = (mask_template * mask
                              for mask_template, mask in zip(mask_templates, masks))
        masked_seed_inputs = [
            np.reshape(masked_seed_input, (-1, ) + masked_seed_input.shape[2:])
            for masked_seed_input in masked_seed_inputs
        ]

        # Predicting masked seed-inputs
        preds = self.model.predict(masked_seed_inputs, batch_size=batch_size)
        preds = (np.reshape(prediction, (channels, -1, prediction.shape[-1]))
                 for prediction in listify(preds))

        # Calculating weights
        weights = ([score(p) for p in prediction] for score, prediction in zip(scores, preds))
        weights = (np.array(w, dtype=np.float32) for w in weights)
        weights = (np.reshape(w, (penultimate_output.shape[0], -1, channels)) for w in weights)
        weights = (np.mean(w, axis=1) for w in weights)
        weights = np.array(list(weights), dtype=np.float32)
        weights = np.sum(weights, axis=0)

        # Generate cam
        cam = K.batch_dot(penultimate_output, weights)
        if activation_modifier is not None:
            cam = activation_modifier(cam)

        if not expand_cam:
            if standardize_cam:
                cam = standardize(cam)
            return cam

        factors = (zoom_factor(cam.shape, X.shape) for X in seed_inputs)
        cam = [zoom(cam, factor) for factor in factors]
        if standardize_cam:
            cam = [standardize(x) for x in cam]
        if len(self.model.inputs) == 1 and not isinstance(seed_input, list):
            cam = cam[0]
        return cam

ScoreCAM = Scorecam


import matplotlib.pyplot as plt
import pandas as pd
import math
import scipy as sc
import numpy as np

data = pd.read_csv('ex1data1.csv', header = None)
x, y  = data[0], data[1]
fy = lambda x: 1.5*x - 10
x1 = sc.linspace(min(x),max(x),10)
y1 = list(map(fy,x1))
fig = plt.figure()

"""
Алгоритма градиентного спуска
"""
def gradient_descent(X, Y, koef, n):
    l = len(x)
    theta0, theta1 = 0, 0
    for i in range(n):
        sum1 = 0
        for i in range(l):
            sum1 += theta0 + theta1 * x[i] - y[i]
        res1 = theta0 - koef * (1 / l) * sum1

        sum2 = 0
        for i in range(l):
            sum2 += (theta0 + theta1 * x[i] - y[i]) * x[i]
        res2 = theta1 - koef * (1 / l) * sum2

        theta0, theta1 = res1, res2

    return theta0, theta1

def sq_error(X,Y,f_x=None):
    squared_error = [];
    for i in range(len(X)):
        squared_error.append((f_x(X[i])-Y[i])**2)
    return sum(squared_error)

print(sq_error(x,y,fy))

pll = plt.subplot(111)
pll.plot(x,y,'b*')
pll.plot(x1,y1,'g-')
plt.show()

"""
По градиентному спуску
"""
x2 = [1, 25]
y2 = [0, 0]
t0, t1 = gradient_descent(x, y, 0.01, len(x))
y2[0] = t0 + x2[0] * t1
y2[1] = t0 + x2[1] * t1
plt.plot(x2, y2, 'r')

"""
Метод polyfit
"""
x1,y1 = [0,22.5],[0,25]
numpy_x = np.array(x)
numpy_y = np.array(y)
numpy_t1, numpy_t0 = np.polyfit(numpy_x, numpy_y, 1)

num_y1 = [0, 0]
num_y1[0] = numpy_t0 + x1[0] * numpy_t1
num_y1[1] = numpy_t0 + x1[1] * numpy_t1
plt.plot(x1, num_y1, 'b')

print(numpy_t0, numpy_t1)

fig1 = plt.plot(x, y, 'g*')
fig2 = plt.plot(x1, y1, 'y')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models
import datetime
from django.utils.encoding import python_2_unicode_compatible
from django.utils import timezone
from django.contrib.auth.models import User

# Create your models here.

@python_2_unicode_compatible     # only if you need to support Python 2
class Post(models.Model):
    #many to one relationship (one user - many posts)
    shown_author = models.CharField(max_length=50, unique=False, default="")
    author = models.ForeignKey(User)
    title = models.CharField(max_length=120, unique=False, default="DefaultTitle")
    main_image = models.ImageField(
            blank=True,
            null=True,
            height_field = "height_field",
            width_field = "width_field")
    height_field = models.IntegerField(default=0)
    width_field = models.IntegerField(default=0)
    category = models.CharField(max_length=50, default="All")
    text = models.TextField(default="")
    views = models.IntegerField(default=0)
    coments = models.IntegerField(default=0)
    likes = models.IntegerField(default=0)
    can_be_published = models.BooleanField(default=True)
    published_date = models.DateTimeField(
            blank=True, null=True,
            auto_now=False, auto_now_add=False)
    creation_date = models.DateTimeField(
            blank=True, null=True,
            auto_now=False, auto_now_add=False)
    tags = models.CharField(max_length=600, default="")

    def publish(self):
        self.published_date = timezone.now()
        self.save()

    def __str__(self):
        return self.title

class MainCounter(models.Model):
    date = models.DateTimeField(auto_now=True)
    views = models.IntegerField(default=0)

class TestGridTable(models.Model):
    date_field = models.DateTimeField(
            blank=True, null=True,
            auto_now=False, auto_now_add=False)
    bool_field = models.BooleanField(default=True)
    int_field = models.IntegerField(default=0)
    char_field = models.CharField(max_length=50, default="All")

#many to one (to do)
class TablesInfo(models.Model):
    table_name = models.CharField(max_length=50, default="")
    date_field = models.DateTimeField(
            blank=True, null=True,
            auto_now=False, auto_now_add=False)
    pass

    '''
    to do
    def get_absolute_url(self):
        return "/post/%s/" %(self.id)
     '''

    #class Meta:
        #ordering = ['-published_date']

'''
#to do
#one post - many users
class Comment(models.Model):
    post = models.ForeignKey(Post, on_delete=models.CASCADE)
    #to do

'''


# ========================================
# DEFINING DYNAMIC PARAMS
# ========================================
# @date created: 20 May 2020
# @authors: Victor
# The file is used for defining the dynamic params.

import json

# Constants which provides default values for dynamic parameters
MAX_JAILED_TERM = ('max_jailed_term', 30)
GOVERNMENT_LEGITIMACY = ('government_legitimacy', 0.5)
MOVEMENT = ('movement', True)
FRAME_INTERVAL = ('frame_interval', 0.1)
INITIAL_COP_DENSITYS = ('initial_cop_density', 0.04)
INITIAL_AGENT_DENSITYS = ('initial_agent_density', 0.7)
VISIONS = ('vision', 7.0)

# Include all parameters here
DYNAMIC_PARAMETERS = [
    MAX_JAILED_TERM,
    GOVERNMENT_LEGITIMACY,
    MOVEMENT,
    FRAME_INTERVAL,
    INITIAL_COP_DENSITYS,
    INITIAL_AGENT_DENSITYS,
    VISIONS
]

class Dynamic_Params:

    def __init__(self, path):
        self.path = path

    # Read and return the params dict from the path
    def read_params(self, path) -> dict:
        with open(path, 'r') as file:
            params = json.load(file)
            return params

# s = Dynamic_Params('dynamic_params.json')
# print(s.read_params('dynamic_params.json'))