#!/usr/bin/env python
import cv2
import numpy as np

distCoeffs = np.zeros((5), dtype=np.float)
print(distCoeffs)

def do_process():
    src = cv2.imread('yolo3.png')
    print('shape: ', src.shape)
    height,width,channel = src.shape
    center = np.array([int(width/2.0), int(height/2.0)])
    print(center)
    # find red markers in horizontal band and vertical band
    # horizontal band
    # the start point is left-up corner of cropped image
    hori_left_start = int(center[0]-height/5.0)
    verti_up_start = int(center[0]-width/5.0)
    hori_left_band = src[int(center[0]-height/5.0):int(center[0]+height/5.0), 10:int(width/2.0)]
    hori_right_band = src[int(center[0]-height/5.0):int(center[0]+height/5.0), int(width/2.0):width-10]
    verti_up_band = src[10:int(height/2.0) , int(center[0]-width/5.0):int(center[0]+width/5.0)]
    verti_down_band = src[int(height/2.0):height-10 , int(center[0]-width/5.0):int(center[0]+width/5.0)]
    # cv2.imshow('hori_left_band',hori_left_band)
    # cv2.imshow('hori_right_band',hori_right_band)
    # cv2.imshow('verti_up_band',verti_up_band)
    # cv2.imshow('verti_down_band',verti_down_band)

    hori_left_hsv = cv2.cvtColor(hori_left_band,cv2.COLOR_BGR2HSV)
    hori_right_hsv = cv2.cvtColor(hori_right_band,cv2.COLOR_BGR2HSV)
    verti_up_hsv = cv2.cvtColor(verti_up_band,cv2.COLOR_BGR2HSV)
    verti_down_hsv = cv2.cvtColor(verti_down_band,cv2.COLOR_BGR2HSV)
    hori_lower_hsv = np.array([0,50,50])
    hori_upper_hsv = np.array([30,255,255])
    verti_lower_hsv = np.array([0,50,50])
    verti_upper_hsv = np.array([100,255,255])

    hori_left_mask = cv2.inRange(hori_left_hsv, hori_lower_hsv, hori_upper_hsv)
    hori_right_mask = cv2.inRange(hori_right_hsv, hori_lower_hsv, hori_upper_hsv)
    verti_up_mask = cv2.inRange(verti_up_hsv, verti_lower_hsv, verti_upper_hsv)
    verti_down_mask = cv2.inRange(verti_down_hsv, verti_lower_hsv, verti_upper_hsv)
    # cv2.imshow('hori_left_mask', hori_left_mask)
    # cv2.imshow('hori_right_mask', hori_right_mask)
    # cv2.imshow('verti_up_mask', verti_up_mask)
    # cv2.imshow('verti_down_mask', verti_down_mask)

    #(B,G,R)=cv2.split(src)
    #sub_BR = cv2.subtract(B,R)
    #cv2.imshow('R',R)
    #cv2.imshow('G',G)
    #cv2.imshow('B',B) # use blue
    #cv2.imshow('sub_BR',sub_BR)
    # B = cv2.threshold(B,150,255,cv2.THRESH_BINARY)[1]
    # B = cv2.threshold(B,150,255,cv2.THRESH_BINARY)[1]
    # B = cv2.threshold(B,120,255,cv2.THRESH_BINARY)[1] # good external contours
    # B = cv2.threshold(B,100,255,cv2.THRESH_BINARY)[1] # good 1 internal circles
    #B = cv2.threshold(B,80,255,cv2.THRESH_BINARY)[1] # good 2 internal circles
    #B = cv2.threshold(B,70,255,cv2.THRESH_BINARY)[1] # not see internal circles
    #cv2.imshow('B_threshold',B)
    #contours, hierarchy = cv2.findContours(B,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
    #contours, hierarchy = cv2.findContours(B,cv2.RETR_CCOMP,cv2.CHAIN_APPROX_NONE)
    # erode and dilate
    erode_kernel = np.ones((2, 2), np.uint8)
    hori_left_erosion = cv2.erode(hori_left_mask, erode_kernel, iterations=1)
    hori_right_erosion = cv2.erode(hori_right_mask, erode_kernel, iterations=1)
    verti_up_erosion = cv2.erode(verti_up_mask, erode_kernel, iterations=1)
    verti_down_erosion = cv2.erode(verti_down_mask, erode_kernel, iterations=1)
    #cv2.imshow('hori_erosion', hori_erosion)
    #cv2.imshow('verti_erosion', verti_erosion)

    dilate_kernel = np.ones((5, 5), np.uint8)
    hori_left_dilate = cv2.dilate(hori_left_erosion, dilate_kernel, iterations=1)
    hori_right_dilate = cv2.dilate(hori_right_erosion, dilate_kernel, iterations=1)
    verti_up_dilate = cv2.dilate(verti_up_erosion, dilate_kernel, iterations=1)
    verti_down_dilate = cv2.dilate(verti_down_erosion, dilate_kernel, iterations=1)
    # cv2.imshow('hori_left_dilate', hori_left_dilate)
    # cv2.imshow('hori_right_dilate', hori_right_dilate)
    # cv2.imshow('verti_up_dilate', verti_up_dilate)
    # cv2.imshow('verti_down_dilate', verti_down_dilate)

    # find contours
    hori_left_contours,hereachy= cv2.findContours(hori_left_dilate,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
    hori_right_contours,hereachy= cv2.findContours(hori_right_dilate,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
    print(len(hori_left_contours))
    print(len(hori_right_contours))

    cv2.drawContours(hori_left_band,hori_left_contours,-1,(0,255,255), thickness=1 )
    cv2.imshow('hori_left_contours',hori_left_band)
    cv2.drawContours(hori_right_band,hori_right_contours,-1,(0,255,255), thickness=1 )
    cv2.imshow('hori_right_contours',hori_right_band)

    verti_up_contours,hereachy = cv2.findContours(verti_up_dilate,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
    verti_down_contours,hereachy = cv2.findContours(verti_down_dilate,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
    print(len(verti_up_contours))
    print(len(verti_down_contours))
    cv2.drawContours(verti_up_band,verti_up_contours,-1,(0,255,255), thickness=1 )
    cv2.imshow('verti_up_contours',verti_up_band)
    cv2.drawContours(verti_down_band,verti_down_contours,-1,(0,255,255), thickness=1 )
    cv2.imshow('verti_down_contours',verti_down_band)

    #for i in hori_contours:

    cv2.waitKey(0)

if __name__ =="__main__":
    do_process()


from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib.messages.views import SuccessMessageMixin
from django.http import HttpResponseRedirect, JsonResponse
from django.shortcuts import get_object_or_404
from django.urls import reverse_lazy, reverse
from django.utils.decorators import method_decorator
from django.views import View
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import ListView, CreateView, DeleteView, UpdateView, TemplateView
from django.utils.translation import ugettext as __

from blogs.forms import BlogPostForm, BlogCategoryForm
from blogs.models import BlogPost, BlogCategory, BlogImage, Template, SiteBlogTemplate
from wresume.utils import SiteAwareView

class BlogPostsView(LoginRequiredMixin, SiteAwareView, ListView):
    template_name = 'blogs/manager/post-list.html'
    extra_context = {'is_blog_posts': True, 'nav_section': 'blog'}
    context_object_name = 'posts'
    paginate_by = 10

    def get_queryset(self):
        return BlogPost.objects.filter(site=self.get_site())

class BlogCategoriesView(LoginRequiredMixin, SiteAwareView, ListView):
    template_name = 'blogs/manager/categories.html'
    extra_context = {'is_categories': True, 'nav_section': 'blog'}
    context_object_name = 'categories'

    def get_queryset(self):
        return BlogCategory.objects.filter(site=self.get_site())

class BlogCategoryDeleteView(LoginRequiredMixin, SuccessMessageMixin, View):
    success_url = reverse_lazy('blogs_public:categories')

    def get_success_message(self, cleaned_data):
        return __('{} has been deleted successfully'.format(cleaned_data['category']))

    def get(self, request, **kwargs):
        cat = get_object_or_404(BlogCategory, pk=kwargs.get('pk'))
        cat.delete()
        messages.success(request, self.get_success_message({'category': cat.category}))
        return HttpResponseRedirect(self.success_url)

class WritePostView(LoginRequiredMixin, SuccessMessageMixin, SiteAwareView, CreateView):
    template_name = 'blogs/manager/write.html'
    extra_context = {'is_write': True, 'nav_section': 'blog'}
    form_class = BlogPostForm
    success_url = reverse_lazy('blogs_public:list')

    def get_form_kwargs(self):
        kwargs = super(WritePostView, self).get_form_kwargs()
        kwargs['site'] = self.get_site()
        return kwargs

    def get_initial(self):
        init = super(WritePostView, self).get_initial()
        init['author'] = self.request.user.id
        init['site'] = self.get_site().id
        return init

    def get_success_message(self, cleaned_data):
        return __('{} has been created'.format(cleaned_data['title']))

    def get_queryset(self):
        return BlogPost.objects.filter(site=self.get_site())

class EditPostView(WritePostView, UpdateView):
    extra_context = {'is_write': True, 'nav_section': 'blog', 'page_title': 'Update Post'}
    success_url = reverse_lazy('blogs_public:list')

    def get_success_message(self, cleaned_data):
        return __('{} has been edited successfully'.format(cleaned_data['title']))

class DeletePostView(EditPostView):
    def get(self, request, *args, **kwargs):
        self.get_object().delete()
        messages.success(request, 'Post has been removed successfully')
        return HttpResponseRedirect(request.GET.get('next', reverse('blogs_public:list')))

class CreateCategoryView(LoginRequiredMixin, SuccessMessageMixin, SiteAwareView, CreateView):
    template_name = 'blogs/manager/create-category.html'
    extra_context = {'is_create_category': True, 'nav_section': 'blog'}
    form_class = BlogCategoryForm
    success_url = reverse_lazy('blogs_public:create-categories')

    def get_initial(self):
        init = super(CreateCategoryView, self).get_initial()
        init['site'] = self.get_site().id
        return init

    def get_success_message(self, cleaned_data):
        return __('{} has been created'.format(cleaned_data['category']))

    def get_queryset(self):
        return BlogCategory.objects.filter(site=self.get_site())

class BlogTemplatesView(LoginRequiredMixin, SuccessMessageMixin, SiteAwareView, ListView):
    extra_context = {'is_templates_list': True, 'nav_section': 'blog'}
    queryset = Template.objects.all()
    template_name = 'blogs/manager/templates.html'
    context_object_name = 'templates'

class ActivateBlogTemplateView(LoginRequiredMixin, SuccessMessageMixin, SiteAwareView, View):
    def get(self, request, pk, *args, **kwargs):
        SiteBlogTemplate.objects.filter(client=self.get_site()).delete()
        SiteBlogTemplate.objects.create(template_id=pk, client=self.get_site())
        template = get_object_or_404(Template, pk=pk)
        messages.success(request, __('%(temp)s is now active for %(site)s') % {'temp': template.name,
                                                                               'site': self.get_site().domain_url})
        return HttpResponseRedirect(request.GET.get('next', reverse('blogs_public:template-list')))

class DeActivateBlogTemplateView(LoginRequiredMixin, SuccessMessageMixin, SiteAwareView, View):
    def get(self, request, pk, *args, **kwargs):
        SiteBlogTemplate.objects.filter(client=self.get_site()).delete()
        template = get_object_or_404(Template, pk=pk)
        messages.success(
            request, __('%(temp)s has been deactivated for %(site)s') % {'temp': template.name,
                                                                         'site': self.get_site().domain_url}
        )
        return HttpResponseRedirect(request.GET.get('next', reverse('blogs_public:template-list')))

@method_decorator(csrf_exempt, name='dispatch')
class HandleEditorUpload(View):
    def post(self, request, *args, **kwargs):
        file = request.FILES.getlist('file')[0]
        bi = BlogImage.objects.create(image=file)
        return JsonResponse({'location': bi.image.url})


import os
import tempfile
from pathlib import Path

import pytest
from cwl_utils import parser_v1_2 as cwlgen

from renku.cli import cli
from renku.core.metadata.database import Database
from renku.core.models.jsonld import write_yaml
from renku.core.plugins.provider import available_workflow_providers
from tests.utils import format_result_exception

def test_workflow_compose(runner, project, run_shell, client):
    """Test renku workflow compose."""
    # Run a shell command with pipe.
    output = run_shell('renku run --name run1 -- echo "a" > output1')

    # Assert expected empty stdout.
    assert b"" == output[0]
    # Assert not allocated stderr.
    assert output[1] is None

    # Run a shell command with pipe.
    output = run_shell("renku run --name run2 -- cp output1 output2")

    # Assert expected empty stdout.
    assert b"" == output[0]
    # Assert not allocated stderr.
    assert output[1] is None

    result = runner.invoke(
        cli,
        [
            "workflow",
            "compose",
            "--map",
            "input_str=@step1.@param1",
            "--map",
            "output_file=run2.@output1",
            "--link",
            "@step1.@output1=@step2.@input1",
            "--set",
            "input_str=b",
            "--set",
            "output_file=other_output.csv",
            "-p",
            "input_str=the input string for the workflow",
            "-p",
            "output_file=the final output file produced",
            "composite_workflow",
            "run1",
            "run2",
        ],
    )

    assert 0 == result.exit_code, format_result_exception(result)

    database = Database.from_path(client.database_path)

    composite_plan = database["plans-by-name"]["composite_workflow"]

    assert composite_plan

    assert len(composite_plan.plans) == 2
    assert len(composite_plan.mappings) == 2

    assert composite_plan.mappings[0].name == "input_str"
    assert composite_plan.mappings[0].default_value == "b"
    assert composite_plan.mappings[0].description == "the input string for the workflow"

    assert composite_plan.mappings[1].name == "output_file"
    assert composite_plan.mappings[1].default_value == "other_output.csv"
    assert composite_plan.mappings[1].description == "the final output file produced"

def test_workflow_show(runner, project, run_shell, client):
    """Test renku workflow show."""
    # Run a shell command with pipe.
    output = run_shell('renku run --name run1 --description "my workflow" --success-code 0 -- echo "a" > output1')

    # Assert expected empty stdout.
    assert b"" == output[0]
    # Assert not allocated stderr.
    assert output[1] is None

    # Run a shell command with pipe.
    output = run_shell("renku run --name run2 -- cp output1 output2")

    # Assert expected empty stdout.
    assert b"" == output[0]
    # Assert not allocated stderr.
    assert output[1] is None

    result = runner.invoke(cli, ["workflow", "show", "run1"])
    assert 0 == result.exit_code, format_result_exception(result)
    assert "run1" in result.output
    assert "output1" in result.output
    assert "my workflow" in result.output

    result = runner.invoke(
        cli,
        [
            "workflow",
            "compose",
            "--description",
            "My composite workflow",
            "--map",
            "input_str=@step1.@param1",
            "--map",
            "output_file=run2.@output1",
            "--link",
            "@step1.@output1=@step2.@input1",
            "--set",
            "input_str=b",
            "--set",
            "output_file=other_output.csv",
            "-p",
            "input_str=the input string for the workflow",
            "-p",
            "output_file=the final output file produced",
            "composite_workflow",
            "run1",
            "run2",
        ],
    )

    assert 0 == result.exit_code, format_result_exception(result)

    result = runner.invoke(cli, ["workflow", "show", "composite_workflow"])

    assert 0 == result.exit_code, format_result_exception(result)
    assert "composite_workflow" in result.output
    assert "input_str" in result.output
    assert "output_file" in result.output
    assert "Links:" in result.output
    assert "Mappings:" in result.output
    assert "My composite workflow" in result.output

def test_workflow_remove_command(runner, project):
    """test workflow remove with builder."""
    workflow_name = "test_workflow"

    result = runner.invoke(cli, ["workflow", "remove", workflow_name])
    assert 2 == result.exit_code

    result = runner.invoke(cli, ["run", "--success-code", "0", "--no-output", "--name", workflow_name, "echo", "foo"])
    assert 0 == result.exit_code

    result = runner.invoke(cli, ["workflow", "remove", "--force", workflow_name])
    assert 0 == result.exit_code

def test_workflow_export_command(runner, project):
    """test workflow export with builder."""
    result = runner.invoke(cli, ["run", "--success-code", "0", "--no-output", "--name", "run1", "touch", "data.csv"])
    assert 0 == result.exit_code

    result = runner.invoke(cli, ["workflow", "export", "run1", "-o", "run1.cwl"])
    assert 0 == result.exit_code
    assert Path("run1.cwl").exists()

    workflow = cwlgen.load_document("run1.cwl")
    assert workflow.baseCommand[0] == "touch"
    assert len(workflow.inputs) == 3
    assert len(workflow.outputs) == 1

def test_workflow_edit(runner, client, run_shell):
    """Test naming of CWL tools and workflows."""

    def _get_plan_id(output):
        return output.split("\n")[0].split(":")[1].strip()

    workflow_name = "run1"
    result = runner.invoke(cli, ["run", "--name", workflow_name, "touch", "data.txt"])
    assert 0 == result.exit_code, format_result_exception(result)

    cmd = ["workflow", "edit", workflow_name, "--name", "first"]
    result = runner.invoke(cli, cmd)
    assert 0 == result.exit_code, format_result_exception(result)

    database = Database.from_path(client.database_path)

    test_plan = database["plans-by-name"][workflow_name]
    first_plan = database["plans-by-name"]["first"]

    assert first_plan
    assert first_plan.name == "first"
    assert first_plan.derived_from == test_plan.id

    cmd = ["workflow", "edit", workflow_name, "--description", "Test workflow"]
    result = runner.invoke(cli, cmd)
    assert 0 == result.exit_code, format_result_exception(result)

    database = Database.from_path(client.database_path)
    first_plan = database["plans"][_get_plan_id(result.stdout)]
    assert first_plan.description == "Test workflow"

    # edit parameter
    cmd = ["workflow", "edit", workflow_name, "--rename-param", "param1=param2"]
    result = runner.invoke(cli, cmd)
    assert 0 != result.exit_code, format_result_exception(result)

    cmd = ["workflow", "edit", workflow_name, "--set", "param1=0"]
    result = runner.invoke(cli, cmd)
    assert 0 == result.exit_code, format_result_exception(result)
    edited_plan_id = _get_plan_id(result.output)

    database = Database.from_path(client.database_path)
    renamed_param_plan = database["plans"][_get_plan_id(result.output)]
    assert len(renamed_param_plan.parameters) > 0

    cmd = ["workflow", "edit", workflow_name, "--rename-param", "param1=param2"]
    result = runner.invoke(cli, cmd)
    assert 0 == result.exit_code, format_result_exception(result)

    database = Database.from_path(client.database_path)
    renamed_param_plan = database["plans"][_get_plan_id(result.output)]
    parameter_names = list(map(lambda x: x.name, renamed_param_plan.parameters))
    assert len(parameter_names) > 0
    assert "param1" not in parameter_names
    assert "param2" in parameter_names
    param2_plan_id = _get_plan_id(result.stdout)

    # edit parameter description
    cmd = ["workflow", "edit", param2_plan_id, "--describe-param", "param1=Test"]
    result = runner.invoke(cli, cmd)
    assert 0 != result.exit_code, format_result_exception(result)

    cmd = ["workflow", "edit", param2_plan_id, "--describe-param", 'param2="Test parameter"']
    result = runner.invoke(cli, cmd)
    assert 0 == result.exit_code, format_result_exception(result)

    database = Database.from_path(client.database_path)
    renamed_param_plan = database["plans"][_get_plan_id(result.output)]
    assert "Test parameter" == renamed_param_plan.parameters[0].description

    # edit parameter mapping
    cmd = ["run", "--name", "run2", "cp", "data.txt", "data2.txt"]
    result = runner.invoke(cli, cmd)
    assert 0 == result.exit_code, format_result_exception(result)

    result = runner.invoke(
        cli,
        [
            "workflow",
            "compose",
            "--description",
            "My composite workflow",
            "--map",
            "input_str=@step1.@param1",
            "--map",
            "output_file=run2.@output1",
            "--link",
            "@step1.@output1=@step2.@input1",
            "--set",
            "input_str=b",
            "--set",
            "output_file=other_output.csv",
            "-p",
            "input_str=the input string for the workflow",
            "-p",
            "output_file=the final output file produced",
            "composite_workflow",
            edited_plan_id,
            "run2",
        ],
    )

    assert 0 == result.exit_code, format_result_exception(result)

    # remove mapping
    cmd = ["workflow", "edit", "composite_workflow", "--map", "output_file="]
    result = runner.invoke(cli, cmd)
    assert 0 == result.exit_code, format_result_exception(result)

    database = Database.from_path(client.database_path)
    edited_composite_plan = database["plans"][_get_plan_id(result.output)]
    assert len(edited_composite_plan.mappings) == 1
    assert edited_composite_plan.mappings[0].mapped_parameters[0].name == "param1"

def test_workflow_show_outputs_with_directory(runner, client, run):
    """Output files in directory are not shown as separate outputs."""
    base_sh = ["bash", "-c", 'DIR="$0"; mkdir -p "$DIR"; ' 'for x in "$@"; do touch "$DIR/$x"; done']

    assert 0 == run(args=["run"] + base_sh + ["output", "foo", "bar"])
    assert (client.path / "output" / "foo").exists()
    assert (client.path / "output" / "bar").exists()

    cmd = ["workflow", "outputs"]
    result = runner.invoke(cli, cmd)
    assert 0 == result.exit_code, format_result_exception(result)
    assert {"output"} == set(result.output.strip().split("\n"))

    result = runner.invoke(cli, cmd + ["output"])
    assert 0 == result.exit_code, format_result_exception(result)
    assert {"output"} == set(result.output.strip().split("\n"))

    result = runner.invoke(cli, cmd + ["output/foo"])
    assert 0 == result.exit_code, format_result_exception(result)
    assert {"output"} == set(result.output.strip().split("\n"))

    result = runner.invoke(cli, cmd + ["output/foo", "output/bar"])
    assert 0 == result.exit_code, format_result_exception(result)
    assert {"output"} == set(result.output.strip().split("\n"))

@pytest.mark.parametrize("provider", available_workflow_providers())
@pytest.mark.parametrize("yaml", [False, True])
@pytest.mark.parametrize(
    "workflows, parameters",
    [
        ([("run", 'echo "a" > output1')], {}),
        ([("run", 'echo "a" > output1')], {"run": {"outputs": ["replaced"]}}),
        ([("run", 'echo "a" > output1')], {"run": {"parameters": ["foo"], "outputs": ["bar"]}}),
        (
            [("run1", "touch data.csv"), ("run2", "wc data.csv > output")],
            {"run1": {"outputs": ["foo"]}, "run2": {"inputs": ["foo"], "outputs": ["bar"]}},
        ),
    ],
)
def test_workflow_execute_command(runner, run_shell, project, capsys, client, provider, yaml, workflows, parameters):
    """test workflow execute."""

    for wf in workflows:
        output = run_shell(f"renku run --name {wf[0]} -- {wf[1]}")
        # Assert expected empty stdout.
        assert b"" == output[0]
        # Assert not allocated stderr.
        assert output[1] is None

    def _execute(args):
        with capsys.disabled():
            try:
                cli.main(
                    args=args,
                    prog_name=runner.get_default_prog_name(cli),
                )
            except SystemExit as e:
                assert e.code in {None, 0}

    if not parameters:
        for wf in workflows:
            execute_cmd = ["workflow", "execute", "-p", provider, wf[0]]
            _execute(execute_cmd)
    else:
        database = Database.from_path(client.database_path)
        for wf in workflows:
            if wf[0] in parameters:
                plan = database["plans-by-name"][wf[0]]
                execute_cmd = ["workflow", "execute", "-p", provider]

                overrides = dict()
                for k, values in parameters[wf[0]].items():
                    for i, v in enumerate(values):
                        overrides[getattr(plan, k)[i].name] = v

                if yaml:
                    fd, values_path = tempfile.mkstemp()
                    os.close(fd)
                    write_yaml(values_path, overrides)
                    execute_cmd += ["--values", values_path]
                else:
                    [execute_cmd.extend(["--set", f"{k}={v}"]) for k, v in overrides.items()]

                execute_cmd.append(wf[0])

                _execute(execute_cmd)

                # check whether parameters setting was effective
                if "outputs" in parameters[wf[0]]:
                    for o in parameters[wf[0]]["outputs"]:
                        assert Path(o).resolve().exists()


import matplotlib.pyplot as plt
import numpy as np
from math import fabs, exp, fmod

class Modeller():
	def __init__(self, data):
		self.data = data

		self.a1 = self.data.get("a1")
		self.b1 = self.data.get("b1")
		self.c1 = self.data.get("c1")
		self.m1 = self.data.get("m1")

		self.a2 = self.data.get("a2")
		self.b2 = self.data.get("b2")
		self.c2 = self.data.get("c2")
		self.m2 = self.data.get("m2")

		self.alpha0 = self.data.get("alpha0")
		self.alphaN = self.data.get("alphaN")

		self.l = self.data.get("l")
		self.T0 = self.data.get("T0")
		self.R = self.data.get("R")

		self.F0_val = self.data.get("F0")
		self.Fmax = self.data.get("Fmax")
		self.tmax = self.data.get("tmax")

		self.period = self.data.get("period")
		self.tu = self.data.get("tu")

		self.h = self.data.get("h")
		self.t = self.data.get("t")

		self.d = (self.alphaN * self.l) / (self.alphaN - self.alpha0)
		self.c_koef = - self.alpha0 * self.d

		self.eps = 1e-4

	def F0(self, T):
		return self.F0_val
		#return 45
		#return (self.Fmax / self.tmax) * T * exp(-(T / self.tmax - 1))
		if T % self.period <= self.tu:
			return (self.Fmax / self.tmax) * T * exp(-(T / self.tmax - 1))
		else:
			return 0
		#nu = 1 / self.period
		#newT = round(fmod(T, nu) * 100) / 100.0
		#if (nu <= 0.01):
		#	newT = 10
		#	return (self.Fmax / self.tmax) * newT * exp(-(newT / self.tmax - 1))
		#return (self.Fmax / self.tmax) * T * exp(-(T / self.tmax - 1))

	def c(self, T):
		res = self.a2 + self.b2 * (T ** self.m2) - (self.c2 / (T ** 2))
		return res
		#return 0

	def f_plus_half(self, x, h, func):
		res = (func(x) + func(x + h)) / 2
		return res

	def f_minus_half(self, x, h, func):
		res = (func(x) + func(x - h)) / 2
		return res

	def old_k(self, x):
		res = self.a1 / (x - self.b1)
		return res

	def k(self, T):
		res = self.old_k(self.x)
		#res = self.a1 * (self.b1 + self.c1 * (T ** self.m1))
		return res

	def alpha(self, x):
		return self.c_koef / (x - self.d)

	def p(self, x):
		return 2 * self.alpha(x) / self.R

	def f(self, x):
		return 2 * self.alpha(x) * self.T0 / self.R

	def __left_boundary_condition(self, T, time):
		h8 = self.h / 8
		h4 = self.h / 4
		h2 = self.h / 2
		c_p12 = self.f_plus_half(T[0], self.t, self.c)
		chi_p12 = self.f_plus_half(T[0], self.t, self.k)
		t_over_h = self.t / self.h

		K0 = h8 * c_p12 + \
			 h4 * self.c(T[0]) + chi_p12 * t_over_h + \
			 self.t * h8 * self.p(h2) + (self.t * self.h) / 4 * self.p(0)

		M0 = h8 * c_p12 - chi_p12 * t_over_h + \
			 self.t * h8 * self.p(h2)

		P0 = h8 * c_p12 * (T[0] + T[1]) + \
			 h4 * self.c(T[0]) * T[0] + self.F0(time) * self.t + \
			 self.t * h8 * (3 * self.f(0) + self.f(self.h))
			 #self.t * h4 * (self.f(0) + self.f_plus_half(T[0], self.t, self.f)) # ?
		return K0, M0, P0

	def __right_boundary_condition(self, T):
		h8 = self.h / 8
		h4 = self.h / 4
		h2 = self.h / 2
		c_m12 = self.f_minus_half(T[-1], self.t, self.c)
		chi_m12 = self.f_minus_half(T[-1], self.t, self.k)

		h8_cm12 = h8 * c_m12

		KN = h4 * self.c(T[-1]) + h8_cm12 + \
			self.t * self.alphaN + self.t * chi_m12 / self.h + \
			h4 * self.t * self.p(self.l) + h8 * self.t * self.p(self.l - h2)

		MN = h8_cm12 - \
	 		 self.t * chi_m12 / self.h + \
	 		 h8 * self.p(self.l - h2)

		PN = h4 * self.c(T[-1]) * T[-1] + h8_cm12 * T[-1] +\
	 		 h8_cm12 * T[-2] + \
	 		 self.t * self.alphaN * self.T0 + \
	 		 h4 * self.t * (self.f(self.l) + self.f(self.l - h2))

		return KN, MN, PN

	def A(self, T):
		return self.t / self.h * self.f_minus_half(T, self.t, self.k)

	def D(self, T):
		return self.t / self.h * self.f_plus_half(T, self.t, self.k)

	def B(self, x, T):
		return self.A(T) + self.D(T) + self.c(T) * self.h + \
			 self.p(x) * self.h * self.t

	def F(self, x, T):
		return self.f(x) * self.h * self.t + self.c(T) * T * self.h

	def progon(self, T, K0, M0, P0, KN, MN, PN):
		epsilon = [0, - M0 / K0]
		eta = [0, P0 / K0]

		x = self.h
		n = 1
		while x + self.h < self.l:
			An = self.A(T[n])
			Bn = self.B(x, T[n])

			newEps = self.D(T[n]) / (Bn - An * epsilon[n])
			epsilon.append(newEps)

			newEta = (self.F(x, T[n]) + An * eta[n]) / (Bn - An * epsilon[n])
			eta.append(newEta)

			x += self.h
			n += 1

		T = [0] * (n + 1)
		T[n] = (PN - MN * eta[n]) / (KN + MN * epsilon[n])

		for i in range(n-1, -1, -1):
			T[i] = epsilon[i + 1] * T[i + 1] + eta[i+1]

		return T

	def check_iter(self, T, T_next):
		# если нагрев замедлилися и изменение температуры за
		# шаг по времени достаточно мало
		max = fabs((T[0] - T_next[0]) / T_next[0])
		for i, j in zip(T, T_next):
			d = fabs(i - j) / j
			if d > max:
				max = d
		return max < 1

	def check_epsilon(self, T, T_next):
		# если максимально изменившийся элемент
		# изменился меньше, чем на eps
		for i, j in zip(T, T_next):

			if fabs((i - j) / j) > self.eps:
				return True
		return False

	def __iterate(self):
		res = []
		n = int(self.l / self.h)
		print(n)
		T = [self.T0] * (n + 1)
		T_new = [0] * (n + 1)
		ti = 0
		res.append(T)
		while True:
			tmp = T
			self.x = self.h
			ti += self.t
			while True:
				K0, M0, P0 = self.__left_boundary_condition(tmp, ti)
				KN, MN, PN = self.__right_boundary_condition(tmp)
				T_new = self.progon(tmp, K0, M0, P0, KN, MN, PN)
				self.x += self.h

				if self.check_iter(tmp, T_new):
					break
				tmp = T_new

			res.append(T_new)
			#ti += self.t
			if not self.check_epsilon(T, T_new):
				break
			T = T_new
		return res, ti

	def compute(self):
		periods = [1]
		for curp in periods:
			result = []
			resulti = []
			self.period = curp
			res, ti = self.__iterate()
			print(ti)
			te = []

			i = 0
			while i < ti:
				te.append(i)
				i += self.t

			i = 0
			xfix = [temp[int(i / self.h)] for temp in res]
			result.append(xfix[:-1])
			resulti.append(te)
		i = 0
		for res, teres in zip(result, resulti):
			#l = 'nu = ' + str(1/periods[i])
			plt.plot(teres, res)
			i += 1

		plt.xlabel("t, sec")
		plt.ylabel("T, K")
		plt.grid()
		plt.legend()
		plt.show()

		'''
		Fmaxes = [10, 15, 25, 35, 45, 50, 75, 100, 125, 150]
		result = []
		resulti = []
		for curFmax in Fmaxes:
			self.Fmax = curFmax
			res, ti = self.__iterate()
			print(ti)
			te = []

			i = 0
			while i < ti:
				te.append(i)
				i += self.t

			i = 0
			xfix = [temp[int(i / self.h)] for temp in res]
			result.append(xfix[:-1])
			resulti.append(te)
		i = 0
		for res, teres in zip(result, resulti):
			l = 'Fmax = ' + str(Fmaxes[i])
			plt.plot(teres, res, label=l)
			i += 1

		plt.xlabel("t, sec")
		plt.ylabel("T, K")
		plt.grid()
		plt.legend()
		plt.show()
		'''
		'''
		a2s = [2.0493, 3, 4, 5]
		result = []
		resulti = []
		for cura in a2s:
			self.a2 = cura
			res, ti = self.__iterate()
			print(ti)
			te = []

			i = 0
			while i < ti:
				te.append(i)
				i += self.t

			i = 0
			xfix = [temp[int(i / self.h)] for temp in res]
			result.append(xfix[:-1])
			resulti.append(te)
		i = 0
		for res, teres in zip(result, resulti):
			l = 'a2 = ' + str(a2s[i])
			plt.plot(teres, res, label=l)
			i += 1

			#xes = [i for i in np.arange(0, self.l, self.h)]
			#ts = [i for i in range(0, int(ti), int(self.t))]
			#ts = [i for i in np.arange(0, ti, self.t)]

			#s = 0
			#p = [j[int(s / self.h)] for j in res]
			#plt.plot(ts, p)
			#s = 0
			#while s < self.l / 3:
			#	p = [j[int(s / self.h)] for j in res]
			#	plt.plot(ts, p[:-1])
			#	s += 0.1

		plt.xlabel("t, sec")
		plt.ylabel("T, K")
		plt.grid()
		plt.legend()
		plt.show()
		'''
		'''
		hes = [0.3, 0.2, 0.1, 0.01]
		#taus = [2, 1, 0.1, 0.05]

		result = []

		for hi in hes:
			self.h = hi
			res, ti = self.__iterate()
			print('h = ', self.h, 'tau = ', self.t)
			i = 0
			xfix = [temp[int(i / self.h)] for temp in res]
			result.append(xfix)

		#print(len(hes))
		#print(len(result))
		count = 0
		print('{:^10}|{:^10}|{:^10}|{:^10}|'.format(hes[0],hes[1],hes[2],hes[3]))
		for i in range(len(result[0])):
			st = ''
			for j in range(len(result)):
				if count % 20 == 0:
					st += '{:^10.3f}|'.format(result[j][i])

				#print('{:^10.3f}|{:^10.3f}|{:^10.3f}|{:^10.3f}'.format())
				#print('  %3.3f  |' % result[j][i], end = '')
			if count % 20 == 0:
				print(st)
			count += 1
		'''


# Example file for Socorro API use with Python

from __future__ import print_function  # to make print available in py3
from pprint import pprint  # pretty print python objects

import datetime
import os
import json
from collections import OrderedDict

from datautils import (getFromAPI, global_defaults, getDataPath,
                       beforeTodayString)

# *** data gathering variables ***

# products to gather data from
products = ['Firefox', 'FennecAndroid']

# for how many days back to get the data
backlog_days = global_defaults['socorrodata_backlog_days']

# *** URLs and paths ***

# Run the actual meat of the script.
def run():
    # Get start and end dates
    day_start = beforeTodayString(days=backlog_days)
    day_end = beforeTodayString(days=1)

    datapath = getDataPath()
    if datapath is None:
        print('ERROR: No data path found, aborting!')
        import sys
        sys.exit(1)
    os.chdir(datapath);

    for product in products:
        fproddata = product + '-crashes-daily.json'

        try:
            with open(fproddata, 'r') as infile:
                print('Read stored ' + product + ' daily data')
                proddata = json.load(infile)
        except IOError:
            proddata = {}

        # Get all active versions for that product.
        ver_results = getFromAPI('ProductVersions', {
            'product': product,
            'active': 'true',
        })
        versions = []
        verinfo = {}
        for ver in ver_results['hits']:
            versions.append(ver['version'])
            verinfo[ver['version']] = {'tfactor': 100 / ver['throttle']}

        print('Fetch daily data for ' + product + ' ' + ', '.join(versions))

        # Get data for those versions and days.
        maxday = None

        results = getFromAPI('CrashesPerAdu', {
            'product': product,
            'versions': versions,
            'from_date': day_start,
            'to_date': day_end,
        })
        for (pver, pvdata) in results['hits'].items():
            for (day, pvd) in pvdata.items():
                ver = pvd['version']
                crashes = pvd['report_count'] * verinfo[ver]['tfactor']
                adu = pvd['adu']
                if crashes or adi:
                    if ver not in proddata:
                        proddata[ver] = {}
                    proddata[ver][day] = {'crashes': crashes, 'adu': adu}
                if maxday is None or maxday < day:
                    maxday = day
        if maxday < day_end:
            print('--- ERROR: Last day retrieved is ' + maxday + ' while yesterday was ' + day_end + '!')

        # Sort and write data back to the file.
        pd_sorted = OrderedDict()
        for version in sorted(proddata.iterkeys()):
           pd_sorted[version] = OrderedDict(sorted(proddata[version].items(), key=lambda t: t[0]))
        with open(fproddata, 'w') as outfile:
            json.dump(pd_sorted, outfile)

# Avoid running the script when e.g. simply importing the file.
if __name__ == '__main__':
    import sys
    sys.exit(run(*sys.argv[1:]))


import json

from flask import Blueprint, render_template, url_for, request, session

from access import verifyUserDecorator
from edit.tableClasses import ItemTableEditUsers, ItemTableEditProducts, createResultPage
from sql.SqlMaster import SqlMaster

editApp = Blueprint('edit', __name__, template_folder='templates')
SqlMaster = SqlMaster(json.load(open('config/dataBaseConfig.json', 'r')), r'./edit/requests')

# region General
@editApp.route('/')
@verifyUserDecorator
def editIndex():
    return render_template('editIndex.html', name=session.get('groupName'))

# endregion

# region Admin
@editApp.route('/users')
@verifyUserDecorator
def editUsers():
    result = SqlMaster.MakeRequest('UserGetAll.sql')
    for user in result:
        idLogin = user['idLogin']
        user['urlUpdate'] = url_for('edit.editUsers') + f'/{idLogin}'
        user['nameUpdate'] = f'Редактировать'
        user['classUpdate'] = f""
        user['urlDelete'] = url_for('edit.editUsers') + f'/delete' + f'/{idLogin}'
        user['nameDelete'] = f'Удалить'
        user['classDelete'] = f"confirmable"

    table = ItemTableEditUsers(result)

    createResultPage(table)

    return render_template('editChild.html', title="Все пользователи", url_for_insert=url_for('edit.editUserInsert'),
                           insertWhat="Добавить пользователя")

@editApp.route('/users/<user>', methods=['GET', 'POST'])
@verifyUserDecorator
def editUserEdit(user):
    if request.method == 'GET':
        result = SqlMaster.MakeRequest('UserGetOne.sql', id=user)[0]
        return render_template('editUserEdit.html', idLogin=result['idLogin'], Login=result['Login'],
                               PasswordL=result['PasswordL'], AccessLevel=result['AccessLevel'])
    else:
        idLogin = int(user)
        Login = request.form.get('Login')
        PasswordL = request.form.get('PasswordL')
        AccessLevel = int(request.form.get('AccessLevel'))
        SqlMaster.MakeUpdateInsert('UserUpdate.sql', idLogin=idLogin, Login=Login, PasswordL=PasswordL,
                                   AccessLevel=AccessLevel)
        return render_template('editUserResult.html')

@editApp.route('/users/add', methods=['GET', 'POST'])
@verifyUserDecorator
def editUserInsert():
    if request.method == 'GET':
        return render_template('editUserInsert.html')
    else:
        Login = request.form.get('Login')
        PasswordL = request.form.get('PasswordL')
        AccessLevel = int(request.form.get('AccessLevel'))
        SqlMaster.MakeUpdateInsert('UserInsert.sql', Login=Login, PasswordL=PasswordL,
                                   AccessLevel=AccessLevel)
        return render_template('editUserResult.html')

@editApp.route('/users/delete/<user>')
@verifyUserDecorator
def editUserDelete(user):
    SqlMaster.MakeUpdateInsert('UserDelete.sql', idLogin=int(user))
    return render_template('editUserDelete.html', idLogin=int(user))

# endregion

# region Worker

@editApp.route('/products')
@verifyUserDecorator
def editProducts():
    result = SqlMaster.MakeRequest('ProductGetAll.sql')
    for product in result:
        idProduct = product['idProduct']
        product['urlUpdate'] = url_for('edit.editProducts') + f'/{idProduct}'
        product['nameUpdate'] = f'Редактировать'
        product['classUpdate'] = f""
        product['urlDelete'] = url_for('edit.editProducts') + f'/delete' + f'/{idProduct}'
        product['nameDelete'] = f'Удалить'
        product['classDelete'] = f"confirmable"
    table = ItemTableEditProducts(result)

    createResultPage(table)

    return render_template('editChild.html', title="Все продукты", url_for_insert=url_for('edit.editProductInsert'),
                           insertWhat="Добавить продукт")

@editApp.route('/products/<product>', methods=['GET', 'POST'])
@verifyUserDecorator
def editProductEdit(product):
    if request.method == 'GET':
        result = SqlMaster.MakeRequest('ProductGetOne.sql', id=product)[0]
        return render_template('editProductEdit.html',
                               idProduct=result['idProduct'],
                               Name=result['Name'],
                               Material=result['Material'],
                               UnitOfMeasurement=result['UnitOfMeasurement'],
                               PricePerUnit=result['PricePerUnit'],
                               ActualQuantity=result['ActualQuantity'],
                               FixationDate=result['FixationDate'],
                               ReservedProduct=result['ReservedProduct'],
                               ReservationDate=result['ReservationDate'])
    else:
        idProduct = int(product)
        Name = request.form.get('Name')
        Material = request.form.get('Material')
        UnitOfMeasurement = request.form.get('UnitOfMeasurement')
        PricePerUnit = request.form.get('PricePerUnit')
        ActualQuantity = request.form.get('ActualQuantity')
        FixationDate = request.form.get('FixationDate')
        ReservedProduct = request.form.get('ReservedProduct')
        ReservationDate = request.form.get('ReservationDate')
        SqlMaster.MakeUpdateInsert('ProductUpdate.sql',
                                   idProduct=idProduct,
                                   Name=Name,
                                   Material=Material,
                                   UnitOfMeasurement=UnitOfMeasurement,
                                   PricePerUnit=PricePerUnit,
                                   ActualQuantity=ActualQuantity,
                                   FixationDate=FixationDate,
                                   ReservedProduct=ReservedProduct,
                                   ReservationDate=ReservationDate)
        return render_template('editProductResult.html')

@editApp.route('/products/add', methods=['GET', 'POST'])
@verifyUserDecorator
def editProductInsert():
    if request.method == 'GET':
        return render_template('editProductInsert.html')
    else:
        Name = request.form.get('Name')
        Material = request.form.get('Material')
        UnitOfMeasurement = request.form.get('UnitOfMeasurement')
        PricePerUnit = request.form.get('PricePerUnit')
        ActualQuantity = request.form.get('ActualQuantity')
        FixationDate = request.form.get('FixationDate')
        ReservedProduct = request.form.get('ReservedProduct')
        ReservationDate = request.form.get('ReservationDate')
        SqlMaster.MakeUpdateInsert('ProductInsert.sql',
                                   Name=Name,
                                   Material=Material,
                                   UnitOfMeasurement=UnitOfMeasurement,
                                   PricePerUnit=PricePerUnit,
                                   ActualQuantity=ActualQuantity,
                                   FixationDate=FixationDate,
                                   ReservedProduct=ReservedProduct,
                                   ReservationDate=ReservationDate)
        return render_template('editProductResult.html')

@editApp.route('/products/delete/<product>')
@verifyUserDecorator
def editProductDelete(product):
    SqlMaster.MakeUpdateInsert('ProductDelete.sql', idProduct=int(product))
    return render_template('editProductDelete.html', idProduct=int(product))

# endregion


import random
import threading
import time

import pytest

import ddtrace

MAX_NAMES = 64

@pytest.mark.parametrize("nb_service", [2, 16, 64, 256])
def test_service_name(nb_service):
    def write_in_subprocess(id_nb):
        time.sleep(random.random())
        ddtrace.config._add_extra_service(f"extra_service_{id_nb}")

    default_remote_config_enabled = ddtrace.config._remote_config_enabled
    ddtrace.config._remote_config_enabled = True

    threads = [threading.Thread(target=write_in_subprocess, args=(i,)) for i in range(nb_service)]
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join()
    ddtrace.config._remote_config_enabled = default_remote_config_enabled
    assert len(ddtrace.config._get_extra_services()) == min(nb_service, MAX_NAMES)


from pyspark import SparkContext, SQLContext
from pyspark.sql.types import StructType
from pyspark.sql.types import StructField
from pyspark.sql.types import StringType, IntegerType

sc = SparkContext()
sqlContext = SQLContext(sc)

####
# 1. Setup (10 points): Download the gbook file and write a function to load it in an RDD & DataFrame
####

# RDD API
# Columns:
# 0: place (string), 1: count1 (int), 2: count2 (int), 3: count3 (int)

# Spark SQL - DataFrame API
rdd = sc.textFile("gbooks")

schema = StructType([
    StructField("word", StringType(), True),
    StructField("count1", StringType(), True),
    StructField("count2", StringType(), True),
    StructField("count3", StringType(), True)
])

df = sqlContext.createDataFrame(rdd.map(lambda x: x.split("\t")), schema)

####
# 4. MapReduce (10 points): List the three most frequent 'word' with their count of appearances
####

# Spark SQL
df.createOrReplaceTempView("MP8")
# sqlContext.sql("SELECT place AS word, COUNT(*) FROM MP8 ORDER BY COUNT(*) DESC").show(3)
sqlContext.sql("SELECT word, COUNT(*) FROM MP8 GROUP BY word ORDER BY COUNT(*) DESC").show(3)

# There are 18 items with count = 425, so could be different
# +---------+--------+
# |     word|count(1)|
# +---------+--------+
# |  all_DET|     425|
# | are_VERB|     425|
# |about_ADP|     425|
# +---------+--------+


import os

import audio
import converter

from splitter import SongSplitter
from youtube import AudioDownloader

def main():
    project_name = input('Please enter a project name: ')
    if not project_name:
        project_name = 'delilah'
    project_root = f'projects/{project_name}'

    youtube_url = input('Please enter a youtube url: ')
    if not youtube_url:
        youtube_url = 'https://www.youtube.com/watch?v=S6XXDw0Mrck'

    model = input(f'Please choose a model. Available models are {list(converter.MODELS.keys())}: ')
    if not model:
        model = 'kanye'

    auto_predict = input('Auto predict? [y/n]: ') == 'y'
    if auto_predict == '':
        auto_predict = False

    f0_method = input(f'f0 method? {converter.F0_METHODS}: ')
    if not f0_method:
        f0_method = 'all'

    transpose = input('Transpose (half steps)?: ')
    if not transpose:
        transpose = 0
    transpose = int(transpose)

    if not os.path.exists(project_root):
        os.makedirs(project_root)

    ad = AudioDownloader(project_root)
    audio_path = ad.download(youtube_url, project_name)

    s = SongSplitter(project_name)
    vocals_path, no_vocals_path = s.split_vocals(audio_path)

    if f0_method == 'all':
        methods = converter.DEFAULT_F0_METHODS
    else:
        methods = [f0_method]

    print(f'Running voice converter with f0_methods {methods} and auto predict {auto_predict}')
    for m in methods:
        vc = converter.VoiceConverter(f0_method=m, auto_predict=auto_predict, transpose=transpose)
        converted_path = vc.convert(model, vocals_path)

        if transpose != 0:
            no_vocals_path = audio.transpose(no_vocals_path, transpose)

        combined_path = os.path.join(project_root, f'{project_name}_{model}_{m}{"_na" if not auto_predict else ""}{"_t" + str(transpose) if transpose != 0 else ""}.wav')
        audio.overlay_audio(converted_path, no_vocals_path, combined_path)

        print(f'Results saved to {combined_path}')

if __name__ == '__main__':
    main()


import typing

import json

import pandas as pd
import pydantic

from retrack.nodes.base import InputConnectionModel, OutputConnectionModel
from retrack.nodes.dynamic.base import BaseDynamicIOModel, BaseDynamicNode

class FlowV0MetadataModel(pydantic.BaseModel):
    value: str
    default: typing.Optional[str] = None

    def parsed_value(self) -> typing.Dict[str, typing.Any]:
        return json.loads(self.value)

class FlowV0OutputsModel(pydantic.BaseModel):
    output_value: OutputConnectionModel

def flow_factory(
    inputs: typing.Dict[str, typing.Any], **kwargs
) -> typing.Type[BaseDynamicNode]:
    input_fields = {}

    for name in inputs.keys():
        input_fields[name] = BaseDynamicNode.create_sub_field(InputConnectionModel)

    inputs_model = BaseDynamicIOModel.with_fields("FlowV0InputsModel", **input_fields)

    models = {
        "inputs": BaseDynamicNode.create_sub_field(inputs_model),
        "outputs": BaseDynamicNode.create_sub_field(FlowV0OutputsModel),
        "data": BaseDynamicNode.create_sub_field(FlowV0MetadataModel),
    }

    BaseFlowV0Model = BaseDynamicNode.with_fields("FlowV0", **models)

    class FlowV0(BaseFlowV0Model):
        def run(self, **kwargs) -> typing.Dict[str, typing.Any]:
            runner = kwargs.get("runner", None)
            if runner is None:
                raise ValueError("Missing runner")

            inputs_in_kwargs = {}

            for name, value in kwargs.items():
                if name.startswith("input_"):
                    inputs_in_kwargs[name[6:]] = value

            response = runner.execute(pd.DataFrame(inputs_in_kwargs))

            return {"output_value": response["output"].values}

    return FlowV0


import numpy
import scipy.sparse as sparse
import scipy.sparse.linalg as linalg

import cluster, movie_lens_data, my_util

class UserProfile(my_util.Model):
    """Model for a single user.
    ::
        User profile:
            profile - [list of genre and tag ids]
            x_factors - vector of coefficients
        Movie data:
            _movie_tags - {movie_id: {tag_id: ln(count) + 1} }
            _tag_counts - {tag id: count}
            _movie_genres - {movie id : set of genre ids}
            _movie_medians - {movie id : movie median rating}
    """

    def __init__(self, movie_genres, movie_ratings, movie_tags, tag_counts,
                 movie_medians):
        """Constructs a user profile.

        :param movie_genres: {movie id : set of genre ids}
        :param movie_ratings: [(movie id, rating)]
        :param movie_tags: {movie_id: {tag_id: ln(count) + 1} }
        :param tag_counts: {tag id: count}
        :param movie_medians: {movie id : movie median rating}
        """
        self._movie_tags = movie_tags
        self._tag_counts = tag_counts
        self._movie_genres = movie_genres
        self._movie_medians = movie_medians

        id_counts = self._count_ids(movie_ratings, movie_genres, movie_tags)
        num_factors = self._decide_num_factors(len(movie_ratings))
        self.profile = self._decide_profile(id_counts, num_factors)

        self.x_factors = self._compute_x_factors(
            movie_ratings, movie_genres, movie_tags, movie_medians,
            tag_counts)

    def _count_ids(self, movie_ratings : list, movie_genres, movie_tags):
        """ Counts the genres and tags encountered in this
        user's movie ratings.

        :param movie_ratings: [(movie id, rating)]
        :param movie_genres: {movie_id: set of genre ids}
        :param movie_tags: {movie_id: {tag_id: ln(count) + 1}}
        :return: {id: count}. The tag id is offset by 1000.
            So a tag id of 10 in the data set will show up as
            a tag id of 1010.
        """
        id_counts = {}
        for movie_id, _ in movie_ratings:

            if movie_id in movie_genres:
                for genre_id in movie_genres[movie_id]:

                    if genre_id not in id_counts:
                        id_counts[genre_id] = 0

                    id_counts[genre_id] += 1

            if movie_id in movie_tags:
                for tag_id in movie_tags[movie_id].keys():

                    if tag_id+1000 not in id_counts:
                        id_counts[tag_id+1000] = 0

                    id_counts[tag_id + 1000] += 1

        return id_counts

    def _decide_num_factors(self, num_ratings):
        """Based on the number of ratings, return number of factors
        to use."""
        # factor profile
        # (number of factors, number of equations(ratings))
        factor_profile = [
            [5, 10], # 5 factors for the first 10 ratings
            [5, 5 * 4],
            [5, 5 * 8],
            [5, 5 * 16],
            [5, 5 * 32],
        ]

        num_factors = 0

        for factors, ratings in factor_profile:
            # determine the multiplier (x2, x4, x8, ...)
            multiplier = int(ratings / factors)

            if num_ratings <= ratings:
                return num_factors + int(num_ratings / multiplier)
            else:
                num_factors += factors
                num_ratings -= ratings

        # max limit reached
        return num_factors

    def _decide_profile(self, id_counts, num_factors):
        """ Decide on which factors (ids) to use for the profile.
        The most common factors (ids) are used.

        :param id_counts: {id: count} shows how often was each
            genre or tag encountered. If the id is < 1000, it's a genre id.
            If id is >= 1000, it's a tag id.
        :param num_factors: number of factors to use
        :return: list of ids
        """
        # build id_count into a list and sort
        count_id_list = [] # list of (count, id)

        for id in id_counts:
            count_id_list.append((id_counts[id], id))

        # Keep just the IDs necessary. There is a bias factor
        # that is not part of the profile, that's why there is
        # a "-1" below.
        count_id_list.sort(reverse=True)
        count_id_list = count_id_list[:num_factors]

        id_list = [x[1] for x in count_id_list]
        return id_list

    def _compute_x_factors(self, movie_ratings, movie_genres,
                           movie_tags, movie_medians, tag_counts):
        """ Compute x_factors using least square fit.

        :param movie_ratings: [(movie id, rating)]
        :param movie_genres: {movie id: set of genre ids}
        :param movie_tags: {movie id: {tag id: ln(count)+1} }
        :param movie_medians: {movie id: movie median rating}
        :param tag_counts: {tag id: count}
        :return: x_factors vector
        """
        # tag_count is optional - but it does make x_factors
        # more standardized

        # build up matrix A
        profile = self.profile
        num_factors = len(profile) + 1
        row = []
        col = []
        data = []

        for i in range(0, len(movie_ratings)):
            movie_id = movie_ratings[i][0]

            for j in range(0, len(profile)):
                factor_id = profile[j]

                if factor_id < 1000:
                    # factor_id is a genre id
                    if movie_id in movie_genres:
                        if factor_id in movie_genres[movie_id]:
                            row.append(i)
                            col.append(j)
                            data.append(1.0)

                else:
                    # factor_id is a tag id
                    tag_id = factor_id - 1000

                    if movie_id in movie_tags:
                        if tag_id in movie_tags[movie_id]:
                            tag_val = movie_tags[movie_id][tag_id] / tag_counts[tag_id]
                            row.append(i)
                            col.append(j)
                            data.append(tag_val)

            # there's always bias as the last factor
            row.append(i)
            col.append(num_factors - 1)
            data.append(1.0)

        m = sparse.coo_matrix((data, (row, col)),
                              shape=(len(movie_ratings), num_factors),
                              dtype=numpy.double)
        A = m.tocsr()

        # build up matrix b
        b = numpy.zeros((len(movie_ratings), 1), dtype=numpy.double)

        for i in range(0, len(movie_ratings)):
            movie_id, rating = movie_ratings[i]
            b[i][0] = rating - movie_medians[movie_id]

        # solve for Ax = b
        result = linalg.lsqr(A, b, iter_lim=1000)
        x = numpy.array([result[0]]).T

        # To detect failure to converge to a solution:
        # if numpy.sum(numpy.abs(x)) < 0.01:

        return x

    def predict(self, movie_id : int):
        """Predicts a rating for the given movie_id. Returns None if
        no prediction can be made."""
        # param movie_tag_scores: {tag id: ln(count) + 1}
        #         :param tag_counts: {tag id: count}
        #         :param movie_genre_ids: a set of genre ids
        #         :param movie_median: median score for the movie

        # movie median is required
        if movie_id not in self._movie_medians:
            return None
        movie_median = self._movie_medians[movie_id]

        # "movie_tag_scores" and "movie_genre_ids" are optional
        movie_tag_scores = None
        if movie_id in self._movie_tags:
            movie_tag_scores = self._movie_tags[movie_id]

        movie_genre_ids = None
        if movie_id in self._movie_genres:
            movie_genre_ids = self._movie_genres[movie_id]

        profile = self.profile # [list of genre and tag ids]

        # build up movie attributes vector
        attr = numpy.zeros((1, len(profile) + 1), dtype=numpy.double)

        for i in range(0, len(profile)):
            id = profile[i]

            if id < 1000:
                # id is genre id
                if movie_genre_ids is not None:
                    if id in movie_genre_ids:
                        attr[0][i] = 1.0
            else:
                # id is tag id
                tag_id = id - 1000
                if movie_tag_scores is not None:
                    if tag_id in movie_tag_scores:
                        attr[0][i] = movie_tag_scores[tag_id] / self._tag_counts[tag_id]

        # the last term is always 1
        attr[0][len(profile)] = 1.0

        # prediction = (attr) dot (x_factors) + median
        return attr.dot(self.x_factors)[0][0] + movie_median

def main():
    length = movie_lens_data.get_input_obj("user_ratings_test_length")

    # Sends a "tag_ls_eval" command to cluster nodes.
    if cluster.cluster_info is None:
        print("Cannot connect to cluster.")
        return

    cluster.send_command({
        "op": "distribute",
        "worker_op": "tag_ls_eval",
        "length": length
    })

    cluster.wait_for_completion()
    cluster.print_status()
    print()

    file_name = "tag_ls_eval_results.bin"
    results = cluster.merge_list_results(file_name)
    user_ids, agreements = zip(*results)

    my_util.print_rank_agreement_results(agreements, "tag least squares")
    print('\a')

if __name__ == "__main__":
    main()


from setuptools import setup
from setuptools import find_packages
from setuptools import Extension
import numpy

with open("README.md", "r") as f:
    documentation = f.read()

setup(
    name         = 'CAMELS_library',
    version      = "0.3",
    author       = 'Francisco Villaescusa-Navarro',
    author_email = 'villaescusa.francisco@gmail.com',
    description  = "Python routines to analyze the data from the simulations of the Cosmology and Astrophysics with MachinE Learning Simulations (CAMELS) project",
    url          = "https://github.com/franciscovillaescusa/CAMELS",
    license      = 'MIT',
    long_description_content_type = "text/markdown",
    long_description = documentation,
    packages=find_packages(where="library/"),
    include_dirs=[numpy.get_include()],
    install_requires=['scipy', 'h5py', 'Pylians', 'tables'],
    package_dir={'':'library/'},
    py_modules=['camels_library']
)


# Load required libraries
import sys
import argparse
import cv2
import math
import os
import numpy as np

# Function to extract desired frames
def video2pic_py(pathIn, pathOut, fps):

    # # Testing
    # pathIn  = "/home/david/Schreibtisch/IMG_0020 genet test.MP4"
    # pathOut = "/home/david/Schreibtisch/Test"
    # fps     = 1

    # Identify original file name
    filename = os.path.basename(pathIn)
    filename = os.path.splitext(filename)[0]

    # Start video
    vidcap = cv2.VideoCapture(pathIn)

    # Get duration of the video in milliseconds
    frame = vidcap.get(cv2.CAP_PROP_FPS)
    count = vidcap.get(cv2.CAP_PROP_FRAME_COUNT)
    duration = count / frame * 1000

    # Read images
    success, image = vidcap.read()
    count          = 0
    # success        = True

    # Extract frames
    while success:

        time = count * (1 / fps) * 1000
        if time > duration:
            break

        # Set video to location of the n-th frame
        vidcap.set(cv2.CAP_PROP_POS_MSEC, time)

        # Read that image
        success, image = vidcap.read()
        if success:
        # print ("Read a new frame: ", success)
            cv2.imwrite(pathOut + "/" + filename + "_Frame_%d.JPG" % count, image)
        count = count + 1

# # Function call
# if __name__ == "__main__":
#     print("aba")
#     a = argparse.ArgumentParser()
#     a.add_argument("--pathIn", help = "path to video")
#     a.add_argument("--pathOut", help = "path to images")
#     a.add_argument("--fps", help = "frames per second", type = int, default = 1)
#     args = a.parse_args()
#     print(args)
#     extractImages(args.pathIn, args.pathOut, args.fps)


import streamlit as st
from PIL import Image
import model

st.set_page_config(page_title="Aerial Drone segmentation")
st.title("Aerial Drone segmentation")

# Developer information
developers = [
    {"name": "Ahmed Adel", "linkedin": "https://www.linkedin.com/in/ahmed23adel/"},
    {"name": "Nour Elding Hammido", "linkedin": "https://www.linkedin.com/in/nourhamedo/"},
    {"name": "Jihad", "linkedin": "https://www.linkedin.com/in/jihad-mahmoud-5804ab18b/"}
]

# Display developer information with links in the sidebar
st.sidebar.markdown("Developed by:")
for developer in developers:
    st.sidebar.markdown(f"[{developer['name']}]({developer['linkedin']})")
st.text("Pick an example image")
examples_img_names = ("000", "001", "002", "003", "004", "005", "006")
image_example_name = st.selectbox("Example Image: ",examples_img_names)

def show_example_image(image_example_name = "000"):
    if image_example_name not in examples_img_names:
        raise Exception("Image not found")
    img_orig_url = r'imgs/{}_orig.jpg'.format(image_example_name)
    img_ground_truth_url = r'imgs/{}_groundTruth.jpg'.format(image_example_name)
    img_mask_pred_url = r'imgs/{}_maskPred.png'.format(image_example_name)
    img_orig = Image.open(img_orig_url)
    mask_ground_truth = Image.open(img_ground_truth_url)
    img_mask_pred = Image.open(img_mask_pred_url)
    st.text("Original image")
    st.image(img_orig, width=600)
    st.text("Mask image original")
    st.image(mask_ground_truth, width=600)
    st.text("Mask image predicted")
    st.image(img_mask_pred, width=600)
show_example_image(image_example_name)

st.header("Now it's time to try your image")
uploaded_file = st.file_uploader("Upload your image", type=["jpg", "png"])
if uploaded_file is not None:
    if uploaded_file.type.split('/')[1] in ['jpg', 'jpeg', 'png']:
        st.write("File uploaded successfully!")
        st.image(uploaded_file, caption='Uploaded Image', width=500)
        image = Image.open(uploaded_file)
        st.subheader('Model prediction')
        mask = model.predict_img(image)
        st.image(mask, caption='mask Image predicted', width=500)

st.header("Model details")
st.text("""
I used PyTorch lightning for this project

Model details:
We used DeepLabV3Plus and resnet34 for feature extraction,\n and for optimization I used Adam, and cross entropy loss.

DeepLabV3Plus:
""")

model_arch = Image.open(r'model_arch.png')
st.image(model_arch, caption='Model architecture', width=500)

st.text("""
I used albumentations for transforms, it generates sunny, rainy effect and many more

For metric i used Accurate, jaccard index(IOS)
""")

# st.image(img, width=600)


#!/usr/bin/env python

"""noreg-training.py: Solution to the second assignment (no regularization)."""
__author__      = "David Bertoldi"

import tensorflow as tf
import torch
import matplotlib.pyplot as plt
import matplotlib as mpl
import numpy as np
import pandas as pd
import os

from mlxtend.plotting import plot_decision_regions
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder
from sklearn.metrics import confusion_matrix
from mlxtend.plotting import plot_confusion_matrix
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, LeakyReLU, Input, GaussianNoise, Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.utils import np_utils
from tensorflow.keras.regularizers import L1, L2
from tensorflow.keras.optimizers import Adam, Nadam, SGD
from tensorflow.keras.initializers import HeNormal, HeUniform

BASE_URL = 'https://github.com/DBertazioli/multi-mnist_custom/raw/master/final_dataset/'
SEED = 42
np.random.seed(SEED)
tf.random.set_seed(SEED)
torch.manual_seed(SEED)

def load_data():
	X_train_path = tf.keras.utils.get_file('X_train.npz', BASE_URL + 'X_train.npz')
	y_train_path = tf.keras.utils.get_file('y_train.npz', BASE_URL + 'y_train.npz')
	X_test_path = tf.keras.utils.get_file('X_test.npz', BASE_URL + 'X_test.npz')
	y_test_path = tf.keras.utils.get_file('y_test.npz', BASE_URL + 'y_test.npz')
	with np.load(X_train_path) as X_train, np.load(y_train_path) as y_train, np.load(X_test_path) as X_test, np.load(y_test_path) as y_test:
		return X_train["arr_0"], y_train["arr_0"], X_test["arr_0"], y_test["arr_0"]

def build_model(input_shape, classes, activation, initializer):
	model = Sequential()
	model.add(Dense(80, activation = activation, kernel_initializer=initializer, input_shape=input_shape))
	model.add(Dense(70, activation = activation,  kernel_initializer=initializer))
	model.add(Dense(classes, activation = "softmax", kernel_initializer=initializer))
	return model

def start():
	X_train, y_train, X_test, y_test = load_data()

	# Separate the test data
	X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=SEED, shuffle=True, stratify = None)

	#scaling
	X_train = X_train.astype('float32') / 255.
	X_val = X_val.astype('float32') / 255.
	X_test = X_test.astype('float32') / 255.

	print(X_test[0].shape)

	#flattening
	X_train = X_train.reshape((X_train.shape[0], np.prod(X_train.shape[1:])))
	X_val = X_val.reshape((X_val.shape[0], np.prod(X_val.shape[1:])))
	X_test = X_test.reshape((X_test.shape[0], np.prod(X_test.shape[1:])))

	#one-hot encoding
	Y_train = np_utils.to_categorical(y_train-1)
	Y_val = np_utils.to_categorical(y_val-1)
	Y_test = np_utils.to_categorical(y_test-1)

	input_dims = np.prod(X_test.shape[1:]) #784
	nb_classes = Y_train.shape[1]

	# configurations
	activation = 'sigmoid'
	optimizer = Adam(learning_rate=1e-03)
	initializer = HeNormal(seed=SEED)
	loss = 'categorical_crossentropy'
	e = 50
	bs = 256

	# model definition
	model = build_model((input_dims,), nb_classes, activation, initializer)
	model.compile(optimizer=optimizer, loss=loss, metrics=['categorical_accuracy'])
	model.summary()

	# Checkpoints
	mcp_save_acc = ModelCheckpoint('./noreg/acc2.hdf5',
									save_best_only=True,
									monitor='val_categorical_accuracy',
									mode='max')
	mcp_save_loss = ModelCheckpoint('./noreg/loss2.hdf5',
									save_best_only=True,
									monitor='val_loss',
									mode='min')

	early_stopping = EarlyStopping(monitor='val_categorical_accuracy',
									patience=100,
									min_delta=0.005,
									mode='max',
									restore_best_weights=True,
									verbose=1)

	# training
	history = model.fit(X_train,Y_train,
						epochs=e,
						batch_size=bs,
						validation_data = (X_val, Y_val),
						callbacks = [mcp_save_acc, mcp_save_loss])

	# plot training & validation accuracy values
	plt.plot(history.history['categorical_accuracy'])
	plt.plot(history.history['val_categorical_accuracy'])
	plt.title('Model accuracy')
	plt.ylabel('Accuracy')
	plt.xlabel('Epoch')
	plt.legend(['Train', 'Test'], loc='upper left')
	plt.savefig(format('./images/noreg/accuracy-{}-{}-{}-{}-{}.png', activation, loss, optimizer, e, bs))

	plt.clf()

	# plot training & validation loss values
	plt.plot(history.history['loss'])
	plt.plot(history.history['val_loss'])
	plt.title('Model loss')
	plt.ylabel('Loss')
	plt.xlabel('Epoch')
	plt.legend(['Train', 'Test'], loc='upper right')
	plt.savefig(format('./images/noreg/loss-{}-{}-{}-{}-{}.png', activation, loss, optimizer, e, bs))

def format(s, activation, loss, optimizer, e, bs):
	return s.format(activation if isinstance(activation, str) else type(activation).__name__,
					loss,
					type(optimizer).__name__,
					e,
					bs)

if __name__ == '__main__':
	start()


from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_moons
import matplotlib.pyplot as plt
import mglearn

# 데이터 로드
X, y = make_moons(n_samples=100, noise=0.25, random_state=3)
X_train, X_test, y_train, y_test = train_test_split(X,y,stratify=y,random_state=42)

# 모델 학습
model = RandomForestClassifier(n_estimators=5, random_state=0)
model.fit(X_train, y_train)
pre = model.predict([X[0]])
print('예측값: ', pre, '실제값: ', y[0])

# 결정 경계 시각화
# 다섯 개의 결정트리 결정 경계
fig, axes = plt.subplots(2, 3, figsize=(20,10) )
for i, (ax, tree) in enumerate( zip( axes.ravel(), model.estimators_ ) ):
    ax.set_title("tree {}".format(i) )
    mglearn.plots.plot_tree_partition(X, y, tree, ax=ax)

# 랜덤포레스트로 만들어진 결정경계
axes[-1, -1].set_title("Random forest")
mglearn.plots.plot_2d_separator(model, X, fill=True, alpha=0.5, ax=axes[-1,-1] )
mglearn.discrete_scatter(X[:,0], X[:,1], y)

plt.show()


# -*- coding: utf-8 -*-
# python36
__author__ = 'wuqili'

# -*- coding: utf-8 -*-
# python36
__author__ = 'wuqili'
# 抓取商品的评论，

import matplotlib
matplotlib.use('Agg')
import requests
import re
import json
import queue
import sys
import os
import numpy as np
from PIL import Image
from threading import Lock, Thread, currentThread
from scrapy.selector import Selector
from urllib.parse import urlencode, urlparse
from snownlp import SnowNLP
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
import mogu_ip
import base64
from io import BytesIO
import matplotlib.pyplot as plt
from urllib.error import URLError
from wordcloud import WordCloud
import jieba

getip = mogu_ip.GetIP()

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'}
lock = Lock()
requests.adapters.DEFAULT_RETRIES = 5

comm_count = {}

class CommentParse:
    def __init__(self):
        self.comm = {}
        self.urlQueue = queue.Queue()
        self.scores = np.empty(0)
        self.ids = []
        self.use_proxy = False

    def get_comments(self):
        raise NotImplementedError

    def build_queue(self):
        raise NotImplementedError

    def get_all_comments(self):
        """使用多线程抓取评论页"""
        # 当当网要重写这个函数
        self.build_queue()
        if self.urlQueue.qsize() > 10:
            self.use_proxy = True
        if not self.urlQueue.empty():
            threads = []
            # 可以调节线程数， 进而控制抓取速度
            thread_num = self.urlQueue.qsize() // 3 + 1
            if thread_num >= 15:
                thread_num = 15
            if thread_num <= 3:
                thread_num = 3
            for i in range(thread_num):
                t = Thread(target=self.get_comments)
                threads.append(t)
            for t in threads:
                t.start()
            for t in threads:
                # 多线程多join的情况下，依次执行各线程的join方法, 这样可以确保主线程最后退出， 且各个线程间没有阻塞
                t.join()
        else:
            raise URLError('商品链接地址不能被识别')

    def comments_scores(self):
        """对评论进行文本分析"""
        for key, value in self.comm.items():
            self.ids.append(key)
            self.scores = np.append(self.scores, value['scores'])
        # 平均值
        mean_score = self.scores.mean().round(2)
        if mean_score in self.scores:
            mean_comm = self.comm[self.ids[int(
                np.argwhere(self.scores == mean_score)[0])]]
        else:
            sorted_scores = sorted(np.append(self.scores, mean_score))
            index = int(np.argwhere(sorted_scores == mean_score))
            near_score = sorted_scores[index + 1]
            mean_comm = self.comm[self.ids[int(
                np.argwhere(self.scores == near_score)[0])]]
        # 最大最小值
        min_score = self.scores.min()
        max_score = self.scores.max()
        min_comm = self.comm[self.ids[self.scores.argmin()]]
        max_comm = self.comm[self.ids[self.scores.argmax()]]
        return mean_score, min_score, max_score, mean_comm, min_comm, max_comm

    def hist_scores(self):
        """对所有评分进行直方图分布分析"""
        fig = plt.figure(figsize=(6, 6))
        plt.rcParams['font.sans-serif'] = ['SimHei']  # 显示中文标签
        plt.hist(
            self.scores,
            bins=20,
            rwidth=0.8,
            facecolor='blue',
            alpha=0.5,
            edgecolor='yellow')
        plt.title('评论打分分布图', fontsize=20)
        plt.xlabel('评论分数', fontsize=16)
        plt.ylabel('评论计数', fontsize=16)
        plt.xticks(fontsize=14)
        plt.yticks(fontsize=14)
        sio = BytesIO()
        plt.savefig(sio, format='png')
        hist_data = "data:image/png;base64," + base64.encodebytes(sio.getvalue()).decode()
        sio.close()
        plt.close()
        return hist_data

    def show_part_scores(self):
        # 分出好评，中评和差评
        good_comments = []
        mid_comments = []
        bad_comments = []
        for key, value in self.comm.items():
            if 70 < value['scores'] <= 100:
                good_comments.append(value)
            elif 40 < value['scores'] <= 70:
                mid_comments.append(value)
            else:
                bad_comments.append(value)
        return good_comments, mid_comments, bad_comments

    def build_wordcloud(self, comments, img_model):
        """对评论内容生成词云"""
        text = ''
        for item in comments:
            text += item['content']
        if not text:
            words = '评论为空'
        else:
            wordlist = jieba.cut(text)
            words = ' '.join(wordlist)
        graph = Image.open(img_model)
        image = np.array(graph)
        wc = WordCloud(
            background_color='white',
            font_path=os.path.join(os.path.dirname(__file__), 'simsun.ttf'),
            mask=image,
            max_words=100)
        wc.generate(words)
        lock.acquire()
        plt.imshow(wc, interpolation='bilinear')
        plt.axis('off')
        wcio = BytesIO()
        plt.savefig(wcio, format='png')
        plt.close()
        lock.release()
        #将图片数据转化成base64
        wc_data = "data:image/png;base64," + base64.encodebytes(wcio.getvalue()).decode()
        wcio.close()
        return wc_data

class JDcomments(CommentParse):
    """获取京东商品的评论"""
    def __init__(self, product_url, timeId):
        super().__init__()
        self.product_url = product_url
        self.pub_time = set()
        self.timeId = timeId

    def build_queue(self):
        """将所有需要爬取的url放入到队列中"""
        # 商品id
        dept, id = re.findall(r'.*?jd\.(\w+?)/(\d+).*', self.product_url)[0]
        if dept == 'hk':
            # 京东全球购
            comm_url = 'https://club.jd.com/productpage/p-{0}-s-0-t-1-p-{1}.html'
        elif dept == 'com':
            # 普通京东购物
            comm_url = 'https://sclub.jd.com/comment/productPageComments.action?productId={0}&score=0&sortType=5&page={1}&pageSize=10'
        else:
            return
        # 爬取评论第一页获得总评论数
        page_1_resp = requests.get(url=comm_url.format(id, 0), headers=headers)
        page_1_text = json.loads(page_1_resp.text)
        total_comms = int(page_1_text['productCommentSummary']['commentCount'])
        page_nums = total_comms // 10 + 1
        if page_nums > 100:
            page_nums = 100
            # 商品评论链接
            # 改变最后一个数字换页，每页10条评论，最多100页
        for i in range(page_nums):
            self.urlQueue.put(comm_url.format(id, i))

    def get_comments(self):
        robot_comms = ['此用户未填写评价内容',
                       '此用户未及时填写评价内容，系统默认好评！', '']
        while True:
            try:
                # 不阻塞的读取队列数据
                url = self.urlQueue.get_nowait()
            except Exception as e:
                print(e)
                break
            try:
                if self.use_proxy:  # 爬取页数少时不使用代理
                    lock.acquire()
                    proxy_dict = getip.get_random_valid_ip()
                    lock.release()
                    if 'https' in url:
                        # http对应的ip只能请求http的url，同理https
                        proxy_dict = {'https': proxy_dict['http'].replace('http', 'https')}
                    response = requests.get(
                        url, headers=headers, proxies=proxy_dict, timeout=8)
                else:
                    response = requests.get(url, headers=headers, timeout=8)
                text = json.loads(response.text)
                comments = text['comments']
                for comment in comments:
                    id = comment['id']
                    create_date = comment['creationTime']
                    if create_date not in self.pub_time:  # 过滤掉重复发表的评论
                        self.pub_time.add(create_date)
                        content = comment['content'].replace('hellip', '')
                        if content not in robot_comms:
                            self.comm[id] = {}
                            self.comm[id]['create_date'] = create_date
                            self.comm[id]['content'] = content
                            self.comm[id]['scores'] = round(
                                (SnowNLP(content).sentiments() * 100), 2)
                            images = comment.get('images', [])
                            small_images, big_images = [], []
                            if images:
                                small_images = ['https:' + i['imgUrl'] for i in images]
                                big_images = [url.replace('n0/s128x96','shaidan/s616x405') for url in small_images]
                            self.comm[id]['small_images'] = small_images
                            self.comm[id]['big_images'] = big_images
                lock.acquire()
                count = len(self.comm)
                comm_count[self.timeId] = count
                lock.release()
                print('Current Thread Name %s, Url: %s ' % (currentThread().name, url))
            except Exception as e:
                print(e)
                self.urlQueue.put(url)

class TMcomments(CommentParse):
    """获取天猫商品的评论"""
    # 要考虑天猫，天猫超市，天猫国际
    def __init__(self, product_url, timeId):
        super().__init__()
        self.product_url = product_url
        self.pub_time = set()
        self.timeId = timeId
        self.count = 0

    def build_queue(self):
        prod_resp = requests.get(self.product_url, headers=headers)
        match = re.match(
            r'.*?g_config.*?itemId:\"(\d+)\",sellerId:\"(\d+)\".*',
            prod_resp.text,
            re.S)
        if not match:
            return
        itemId = match.group(1)
        sellerId = match.group(2)
        comm_url = 'https://rate.tmall.com/list_detail_rate.htm?itemId={0}&sellerId={1}&currentPage={2}'
        while True:
            page_1_resp = requests.get(comm_url.format(itemId, sellerId, 1), headers=headers)
            if 'rateDetail' in page_1_resp.text:
                break
        try:
            info = json.loads('{' + page_1_resp.text.strip() + '}')
        except BaseException:
            text = re.match('.*?\(({.*})\).*',page_1_resp.text.strip()).group(1)
            info = json.loads(text)
        total_page = int(info['rateDetail']['paginator']['lastPage'])
        for i in range(1, total_page + 1):
            self.urlQueue.put(comm_url.format(itemId, sellerId, i))

    def get_comments(self):
        robot_comms = ['此用户没有填写评论!', '']
        while True:
            try:
                # 不阻塞的读取队列数据
                url = self.urlQueue.get_nowait()
            except Exception as e:
                print(e)
                break
            try:
                if self.use_proxy:  # 爬取页数少时不使用代理
                    lock.acquire()
                    proxy_dict = getip.get_random_valid_ip()
                    lock.release()
                    if 'https' in url:
                        proxy_dict = {'https': proxy_dict['http'].replace('http', 'https')}
                    response = requests.get(url, headers=headers, proxies=proxy_dict, timeout=8)
                else:
                    response = requests.get(url, headers=headers, timeout=8)
                try:
                    text = json.loads('{' + response.text.strip() + '}')
                except BaseException:
                    text = re.match('.*?\(({.*})\).*', response.text.strip()).group(1)
                    text = json.loads(text)
                comments = text['rateDetail']['rateList']
                for comment in comments:
                    id = comment['id']
                    create_date = comment['rateDate']
                    if create_date not in self.pub_time:
                        self.pub_time.add(create_date)
                        content = comment['rateContent'].replace('hellip', '')
                        if content not in robot_comms:
                            self.comm[id] = {}
                            self.comm[id]['create_date'] = create_date
                            self.comm[id]['content'] = content
                            images = comment.get('pics', [])
                            small_images, big_images = [], []
                            if images:
                                small_images = ['https:' + i + '_40x40.jpg' for i in images]
                                big_images = ['https:' + i + '_400x400.jpg' for i in images]
                            self.comm[id]['small_images'] = small_images
                            self.comm[id]['big_images'] = big_images
                            # 获取追加评论
                            append_comm = comment.get('appendComment', '')
                            if append_comm:
                                self.comm[id]['content'] += append_comm['content']
                                self.comm[id]['small_images'] += ['https:' + i + '_40x40.jpg' for i in append_comm['pics']]
                                self.comm[id]['big_images'] += ['https:' + i + '_400x400.jpg' for i in append_comm['pics']]
                            self.comm[id]['scores'] = round((SnowNLP(self.comm[id]['content']).sentiments() * 100), 2)
                lock.acquire()
                count = len(self.comm)
                comm_count[self.timeId] = count
                lock.release()
                print('Current Thread Name %s, Url: %s ' %
                      (currentThread().name, url[-10:]))
            except Exception as e:
                print(e)
                self.urlQueue.put(url)

class AMScomments(CommentParse):
    """获取亚马逊商品评论"""

    def __init__(self, product_url, timeId):
        super().__init__()
        self.product_url = product_url
        self.timeId = timeId
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
            'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'}

    def build_queue(self):
        try:
            prod_resp = requests.get(self.product_url, headers=self.headers)
            if 200 <= prod_resp.status_code <= 300:
                selector = Selector(prod_resp)
                total_comms = int(selector.css('.a-size-medium.totalReviewCount::text').extract()[0].replace(',', ''))
            else:
                print('请求商品链接失败：{}'.format(prod_resp.status_code))
                return
        except BaseException:
            return
        total_pages = (total_comms // 10) + 1  # 每次请求20条好了
        comm_url = 'https://www.amazon.cn/hz/reviews-render/ajax/reviews/get/ref=cm_cr_othr_d_paging_btm_{0}'
        for i in range(1, total_pages + 1):
            self.urlQueue.put(comm_url.format(i))

    def get_comments(self):
        while True:
            try:
                # 不阻塞的读取队列数据
                url = self.urlQueue.get_nowait()
            except Exception as e:
                print(e)
                break
            try:
                id = re.match(
                    r'.*(dp|product)/(.+?)[/?].*',
                    self.product_url).group(2)
                post_data = {
                    'reviewerType': 'all_reviews',
                    'filterByStar': 'all_stars',
                    'pageNumber': url.split('_')[-1],
                    'pageSize': '10',
                    'asin': id
                }
                if self.use_proxy:  # 爬取页数少时不使用代理
                    lock.acquire()
                    proxy_dict = getip.get_random_valid_ip()
                    lock.release()
                    if 'https' in url:
                        # http对应的ip只能请求http的url，同理https
                        proxy_dict = {'https': proxy_dict['http'].replace('http', 'https')}
                    response = requests.post(
                        url=url,
                        data=urlencode(post_data),
                        headers=self.headers,
                        proxies=proxy_dict,
                        timeout=8)
                else:
                    response = requests.post(url=url, data=urlencode(
                        post_data), headers=self.headers, timeout=8)
                comments = response.text.split('&&&')
                for comment in comments[3:-4]:
                    comment = comment.strip().replace('["append","#cm_cr-review_list","', '')[:-2]
                    comment = comment.replace('\\', '')
                    selector = Selector(text=comment)
                    id = selector.css('.a-section.review::attr(id)').extract()[0]
                    self.comm[id] = {}
                    self.comm[id]['create_date'] = selector.css('span.review-date::text').extract()[0][2:]
                    content = '\n'.join(selector.css('.a-size-base.review-text::text').extract())
                    if content == '':
                        self.comm.pop(id)
                        continue
                    self.comm[id]['content'] = content
                    self.comm[id]['scores'] = round(
                        (SnowNLP(content).sentiments() * 100), 2)
                    small_images = selector.css('div.review-image-tile-section img::attr(src)').extract()
                    self.comm[id]['small_images'] = small_images
                    big_images = []
                    if small_images:
                        big_images = [url.replace('jpg', '_SY88.jpg') for url in small_images]
                    self.comm[id]['big_images'] = big_images
                lock.acquire()
                count = len(self.comm)
                comm_count[self.timeId] = count
                lock.release()
                print('Current Thread Name %s, Url: %s ' % (currentThread().name, url[-10:]))
            except Exception as e:
                print(e)
                print(url)
                self.urlQueue.put(url)

class SNcomments(CommentParse):
    """获取苏宁商品的评论"""

    def __init__(self, product_url, timeId):
        super().__init__()
        self.product_url = product_url
        self.timeId = timeId

    def build_queue(self):
        id_1, id_2 = re.findall(r'.*/(\d+)/(\d+)\.html', self.product_url)[0]
        if not id_1 or not id_2:
            return
        id_2 = id_2.rjust(18, '0')  # 左边补0至18位
        first_page = 'https://review.suning.com/cmmdty_review/general-{0}-{1}-1-total.htm'.format(
            id_2, id_1)
        try:
            page_1_resp = requests.get(first_page, headers=headers)
            selector = Selector(page_1_resp)
            total_comms = int(selector.css('li[data-type="total"]::attr(data-num)').extract_first(default=0))
        except Exception as e:
            print(e)
            return
        page_nums = total_comms // 10 + 1
        if page_nums > 50:
            page_nums = 50
        comm_url = 'https://review.suning.com/ajax/review_lists/general-{0}-{1}-total-{2}-default-10-----reviewList.htm?callback=reviewList'
        # 评论最多50页
        for page in range(1, page_nums + 1):
            self.urlQueue.put(comm_url.format(id_2, id_1, page))

    def get_comments(self):
        robot_comms = ['买家未及时做出评价，系统默认好评！', '']
        while True:
            try:
                # 不阻塞的读取队列数据
                url = self.urlQueue.get_nowait()
            except Exception as e:
                print(e)
                break
            try:
                if self.use_proxy:  # 爬取页数少时不使用代理
                    lock.acquire()
                    proxy_dict = getip.get_random_valid_ip()
                    lock.release()
                    if 'https' in url:
                        # http对应的ip只能请求http的url，同理https
                        proxy_dict = {'https': proxy_dict['http'].replace('http', 'https')}
                    response = requests.get(url, headers=headers, proxies=proxy_dict, timeout=8)
                else:
                    response = requests.get(url, headers=headers, timeout=8)
                text = re.match(r'reviewList\((.*)\)', response.text).group(1)
                data = json.loads(text)
                comments = data['commodityReviews']
                for comment in comments:
                    id = comment['commodityReviewId']
                    content = comment['content']
                    if content not in robot_comms:
                        self.comm[id] = {}
                        self.comm[id]['content'] = content
                        self.comm[id]['scores'] = round(
                            (SnowNLP(content).sentiments() * 100), 2)
                        self.comm[id]['create_date'] = comment['publishTime']
                        images_info = comment.get('picVideInfo', {})
                        if images_info:
                            small_images = ['https:' + i['url'] + '_100x100.jpg' for i in images_info['imageInfo']]
                            big_images = ['https:' + i['url'] + '_400x400.jpg' for i in images_info['imageInfo']]
                        else:
                            small_images = []
                            big_images = []
                        self.comm[id]['small_images'] = small_images
                        self.comm[id]['big_images'] = big_images
                lock.acquire()
                count = len(self.comm)
                comm_count[self.timeId] = count
                lock.release()
                print('Current Thread Name %s, Url: %s ' % (currentThread().name, url))
            except Exception as e:
                print(e)
                self.urlQueue.put(url)

class DDcomments(CommentParse):
    """获取当当商品的评论"""
    # 当当有防爬，要替换ip

    def __init__(self, product_url, timeId):
        super().__init__()
        self.product_url = product_url
        self.timeId = timeId
        self.pageIndex = 10000

    def build_queue(self):
        # 获取参数categoryPath
        try:
            prod_resp = requests.get(url=self.product_url, headers=headers)
            categoryPath = re.match(
                r'.*?categoryPath":"(.*?)".*',
                prod_resp.text,
                re.S).group(1)
            id = re.match(r'.*?(\d+).*', self.product_url).group(1)
        except Exception as e:
            print(e)
            return
        comm_url = 'http://product.dangdang.com/index.php?r=comment%2Flist&productId={0}&categoryPath={1}&mainProductId={0}&pageIndex={2}'
        try:
            page_1_resp = requests.get(
                url=comm_url.format(
                    id, categoryPath, 1), headers=headers)
            page_1_text = json.loads(page_1_resp.text)
            page_nums = int(page_1_text['data']['list']['summary']['pageCount'])
        except Exception as e:
            print(e)
            return
        for page in range(1, page_nums + 1):
            self.urlQueue.put(comm_url.format(id, categoryPath, page))

    def get_comments(self):
        while True:
            try:
                # 不阻塞的读取队列数据
                url = self.urlQueue.get_nowait()
                if int(url.split('=')[-1]) > self.pageIndex:
                    continue
            except Exception as e:
                break
            try:
                if self.use_proxy:  # 爬取页数少时不使用代理
                    lock.acquire()
                    proxy_dict = getip.get_random_valid_ip()
                    lock.release()
                    if 'https' in url:
                        # http对应的ip只能请求http的url，同理https
                        proxy_dict = {'https': proxy_dict['http'].replace('http', 'https')}
                    response = requests.get(
                        url, headers=headers, proxies=proxy_dict, timeout=8)
                else:
                    response = requests.get(
                        url, headers=headers, timeout=8)  # 要设timeout,否则最后join时会阻塞
                text = json.loads(response.text)['data']['list']['html']
                selector = Selector(text=text)
                comments = selector.css('.comment_items.clearfix')
                if not comments.extract():
                    self.pageIndex = int(url.split('=')[-1])
                for comment in comments:
                    id = comment.css('div.support::attr(data-comment-id)').extract()[0]
                    self.comm[id] = {}
                    self.comm[id]['create_date'] = comment.css('.starline.clearfix span::text').extract()[0]
                    content_list = comment.css('.describe_detail a::text, .describe_detail span::text').extract()
                    self.comm[id]['content'] = ''.join(
                        [content for content in content_list if content not in ['初评', '追评']])
                    if self.comm[id]['content'] == '':
                        self.comm.pop(id)
                        continue
                    self.comm[id]['scores'] = round((SnowNLP(self.comm[id]['content']).sentiments() * 100), 2)
                    self.comm[id]['small_images'] = selector.css('.pic_show.clearfix img::attr(src)').extract()
                    self.comm[id]['big_images'] = selector.css('.pic_show.clearfix img::attr(data-big-pic)').extract()
                lock.acquire()
                count = len(self.comm)
                comm_count[self.timeId] = count
                lock.release()
                print('Current Thread Name %s, Url: %s ' % (currentThread().name, url))
            except Exception as e:
                print(e)
                self.urlQueue.put(url)

def main(url, timeId):
    import time
    start_time = time.time()
    comm_count[timeId] = 0
    host = urlparse(url).netloc
    name = host.split('.')[-2]
    name_dict = {'jd': JDcomments, 'tmall': TMcomments,
                 'amazon': AMScomments, 'suning': SNcomments,
                 'dangdang': DDcomments}
    web_name = name_dict[name]
    web_comment = web_name(url, timeId)
    web_comment.get_all_comments()
    spider_end = time.time()
    print("爬取结束，开始分析，爬取耗时：%s" % (spider_end - start_time))
    comm_count[timeId] = 10000
    mean_score, min_score, max_score, mean_comm, min_comm, max_comm = web_comment.comments_scores()
    hist_imgs = web_comment.hist_scores()
    good_comments, mid_comments, bad_comments = web_comment.show_part_scores()
    analyse_end = time.time()
    print("分析结束，开始生成词云，分析耗时：%s" % (analyse_end - spider_end))
    good_wc = web_comment.build_wordcloud(
        good_comments, os.path.join(
            os.path.dirname(__file__), 'good.jpg'))
    mid_wc = web_comment.build_wordcloud(
        mid_comments, os.path.join(
            os.path.dirname(__file__), 'mid.jpg'))
    bad_wc = web_comment.build_wordcloud(
        bad_comments, os.path.join(
            os.path.dirname(__file__), 'bad.jpg'))
    wc_end = time.time()
    print("生成词云结束，耗时：%s" % (wc_end - analyse_end))
    data = {'mean_score': mean_score, 'mean_comm': mean_comm,
            'min_score': min_score, 'min_comm': min_comm,
            'max_score': max_score, 'max_comm': max_comm,
            'hist_imgs': hist_imgs,
            'good_comments': good_comments,
            'mid_comments': mid_comments,
            'bad_comments': bad_comments,
            'good_wc': good_wc,
            'mid_wc': mid_wc,
            'bad_wc': bad_wc,
            }

    print('总耗时：{} s'.format(time.time() - start_time))
    comm_count.pop(timeId)
    return (json.dumps(data))