# -*- coding: utf-8 -*-
"""
Created on Wed Jul 26 23:40:59 2017

@author: gilberto_titericz
"""
#https://competitions.codalab.org/competitions/16652

import numpy as np
import pandas as pd

from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import mean_squared_error, roc_auc_score
from sklearn.externals import joblib
import xgboost as xgb

df = pd.read_csv("../data/data_train.csv", header=None, names=['country','sku_id','title','category_lvl_1','category_lvl_2','category_lvl_3','short_description','price','product_type'])
df['conciseness'] = np.loadtxt("../data/conciseness_train.labels", dtype=int)
df['clarity']     = np.loadtxt("../data/clarity_train.labels", dtype=int)

X, X_test = joblib.load('../features/all.dmp')

YSTRATIFIED = df['conciseness'].values

#Train conciseness
CV = StratifiedKFold(n_splits=10, shuffle=True, random_state=2017)
y = df['conciseness'].values
predtrain1 = np.zeros( (X.shape[0],2) )
predtest1  = []
for train_index, test_index in CV.split(X, YSTRATIFIED ):

    dtrain = xgb.DMatrix( X[train_index], label=y[train_index])
    dvalid = xgb.DMatrix( X[test_index ], label=y[test_index ])
    dtest  = xgb.DMatrix( X_test )

    param = {'max_depth':0, 'subsample':0.80, 'colsample_bytree':0.80, 'learning_rate':0.015, 'silent':1, 'objective':'reg:linear', 'nthread':8 }
    param['tree_method'] = 'hist'
    param['max_leaves'] = 128
    param['max_bin'] = 128
    param['min_child_weight'] = 2
    param['grow_policy'] = 'lossguide'
    param['eval_metric'] = 'rmse'
    evallist  = [(dvalid,'eval')]
    bst = xgb.train( param, dtrain, 800, evallist, verbose_eval=50 )

    predtrain1[test_index,0] = bst.predict( dvalid  )
    #print np.sqrt( mean_squared_error( y[test_index],predtrain1[test_index,0] )  ), roc_auc_score( y[test_index],predtrain1[test_index,0] )

    predtest1.append( bst.predict( dtest ) )

#print np.sqrt( mean_squared_error( y,predtrain1[:,0] )  ), roc_auc_score( y,predtrain1[:,0] )

#Train Clarity
CV = StratifiedKFold(n_splits=10, shuffle=True, random_state=2017)
y = df['clarity'].values
predtest2  = []
for train_index, test_index in CV.split(X, YSTRATIFIED ):

    dtrain = xgb.DMatrix( X[train_index], label=y[train_index])
    dvalid = xgb.DMatrix( X[test_index ], label=y[test_index ])
    dtest  = xgb.DMatrix( X_test )

    param = {'max_depth':0, 'subsample':0.85, 'colsample_bytree':0.75, 'learning_rate':0.0075, 'silent':1, 'objective':'reg:linear', 'nthread':8 }
    param['tree_method'] = 'hist'
    param['max_leaves'] = 128
    param['max_bin'] = 150
    param['min_child_weight'] = 1
    param['grow_policy'] = 'lossguide'
    param['eval_metric'] = 'rmse'
    evallist  = [(dvalid,'eval')]
    bst = xgb.train( param, dtrain, 600, evallist, verbose_eval=50 )

    predtrain1[test_index,1] = bst.predict( dvalid  )
#    print np.sqrt( mean_squared_error( y[test_index],predtrain1[test_index,1] )  ), roc_auc_score( y[test_index],predtrain1[test_index,0] )

    predtest2.append( bst.predict( dtest ) )

#print np.sqrt( mean_squared_error( y,predtrain1[:,1] )  ), roc_auc_score( y,predtrain1[:,1] )

y = df[ ['conciseness','clarity'] ].values
#print np.sqrt( mean_squared_error( y,predtrain1 )  )

#Write Train Validation and Test predictions to file
df                = pd.read_csv("../data/data_train.csv", header=None, names=['country','sku_id','title','category_lvl_1','category_lvl_2','category_lvl_3','short_description','price','product_type'])
df['conciseness'] = np.loadtxt("../data/conciseness_train.labels", dtype=int)
df['clarity']     = np.loadtxt("../data/clarity_train.labels", dtype=int)
df = df[ ['sku_id','conciseness','clarity'] ]
predtrain1[ predtrain1>1.0 ] = 1.0
predtrain1[ predtrain1<0.0 ] = 0.0
df[ 'giba_conc_xgb1' ] = predtrain1[:,0]
df[ 'giba_clar_xgb1' ] = predtrain1[:,1]
df.head()
df.to_csv( '../pred/L1_giba_xgb1.FoldSet0.TRAINSET.csv', index=False )
#print np.sqrt( mean_squared_error( y,predtrain1 )  )

fold=0
for fold in range(10):
    pred1 = np.copy(predtest1[fold])
    pred1[ pred1>1.0 ] = 1.0
    pred1[ pred1<0.0 ] = 0.0
    pred2 = np.copy(predtest2[fold])
    pred2[ pred2>1.0 ] = 1.0
    pred2[ pred2<0.0 ] = 0.0
    df                = pd.read_csv("../data/data_test.csv", header=None, names=['country','sku_id','title','category_lvl_1','category_lvl_2','category_lvl_3','short_description','price','product_type'])
    df = df[ ['sku_id'] ]
    df['conciseness'] = 0
    df['clarity'] = 0
    df[ 'giba_conc_xgb1' ] = pred1
    df[ 'giba_clar_xgb1' ] = pred2
    df.head()
    df.to_csv( '../pred/L1_giba_xgb1.FoldSet0.TESTSET.fold'+str(fold)+'.csv', index=False )


import pathlib
import dyn2py

INPUT_DIR = "tests/input_files"
OUTPUT_DIR = "tests/output_files"
TEMP_DIR = "tests/temp_files"

def cleanup_dirs():
    for p in [OUTPUT_DIR, TEMP_DIR]:
        the_dir = pathlib.Path(p)
        if the_dir.exists():
            for f in the_dir.iterdir():
                f.unlink()
        else:
            the_dir.mkdir()

def extract_single_node_dyn(modify_py: bool = False):
    """Extract python from single_node.dyn
        File will be here: f"{OUTPUT_DIR}/single_node_1c5d99792882409e97e132b3e9f814b0.py"
        Modified file will be here: f"{OUTPUT_DIR}/single_node_mod.py"

    Args:
        modify_py (bool, optional): Also do some changes on the exported file. Defaults to False.

    """
    cleanup_dirs()

    # Extract py:
    options = dyn2py.Options(python_folder=OUTPUT_DIR)
    dyn = dyn2py.DynamoFile(f"{INPUT_DIR}/single_node.dyn")
    pythonfiles = dyn.extract_python(options)
    pythonfiles[0].write()

    dyn2py.File.open_files.clear()

    if modify_py:
        # Open the extracted file and replace a string:
        with open(f"{OUTPUT_DIR}/single_node_1c5d99792882409e97e132b3e9f814b0.py") as orig_py, \
                open(f"{OUTPUT_DIR}/single_node_mod.py", "w") as mod_py:
            for line in orig_py:
                if "asd_string" in line:
                    line = line.replace("asd_string", "qwe_string")
                mod_py.write(line)


import os
from flask import Flask, request, jsonify, abort
from sqlalchemy import exc
import json
from flask_cors import CORS

from .database.models import db_drop_and_create_all, setup_db, Drink
from .auth.auth import AuthError, requires_auth

app = Flask(__name__)
setup_db(app)
CORS(app)

db_drop_and_create_all()

## ROUTES

#get drink
@app.route('/drinks', methods= ['GET'])
def get_drinks():
    selection = Drink.query.all()
    drinks = [drink.short() for drink in selection]

    if (drinks):
        return jsonify({
            'success': True,
            'drinks': drinks,
        })
    else:
        abort(404)

#get drink detail
@app.route('/drinks-detail', methods= ['GET'])
@requires_auth('get:drinks-detail')
def get_drinks_detail(payload):
    selection = Drink.query.all()
    drinks = [drink.long() for drink in selection]

    if (drinks):
        return jsonify({
            'success': True,
            'drinks': drinks,
        })
    else:
        abort(404)

#create drink
@app.route('/drinks', methods = ['POST'])
@requires_auth('post:drinks')
def create_drink(payload):
    body = request.get_json()

    try:
        new_title = body.get('title', None)
        new_recipe = body.get('recipe')
        # Adjust the input form of the recipe to match the form stored in the database
        # The postman form is a dictionary and needs to be put in a list first
        # Whereas, the browser form is in a list already and doens't need that
        if isinstance(new_recipe, dict):
            new_recipe = [new_recipe]
        new_recipe = json.dumps(new_recipe) #Convert to string to be stored in the database

        new_drink = Drink(title=new_title, recipe=new_recipe)
        new_drink.insert()

        drink = [new_drink.long()]

        return jsonify({
            'success': True,
            'drinks': drink,
            })
    except:
        abort(422)

#update drink
@app.route('/drinks/<id>', methods=['PATCH'])
@requires_auth('patch:drinks')
def update_drink(payload, id):
    body = request.get_json()

    try:
        drink = Drink.query.filter(Drink.id == id).one_or_none()
        if drink is None:
            abort(404)

        updated_title = body.get('title')
        if updated_title:
            drink.title = updated_title

        updated_recipe = body.get('recipe')
        if updated_recipe:
            if isinstance(updated_recipe, dict):
                updated_recipe = [updated_recipe]
            drink.recipe = json.dumps(updated_recipe)   #Convert to string to be stored in the database

        drink.update()

        drink = [drink.long()]

        return jsonify({
            'success': True,
            'drinks': drink,
            })
    except:
      abort(400)

#delete drink
@app.route('/drinks/<id>', methods=['DELETE'])
@requires_auth('delete:drinks')
def delete_drink(payload, id):
    try:
        drink = Drink.query.filter(Drink.id == id).one_or_none()

        if drink is None:
            abort(404)

        drink.delete()

        return jsonify({
            'success': True,
            'delete': id,
        })
    except:
        abort(422)

## Error Handling

@app.errorhandler(422)
def unprocessable(error):
    return jsonify({
                    "success": False,
                    "error": 422,
                    "message": "unprocessable"
                    }), 422

@app.errorhandler(404)
def unprocessable(error):
    return jsonify({
                    "success": False,
                    "error": 404,
                    "message": "resource not found"
                    }), 404

@app.errorhandler(AuthError) #Whenever raised in auth.py
def unauthorized(error):
    return jsonify({
                    "success": False,
                    "error": error.status_code,
                    "message": error.error['description']
                    }), error.status_code


from django.shortcuts import render, get_object_or_404, redirect
from django.http import HttpResponse
from .models import News, Category
from .forms import NewsForm
from django.urls import reverse_lazy
from django.views.generic import ListView, DetailView, CreateView
from news.utils import MyMixin
from django.contrib.auth.mixins import LoginRequiredMixin
# Create your views here.

class HomeNews(MyMixin, ListView ):
    model = News
    template_name = 'news/home_news_list.html'
    context_object_name = 'news'
    mixin_prop = 'hello world'
    #extra_context = {'title': 'Главная страница'}

    def get_context_data(self, *, object_list=None, **kwargs):
        context = super().get_context_data(**kwargs)
        context['title'] = self.get_upper('Главная страница')
        context['mixin_prop'] = self.get_prop()
        return context

    def get_queryset(self):
        return News.objects.filter(is_published=True)

class NewsByCategory(ListView, MyMixin):
    model = News
    template_name = 'news/home_news_list.html'
    context_object_name = 'news'
    allow_empty = False

    def get_queryset(self):
        return News.objects.filter(category_id=self.kwargs['category_id'], is_published=True)

    def get_context_data(self, *, object_list=None, **kwargs):
        context = super().get_context_data(**kwargs)
        context['title'] = self.get_upper(Category.objects.get(pk=self.kwargs['category_id']))
        return context

class ViewNews(DetailView):
    model = News
    context_object_name = 'news_item'
    #template_name = 'news/news_detail.html'
    #pk_url_kwarg = 'news_id'

class CreateNews(LoginRequiredMixin, CreateView):
    form_class = NewsForm
    template_name = 'news/add_news.html'
    success_url = reverse_lazy('home')
    raise_exception = True             #Генерация ошибки при попытке зайти по адресу создания новости
                                       #будучи не авторизованным

# def index(request):
#     news = News.objects.all()
#     return render(request, 'news/index.html', {'news': news,
#                                                'title': 'Список новостей',
#                                                })
def get_category(request, category_id):
    news = News.objects.filter(category_id=category_id)
    category = Category.objects.get(pk=category_id)
    return render(request, 'news/category.html', {'news': news,
                                               'category': category
                                               })

#def view_news(request, news_id):
#    #news_item = News.objects.get(pk=news_id)
#    news_item = get_object_or_404(News, pk=news_id)
#    return render(request, 'news/view_news.html', {"news_item": news_item})

# def add_news(request):
#     if request.method == 'POST':
#         form = NewsForm(request.POST)    #Забираем из формы данные
#         if form.is_valid():
#             #print(form.cleaned_data)
#             #News.objects.create(**form.cleaned_data)  # Операция распаковки словаря в Python
#             #return redirect('home')
#             news = form.save()
#             return redirect(news)
#     else:
#         form = NewsForm()
#     return render(request, 'news/add_news.html', {'form': form})


# -*- coding: utf-8 -*-
"""
Created on Mon Nov 18 16:26:32 2019

@author: sdorle
"""
# =============================================================================
# Concatenating all segments
# =============================================================================
from moviepy.editor import VideoFileClip, clips_array, concatenate_videoclips
# creating array for storing all the segments
all_video_clips = []
import glob, os
#os.chdir("/Output") # To change directory
for file in glob.glob("*.mpg"):
    print(file)
    print(type(file))
    all_video_clips.append(VideoFileClip(file))

# Full movie after combining all segments
concatenated_clip = concatenate_videoclips(all_video_clips)
#final_clip.write_videofile("my_concatenation_demo.mp4")

# Importing mp4 file of graph animation
graph_clip = VideoFileClip(r"Output\\Phobia_mp4.mp4")

# Combining Movie file and Graph file in split screen
final_clip = clips_array([[concatenated_clip, graph_clip]])

# Exporting file in mp4 format
final_clip.write_videofile("Output\\Movie_graph_4.mp4")#, fps = 30)

# For saving specific frame as image
np_frame = final_clip.get_frame(2) # get the frame at t=2 seconds

final_clip.save_frame('my_image.png', t=400)

'''
# =============================================================================
# Concatenating clips one after other
# =============================================================================
from moviepy.editor import VideoFileClip, concatenate_videoclips
clip1 = VideoFileClip("demo2.mp4") # Importing 1st video file
clip2 = VideoFileClip("518059_11_1.mpg") # importing second video file
final_clip = concatenate_videoclips([clip1,clip2]) # concatenating both vi
final_clip.write_videofile("my_concatenation.mp4") # Exporting results to mp4 file

# =============================================================================
# Screen Split example
# =============================================================================

from moviepy.editor import VideoFileClip, clips_array, vfx

clip1 = VideoFileClip("517695_01.mpg").subclip(50,80)#
clip2 = VideoFileClip(r"517695_02.mpg").subclip(50,80)

# Concatenating both video files
final_clip = clips_array([[clip1, clip2]])
#final_clip.resize(width=480).write_videofile("my_stack_2.mp4")

# Writing/exporting video file
final_clip.write_videofile("my_stack_5.mp4")
'''


"""
1014. Best Sightseeing Pair
You are given an integer array values where values[i] represents the value of the ith sightseeing spot. Two sightseeing spots i and j have a distance j - i between them.

The score of a pair (i < j) of sightseeing spots is values[i] + values[j] + i - j: the sum of the values of the sightseeing spots, minus the distance between them.

Return the maximum score of a pair of sightseeing spots.

Example1:
Input: values = [8,1,5,2,6]
Output: 11
Explanation: i = 0, j = 2, values[i] + values[j] + i - j = 8 + 5 + 0 - 2 = 11

Example2:
Input: values = [1,2]
Output: 2

Constraints:
2 <= values.length <= 5 * 10^4
1 <= values[i] <= 1000
"""

"""
Note:
1. SuffixMax: O(n) time | O(n) space - where n is the length of values
(1) values[i] + values[j] + i - j = (values[i]+i) + (values[j]-j)
(2) Build a SuffixMax array
(3) iterate each number, and get the Max values[j]-j from position i, and update the target
"""

from typing import List
class Solution:
    def maxScoreSightseeingPair(self, values: List[int]) -> int:
        n = len(values)
        plus = [0] * n
        minus = [0] * n
        for i in range(n):
            plus[i] = values[i]+i
            minus[i] = values[i]-i

        suffixMinusMax = [-float("inf")] * n
        suffixMinusMax[-1] = minus[-1]
        for i in range(n-2, -1, -1):
            suffixMinusMax[i] = max(suffixMinusMax[i+1], minus[i])

        maxVal = -float("inf")
        for i in range(n-1):
            maxVal = max(maxVal, plus[i]+suffixMinusMax[i+1])
        return maxVal

# Unit Tests
import unittest
funcs = [Solution().maxScoreSightseeingPair]
class TestMaxScoreSightseeingPair(unittest.TestCase):
    def testMaxScoreSightseeingPair1(self):
        for func in funcs:
            values = [8,1,5,2,6]
            self.assertEqual(func(values=values), 11)

    def testMaxScoreSightseeingPair2(self):
        for func in funcs:
            values = [1,2]
            self.assertEqual(func(values=values), 2)

if __name__ == "__main__":
    unittest.main()


"""This module provides H&E stain normalization tools, with numpy, PyTorch,
and Tensorflow implementations for several stain normalization methods.

Overview
--------

The main normalizer interface, :class:`slideflow.norm.StainNormalizer`, offers
efficient numpy implementations for the Macenko, Reinhard, Reinhard-Fast,
Reinhard (masked), and Vahadane H&E stain normalization algorithms, as well
as an HSV colorspace stain augmentation method. This normalizer can convert
images to and from Tensors, numpy arrays, and raw JPEG/PNG images.

In addition to these numpy implementations, PyTorch-native and Tensorflow-native
implementations are also provided, which offer performance improvements
and/or vectorized application. The native normalizers are found in
``slideflow.norm.tensorflow`` and ``slideflow.norm.torch``, respectively.
Tensorflow-native normalizer methods include Macenko, Reinhard, and
Reinhard-fast. Torch-native normalizer methods include Reinhard and
Reinhard-fast.

The Vahadane normalizer has two numpy implementations available: SPAMS
(``vahadane_spams``) and sklearn (``vahadane_sklearn``). As of version 1.2.3,
the sklearn implementation will be used if unspecified (``method='vahadane'``).

Performance
-----------

The Numpy implementations contain all functions necessary for normalizing
Tensors from both Tensorflow and PyTorch, but may be slower than backend-native
implementations when available. Performance benchmarks for the normalizer
implementations are given below:

.. list-table:: **Performance Benchmarks** (256 x 256 images, Slideflow 1.2.3, benchmarked on 3960X and A100 40GB)
    :header-rows: 1

    * -
      - Tensorflow backend
      - PyTorch backend
    * - macenko
      - 1,295 img/s (**native**)
      - 1,265 img/s (**native**)
    * - reinhard
      - 1,536 img/s (**native**)
      - 2,246 img/s (**native**)
    * - reinhard_fast
      - 8,599 img/s (**native**)
      - 2,832 img/s (**native**)
    * - reinhard_mask
      - 1,537 img/s (**native**)
      - 2,246 img/s
    * - reinhard_fast_mask
      - 7,885 img/s (**native**)
      - 2,719 img/s
    * - vahadane_spams
      - 0.7 img/s
      - 2.2 img/s
    * - vahadane_sklearn
      - 0.9 img/s
      - 1.0 img/s

Use :func:`slideflow.norm.autoselect` to get the fastest available normalizer
for a given method and active backend (Tensorflow/PyTorch).
"""

from __future__ import absolute_import

import os
import sys
import multiprocessing as mp
from io import BytesIO
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union

import cv2
import numpy as np
import slideflow as sf
from PIL import Image
from slideflow import errors
from slideflow.dataset import Dataset
from slideflow.util import detuple, log
from rich.progress import Progress

if TYPE_CHECKING:
    import tensorflow as tf
    import torch

from slideflow.norm import (augment, macenko, reinhard, vahadane)

class StainNormalizer:

    vectorized = False
    normalizers = {
        'macenko':  macenko.MacenkoNormalizer,
        'reinhard': reinhard.ReinhardNormalizer,
        'reinhard_fast': reinhard.ReinhardFastNormalizer,
        'reinhard_mask': reinhard.ReinhardMaskNormalizer,
        'reinhard_fast_mask': reinhard.ReinhardFastMaskNormalizer,
        'vahadane': vahadane.VahadaneSpamsNormalizer,
        'vahadane_sklearn': vahadane.VahadaneSklearnNormalizer,
        'vahadane_spams': vahadane.VahadaneSpamsNormalizer,
        'augment': augment.AugmentNormalizer
    }  # type: Dict[str, Any]

    def __init__(self, method: str, **kwargs) -> None:
        """H&E Stain normalizer supporting various normalization methods.

        The stain normalizer supports numpy images, PNG or JPG strings,
        Tensorflow tensors, and PyTorch tensors. The default ``.transform()``
        method will attempt to preserve the original image type while minimizing
        conversions to and from Tensors.

        Alternatively, you can manually specify the image conversion type
        by using the appropriate function. For example, to convert a Tensor
        to a normalized numpy RGB image, use ``.tf_to_rgb()``.

        Args:
            method (str): Normalization method. Options include 'macenko',
                'reinhard', 'reinhard_fast', 'reinhard_mask',
                'reinhard_fast_mask', 'vahadane', 'vahadane_spams',
                'vahadane_sklearn', and 'augment'.

        Keyword args:
            stain_matrix_target (np.ndarray, optional): Set the stain matrix
                target for the normalizer. May raise an error if the normalizer
                does not have a stain_matrix_target fit attribute.
            target_concentrations (np.ndarray, optional): Set the target
                concentrations for the normalizer. May raise an error if the
                normalizer does not have a target_concentrations fit attribute.
            target_means (np.ndarray, optional): Set the target means for the
                normalizer. May raise an error if the normalizer does not have
                a target_means fit attribute.
            target_stds (np.ndarray, optional): Set the target standard
                deviations for the normalizer. May raise an error if the
                normalizer does not have a target_stds fit attribute.

        Raises:
            ValueError: If the specified normalizer method is not available.

        Examples
            Normalize a numpy image using the default fit.

                >>> import slideflow as sf
                >>> macenko = sf.norm.StainNormalizer('macenko')
                >>> macenko.transform(image)

            Fit the normalizer to a target image (numpy or path).

                >>> macenko.fit(target_image)

            Fit the normalizer to all images in a Dataset.

                >>> dataset = sf.Dataset(...)
                >>> macenko.fit(dataset)

            Normalize an image and convert from Tensor to RGB.

                >>> macenko.tf_to_rgb(image)

            Normalize images during DataLoader pre-processing.

                >>> dataset = sf.Dataset(...)
                >>> dataloader = dataset.torch(..., normalizer=macenko)
                >>> dts = dataset.tensorflow(..., normalizer=macenko)

        """
        if method not in self.normalizers:
            raise ValueError(f"Unrecognized normalizer method {method}")

        self.method = method
        self.n = self.normalizers[method]()

        if kwargs:
            self.n.fit(**kwargs)

    def __repr__(self):
        base = "{}(\n".format(self.__class__.__name__)
        base += "  method = {!r},\n".format(self.method)
        for fit_param, fit_val in self.get_fit().items():
            base += "  {} = {!r},\n".format(fit_param, fit_val)
        base += ")"
        return base

    def _torch_transform(self, inp: "torch.Tensor") -> "torch.Tensor":
        """Normalize a torch uint8 image (CWH), via intermediate
        conversion to WHC.

        Args:
            inp (torch.Tensor): Image, uint8. Images are normalized in
                W x H x C space. Images provided as C x W x H will be
                auto-converted and permuted back after normalization.

        Returns:
            torch.Tensor:   Image, uint8.

        """
        import torch
        from slideflow.io.torch import cwh_to_whc, whc_to_cwh

        if len(inp.shape) == 4:
            return torch.stack([self._torch_transform(img) for img in inp])
        elif inp.shape[0] == 3:
            # Convert from CWH -> WHC (normalize) -> CWH
            return whc_to_cwh(torch.from_numpy(self.rgb_to_rgb(cwh_to_whc(inp).cpu().numpy())))
        else:
            return torch.from_numpy(self.rgb_to_rgb(inp.cpu().numpy()))

    def fit(
        self,
        arg1: Optional[Union[Dataset, np.ndarray, str]],
        batch_size: int = 64,
        num_threads: Union[str, int] = 'auto',
        **kwargs,
    ) -> None:
        """Fit the normalizer to a target image or dataset of images.

        Args:
            arg1: (Dataset, np.ndarray, str): Target to fit. May be a numpy
                image array (uint8), path to an image, or a Slideflow Dataset.
                If a Dataset is provided, will average fit values across
                all images in the dataset.
            batch_size (int, optional): Batch size during fitting, if fitting
                to dataset. Defaults to 64.
            num_threads (Union[str, int], optional): Number of threads to use
                during fitting, if fitting to a dataset. Defaults to 'auto'.
        """

        # Fit to a dataset
        if isinstance(arg1, Dataset):
            # Set up thread pool
            if num_threads == 'auto':
                num_threads = os.cpu_count()  # type: ignore
            log.debug(f"Setting up pool (size={num_threads}) for norm fitting")
            log.debug(f"Using normalizer batch size of {batch_size}")
            pool = mp.dummy.Pool(num_threads)  # type: ignore

            dataset = arg1
            if sf.backend() == 'tensorflow':
                dts = dataset.tensorflow(
                    None,
                    batch_size,
                    standardize=False,
                    infinite=False
                )
            elif sf.backend() == 'torch':
                dts = dataset.torch(
                    None,
                    batch_size,
                    standardize=False,
                    infinite=False,
                    num_workers=8
                )
            all_fit_vals = []  # type: ignore
            pb = Progress(transient=True)
            task = pb.add_task('Fitting normalizer...', total=dataset.num_tiles)
            pb.start()
            for img_batch, slide in dts:
                if sf.model.is_torch_tensor(img_batch):
                    img_batch = img_batch.permute(0, 2, 3, 1)  # BCWH -> BWHC

                mapped = pool.imap(lambda x: self.n.fit(x.numpy()), img_batch)
                for fit_vals in mapped:
                    if all_fit_vals == []:
                        all_fit_vals = [[] for _ in range(len(fit_vals))]
                    for v, val in enumerate(fit_vals):
                        all_fit_vals[v] += [np.squeeze(val)]
                pb.advance(task, batch_size)
            pb.stop()
            self.n.set_fit(*[np.array(v).mean(axis=0) for v in all_fit_vals])
            pool.close()

        # Fit to numpy image
        elif isinstance(arg1, np.ndarray):
            self.n.fit(arg1)

        # Fit to a preset
        elif isinstance(arg1, str) and arg1 in ('v1', 'v2'):
            self.n.fit_preset(arg1)

        # Fit to a path to an image
        elif isinstance(arg1, str):
            self.src_img = cv2.cvtColor(cv2.imread(arg1), cv2.COLOR_BGR2RGB)
            self.n.fit(self.src_img)

        elif arg1 is None and kwargs:
            self.set_fit(**kwargs)

        else:
            raise ValueError(f'Unrecognized args for fit: {arg1}')

        log.debug('Fit normalizer: {}'.format(
            ', '.join([f"{fit_key} = {fit_val}"
            for fit_key, fit_val in self.get_fit().items()])
        ))

    def get_fit(self, as_list: bool = False):
        """Get the current normalizer fit.

        Args:
            as_list (bool). Convert the fit values (numpy arrays) to list
                format. Defaults to False.

        Returns:
            Dict[str, np.ndarray]: Dictionary mapping fit parameters (e.g.
            'target_concentrations') to their respective fit values.
        """
        _fit = self.n.get_fit()
        if as_list:
            return {k: v.tolist() for k, v in _fit.items()}
        else:
            return _fit

    def set_fit(self, **kwargs) -> None:
        """Set the normalizer fit to the given values.

        Keyword args:
            stain_matrix_target (np.ndarray, optional): Set the stain matrix
                target for the normalizer. May raise an error if the normalizer
                does not have a stain_matrix_target fit attribute.
            target_concentrations (np.ndarray, optional): Set the target
                concentrations for the normalizer. May raise an error if the
                normalizer does not have a target_concentrations fit attribute.
            target_means (np.ndarray, optional): Set the target means for the
                normalizer. May raise an error if the normalizer does not have
                a target_means fit attribute.
            target_stds (np.ndarray, optional): Set the target standard
                deviations for the normalizer. May raise an error if the
                normalizer does not have a target_stds fit attribute.
        """
        self.n.set_fit(**{k:v for k, v in kwargs.items() if v is not None})

    def transform(
        self,
        image: Union[str, bytes, np.ndarray, "tf.Tensor", "torch.Tensor"]
    ) -> Union[str, bytes, np.ndarray, "tf.Tensor", "torch.Tensor"]:
        """Normalize a target image, attempting to preserve the original type.

        Args:
            image (np.ndarray, tf.Tensor, or torch.Tensor): Image.

        Returns:
            Normalized image of the original type.
        """
        if isinstance(image, (str, bytes)):
            raise ValueError("Unable to auto-transform bytes or str; please "
                             "use .png_to_png() or .jpeg_to_jpeg().")
        if 'tensorflow' in sys.modules:
            import tensorflow as tf
            if isinstance(image, tf.Tensor):
                return self.tf_to_tf(image)
        if 'torch' in sys.modules:
            import torch
            if isinstance(image, torch.Tensor):
                return self.torch_to_torch(image)
        if isinstance(image, np.ndarray):
            return self.rgb_to_rgb(image)
        raise ValueError(f"Unrecognized image type {type(image)}; expected "
                         "np.ndarray, tf.Tensor, or torch.Tensor")

    def jpeg_to_jpeg(
        self,
        jpeg_string: Union[str, bytes],
        quality: int = 100
    ) -> bytes:
        """Normalize a JPEG image, returning a JPEG image.

        Args:
            jpeg_string (str, bytes): JPEG image data.
            quality (int, optional): Quality level for creating the resulting
                normalized JPEG image. Defaults to 100.

        Returns:
            bytes:  Normalized JPEG image.
        """
        cv_image = self.jpeg_to_rgb(jpeg_string)
        with BytesIO() as output:
            Image.fromarray(cv_image).save(
                output,
                format="JPEG",
                quality=quality
            )
            return output.getvalue()

    def jpeg_to_rgb(self, jpeg_string: Union[str, bytes]) -> np.ndarray:
        """Normalize a JPEG image, returning a numpy uint8 array.

        Args:
            jpeg_string (str, bytes): JPEG image data.

        Returns:
            np.ndarray: Normalized image, uint8, W x H x C.
        """
        cv_image = cv2.imdecode(
            np.fromstring(jpeg_string, dtype=np.uint8),
            cv2.IMREAD_COLOR
        )
        cv_image = cv2.cvtColor(cv_image, cv2.COLOR_BGR2RGB)
        return self.n.transform(cv_image)

    def png_to_png(self, png_string: Union[str, bytes]) -> bytes:
        """Normalize a PNG image, returning a PNG image.

        Args:
            png_string (str, bytes): PNG image data.

        Returns:
            bytes: Normalized PNG image.
        """
        cv_image = self.png_to_rgb(png_string)
        with BytesIO() as output:
            Image.fromarray(cv_image).save(output, format="PNG")
            return output.getvalue()

    def png_to_rgb(self, png_string: Union[str, bytes]) -> np.ndarray:
        """Normalize a PNG image, returning a numpy uint8 array.

        Args:
            png_string (str, bytes): PNG image data.

        Returns:
            np.ndarray: Normalized image, uint8, W x H x C.
        """
        return self.jpeg_to_rgb(png_string)  # It should auto-detect format

    def rgb_to_rgb(self, image: np.ndarray) -> np.ndarray:
        """Normalize a numpy array (uint8), returning a numpy array (uint8).

        Args:
            image (np.ndarray): Image (uint8).

        Returns:
            np.ndarray: Normalized image, uint8, W x H x C.
        """
        return self.n.transform(image)

    def tf_to_rgb(self, image: "tf.Tensor") -> np.ndarray:
        """Normalize a tf.Tensor (uint8), returning a numpy array (uint8).

        Args:
            image (tf.Tensor): Image (uint8).

        Returns:
            np.ndarray: Normalized image, uint8, W x H x C.
        """
        return self.rgb_to_rgb(np.array(image))

    def tf_to_tf(
        self,
        image: Union[Dict, "tf.Tensor"],
        *args: Any
    ) -> Tuple[Union[Dict, "tf.Tensor"], ...]:
        """Normalize a tf.Tensor (uint8), returning a numpy array (uint8).

        Args:
            image (tf.Tensor, Dict): Image (uint8) either as a raw Tensor,
                or a Dictionary with the image under the key 'tile_image'.
            args (Any, optional): Any additional arguments, which will be passed
                and returned unmodified.

        Returns:
            A tuple containing the normalized tf.Tensor image (uint8,
            W x H x C) and any additional arguments provided.
        """
        import tensorflow as tf

        if isinstance(image, dict):
            image['tile_image'] = tf.py_function(
                self.tf_to_rgb,
                [image['tile_image']],
                tf.int32
            )
        elif len(image.shape) == 4:
            image = tf.stack([self.tf_to_tf(_i) for _i in image])
        else:
            image = tf.py_function(self.tf_to_rgb, [image], tf.int32)
        return detuple(image, args)

    def torch_to_torch(
        self,
        image: Union[Dict, "torch.Tensor"],
        *args
    ) -> Tuple[Union[Dict, "torch.Tensor"], ...]:
        """Normalize a torch.Tensor (uint8), returning a numpy array (uint8).

        Args:
            image (torch.Tensor, Dict): Image (uint8) either as a raw Tensor,
                or a Dictionary with the image under the key 'tile_image'.
            args (Any, optional): Any additional arguments, which will be passed
                and returned unmodified.

        Returns:
            A tuple containing

                np.ndarray: Normalized tf.Tensor image, uint8, C x W x H.

                args (Any, optional): Any additional arguments provided, unmodified.
        """
        if isinstance(image, dict):
            to_return = {
                k: v for k, v in image.items()
                if k != 'tile_image'
            }
            to_return['tile_image'] = self._torch_transform(image['tile_image'])
            return detuple(to_return, args)
        else:
            return detuple(self._torch_transform(image), args)

def autoselect(
    method: str,
    source: Optional[str] = None,
    **kwargs
) -> StainNormalizer:
    """Select the best normalizer for a given method, and fit to a given source.

    If a normalizer method has a native implementation in the current backend
    (Tensorflow or PyTorch), the native normalizer will be used.
    If not, the default numpy implementation will be used.

    Currently, the PyTorch-native normalizers are NOT used by default, as they
    are slower than the numpy implementations. Thus, with the PyTorch backend,
    all normalizers will be the default numpy implementations.

    Args:
        method (str): Normalization method. Options include 'macenko',
            'reinhard', 'reinhard_fast', 'reinhard_mask', 'reinhard_fast_mask',
            'vahadane', 'vahadane_spams', 'vahadane_sklearn', and 'augment'.
        source (str, optional): Path to a source image. If provided, will
            fit the normalizer to this image. Defaults to None.

    Returns:
        StainNormalizer:    Initialized StainNormalizer.
    """

    if sf.backend() == 'tensorflow':
        import slideflow.norm.tensorflow
        BackendNormalizer = sf.norm.tensorflow.TensorflowStainNormalizer
    elif sf.backend() == 'torch' and method == 'macenko':
        import slideflow.norm.torch
        BackendNormalizer = sf.norm.torch.TorchStainNormalizer  # type: ignore
    elif sf.backend() == 'torch':
        BackendNormalizer = StainNormalizer  # type: ignore
    else:
        raise errors.UnrecognizedBackendError

    if method in BackendNormalizer.normalizers:
        normalizer = BackendNormalizer(method, **kwargs)
    else:
        normalizer = StainNormalizer(method, **kwargs)  # type: ignore

    if source is not None and source != 'dataset':
        normalizer.fit(source)

    return normalizer

{'torch': 'torch', 'cwh_to_whc': 'slideflow.io.torch.cwh_to_whc', 'whc_to_cwh': 'slideflow.io.torch.whc_to_cwh', 'tf': 'tensorflow'}


import subprocess
from blkinfo import BlkDiskInfo # fork of genalt/blkinfo see https://github.com/macOS-KVM/blkinfo/releases/tag/0.2.0
from .storage_manager_exception import StorageManagerException
from .convertsize import convertSizeUnit
import time
import os
from pyfstab import Fstab, Entry

def deletePartition(disk, partition):
    try:
        subprocess.check_output(['parted', disk, 'rm', str(partition)])
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to delete partition {partition} on disk {disk}") from e

def formatPartition(path, fstype):
    try:
        if fstype == 'xfs':
            subprocess.check_output(['mkfs.xfs', '-f', path])
        else:
            subprocess.check_output(['mkfs', '-t', fstype, path])
        # wait for the partition to be formatted
        while find_uuid(path) == None:
            time.sleep(0.1)
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to format partition {path} with fstype {fstype}") from e

def findUsedSpace(path):
    try:
        used_space = subprocess.check_output(['df', '-BM', path])
        output = used_space.decode('utf-8')
        used_space = output.split('\n')[1].split()[2]
        capacity = output.split('\n')[1].split()[1]
        free = output.split('\n')[1].split()[3]
        return [int(used_space[:-1]), int(capacity[:-1]), int(free[:-1])]
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to find used space for partition {path}") from e

def find_uuid(path):
    try:
        uuid_process = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', path])
        uuid = uuid_process.decode('utf-8').strip()
        return uuid if uuid != "" else None
    except subprocess.CalledProcessError as e:
        return None

def find_statupmount(uuid):
    try:
        with open("/etc/fstab", "r") as f:
            fstab = Fstab().read_file(f)
        # find the fstab entry with the same uuid
        for entry in fstab.entries:
            if entry.device_tag_type == "UUID" and entry.device_tag_value == uuid:
                return entry.dir
        return None
    except Exception as e:
        raise StorageManagerException(f"Failed to get startup mount for device {uuid}") from e

def remove_startupmount(uuid=None, path=None):
    if uuid == None and path == None:
        raise StorageManagerException("Either uuid or path must be specified")
    try:
        found = False
        with open("/etc/fstab", "r") as f:
            fstab = Fstab().read_file(f)
        for entry in fstab.entries:
            if uuid != None:
                if entry.device_tag_type == "UUID" and entry.device_tag_value == uuid:
                    fstab.entries.remove(entry)
                    found = True

            elif path != None:
                if entry.device_tag_value == path:
                    fstab.entries.remove(entry)
                    found = True
        if found:
            with open("/etc/fstab", "w") as f:
                fstab.write_file(f)
    except Exception as e:
        raise StorageManagerException(f"Failed to remove startup mount for device {uuid}") from e

def add_startupmount(uuid=None, path=None, mountpoint=None, fstype='ext4'):
    if uuid == None and path == None:
        raise StorageManagerException("Either uuid or path must be specified")
    if mountpoint == None:
        raise StorageManagerException("Mountpoint must be specified")
    if uuid == None:
        fstab_entry = Entry(
            path,
            mountpoint,
            fstype,
            "defaults",
            0,
            0
        )
    else:
        fstab_entry = Entry(
            "UUID=" + uuid,
            mountpoint,
            fstype,
            "defaults",
            0,
            0
        )

    try:
        with open("/etc/fstab", "r") as f:
            fstab = Fstab().read_file(f)
        fstab.entries.append(fstab_entry)
        with open("/etc/fstab", "w") as f:
            fstab.write_file(f)
    except Exception as e:
        raise StorageManagerException(f"Failed to add startup mount for device {uuid}") from e

def lookupPartition(uuid):
    # get the partition from get()
    partition = None
    for disk in get():
        for partition in disk['partitions']:
            if partition['uuid'] == uuid:
                return partition
    if partition == None:
        raise StorageManagerException(f"Failed to lookup partition with uuid {uuid}")

def mountPartition(uuid, mountpoint):
    # lookup the partition
    partition = lookupPartition(uuid)

    # create the mountpoint if it doesn't exist
    if not os.path.exists(mountpoint):
        os.makedirs(mountpoint)

    # add the mountpoint to /etc/fstab
    if find_statupmount(uuid) == None:
        add_startupmount(uuid=uuid, mountpoint=mountpoint, fstype=partition['fstype'])
    else:
        remove_startupmount(uuid=uuid)
        add_startupmount(uuid=uuid, mountpoint=mountpoint, fstype=partition['fstype'])

    # reload systemd and mount the partition
    try:
        subprocess.check_output(['systemctl', 'daemon-reload'])
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to reload systemd") from e
    try:
        subprocess.check_output(['mount', '-a'])
        # wait for the partition to be mounted
        while not os.path.ismount(mountpoint):
            time.sleep(0.1)
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to mount partition {uuid} at {mountpoint}") from e

def unmountPartition(uuid):
    remove_startupmount(uuid=uuid)
    try:
        subprocess.check_output(['umount', f'/dev/disk/by-uuid/{uuid}'])
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to unmount partition {uuid}") from e

def checkPartitioningScheme(diskpath):
    # run parted /dev/sdc print
    # if the output contains "Partition Table: unknown", then return False
    # else return True
    try:
        output = subprocess.check_output(['parted', diskpath, 'print'])
        output = output.decode('utf-8')
        # the 4th line of the output contains the partition table type
        if output.split('\n')[3].split(':')[1].strip() == 'unknown':
            return False
        else:
            return True
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to check partitioning scheme on disk {diskpath}") from e

def createPartition(diskpath, fstype):
    # check if the disk has a partition table
    if not checkPartitioningScheme(diskpath):
        # create a partition table
        try:
            subprocess.check_output(['parted', diskpath, 'mklabel', 'gpt'])
            # wait for the partition table to be created
            while not checkPartitioningScheme(diskpath):
                time.sleep(0.1)
        except subprocess.CalledProcessError as e:
            raise StorageManagerException(f"Failed to create partition table on disk {diskpath}") from e

    # create a partition using parted
    try:
        subprocess.check_output(['parted', diskpath, 'mkpart', 'primary', '0%', '100%'])
        # wait for the partition to be created
        while not os.path.exists(diskpath + '1'):
            time.sleep(0.1)
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to create partition on disk {diskpath}") from e

    # format the partition using formatPartition()
    formatPartition(diskpath + '1', fstype)

def wipeDisk(path):
    try:
        subprocess.check_output(['wipefs', '-a', path])
    except subprocess.CalledProcessError as e:
        raise StorageManagerException(f"Failed to wipe disk {path}") from e

def get():
    disk_list = []
    blk_info = BlkDiskInfo().get_disks(ignore_raid_parents=False)
    for disk in blk_info:
        # skip all devices that are not disks and zram devices
        if disk['type'] == 'disk' and not disk['name'].startswith('zram'):
            size = disk['size']
            diskpath = f"/dev/{disk['name']}"
            disk_has_partitiontable = True
            disk_type = 'individual' # disk type can be raid_member, individual, or system

            if disk['fstype'] == 'linux_raid_member':
                disk_type = 'raid_member'

            partitions = []
            for index, partition in enumerate(disk['children']):
                partition_fstype = partition['fstype']
                partition_name = partition['name']
                partition_mount = partition['mountpoint']
                partition_size = partition['size']
                partition_path = f"/dev/{partition_name}"
                partition_uuid = find_uuid(partition_path)
                partition_used = findUsedSpace(partition_path)[0]
                if partition_fstype == 'linux_raid_member':
                    disk_type = 'raid_member'
                    continue
                elif partition_fstype == '':
                    partition_fstype = 'unformatted'

                if partition_mount.startswith('/boot'):
                    disk_type = 'system'

                partitions.append({
                    "number": index+1,
                    "fstype": partition_fstype,
                    "uuid": partition_uuid,
                    "name": partition_name,
                    "mount": partition_mount,
                    "path": partition_path,
                    "parent": diskpath,
                    "used": None if partition_used == 0 else convertSizeUnit(size=partition_used, from_unit="MB", mode="str", round_state=True, round_to=None),
                    "size": convertSizeUnit(size=int(partition_size), from_unit="B", mode="str", round_state=True, round_to=None),
                    })

            disk_list.append({
                'name': disk['name'],
                'model': disk['model'],
                'disktype': disk_type,
                'size': convertSizeUnit(size=int(size), from_unit="B", mode="str", round_state=True, round_to=None),
                'path': diskpath,
                'serial': disk['serial'],
                'partitions': partitions,
                'has_partitiontable':  disk_has_partitiontable,
            })
    return disk_list


#Найти для сортов Gew[üu]rztraminer, Riesling, Merlot, Madera, Tempranillo, Red Blend следующую информацию:
#
#average_price
#min_price
#max_price
#most_common_region где больше всего вин этого сорта производят ?
#most_common_country
#avarage_score

import json

with open('winedata_full.json', 'r') as json.file:
    wine_data = json.load(json.file)

def average_price(name):
    total = 0
    counter = 0
    for wine in wine_data:
        if wine['designation'] == name:
            if wine['price'] is None:
                continue
            counter += 1
            total += wine['price']
    if counter == 0:
        return 'There is no such wine as', name
    return total / counter

def min_price(name):
    min_price = 10 ** 100
    counter = 0
    for wine in wine_data:
        if wine['designation'] == name:
            counter += 1
            if wine['price'] is None:
                continue
            if wine['price'] < min_price:
                min_price = wine['price']
    if counter == 0:
        return 'There is no such wine as', name
    return min_price

def max_price(name):
    counter_wine = 0
    max_price = 0
    for wine in wine_data:
        if wine['designation'] == name:
            counter_wine += 1
            if wine['price'] is None:
                continue
            if wine['price'] > max_price:
                max_price = wine['price']
    if counter_wine == 0:
        return 'There is no such wine as', name
    return max_price

def most_common_region(name):
    from collections import Counter
    counter_wine = 0
    counter = Counter()
    most_common_region = []
    for wine in wine_data:
        if wine['designation'] != name:
            continue
        if wine['region_1'] is None:
            continue
        else:
            counter_wine += 1
            counter[wine['region_1']] += 1

    if counter_wine == 0:
        return 'There is no such wine as', name

    most_occur = counter.most_common()

    if len(most_occur) == 1:
        return [most_occur[0][0]]

    for element_wine in range(len(most_occur) - 1):
        if most_occur[0][1] == most_occur[element_wine][1]:
            most_common_region.append(most_occur[element_wine][0])

    return most_common_region

def most_common_country(name):
    from collections import Counter
    counter_wine = 0
    counter = Counter()
    most_common_country = []
    for wine in wine_data:
        if wine['designation'] != name:
            continue
        if wine['country'] is None:
            continue
        else:
            counter_wine += 1
            counter[wine['country']] += 1

    if counter_wine == 0:
        return 'There is no such wine as', name

    most_occur = counter.most_common()

    if len(most_occur) == 1:
        return [most_occur[0][0]]

    for element_wine in range(len(most_occur) - 1):
        if most_occur[0][1] == most_occur[element_wine][1]:
            most_common_country.append(most_occur[element_wine][0])

    return most_common_country

def avarage_score(name):
    total_points = 0
    counter = 0
    for wine in wine_data:
        if wine['designation'] == name:
            if wine['points'] is None:
                continue
            counter += 1
            total_points += int(wine['points'])
    if counter == 0:
        return 'There is no such wine as', name
    return total_points / counter

for wine in ('Gewüurztraminer', 'Gewuurztraminer', 'Riesling', 'Merlot', 'Madera', 'Tempranillo', 'Red Blend'):
    print('average_price of', 'wine', wine, 'is', average_price(wine))
    print('min_price_price of', 'wine', wine, 'is', min_price(wine))
    print('max_price of', 'wine', wine, 'is', max_price(wine))
    print('most_common_region of', 'wine', wine, 'is', most_common_region(wine))
    print('most_common_country of', 'wine', wine, 'is', most_common_country(wine))
    print('avarage_score of', 'wine', wine, 'is', avarage_score(wine))


"""init

Revision ID: 605c9198c69d
Revises:
Create Date: 2021-06-01 15:13:49.210911

"""
from alembic import op
import sqlalchemy as sa
import lore

# revision identifiers, used by Alembic.
revision = '605c9198c69d'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('campaigns',
    sa.Column('campaign_id', sa.Integer(), nullable=False),
    sa.Column('stub', lore.stub.stub.Stub(), nullable=False),
    sa.Column('name', sa.String(), nullable=True),
    sa.Column('created_on', sa.DateTime(), nullable=True),
    sa.Column('root_node_id', sa.Integer(), nullable=True),
    sa.Column('owner_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['owner_id'], ['users.user_id'], ),
    sa.ForeignKeyConstraint(['root_node_id'], ['pages.page_id'], ),
    sa.PrimaryKeyConstraint('campaign_id')
    )
    op.create_index(op.f('ix_campaigns_stub'), 'campaigns', ['stub'], unique=True)
    op.create_table('pages',
    sa.Column('page_id', sa.Integer(), nullable=False),
    sa.Column('public_id', sa.String(), nullable=True),
    sa.Column('permalink', sa.String(), nullable=True),
    sa.Column('title', sa.String(), nullable=False),
    sa.Column('stub', lore.stub.stub.Stub(), nullable=False),
    sa.Column('icon', sa.String(), nullable=True),
    sa.Column('parent_id', sa.Integer(), nullable=True),
    sa.Column('depth', sa.Integer(), nullable=True),
    sa.Column('campaign_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.campaign_id'], ),
    sa.ForeignKeyConstraint(['parent_id'], ['pages.page_id'], ),
    sa.PrimaryKeyConstraint('page_id')
    )
    op.create_index(op.f('ix_pages_permalink'), 'pages', ['permalink'], unique=False)
    op.create_index(op.f('ix_pages_public_id'), 'pages', ['public_id'], unique=False)
    op.create_index(op.f('ix_pages_stub'), 'pages', ['stub'], unique=False)
    op.create_table('users',
    sa.Column('user_id', sa.Integer(), nullable=False),
    sa.Column('username', sa.String(length=64), nullable=True),
    sa.Column('email', sa.String(length=120), nullable=True),
    sa.Column('password_hash', sa.String(length=128), nullable=True),
    sa.Column('last_seen', sa.DateTime(), nullable=True),
    sa.Column('is_activated', sa.Boolean(), nullable=True),
    sa.PrimaryKeyConstraint('user_id')
    )
    op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=True)
    op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True)
    op.create_table('aliases',
    sa.Column('alias_id', sa.Integer(), nullable=False),
    sa.Column('stub', sa.String(), nullable=False),
    sa.Column('page_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['page_id'], ['pages.page_id'], ),
    sa.PrimaryKeyConstraint('alias_id'),
    sa.UniqueConstraint('stub'),
    sa.UniqueConstraint('stub')
    )
    op.create_table('campaign_membership',
    sa.Column('campaign_id', sa.Integer(), nullable=False),
    sa.Column('user_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.campaign_id'], ),
    sa.ForeignKeyConstraint(['user_id'], ['users.user_id'], ),
    sa.PrimaryKeyConstraint('campaign_id', 'user_id')
    )
    op.create_table('page_links',
    sa.Column('page_from', sa.Integer(), nullable=False),
    sa.Column('page_to', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['page_from'], ['pages.page_id'], ),
    sa.ForeignKeyConstraint(['page_to'], ['pages.page_id'], ),
    sa.PrimaryKeyConstraint('page_from', 'page_to')
    )
    op.create_table('paragraphs',
    sa.Column('paragraph_id', sa.Integer(), nullable=False),
    sa.Column('page_id', sa.Integer(), nullable=True),
    sa.Column('title', sa.String(), nullable=True),
    sa.Column('body', sa.String(), nullable=True),
    sa.Column('order', sa.Integer(), nullable=True),
    sa.Column('edited_by_id', sa.Integer(), nullable=True),
    sa.Column('edited_on', sa.DateTime(), nullable=True),
    sa.Column('edit_comment', sa.String(), nullable=True),
    sa.ForeignKeyConstraint(['edited_by_id'], ['users.user_id'], ),
    sa.ForeignKeyConstraint(['page_id'], ['pages.page_id'], ),
    sa.PrimaryKeyConstraint('paragraph_id')
    )
    op.create_table('tags',
    sa.Column('tag_id', sa.Integer(), nullable=False),
    sa.Column('tag_name', sa.String(), nullable=False),
    sa.Column('parent_tag_id', sa.Integer(), nullable=True),
    sa.Column('campaign_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.campaign_id'], ),
    sa.ForeignKeyConstraint(['parent_tag_id'], ['tags.tag_id'], ),
    sa.PrimaryKeyConstraint('tag_id'),
    sa.UniqueConstraint('tag_name'),
    sa.UniqueConstraint('tag_name')
    )
    op.create_table('page_tags',
    sa.Column('page_id', sa.Integer(), nullable=False),
    sa.Column('tag_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['page_id'], ['pages.page_id'], ),
    sa.ForeignKeyConstraint(['tag_id'], ['tags.tag_id'], ),
    sa.PrimaryKeyConstraint('page_id', 'tag_id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('page_tags')
    op.drop_table('tags')
    op.drop_table('paragraphs')
    op.drop_table('page_links')
    op.drop_table('campaign_membership')
    op.drop_table('aliases')
    op.drop_index(op.f('ix_users_username'), table_name='users')
    op.drop_index(op.f('ix_users_email'), table_name='users')
    op.drop_table('users')
    op.drop_index(op.f('ix_pages_stub'), table_name='pages')
    op.drop_index(op.f('ix_pages_public_id'), table_name='pages')
    op.drop_index(op.f('ix_pages_permalink'), table_name='pages')
    op.drop_table('pages')
    op.drop_index(op.f('ix_campaigns_stub'), table_name='campaigns')
    op.drop_table('campaigns')
    # ### end Alembic commands ###


import json

from sseclient import SSEClient
from kafka import KafkaProducer
from kafka.admin import KafkaAdminClient, NewTopic

bootstrap_servers = ['localhost:19090', 'localhost:19091', 'localhost:19092']

kafka_client = KafkaAdminClient(
    bootstrap_servers=bootstrap_servers,
    client_id='PythonClient'
)

# verify if topics exists
topic = 'wikimedia'
current_topics = kafka_client.list_topics()

if topic not in current_topics:
    new_topic = NewTopic(
        name=topic,
        num_partitions=3,
        replication_factor=2,
    )

    kafka_client.create_topics(new_topics=[new_topic])

url = 'https://stream.wikimedia.org/v2/stream/recentchange'
client = SSEClient(url)

producer = KafkaProducer(bootstrap_servers=bootstrap_servers)

for msg in client:
    outputMsg = msg.data
    if len(outputMsg) > 2:
        outputJS = json.loads(outputMsg)
        print("Send ", outputJS)
        producer.send(topic=topic, value=json.dumps(outputJS).encode())


# for a 48mm wide tape,
# for the right side of the tape connect the sensors  9, 10 and 11 (or 8, 10 and 12) of the QTRX to pins 4, 5 and 6 of the arduino
# for the left side of the tape connect the sensors  21, 22, and 23 (or 20, 22 and 24) of the QTRX to pins 7, 8 and 9 of the arduino
# connect the sensors 1 and 31 to pins 3 and 10 of the arduino

import serial
import serial.tools.list_ports

line_loss_threshold = 1500 # depends on conditions (scotch/ground color)

Data = serial.Serial("/dev/ttyACM0", 9600)

def read_line():
    """
        returns an array containing the values returned by each of the 8 connected sensors
    """
    line = Data.readline().decode('utf-8')
    line = line.split()

    for i in range (len(line)) :
        line[i] = float(line[i])
    #print(line)
    return line

line_right = 0
line_left = 0
loose_right = 0
loose_left = 0

def intensity():
    """
        returns the average of the intensity measured by the sensors on the left and right of the tape,
        as well as the intensity measured by the sensors on the left and right end of the sensor bar
    """
    line = read_line(line_left, line_right, loose_left, loose_right)
    if len(line) == 9 :       # allows to avoid some reading errors
        line_left = line[4]+2*line[5]+line[6]
        line_right = line[1]+2*line[2]+line[3]
        loose_left = line[7]
        loose_right = line[0]
    return line_left, line_right, loose_left, loose_right

def run():
    line_left, line_right, loose_left, loose_right = intensity(line_left, line_right, loose_left, loose_right)
    dif = line_right - line_left
    add = line_right + line_left

    if (add > line_loss_threshold) & (dif > 0) : # allows you to know if you have not lost the line of sight
        print ("Go right", dif, line_left, line_right, add)
    elif (add > line_loss_threshold) & (dif < 0) :
        print ("Go left", dif, line_left, line_right, add)

    elif (add < line_loss_threshold) & (loose_left > loose_right): # if the line is lost
        print ("Lost line, go left")
    elif (add < line_loss_threshold) & (loose_left < loose_right):
        print ("Lost line, go right")

while True :
    run()

Data.close()


#!/usr/bin/python3

"""fetches https://alx-intranet.hbt/status"""

import sys
import requests

if __name__ == "__main__":
    url = sys.argv[1]

    r = requests.get(url)
    if r.status_code >= 400:
        print("Error code: {}".format(r.status_code))
    else:
        print(r.text)


import asyncio
from scrapli.driver.core import AsyncIOSXEDriver
from scrapli import Scrapli
from datetime import datetime
from pathlib import Path
from devices import DEVICES
from commands import COMMANDS
import logging

SUCCESS = 55
logging.addLevelName(SUCCESS , 'SUCCESS')

logging.basicConfig(
    filename="process-logging.log",
    filemode="a",
    format="%(asctime)s:%(name)s - %(message)s",
    datefmt= "%d-%m-%Y %H:%M:%S",
    level=logging.ERROR)

logger=logging.getLogger()

today = datetime.today().date()

async def collect_commands(hostname, device):
    conn = AsyncIOSXEDriver(**device)
    try:
        await conn.open()
    except Exception as error:
        logger.error(f"ERROR: {hostname} - Failed to connect device\n{error}")
    else:
        config_dir = f"archive/{today}/{hostname}"
        Path(config_dir).mkdir(parents=True, exist_ok=True)
        for feature,commands in COMMANDS.items():
            response = await conn.send_commands(commands, strip_prompt=False)
            if response.failed:
                logger.error(f"ERROR: {hostname} - Failed to get command(s)\n{response.result}")
            else:
                filename = f"{config_dir}/{feature}.txt"
                with open(filename, "w") as file:
                    file.write(response.result)
                    file.close()
                logger.log(SUCCESS, f"SUCCESS: {hostname} - Commands collected.\n{commands}")

        await conn.close()

async def main():
    coroutines = [collect_commands(hostname, device) for hostname, device in DEVICES.items()]
    results = await asyncio.gather(*coroutines)

if __name__ == "__main__":
    asyncio.get_event_loop().run_until_complete(main())


#!/usr/bin/env python
# coding: utf-8
#保持直行
# In[ ]:

import cv2 as cv
import numpy as np
import time
import json

#识别标识的依赖
import os
import sys
import cv2 as cv
import time
import ctypes
import numpy as np
import bwtricar
import pycuda.autoinit
import pycuda.driver as cuda
import matplotlib.pyplot as plt

import aicar
import uff
import tensorrt as trt
import graphsurgeon as gs
from config import model_ssd_mobilenet_v1_aicar as model

######################
plt.rcParams['figure.dpi'] = 120 #分辨率
cap=cv.VideoCapture(0)
maxWidth = 640
maxHeight = 480
cap.set(3,maxWidth)
cap.set(4,maxHeight)
ctypes.CDLL("lib/libflattenconcat.so")
COCO_LABELS = aicar.COCO_CLASSES_LIST
# initialize
TRT_LOGGER = trt.Logger(trt.Logger.INFO)
trt.init_libnvinfer_plugins(TRT_LOGGER, '')
runtime = trt.Runtime(TRT_LOGGER)
# create engine
with open(model.TRTbin, 'rb') as f:
    buf = f.read()
    engine = runtime.deserialize_cuda_engine(buf)
host_inputs  = []
cuda_inputs  = []
host_outputs = []
cuda_outputs = []
bindings = []
stream = cuda.Stream()

for binding in engine:
    size = trt.volume(engine.get_binding_shape(binding)) * engine.max_batch_size
    host_mem = cuda.pagelocked_empty(size, np.float32)
    cuda_mem = cuda.mem_alloc(host_mem.nbytes)

    bindings.append(int(cuda_mem))
    if engine.binding_is_input(binding):
        host_inputs.append(host_mem)
        cuda_inputs.append(cuda_mem)
    else:
        host_outputs.append(host_mem)
        cuda_outputs.append(cuda_mem)
context = engine.create_execution_context()

def detectObject(image):
    ori = image
    image = cv.resize(ori, (model.dims[2],model.dims[1]))
    image = (2.0/255.0) * image - 1.0
    image = image.transpose((2, 0, 1))
    np.copyto(host_inputs[0], image.ravel())
    cuda.memcpy_htod_async(cuda_inputs[0], host_inputs[0], stream)
    context.execute_async(bindings=bindings, stream_handle=stream.handle)
    cuda.memcpy_dtoh_async(host_outputs[1], cuda_outputs[1], stream)
    cuda.memcpy_dtoh_async(host_outputs[0], cuda_outputs[0], stream)
    stream.synchronize()
    output = host_outputs[0]
    height, width, channels = ori.shape
    retObj = []
    for i in range(int(len(output)/model.layout)):
        prefix = i*model.layout
        index = int(output[prefix+0])
        label = int(output[prefix+1])
        conf  = output[prefix+2]
        lx  = int(output[prefix+3]*width)
        ly  = int(output[prefix+4]*height)
        ux  = int(output[prefix+5]*width)
        uy  = int(output[prefix+6]*height)
        if(conf > 0.7):
            thisObj = {
                'box':[[xmin,ymin],[xmax,ymax]],
                'label':COCO_LABELS[label],
                'score':conf
            }
            retObj.append(thisObj)

            cv.rectangle(ori, (xmin,ymin), (xmax, ymax), (0,0,255),3)
            cv.putText(ori, COCO_LABELS[label],(xmin+10,ymin+10), cv.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv.LINE_AA)
    return retObj

##############################
# In[ ]:

from periphery import Serial

# In[ ]:

def displayImg(name, ori, imgType = 'rgb'):
    displayImg = ori
    if(imgType == 'rgb'):
        displayImg = cv.cvtColor(ori, cv.COLOR_RGB2BGR)
    #cv.imshow(name, displayImg)
    #cv.waitKey(1)

# In[ ]:

#识别标识
def detectTrafficSign(image):
    ori = image
    image = cv.resize(ori, (model.dims[2],model.dims[1]))
    image = (2.0/255.0) * image - 1.0
    image = image.transpose((2, 0, 1))
    np.copyto(host_inputs[0], image.ravel())
    cuda.memcpy_htod_async(cuda_inputs[0], host_inputs[0], stream)
    context.execute_async(bindings=bindings, stream_handle=stream.handle)
    cuda.memcpy_dtoh_async(host_outputs[1], cuda_outputs[1], stream)
    cuda.memcpy_dtoh_async(host_outputs[0], cuda_outputs[0], stream)
    stream.synchronize()
    output = host_outputs[0]
    height, width, channels = ori.shape
    retObj = []
    for i in range(int(len(output)/model.layout)):
        prefix = i*model.layout
        index = int(output[prefix+0])
        label = int(output[prefix+1])
        conf  = output[prefix+2]
        lx  = int(output[prefix+3]*width)
        ly  = int(output[prefix+4]*height)
        ux  = int(output[prefix+5]*width)
        uy  = int(output[prefix+6]*height)
        if(conf > 0.7):
            thisObj = {
                'box':[[xmin,ymin],[xmax,ymax]],
                'label':COCO_LABELS[label],
                'score':conf
            }
            retObj.append(thisObj)

            cv.rectangle(ori, (xmin,ymin), (xmax, ymax), (0,0,255),3)
            cv.putText(ori, COCO_LABELS[label],(xmin+10,ymin+10), cv.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv.LINE_AA)
    return retObj

def detectLane(rgbImg, roiPoints = None):
    # Global variable
    maxHeight = rgbImg.shape[0]
    maxWidth = rgbImg.shape[1]
    kernel = np.ones((5, 5), np.uint8)

    displayImg('rgbImg', rgbImg)

    # 11 55
    lowerYellow = np.array([11, threshold1, threshold2])
    upperYellow = np.array([35, 255, 255])

    lowerWhite = np.array([0, 0, 255-threshold3])
    upperWhite = np.array([255, threshold3, 255])

    # Init
    hsvImg = cv.cvtColor(rgbImg, cv.COLOR_RGB2HSV)

    # Color spilt
    maskYellow = cv.inRange(hsvImg, lowerYellow, upperYellow)
    maskYellow = cv.morphologyEx(maskYellow, cv.MORPH_OPEN, kernel)

    displayImg('maskYellow', maskYellow, 'gray')

    maskWhite = cv.inRange(hsvImg, lowerWhite, upperWhite)
    maskWhite = cv.morphologyEx(maskWhite, cv.MORPH_OPEN, kernel)

    displayImg('maskWhite', maskWhite, 'gray')

    # ROI area
    if(type(roiPoints) == type(None)):
        roiPoints = defaultRoiPoints

    roiMask = np.zeros((maxHeight, maxWidth), np.uint8)
    cv.fillPoly(roiMask, [roiPoints], (255, 0, 0))

    displayImg('roiMask', roiMask, 'gray')

    needQuickly = False

    # Yellow line
    maskYellow = cv.bitwise_and(maskYellow, roiMask)
    yellowLineEdges = cv.Canny(maskYellow, 20, 60)

    # displayImg('yellowLineEdges', yellowLineEdges, 'gray')

    yellowLines = cv.HoughLinesP(yellowLineEdges, 4, np.pi/180, 30,minLineLength=40, maxLineGap=180)

    yellowBottomX = -1
    if type(yellowLines) != type(None):
        yellowKAvg = 0

        cnt = 0
        for line in yellowLines:
            x1, y1, x2, y2 = line[0]
            if(x2 != x1):
                yellowKAvg += 1.0 * (y2-y1) / (x2-x1)
                cnt += 1
        if(cnt != 0):
            yellowKAvg /= cnt

            if(abs(yellowKAvg) < 0.55):
                needQuickly = True

            _, yellowLineContours, _ = cv.findContours(yellowLineEdges, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)
            yellowMID = 0
            yellowMA = 0
            for i in range(len(yellowLineContours)):
                tmp = cv.contourArea(yellowLineContours[i])
                if(tmp > yellowMA):
                    yellowMA = tmp
                    yellowMID = i

            yellowM = cv.moments(yellowLineContours[i])
            yellowBottomX = 1
            if(yellowM['m00']!=0):
                yellowCX = int(yellowM['m10']/yellowM['m00'])
                yellowCY = int(yellowM['m01']/yellowM['m00'])
                yellowB = yellowCY - yellowKAvg * yellowCX

                if((maxHeight - yellowB) / yellowKAvg != float("inf")):
                    yellowBottomX = (int)((maxHeight - yellowB) / yellowKAvg)

    # White line
    maskWhite = cv.bitwise_and(maskWhite, roiMask)
    whiteLineEdges = cv.Canny(maskWhite, 20, 60)

    # displayImg('whiteLineEdges', whiteLineEdges, 'gray')

    whiteLines = cv.HoughLinesP(whiteLineEdges, 4, np.pi/180, 30,minLineLength=40, maxLineGap=180)

    whiteBottomX = -1
    if type(whiteLines) != type(None):

        whiteKAvg = 0

        cnt = 0
        for line in whiteLines:
            x1, y1, x2, y2 = line[0]
            if(x2 != x1):
                whiteKAvg += 1.0 * (y2-y1) / (x2-x1)
                cnt += 1
        if(cnt != 0):
            whiteKAvg /= cnt

            if(abs(whiteKAvg) < 0.55):
                needQuickly = True

            _, whiteLineContours, _ = cv.findContours(whiteLineEdges, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)
            whiteMID = 0
            whiteMA = 0
            for i in range(len(whiteLineContours)):
                tmp = cv.contourArea(whiteLineContours[i])
                if(tmp > whiteMA):
                    whiteMA = tmp
                    whiteMID = i

            whiteM = cv.moments(whiteLineContours[i])
            whiteBottomX = 1
            if(whiteM['m00']!=0):
                whiteCX = int(whiteM['m10']/whiteM['m00'])
                whiteCY = int(whiteM['m01']/whiteM['m00'])
                whiteB = whiteCY - whiteKAvg * whiteCX

                if((maxHeight - whiteB) / whiteKAvg != float("inf")):
                    whiteBottomX = (int)((maxHeight - whiteB) / whiteKAvg)

    debugImg = cv.bitwise_or(maskYellow, maskWhite)
    displayImg('debugImg', debugImg, 'gray')

    # turn left
    if(yellowBottomX == -1 and whiteBottomX != -1):
        if(needQuickly):
            return 'leftQuickly'
        else:
            return 'left'

    # turn right
    if(yellowBottomX != -1 and whiteBottomX == -1):
        if(needQuickly):
            return 'rightQuickly'
        else:
            return 'right'

    # go straight
    return 'straight'

    #speedup
    if detectTrafficSign(rgbImg) == 'limit80':
        return 'speedup'
    else:
        return 'straight'

    #slowdown
    if detectTrafficSign(rgbImg) == 'limit40':
        return 'slowdown'
    else:
        return 'straight'

# In[ ]:

delayTIme = 0.1

# Image
maxHeight = 480
maxWidth = 640

# light
blight = 1.3
bweight = 25

# color
threshold1 = 80
threshold2 = 80
threshold3 = 70

# default ROI
defaultRoiPoints = np.array([[maxWidth / 2 - maxWidth * 3 / 8, maxHeight],
                              [maxWidth / 2  + maxWidth * 3 / 8, maxHeight],
                              [maxWidth / 2 + maxWidth / 8, maxHeight / 2 - maxHeight / 7],
                              [maxWidth / 2 - maxWidth / 8, maxHeight / 2 - maxHeight / 7]], np.int32)

# In[ ]:

# Serial
currentTime = time.time()
serial = Serial('/dev/ttyTHS1', 115200)
commandJson = {'o':1, 'v': 0, 'c':0, 'd':0, 'r':0, 'a':0}

# In[ ]:

cap = cv.VideoCapture(0)
cap.set(3,maxWidth)
cap.set(4,maxHeight)

# In[ ]:

def controlTricar(command):
    global currentTime
    global commandJson

    if(command == 'left'):
        commandJson['o'] = 0
        commandJson['v'] = 29
        commandJson['c'] = 0
        commandJson['d'] = 0
        commandJson['r'] = 10000
        commandJson['a'] = 0
    elif(command == 'leftQuickly'):
        commandJson['o'] = 0
        commandJson['v'] = 10
        commandJson['c'] = 0
        commandJson['d'] = 0
        commandJson['r'] = 7000
        commandJson['a'] = 0
    elif(command == 'right'):
        commandJson['o'] = 0
        commandJson['v'] = 29
        commandJson['c'] = 0
        commandJson['d'] = 1
        commandJson['r'] = 10000
        commandJson['a'] = 0
    elif(command == 'rightQuickly'):
        commandJson['o'] = 0
        commandJson['v'] = 10
        commandJson['c'] = 0
        commandJson['d'] = 1
        commandJson['r'] = 7000
        commandJson['a'] = 0
    elif(command == 'straight'):
        commandJson['o'] = 0
        commandJson['v'] = 35
        commandJson['c'] = 0
        commandJson['d'] = 0
        commandJson['r'] = 0
        commandJson['a'] = 0
    elif(command == 'stop'):
        commandJson['o'] = 0
        commandJson['v'] = 0
        commandJson['c'] = 0
        commandJson['d'] = 0
        commandJson['r'] = 0
        commandJson['a'] = 0
    elif(command == 'turnaround'):
        commandJson['o'] = 0
        commandJson['v'] = 0
        commandJson['c'] = 0
        commandJson['d'] = 1
        commandJson['r'] = 1000
        commandJson['a'] = 180
    elif(command == 'speedup'):
        commandJson['o'] = 0
        commandJson['v'] = 60
        commandJson['c'] = 0
        commandJson['d'] = 0
        commandJson['r'] = 0
        commandJson['a'] = 0
    elif(command == 'slowdown'):
        commandJson['o'] = 0
        commandJson['v'] = 10
        commandJson['c'] = 0
        commandJson['d'] = 0
        commandJson['r'] = 0
        commandJson['a'] = 0

# In[ ]:

def sendCommand():
    serial.write( bytes(json.dumps(commandJson), encoding = "utf8"))

# In[ ]:

def gogogo():
    global currentTime

    while 1:
        # Auto send commdand with none stop
        if(time.time() - currentTime > delayTIme):
            sendCommand()
            currentTime = time.time()

        ret,laneFrame=cap.read()

        if(ret):
            laneFrame = (laneFrame.astype(np.float32) * blight + bweight)
            laneFrame[laneFrame > 255] = 255
            laneFrame = laneFrame.astype(np.uint8)

            laneImage = cv.cvtColor(laneFrame, cv.COLOR_BGR2RGB)
            lineCommand = detectLane(laneImage)
            controlTricar(lineCommand)

# In[ ]:

print('准备开车')
gogogo()

# In[ ]:

controlCar('stop')
sendCommand()


from urllib.request import urlopen
from bs4 import BeautifulSoup

#world development indicator: education inputs
url = 'http://wdi.worldbank.org/table/2.7'
html = urlopen(url)

soup = BeautifulSoup(html.read(), 'html.parser')

#existem dois elementos table com class indicators-table (uma no topo, e uma com os dados)
tabela = soup.findAll('table', {'class': 'indicators-table'})[1]

#cria uma lista com todas as linhas (tr) da tabela de dados
linhas = tabela.findAll('tr')

#cria um dicionario vazio de duas dimensoes {'pai': {'investimento':investimento, 'aluno_por_professor':qtde}}
dict = {}
#cria um dicionario vazio {'pais':investimento}
dict_investimento = {}

for linha in linhas:
    #recupera o conteúdo da primeira coluna com a class 'country'
    country = linha.find('td', { 'class' : 'country'}).text

    dict[country] = {}

    investimento = linha.findAll('td')[7].text
    dict[country]['investimento'] = investimento

    aluno_por_professor_pri = linha.findAll('td')[11].text
    dict[country]['alunos_por_professor_primario'] = aluno_por_professor_pri

    investimento_float = investimento
    if investimento_float == '..':
        investimento_float = '0'

    dict_investimento[country] = float(investimento_float)

print('Afghanistan', dict['Afghanistan']['investimento'], dict['Afghanistan']['alunos_por_professor_primario'])
print('Brasil', dict['Brazil']['investimento'], dict['Brazil']['alunos_por_professor_primario'])
print('Suíca', dict['Switzerland']['investimento'], dict['Switzerland']['alunos_por_professor_primario'])
print('Alemanha', dict['Germany']['investimento'], dict['Germany']['alunos_por_professor_primario'])
print('Japão', dict['Japan']['investimento'], dict['Japan']['alunos_por_professor_primario'])
print('Áustria - Investimento {} Qtde alunos por professor {}'.format(dict['Austria']['investimento'], dict['Austria']['alunos_por_professor_primario']))

#ordena os países pelo investimento, em ordem descrescente
paisesOrdenados = sorted(dict_investimento, key=dict_investimento.get, reverse=True)
print('País com maior investimento {}'.format(paisesOrdenados[0]))

#itera a lista ordenada exibindo a posicao ocupada por cada pais
for pais in paisesOrdenados:
    indice = paisesOrdenados.index(pais) + 1
    print(indice, pais)

#exibe a posicao ocupada pelo Brasil
print('Posição ocupada pelo Brasil: ', paisesOrdenados.index('Brazil') + 1)


from sql import db
from sqlalchemy.dialects.mysql import INTEGER
from sqlalchemy import ForeignKey

class BudgetItem(db.Model):
	budgetitem_id = db.Column(db.Integer, nullable = False, primary_key=True)
	budgetitem_code = db.Column(db.String(4), nullable = False)
	budgetgroup_id = db.Column(db.Integer, ForeignKey('budget_group.budgetgroup_id'), nullable = False)
	budgetitem_name = db.Column(db.String(255), nullable = False)
	financial_year = db.Column(INTEGER(4), nullable = False)
	expenditure_budgeted = db.Column(db.Numeric(precision=10,scale=2), nullable = False, default=0.00)
	revenue_budgeted = db.Column(db.Numeric(precision=10,scale=2), nullable = False, default=0.00)
	expenditure_confirmed = db.Column(db.Numeric(precision=10,scale=2), nullable = False, default=0.00)
	revenue_confirmed = db.Column(db.Numeric(precision=10,scale=2), nullable = False, default=0.00)

class BudgetGroup(db.Model):
	budgetgroup_id = db.Column(db.Integer, nullable = False, primary_key=True)
	budgetgroup_name = db.Column(db.String(255), nullable = False)

class Settings(db.Model):
	key = db.Column(db.String(63), nullable = False, primary_key=True)
	value = db.Column(db.String(255), nullable = False)


#!/usr/bin/env python
# coding=utf-8

import requests
from bs4 import BeautifulSoup

URL = 'http://210.44.176.116/cjcx/zcjcx_list.php'

def get_grade_list(studentno):
    data = {'post_xuehao': studentno}
    res = requests.post(URL, data=data)
    soup = BeautifulSoup(res.text)

    # 获得学生信息
    try:
        info = {
            'no': studentno,
            'name': soup.body.table.find_all('table')[0].find_all('tr')[1].find_all('td')[1].text,
            'class': soup.body.table.find_all('table')[0].find_all('tr')[1].find_all('td')[6].text
        }
    except:
        return None

    # 获得学生成绩单
    gradelist = []
    len = 0
    for grade in soup.body.table.find_all('table')[2].find_all('tr'):
        if len == 0:
            len += 1
            continue

        # 跳过双专业
        way = grade.find_all('td')[8].text.strip()
        if way.isalpha():
            continue

        tmp = {
            'id': grade.find_all('td')[0].text.strip(),
            'schoolyear': grade.find_all('td')[1].text.strip(),
            'semester': grade.find_all('td')[2].text.strip(),
            'class': grade.find_all('td')[3].text.strip(),
            'name': grade.find_all('td')[5].text.strip(),
            'point': grade.find_all('td')[7].text.strip(),
            'grade': grade.find_all('td')[9].text.strip()
        }

        update = grade.find_all('td')[10].text.strip()
        if update is not None and update.isdigit():
            tmp['grade'] = update

        gradelist.append(tmp)

    return {
        'info': info,
        'gradelist': gradelist
    }


import torch
import torchvision.transforms as transforms
import torch.nn.functional as F

class CNNClassifier(torch.nn.Module):
    def __init__(self):
        super().__init__()

        self.conv_block1 = torch.nn.Sequential(
            torch.nn.Conv2d(3, 16, kernel_size=3),
            torch.nn.Conv2d(16, 16, kernel_size=3),
            torch.nn.BatchNorm2d(16),
            torch.nn.MaxPool2d(kernel_size = 2, stride = 2),
            torch.nn.ReLU()
        )

        self.conv_block2 = torch.nn.Sequential(
            torch.nn.Conv2d(16, 32, kernel_size=3),
            torch.nn.Conv2d(32, 32, kernel_size=3),
            torch.nn.BatchNorm2d(32),
            torch.nn.MaxPool2d(kernel_size = 2, stride = 2),
            torch.nn.ReLU()
        )

        self.conv_block3 = torch.nn.Sequential(
            torch.nn.Conv2d(32, 64, kernel_size=3),
            torch.nn.Conv2d(64, 64, kernel_size=3),
            torch.nn.BatchNorm2d(64),
            torch.nn.MaxPool2d(kernel_size = 2, stride = 2),
            torch.nn.ReLU()
        )

        self.conv_block4 = torch.nn.Sequential(
            torch.nn.Conv2d(64, 128, kernel_size=3),
            torch.nn.Conv2d(128, 128, kernel_size=3),
            torch.nn.BatchNorm2d(128),
            torch.nn.MaxPool2d(kernel_size = 2, stride = 2),
            torch.nn.ReLU()
        )

        self.linear = torch.nn.Linear(12800, 64)
        self.fc = torch.nn.Linear(64, 3)

    def forward(self, x):
        """
        @x: torch.Tensor((B,3,150,200))
        @return: torch.Tensor((B,3))
        """
        # Normalize images for CNN by using grayscale
        # augumentation = transforms.Compose([
        #     transforms.Grayscale(num_output_channels=1)])

        x = self.conv_block1(x)
        x = self.conv_block2(x)
        x = self.conv_block3(x)
        x = self.conv_block4(x)

        x = x.reshape(x.size(0), -1)

        x = self.linear(F.relu(x))
        x = self.fc(x)
        return x.view(-1, 3, 1)

        # Inspiration taken from following URLs:
        # https://blog.paperspace.com/writing-cnns-from-scratch-in-pytorch/
        # https://www.kaggle.com/code/reukki/pytorch-cnn-tutorial-with-cats-and-dogs

def save_model(model):
    from torch import save
    from os import path
    #if isinstance(model, CNNClassifier):
    return save(model.state_dict(), path.join("weights", 'reshapedcnn.th'))
    #raise ValueError("model type '%s' not supported!"%str(type(model)))

def load_model():
    from torch import load
    from os import path
    r = CNNClassifier()
    r.load_state_dict(load(path.join("weights", 'reshapedcnn.th'), map_location='cuda:1'))
    return r


from comiee import overload
from public.config import data_path
from enum import IntEnum
import os
import logging

LOG_PATH = data_path('log')

class LogLevel(IntEnum):
    CRITICAL = logging.CRITICAL
    FATAL = logging.FATAL
    ERROR = logging.ERROR
    WARNING = logging.WARNING
    WARN = logging.WARN
    INFO = logging.INFO
    DEBUG = logging.DEBUG
    NOTSET = logging.NOTSET

@overload
def get_logger(name, level) -> logging.Logger:
    return get_logger(name, level, name + '.txt', level)

@overload
def get_logger(name, level, file_level) -> logging.Logger:
    return get_logger(name, level, name + '.txt', file_level)

@overload
def get_logger(name, level, file_name, file_level) -> logging.Logger:
    logger = logging.getLogger(name)
    logger.setLevel(LogLevel.DEBUG)
    formatter = logging.Formatter('[%(asctime)s] %(name)s %(levelname)s: %(message)s')
    ch = logging.StreamHandler()
    ch.setLevel(level)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    fh = logging.FileHandler(os.path.join(LOG_PATH, file_name), encoding='utf-8')
    fh.setLevel(file_level)
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    return logger

# 底层的client和server使用logger
client_logger: logging.Logger = get_logger('client', LogLevel.INFO, LogLevel.DEBUG)
server_logger: logging.Logger = get_logger('server', LogLevel.DEBUG)

# 服务器各组件使用的logger
master_server_logger: logging.Logger = get_logger('masterServer', LogLevel.INFO, LogLevel.DEBUG)
admin_logger: logging.Logger = get_logger('admin', LogLevel.DEBUG)

# bot客户端各组件使用的logger
bot_client_logger: logging.Logger = get_logger('botClient', LogLevel.DEBUG)
bot_logger: logging.Logger = get_logger('bot', LogLevel.DEBUG)

# 公共部分使用的logger
public_logger: logging.Logger = get_logger('public', LogLevel.DEBUG)


import time
from selenium import webdriver
import os
import requests
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException

# Set the path to the ChromeDriver executable
chrome_driver_path = r"C:\Users\Tahsin\Desktop\Proje\trendyol-scrapping\venv\chromedriver.exe"

# Create a Service object with the ChromeDriver path
service = Service(chrome_driver_path)

# Start the ChromeDriver service
service.start()

# Create a WebDriver instance
driver = webdriver.Chrome(service=service)

driver.get('https://www.trendyol.com/magaza/bilsen-store-m-207964?sst=0')
time.sleep(2)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

div_element = driver.find_element(by=By.XPATH, value='//*[@id="seller-store"]/div/div[2]/div/div[4]')

# Find all the anchor tags within the div element
link_elements = div_element.find_elements(By.TAG_NAME, 'a')

# Extract the href attribute from each anchor tag and store them in a list
links = [link.get_attribute('href') for link in link_elements]
time.sleep(2)
# Visit each link
folder_counter = 0
# Visit each link
for link in links:
    driver.get(link)

    time.sleep(2)
    # Extract the product ID from the link
    product_id = str(folder_counter)

    # Create a folder for the product ID
    folder_path = f"./{product_id}"
    os.makedirs(folder_path, exist_ok=True)

    # Get product name
    name_element = driver.find_element(By.XPATH, '//*[@id="product-detail-app"]/div/div[2]/div[1]/div[2]/div[2]/div[1]/div/div/div[1]/h1')
    product_name = name_element.text

    # Increment the folder counter
    folder_counter += 1

    # Get product price
    try:
        price_element = driver.find_element(By.XPATH, '//*[@id="product-detail-app"]/div/div[2]/div[1]/div[2]/div[2]/div[1]/div/div/div[4]/div/div/span')
        price = price_element.text
        # Process the price value here
    except NoSuchElementException:
        price = "Price not available"
    time.sleep(2)
    try:
        discount_element = driver.find_element(By.XPATH, '//*[@id="product-detail-app"]/div/div[2]/div[1]/div[2]/div[2]/div[1]/div/div/div[4]/div/div/div/div[2]/span[2]')
        discount = discount_element.text
        # Process the discount value here
    except NoSuchElementException:
        discount = "Discount not available"
        # Handle the case when discount is not found

    time.sleep(2)
    # Get product details
    details_element = driver.find_element(By.XPATH, '//section[1]/div[1]/div[1]/div[1]/ul[1]')
    product_details = details_element.text

    # Save product details to a text file
    txt_file_path = f"{folder_path}/{product_id}.txt"
    with open(txt_file_path, 'w', encoding='utf-8') as f:
        f.write(f"Name: {product_name}\n")
        f.write(f"Price: {price}\n")
        f.write(f"if discount: {discount}\n")
        f.write(f"Link: {link}\n")
        f.write(f"Details:\n{product_details}\n")
    time.sleep(2)
    # Get product photos
    try:
        # Find the parent div element
        parent_div = driver.find_element(By.CLASS_NAME, 'styles-module_slider__o0fqa')

        # Find all the child div elements
        child_divs = parent_div.find_elements(By.TAG_NAME, 'div')

        # Click each child div
        for index, div in enumerate(child_divs):
            div.click()
            time.sleep(0.25)
            try:
                # Find the photo element
                photo_element = driver.find_element(By.XPATH, '//*[@id="product-detail-app"]/div/div/div/div/div/div/div/div/img[1]')
                photo_url = photo_element.get_attribute("src")

                # Set the file path to save the photo with an incremented name
                photo_file_path = f"{folder_path}/{product_id}_photo{index+1}.jpg"

                # Download the photo
                response = requests.get(photo_url)
                with open(photo_file_path, 'wb') as f:
                    f.write(response.content)

                print("Photo downloaded successfully.")
            except NoSuchElementException:
                print("Photo element not found.")
            # Add your desired actions after clicking each div

        print("Divs clicked successfully.")
    except NoSuchElementException:
        print("Parent div element not found.")
    time.sleep(2)
driver.quit()


from rest_framework.serializers import ModelSerializer

from apps.recipe.models.biscuit_recipe import BiscuitRecipe
from api.biscuit.serializers.biscuit import BiscuitModelSerializer
from api.product.serializers.product import ProductModelSerializer

class BiscuitRecipeSerializer(ModelSerializer):
    class Meta:
        model = BiscuitRecipe
        fields = (
            'biscuit',
            'product',
            'value',
            'unit_of_measurement'
        )

class BiscuitRecipeDetailSerializer(ModelSerializer):
    biscuit = BiscuitModelSerializer(read_only=True, many=False)
    product = ProductModelSerializer(read_only=True, many=False)

    class Meta:
        model = BiscuitRecipe
        fields = (
            'biscuit',
            'product',
            'value',
            'unit_of_measurement'
        )


"""
Author: Aaron Hakrider
Date: 9/27/18
Worked with Joy Shaffer for assistance

"""

from LFSR_LAB import lfsr
import argparse

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("n_bits", type=int, help="number of bits")
    args = parser.parse_args()

    # call my program
    output = my_program(args.n_bits)
    print(output)
    print("percentage: ", output * 100)

def my_program(n_bits):
    """
    The program should take, as positional argument,
     the number of bits to generate and print out the percentage of 1s in the sequence.
     (0, 1, 2, 7, 128)
    :param n_bits: number of bits to generate and print out
    :return:
    """
    c = 1 | (1 << 1) | (1 << 2) | (1 << 7)
    s = int('A' * 32, 16)  # hex number

    mylfsr = lfsr.LFSR(c, s)

    count_ones = 0
    for _ in range(n_bits):
        count = mylfsr.cycle()
        if count == 1:
            count_ones += 1

    return count_ones / n_bits

# Standard boilerplate to call the main function, if executed
if __name__ == '__main__':
    main()


import openai
import chromadb

from dotenv import load_dotenv

from .variables import OPENAI_API_KEY

from langchain.vectorstores import Chroma
from langchain.docstore.document import Document
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter

load_dotenv()

openai.api_key = OPENAI_API_KEY

HOST = "35.86.173.4"
PORT = 8000

client = chromadb.HttpClient(host=HOST, port=PORT)

class MissingOrganization(Exception):
    """
    Exception raised for errors for missing database.

    Attributes:
        message -- explanation of the error
    """

    def __init__(self, message="Organization name not in database!!!"):
        self.message = message
        super().__init__(self.message)

def create_organization(org_name):
    try:
        client = chromadb.HttpClient(host=HOST, port=PORT)
        collection = client.create_collection(name=org_name)

        return {
            'status': 200,
            'log': "Successful!!!"
        }
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }

def text_to_docs(ids, metadatas, text):
    """Converts a string or list of strings to a list of Documents with metadata."""

    if isinstance(text, str):
        # Take a single string as one page
        text = [text]

    page_docs = [Document(page_content=page) for page in text]

    # Add page numbers as metadata
    for i, doc in enumerate(page_docs):
        doc.metadata["page"] = i + 1

    # Split pages into chunks
    doc_chunks = []
    doc_id = []
    _metadatas = []

    for _id, meta, doc in zip(ids, metadatas, page_docs):
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=700,
            separators=["\n\n", "\n", ".", "!", "?"],
            chunk_overlap=0,
        )

        chunks = text_splitter.split_text(doc.page_content)
        meta['id_source'] = _id
        for i, chunk in enumerate(chunks):
            doc = Document(
                page_content=chunk, metadata={"page": doc.metadata["page"], "chunk": i}
            )
            doc.metadata["source"] = f"{doc.metadata['page']}-{doc.metadata['chunk']}"

            doc_chunks.append(doc.page_content)
            doc_id.append(_id + f"_{i}")
            _metadatas.append(meta)

    return doc_id, _metadatas, doc_chunks

def add_embedding(input_data):
    """Adds embedding to multiple documents from the database"""

    if (len(input_data['contents']) == len(input_data['ids'])) and (
            len(input_data['ids']) == len(input_data['metadatas'])):
        for i in range(len(input_data['contents'])):
            input_data['contents'][i] += f"\n||||\nMetadata Information: {str(input_data['metadatas'][i])}"

        input_data['ids'], input_data['metadatas'], input_data['contents'] = text_to_docs(
            input_data['ids'], input_data['metadatas'], input_data['contents']
        )
        input_data['embeddings'] = OpenAIEmbeddings().embed_documents(input_data['contents'])

        return input_data
    else:
        raise ValueError("Input mis-match!!!")

def add_data_to_vector_db(data_dict):
    """
    Receives data as input and adds that to the vector database

    Creates a vector database collection for each organization, and
    adds to existing organizations
    """

    try:
        data_dict = add_embedding(data_dict)

        owners = data_dict['org']
        collection = client.get_or_create_collection(name=owners)

        for i, (metadata, ids) in enumerate(zip(
                data_dict['metadatas'],
                data_dict['ids']
        )):
            metadata['source'] = ids

        collection.add(
            embeddings=data_dict['embeddings'],
            documents=data_dict['contents'],
            metadatas=data_dict['metadatas'],
            ids=[owners + i for i in map(str, data_dict['ids'])]
        )

        return {
            'status': 200,
            'log': "Successful!!!"
        }
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }

def get_vectorstore(org_name):
    embedding_function = OpenAIEmbeddings(
        model="text-embedding-ada-002",
        chunk_size=5000
    )

    try:
        client.get_collection(name=org_name)
    except:
        raise MissingOrganization

    return Chroma(client=client, collection_name=org_name, embedding_function=embedding_function)

def delete_organization(organization_name):
    """
    Deletion of organization from database
    """

    try:
        client.delete_collection(name=organization_name)

        return {
            'status': 200,
            'log': "Successful"
        }
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }

def modify_organization(prev_organization_name, new_organization_name):
    """
    Modification of organizations name
    """

    try:
        collection = client.get_collection(name=prev_organization_name)
        collection.modify(name=new_organization_name)
        client.delete_collection(name=prev_organization_name)

        return {
            'status': 200,
            'log': "Successful"
        }
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }

def get_emb_form_vector_db(owner, ids, return_only_emb=True):
    """Retrieve documents from vector database using their ids"""

    try:
        collection = client.get_or_create_collection(name=owner)

        if len(ids) > 1:
            result = collection.get(
                include=['embeddings', 'documents'],
                where={'$or': [{'id_source': _id} for _id in ids]}
            )
        else:
            result = collection.get(
                include=['embeddings', 'documents'],
                where={'id_source': ids[0]}
            )

        if return_only_emb:
            return result['embeddings']
        else:
            return result['documents'], result['embeddings']
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }