from flask import Flask, request, jsonify, render_template_string
import requests
from moviepy.editor import VideoFileClip
from PIL import Image
import base64
import io

app = Flask(__name__)

def get_video_info(url):
    try:
        response = requests.head(url)
        if 'content-type' in response.headers:
            mime_type = response.headers['content-type']
            extension = mime_type.split("/")[-1]
        else:
            mime_type = "Unknown"
            extension = "Unknown"

        content_length = response.headers.get("content-length", "Unknown")
        content_encoding = response.headers.get("content-encoding", "Unknown")
        server = response.headers.get("server", "Unknown")
        last_modified = response.headers.get("last-modified", "Unknown")
        content_disposition = response.headers.get("content-disposition", "Unknown")
        content_language = response.headers.get("content-language", "Unknown")

        video_duration = "N/A"
        resolution = "N/A"
        fps = "N/A"
        estimated_frames = "N/A"
        thumbnail_base64 = None

        if mime_type.startswith("video"):
            try:
                video_clip = VideoFileClip(url)
                video_duration = {
                    "seconds": video_clip.duration,
                    "minutes": video_clip.duration / 60
                }
                resolution = {
                    "width": video_clip.size[0],
                    "height": video_clip.size[1]
                }
                fps = video_clip.fps

                estimated_frames = "{:,}".format(int(video_clip.duration * fps))

                thumbnail = video_clip.get_frame(50)
                with Image.fromarray(thumbnail) as img:
                    img_byte_array = io.BytesIO()
                    img.save(img_byte_array, format="JPEG")
                    thumbnail_base64 = base64.b64encode(img_byte_array.getvalue()).decode()

            except Exception as e:
                video_duration = "Unknown"
                resolution = "Unknown"
                fps = "Unknown"
                estimated_frames = "Unknown"

        video_info = {
            "url": url,
            "mime_type": mime_type,
            "extension": extension,
            "content_length": content_length,
            "content_encoding": content_encoding,
            "server": server,
            "last_modified": last_modified,
            "content_disposition": content_disposition,
            "content_language": content_language,
            "video_duration": video_duration,
            "resolution": resolution,
            "fps": fps,
            "estimated_frames": estimated_frames,
            "thumbnail_base64": thumbnail_base64
        }
        return video_info

    except Exception as e:
        return {"error": str(e)}

@app.route('/video_info', methods=['POST'])
def video_info_api():
    data = request.get_json()

    if 'url' not in data:
        return jsonify({"error": "URL parameter missing"}), 400

    video_url = data['url']
    video_info = get_video_info(video_url)
    return jsonify(video_info)
@app.route('/', methods=['GET', 'POST'])
@app.route('/video_info/form', methods=['GET', 'POST'])
def video_info_form():
    video_info = None

    if request.method == 'POST':
        url = request.form.get('url')
        video_info = get_video_info(url)

    template = """
    <!DOCTYPE html>

        <title>Video Information Form</title>

Video Information Form

            <form method="POST" action="/video_info/form">
                Video URL:
                <input type="text" name="url" id="url" value="http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4" required>
                <input type="submit" value="Submit">
            </form>
            {% if video_info %}

Video Information:

Video URL: {{ video_info.url }}

MIME Type: {{ video_info.mime_type }}

Extension: {{ video_info.extension }}

Content Length: {{ video_info.content_length }} bytes

Content Encoding: {{ video_info.content_encoding }}

Content Disposition: {{ video_info.content_disposition }}

Content Language: {{ video_info.content_language }}

Video Duration (seconds): {{ video_info.video_duration.seconds }} seconds

Video Duration (minutes): {{ video_info.video_duration.minutes }} minutes

Resolution (Width): {{ video_info.resolution.width }} pixels

Resolution (Height): {{ video_info.resolution.height }} pixels

FPS (Frames per Second): {{ video_info.fps }} FPS

Estimated Frames: {{ video_info.estimated_frames }} frames

            {% if video_info.thumbnail_base64 %}

            {% endif %}
            {% endif %}

    """

    return render_template_string(template, video_info=video_info)

if __name__ == '__main__':
    app.run(debug=True)


import datetime as dt

from django.core.validators import MaxValueValidator, MinValueValidator
from django.shortcuts import get_object_or_404
from rest_framework import serializers
from rest_framework.serializers import ValidationError
from django.contrib.auth import get_user_model
from django.contrib.auth.validators import UnicodeUsernameValidator

from reviews.models import Category, Comment, Genre, Review, Title
from users.validators import username_validator

User = get_user_model()

class CategorySerializer(serializers.ModelSerializer):
    """Класс сериализатор категории."""

    class Meta:
        fields = ('name', 'slug',)
        model = Category

class GenreSerializer(serializers.ModelSerializer):
    """Класс сериализатор жанра."""

    class Meta:
        fields = ('name', 'slug')
        model = Genre

class TitleListSerializer(serializers.ModelSerializer):
    """Класс сериализатор получения списка произведений."""

    genre = GenreSerializer(many=True, read_only=True)
    category = CategorySerializer(read_only=True)
    rating = serializers.IntegerField(read_only=True)

    class Meta:
        model = Title
        fields = '__all__'
        read_only_fields = ('id', 'name',
                            'year', 'description')

class CommentSerializer(serializers.ModelSerializer):
    """Сериалайзер комментариев."""

    author = serializers.SlugRelatedField(
        read_only=True,
        slug_field='username'
    )

    class Meta:
        model = Comment
        fields = ('id', 'author', 'pub_date', 'text')
        read_only_fields = ('id', 'pub_date')

class TitleCreateSerializer(serializers.ModelSerializer):
    """Класс сериализатор создания произведений."""

    category = serializers.SlugRelatedField(
        slug_field='slug',
        many=False,
        queryset=Category.objects.all()
    )
    genre = serializers.SlugRelatedField(
        slug_field='slug',
        many=True,
        required=False,
        queryset=Genre.objects.all()
    )

    class Meta:
        model = Title
        fields = '__all__'

    def year_validate(self, value):
        current_year = dt.date.today().year
        if value > current_year:
            raise serializers.ValidationError(
                'Год выпуска не может быть больше текущего'
            )
        return value

    def to_representation(self, instance):
        return TitleListSerializer(instance, context=self.context).data

class ReviewSerializer(serializers.ModelSerializer):
    """Сериализатор модели User."""

    title = serializers.PrimaryKeyRelatedField(
        read_only=True,
    )
    author = serializers.SlugRelatedField(
        default=serializers.CurrentUserDefault(),
        slug_field='username',
        read_only=True
    )
    score = serializers.IntegerField(
        validators=[
            MinValueValidator(1, 'Оценка должна быть не меньше 1.'),
            MaxValueValidator(10, 'Оценка должна быть не больше 10.')
        ],
    )

    class Meta:
        model = Review
        fields = '__all__'

    def validate(self, data):
        if self.context['request'].method == 'POST':
            title_id = self.context.get('view').kwargs.get('title_id')
            title = get_object_or_404(Title, id=title_id)
            request = self.context['request']
            author = request.user
            if Review.objects.filter(title=title, author=author).exists():
                raise serializers.ValidationError('Вы уже оставили свой отзыв'
                                                  'к этому произведению!')
        return data

class SignUpSerializer(serializers.Serializer):
    username = serializers.CharField(
        validators=(
            UnicodeUsernameValidator(),
        ),
        max_length=150
    )
    email = serializers.EmailField(
        max_length=254
    )

    def validate_username(self, username):
        if username.lower() == 'me':
            raise serializers.ValidationError('Недопустимое имя пользователя.')
        return username

    def validate(self, data):
        suspect_email = User.objects.filter(email=data['email']).exists()
        suspect_user = User.objects.filter(username=data['username']).exists()
        if User.objects.filter(username=data['username'],
                               email=data['email']).exists():
            return data
        if suspect_user:
            raise ValidationError('Это имя уже занято')
        if suspect_email:
            raise ValidationError('Эта почта уже зарегистрирована')
        return data

    class Meta:
        model = User
        fields = ('email',
                  'username',
                  )

class TokenSerializer(serializers.Serializer):
    confirmation_code = serializers.CharField(
        max_length=32,
    )
    username = serializers.CharField(max_length=150)

class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ('username', 'email',
                  'first_name', 'last_name', 'bio', 'role',)

    def validate_username(self, value):
        return username_validator(value)

class UserRestrictedSerializer(UserSerializer):
    username = serializers.CharField(
        max_length=150,
        required=True,
        validators=[
            username_validator,
        ]
    )
    email = serializers.EmailField(
        required=True,
        max_length=254
    )

    class Meta(UserSerializer.Meta):
        read_only_fields = ('username', 'email', 'role')


from time import sleep

import wrapt

from azdashboard.lib.azurelib.core.errors import AzureException
# from utils.vt4log import getLog

def handle_error(error, suppress_list):
    """Creates a simpler error object"""

    # log = getLog("azurelib")
    error = AzureException(error)
    if error.number in suppress_list:
        # log.info("Error suppresed.", suppresed_error=error.number)
        return None
    # log.error(error.message, http_code=error.number, exc=error)
    return error

def azure_error(no_of_retries=10, retry_timing=60, suppress=[]):
    @wrapt.decorator
    def retry_error(wrapped, instance, args, kwargs):
        retriable = True
        retry_no = 0
        while retry_no < no_of_retries and retriable:
            try:
                return wrapped(*args, **kwargs)
            except Exception as err:
                exc = handle_error(err, suppress)
                if not exc:
                    return False
                if exc.retriable and retry_no < no_of_retries:
                    retriable = True
                    retry_no += 1
                    sleep(retry_timing)
                else:
                    raise exc
    return retry_error


#!/usr/bin/env python

import os, popen2, time
from socket import *

def main0():
    cin, cout = popen2.popen2('openssl s_server')
    cout.write('Q\n')
    cout.flush()
    s = socket(AF_INET, SOCK_STREAM)
    s.connect(('', 4433))
    s.close()

def main():
    pid = os.fork()
    if pid:
        time.sleep(1)
        os.kill(pid, 1)
        os.waitpid(pid, 0)
    else:
        os.execvp('openssl', ('s_server',))

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
from typing import Dict, Iterable, List, Optional, Union

import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn.preprocessing import _data as sklearn_preprocessing_data

from snowflake import snowpark
from snowflake.ml._internal import telemetry
from snowflake.ml.modeling.framework import _utils, base

class StandardScaler(base.BaseTransformer):
    r"""
    Standardizes features by removing the mean and scaling to unit variance. Values must be of float type.

    For more details on what this transformer does, see [sklearn.preprocessing.StandardScaler]
    (https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html).

    Args:
        with_mean: bool, default=True
            If True, center the data before scaling.

        with_std: bool, default=True
            If True, scale the data unit variance (i.e. unit standard deviation).

        input_cols: Optional[Union[str, List[str]]], default=None
            The name(s) of one or more columns in a DataFrame containing a feature to be scaled.

        output_cols: Optional[Union[str, List[str]]], default=None
            The name(s) of one or more columns in a DataFrame in which results will be stored. The number of
            columns specified must match the number of input columns.

        passthrough_cols: Optional[Union[str, List[str]]], default=None
            A string or a list of strings indicating column names to be excluded from any
            operations (such as train, transform, or inference). These specified column(s)
            will remain untouched throughout the process. This option is helpful in scenarios
            requiring automatic input_cols inference, but need to avoid using specific
            columns, like index columns, during training or inference.

        drop_input_cols: Optional[bool], default=False
            Remove input columns from output if set True. False by default.

    Attributes:
        scale_: Optional[Dict[str, float]] = {}
            Dictionary mapping input column names to relative scaling factor to achieve zero mean and unit variance.
            If a variance is zero, unit variance could not be achieved, and the data is left as-is, giving a scaling
            factor of 1. None if with_std is False.

        mean_: Optional[Dict[str, float]] = {}
            Dictionary mapping input column name to the mean value for that feature. None if with_mean is False.

        var_: Optional[Dict[str, float]] = {}
            Dictionary mapping input column name to the variance for that feature. Used to compute scale_. None if
            with_std is False
    """

    def __init__(
        self,
        *,
        with_mean: bool = True,
        with_std: bool = True,
        input_cols: Optional[Union[str, Iterable[str]]] = None,
        output_cols: Optional[Union[str, Iterable[str]]] = None,
        passthrough_cols: Optional[Union[str, Iterable[str]]] = None,
        drop_input_cols: Optional[bool] = False,
    ) -> None:
        """
        Standardize features by removing the mean and scaling to unit variance.

        Args:
            with_mean: If True, center the data before scaling.
                This does not work (and will raise an exception) when attempted on
                sparse matrices, because centering them entails building a dense
                matrix which in common use cases is likely to be too large to fit in
                memory.
            with_std: If True, scale the data to unit variance (or equivalently,
                unit standard deviation).
            input_cols: Single or multiple input columns.
            output_cols: Single or multiple output columns.
            passthrough_cols: A string or a list of strings indicating column names to be excluded from any
                operations (such as train, transform, or inference). These specified column(s)
                will remain untouched throughout the process. This option is helful in scenarios
                requiring automatic input_cols inference, but need to avoid using specific
                columns, like index columns, during in training or inference.
            drop_input_cols: Remove input columns from output if set True. False by default.

        Attributes:
            scale_: dict {column_name: value} or None
                Per feature relative scaling of the data to achieve zero mean and unit
                variance. If a variance is zero, we can't achieve unit variance, and
                the data is left as-is, giving a scaling factor of 1. `scale_` is equal
                to `None` when `with_std=False`.
            mean_: dict {column_name: value} or None
                The mean value for each feature in the training set.
                Equal to ``None`` when ``with_mean=False``.
            var_: dict {column_name: value} or None
                The variance for each feature in the training set. Used to compute
                `scale_`. Equal to ``None`` when ``with_std=False``.
        """
        self.with_mean = with_mean
        self.with_std = with_std

        self.scale_: Optional[Dict[str, float]] = {} if with_std else None
        self.mean_: Optional[Dict[str, float]] = {} if with_mean else None
        self.var_: Optional[Dict[str, float]] = {} if with_std else None

        self.custom_states: List[str] = []
        if with_mean:
            self.custom_states.append(_utils.NumericStatistics.MEAN)
        if with_std:
            self.custom_states.append(_utils.NumericStatistics.VAR_POP)
            self.custom_states.append(_utils.NumericStatistics.STDDEV_POP)

        super().__init__(drop_input_cols=drop_input_cols, custom_states=self.custom_states)

        self.set_input_cols(input_cols)
        self.set_output_cols(output_cols)
        self.set_passthrough_cols(passthrough_cols)

    def _reset(self) -> None:
        """
        Reset internal data-dependent state of the scaler, if necessary.
        __init__ parameters are not touched.
        """
        super()._reset()
        if hasattr(self, "scale_"):
            self.scale_ = {} if self.with_std else None
        if hasattr(self, "mean_"):
            self.mean_ = {} if self.with_mean else None
        if hasattr(self, "var_"):
            self.var_ = {} if self.with_std else None

    @telemetry.send_api_usage_telemetry(
        project=base.PROJECT,
        subproject=base.SUBPROJECT,
    )
    def fit(self, dataset: Union[snowpark.DataFrame, pd.DataFrame]) -> "StandardScaler":
        """
        Compute mean and std values of the dataset.

        Args:
            dataset: Input dataset.

        Returns:
            Fitted scaler.
        """
        super()._check_input_cols()
        super()._check_dataset_type(dataset)
        self._reset()

        if isinstance(dataset, pd.DataFrame):
            self._fit_sklearn(dataset)
        else:
            self._fit_snowpark(dataset)

        self._is_fitted = True
        return self

    def _fit_sklearn(self, dataset: pd.DataFrame) -> None:
        dataset = self._use_input_cols_only(dataset)
        sklearn_scaler = self._create_unfitted_sklearn_object()
        sklearn_scaler.fit(dataset[self.input_cols])

        for i, input_col in enumerate(self.input_cols):
            if self.mean_ is not None:
                self.mean_[input_col] = float(sklearn_scaler.mean_[i])
            if self.scale_ is not None:
                self.scale_[input_col] = float(sklearn_scaler.scale_[i])
            if self.var_ is not None:
                self.var_[input_col] = float(sklearn_scaler.var_[i])

    def _fit_snowpark(self, dataset: snowpark.DataFrame) -> None:
        computed_states = self._compute(dataset, self.input_cols, self.custom_states)

        # assign states to the object
        for input_col in self.input_cols:
            numeric_stats = computed_states[input_col]

            if self.mean_ is not None:
                self.mean_[input_col] = float(numeric_stats[_utils.NumericStatistics.MEAN])

            if self.var_ is not None:
                self.var_[input_col] = float(numeric_stats[_utils.NumericStatistics.VAR_POP])

            if self.scale_ is not None:
                self.scale_[input_col] = sklearn_preprocessing_data._handle_zeros_in_scale(
                    float(numeric_stats[_utils.NumericStatistics.STDDEV_POP])
                )

    @telemetry.send_api_usage_telemetry(
        project=base.PROJECT,
        subproject=base.SUBPROJECT,
    )
    def transform(self, dataset: Union[snowpark.DataFrame, pd.DataFrame]) -> Union[snowpark.DataFrame, pd.DataFrame]:
        """
        Perform standardization by centering and scaling.

        Args:
            dataset: Input dataset.

        Returns:
            transformed_dataset: Output dataset.
        """
        self._enforce_fit()
        super()._check_input_cols()
        super()._check_output_cols()
        super()._check_dataset_type(dataset)

        if isinstance(dataset, snowpark.DataFrame):
            output_df = self._transform_snowpark(dataset)
        else:
            output_df = self._transform_sklearn(dataset)

        return self._drop_input_columns(output_df) if self._drop_input_cols is True else output_df

    def _transform_snowpark(self, dataset: snowpark.DataFrame) -> snowpark.DataFrame:
        """
        Perform standardization by centering and scaling on
        Snowpark dataframe.

        Args:
            dataset: Input dataset.

        Returns:
            Output dataset.
        """
        passthrough_columns = [c for c in dataset.columns if c not in self.output_cols]
        output_columns = []
        for _, input_col in enumerate(self.input_cols):
            output_column = dataset[input_col]

            if self.mean_ is not None:
                output_column = output_column - self.mean_[input_col]
            if self.scale_ is not None:
                output_column = output_column / self.scale_[input_col]

            output_columns.append(output_column)

        transformed_dataset: snowpark.DataFrame = dataset.with_columns(self.output_cols, output_columns)
        # Reorder columns. Passthrough columns are added at the right to the output of the transformers.
        transformed_dataset = transformed_dataset[self.output_cols + passthrough_columns]
        return transformed_dataset

    def _create_unfitted_sklearn_object(self) -> preprocessing.StandardScaler:
        return preprocessing.StandardScaler(with_mean=self.with_mean, with_std=self.with_std)

    def _create_sklearn_object(self) -> preprocessing.StandardScaler:
        """
        Get an equivalent sklearn StandardScaler.

        Returns:
            The Sklearn StandardScaler.
        """
        scaler = self._create_unfitted_sklearn_object()
        if self._is_fitted:
            scaler.scale_ = self._convert_attribute_dict_to_ndarray(self.scale_, np.float64)
            scaler.mean_ = self._convert_attribute_dict_to_ndarray(self.mean_, np.float64)
            scaler.var_ = self._convert_attribute_dict_to_ndarray(self.var_, np.float64)
        return scaler


import pandas as pd
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score, roc_curve, auc
from sklearn.decomposition import PCA
from sklearn.naive_bayes import GaussianNB
import matplotlib.pyplot as plt
import time

# 读取数据文件
file_path = "../data/abalone.data"  # 替换为实际文件路径
column_names = ["Sex", "Length", "Diameter", "Height", "Whole weight",
                "Shucked weight", "Viscera weight", "Shell weight", "Rings"]
data = pd.read_csv(file_path, header=None, names=column_names)

# 使用LabelEncoder将标签编码为数字
label_encoder = LabelEncoder()
data['Sex'] = label_encoder.fit_transform(data['Sex'])

# 提取特征列和标签列
X = data.drop("Sex", axis=1)
y_true = data["Sex"]

# 使用PCA进行降维
pca = PCA(n_components=None)  # 设置为None，表示保留所有主成分
X_pca = pca.fit_transform(X)

# 查看PCA保留的维度
num_components = pca.n_components_
print("Number of PCA Components:", num_components)
explained_variance_ratios = pca.explained_variance_ratio_
print("Explained Variance Ratios:", explained_variance_ratios)
# 打印各主成分的标签
for i in range(num_components):
    print("PCA Component {} Label: {}".format(i + 1, column_names[i + 1]))

# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X_pca, y_true, test_size=0.2, random_state=42)
print("X_test", X_test)
print("y_test", y_test)
# 使用朴素贝叶斯分类器（这里使用高斯朴素贝叶斯）

# 设置朴素贝叶斯分类器的参数范围
param_grid = {'var_smoothing': [1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 0.1]}

# 使用GridSearchCV进行交叉验证和参数选择
nb_classifier = GaussianNB()
grid_search = GridSearchCV(nb_classifier, param_grid, cv=5, scoring='accuracy')

# 计算训练时间
start_time = time.time()
grid_search.fit(X_train, y_train)
training_time = time.time() - start_time

# 输出最佳参数和交叉验证准确率
print("Best Parameters:", grid_search.best_params_)
print("Cross-Validation Accuracy:{:.2f}%".format(round((grid_search.best_score_ * 100), 2)))
print("Training Time with Cross-Validation: {:.4f} seconds".format(training_time))

# 在测试集上进行预测
start_time = time.time()
y_pred = grid_search.best_estimator_.predict(X_test)
testing_time = time.time() - start_time
print("Testing Time: {:.4f} seconds".format(testing_time))

# 计算准确率
accuracy = accuracy_score(y_test, y_pred)
print("Classifier Accuracy:{:.2f}%".format(accuracy * 100))

# 计算混淆矩阵
conf_matrix = confusion_matrix(y_test, y_pred)
print("confusion_matrix", conf_matrix)

# 显示混淆矩阵
disp = ConfusionMatrixDisplay(confusion_matrix=conf_matrix, display_labels=label_encoder.classes_)
disp.plot(cmap='viridis', values_format='d')
plt.title('Naive Bayes Confusion Matrix')
# plt.savefig('NaiveBayesConfusionMatrix.png')
plt.show()

# 计算训练和测试的计算复杂度（近似值）
training_complexity = X_train.shape[1] * X_train.shape[0]
testing_complexity = X_test.shape[1] * X_test.shape[0]
print("Approximate Training Complexity: O({})".format(training_complexity))
print("Approximate Testing Complexity: O({})".format(testing_complexity))

# 绘制ROC曲线
y_probs = grid_search.best_estimator_.predict_proba(X_test)

# 为每个类别计算ROC曲线和AUC值
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(len(label_encoder.classes_)):
    fpr[i], tpr[i], _ = roc_curve(y_test == i, y_probs[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])
print("roc_auc", roc_auc)
print("fpr", fpr)
print("tpr", tpr)

# 绘制所有类别的ROC曲线
plt.figure(figsize=(10, 6))
for i in range(len(label_encoder.classes_)):
    plt.plot(fpr[i], tpr[i], label=f'Class {label_encoder.classes_[i]} (AUC = {roc_auc[i]:.2f})')

# 绘制对角线
plt.plot([0, 1], [0, 1], 'k--', lw=2)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])

# 设置图例
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Naive Bayes (ROC) Curve')
plt.legend(loc="lower right")
# plt.savefig('NaiveBayesROC.png')
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Wed Jan 19 11:49:20 2022

@author: robyates
"""

"""
mass_checks.py
  ;Script to check:
  ;a) how many NaNs and negatives there are in key mass properties,
  ;b) if sub-component masses add up to component masses properly (e.g. SUM(ColdGas_elements) = ColdGas, etc)
  ;
  ;Rob Yates 19-01-2022
  ;
  ;08-11-22: This version was adapted for use at the L-Galaxies workshop 2022
  ;
"""

#Base packages:
import numpy as np
import matplotlib.pyplot as plt

#################
def mass_checks(G_lgal, COSMOLOGY, SIMULATION, FILE_TYPE, STRUCT_TYPE, MODEL, VERSION, SAMPLE_TYPE, plots=None) :
    #Properties to check:
    prop_name = ['ColdGas','BulgeMass','DiskMass', \
                 'MetalsBulgeMass_SNII','MetalsBulgeMass_SNIa','MetalsBulgeMass_AGB', \
                 'MetalsDiskMass_SNII','MetalsDiskMass_SNIa','MetalsDiskMass_AGB', \
                 'MetalsDiskMassRings_SNII','MetalsDiskMassRings_SNIa','MetalsDiskMassRings_AGB', \
                 'ColdGas_elements']
    prop = [G_lgal['ColdGas'],G_lgal['BulgeMass'],G_lgal['DiskMass'], \
            G_lgal['MetalsBulgeMass'][:,0],G_lgal['MetalsBulgeMass'][:,1],G_lgal['MetalsBulgeMass'][:,2], \
            G_lgal['MetalsDiskMass'][:,0],G_lgal['MetalsDiskMass'][:,1],G_lgal['MetalsDiskMass'][:,2], \
            G_lgal['MetalsDiskMassRings'][:,:,0],G_lgal['MetalsDiskMassRings'][:,:,1],G_lgal['MetalsDiskMassRings'][:,:,2], \
            G_lgal['ColdGas_elements']]

    ##############
    #Check for NaNs:
    print("\n------------------")
    print("Check for NaNs:")
    print("------------------")
    for ii in range(len(prop)) :
        nans = np.count_nonzero(np.isnan(prop[ii]))
        if nans == 0 :
            print(prop_name[ii], ":", nans, "/", prop[ii].size)
        else :
            print("*************************************")
            print("WARNING: NaNs in", prop_name[ii], "=", nans, "/", prop[ii].size)
            print("*************************************")

    ##############
    #Check for negatives:
    print("\n------------------")
    print("Check for < 0:")
    print("------------------")
    for ii in range(len(prop)) :
        negs = np.sum(prop[ii] < 0.)
        if negs == 0 :
            print(prop_name[ii], ":", negs, "/", prop[ii].size)
        else :
            var = prop[ii]
            varneg = var[var < 0.]
            print("*************************************")
            print("WARNING: Negatives in", prop_name[ii], ":", negs, "/", prop[ii].size)
            print("Min =", np.min(varneg), " | ", "Max =", np.max(varneg))
            print("*************************************")

    ##############
    #Check for sums:
    print("\n------------------")
    print("Check sub-component masses:")
    print("------------------")
    #Mass sum properties:
    prop1_name = ['ColdGas','DiskMass','MetalsColdGas','MetalsDiskMass','MetalsBulgeMass', \
                  'ColdGas','DiskMass','BulgeMass','HotGas','HaloStars', \
                  'MetalsColdGas','MetalsDiskMass','MetalsBulgeMass', \
                  'MetalsHotGas','MetalsHaloStars', \
                  'Gal number', 'Gal number', 'Gal number', \
                  'Gal number', 'Gal number', 'Gal number', \
                  'Gal number', 'Gal number', 'Gal number']
    prop1 = [G_lgal['ColdGas'],G_lgal['DiskMass'],np.nansum(G_lgal['MetalsColdGas'], axis=1),np.nansum(G_lgal['MetalsDiskMass'], axis=1),np.nansum(G_lgal['MetalsBulgeMass'], axis=1), \
             G_lgal['ColdGas'],G_lgal['DiskMass'],G_lgal['BulgeMass'],G_lgal['HotGas'],G_lgal['ICM'], \
             np.nansum(G_lgal['MetalsColdGas'], axis=1),np.nansum(G_lgal['MetalsDiskMass'], axis=1),np.nansum(G_lgal['MetalsBulgeMass'], axis=1), \
             np.nansum(G_lgal['MetalsHotGas'], axis=1),np.nansum(G_lgal['MetalsICM'], axis=1), \
             np.arange(len(G_lgal['DiskMass'])), np.arange(len(G_lgal['DiskMass'])), np.arange(len(G_lgal['DiskMass'])), \
             np.arange(len(G_lgal['BulgeMass'])), np.arange(len(G_lgal['BulgeMass'])), np.arange(len(G_lgal['BulgeMass'])), \
             np.arange(len(G_lgal['ICM'])), np.arange(len(G_lgal['ICM'])), np.arange(len(G_lgal['ICM']))]
    prop2_name = ['ColdGasRings','DiskMassRings','MetalsColdGasRings','MetalsDiskMassRings', 'MetalsBulgeMassRings', \
                  'ColdGasElements','DiskMassElements','BulgeMassElements','HotGasElements','HaloStarElements', \
                  'MetalsColdGasElements','MetalsDiskMassElements','MetalsBulgeMassElements', \
                  'MetalsHotGasElements','MetalsHaloStarsElements', \
                  'MetalsDiskMassSNII','MetalsDiskMassSNIa','MetalsDiskMassAGB', \
                  'MetalsBulgeMassSNII','MetalsBulgeMassSNIa','MetalsBulgeMassAGB', \
                  'MetalsHaloStarsSNII','MetalsHaloStarsSNIa','MetalsHaloStarsAGB']
    prop2 = [G_lgal['ColdGasRings'],G_lgal['DiskMassRings'],np.nansum(G_lgal['MetalsColdGasRings'], axis=2),np.nansum(G_lgal['MetalsDiskMassRings'], axis=2),np.nansum(G_lgal['MetalsBulgeMassRings'], axis=2), \
             G_lgal['ColdGas_elements'],G_lgal['DiskMass_elements'],G_lgal['BulgeMass_elements'],G_lgal['HotGas_elements'],G_lgal['ICM_elements'],\
             G_lgal['ColdGas_elements'][:,2:],G_lgal['DiskMass_elements'][:,2:],G_lgal['BulgeMass_elements'][:,2:], \
             G_lgal['HotGas_elements'][:,2:],G_lgal['ICM_elements'][:,2:], \
             G_lgal['MetalsDiskMass'][:,0],G_lgal['MetalsDiskMass'][:,1],G_lgal['MetalsDiskMass'][:,2], \
             G_lgal['MetalsBulgeMass'][:,0],G_lgal['MetalsBulgeMass'][:,1],G_lgal['MetalsBulgeMass'][:,2], \
             G_lgal['MetalsICM'][:,0],G_lgal['MetalsICM'][:,1],G_lgal['MetalsICM'][:,2]]

    if plots :
        fig = plt.figure(figsize=(15,38)) #15,25
        plt.rc('font', family='serif', size='14.0')
        rowno = 9 #6
        colno = 3

    for ii in range(len(prop1)) :
        theprop1 = prop1[ii]
        if (prop1_name[ii] != 'Gal number')         :
            theprop2 = np.nansum(prop2[ii], axis=1)
            if (ii == 7) | (ii == 8) : #If prop1 or prop2 include a multiple of G_lgal['H2fraction']
                ww = np.where((theprop1 > 100.0) & (theprop2 > 100.0) & (G_lgal['H2fraction'] > 1.e-04))
                theprop3 = G_lgal['H2fraction'][ww]
            else :
                ww = np.where((theprop1 > 100.0) & (theprop2 > 100.0)) #Only consider galaxies with whole-component mass of >100 Msun
            theprop1 = theprop1[ww]
            theprop2 = theprop2[ww]

            if np.allclose(theprop1, theprop2) :
                print("SUM(", prop2_name[ii], ") !=", prop1_name[ii], ": 0 /", prop1[ii].size)#
            else :
                num = np.count_nonzero(theprop1 != theprop2)
                vardiff = theprop1 - theprop2
                vardiff = abs(vardiff[vardiff != 0.0])
                vardifffrac = (theprop1 - theprop2) / theprop1
                vardifffrac = abs(vardifffrac[vardifffrac != 0.0])
                print("*************************************")
                print("WARNING: SUM(", prop2_name[ii], ") !=", prop1_name[ii], ":", num, "/", theprop1.size)
                print("Min diff =", np.min(vardiff), " | ", "Mean diff =", np.mean(vardiff), " | ", "Max diff =", np.max(vardiff))
                print("Min diff frac =", np.min(vardifffrac), " | ", "Mean diff frac =", np.mean(vardifffrac), " | ", "Max diff frac =", np.max(vardifffrac))
                print("*************************************")
        else :
            theprop2 = prop2[ii]

        if plots :
            #Set-up plot:
            plt.subplot(rowno, colno, ii+1)
            if (prop1_name[ii] != 'Gal number') :
                xlimits = np.array([0.0,np.max([np.max(theprop1),np.max(theprop2)])])
                ylimits = np.array([0.0,np.max([np.max(theprop1),np.max(theprop2)])])
                plt.plot(xlimits, ylimits, linestyle='--', color='grey')
            else :
                xlimits = np.array([np.min(theprop1),np.max(theprop1)])
                ylimits = np.array([np.min(theprop2),np.max(theprop2)])
                plt.plot(xlimits, np.array([0.0,0.0]), linestyle='--', color='grey')
            #fig = plt.figure(figsize=(5,5))
            plt.xlabel(prop1_name[ii], fontsize=12)
            plt.ylabel(prop2_name[ii], fontsize=12)
            plt.xlim(xlimits)
            plt.ylim(ylimits)
            plt.tick_params(direction="in", top=True, bottom=True, left=True, right=True)
            plt.scatter(theprop1, theprop2, marker='o')
            if ii == 0 :
                plt.text(0.25, 0.92, ['Mass checks', 'z=0', str(len(G_lgal))+' gals', COSMOLOGY, SIMULATION, FILE_TYPE, MODEL, VERSION, SAMPLE_TYPE], transform=fig.transFigure, horizontalalignment='left', color='blue')

    print('Mass checks done')


"""ImportJson Migration."""

from masoniteorm.migrations import Migration
from masoniteorm.query import QueryBuilder

import json
from os import path

class ImportJson(Migration):
    def up(self):
        """
        Run the migrations.
        """

        if not path.exists('db.json'):
            return

        guilds = QueryBuilder().on(self.connection).table('guilds')
        text_channels = QueryBuilder().on(self.connection).table('text_channels')

        with open('db.json', encoding='utf-8') as f:
            data = json.load(f)

        for guild_id, guild_content in data["guilds"].items():
            guilds.create({
                "id": guild_id
            })
            for channel_id, channel_content in guild_content["channels"].items():
                text_channels.create({
                    "id": channel_id,
                    "guild_id": guild_id,
                    "fix_twitter": channel_content["fixtweet"]
                })

    def down(self):
        """
        Revert the migrations.
        """
        return


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Author: Fang Zhang <thuzhf@gmail.com>

import sys,os,json,gzip,math,time,datetime,random,copy
import functools,itertools,requests,pickle,configparser
import argparse,logging,uuid,shutil
from collections import defaultdict as dd
import multiprocessing as mp
import numpy as np
import regex as re
re.DEFAULT_VERSION = re.VERSION1
import gensim

from FastKATE.utils.multiprocess import multiprocess_dir, multiprocess_file
from FastKATE.utils.logger import simple_logger
logger = simple_logger(__name__, 'FastKATE/log/')

from nltk.stem import WordNetLemmatizer
lemmatize = WordNetLemmatizer().lemmatize
lemmatize('')
from spacy.en import English
nlp = English()

# global variables are defined here
config = configparser.ConfigParser()
config_file = 'FastKATE/config/word2vec.cfg'
config.read(config_file)
section = config['global']

SIZE = eval(section['SIZE'])
WINDOW = eval(section['WINDOW'])
MIN_COUNT = eval(section['MIN_COUNT'])
WORKERS = eval(section['WORKERS'])
SAMPLE = eval(section['SAMPLE'])

MIN_WORD_LENGTH = eval(section['MIN_WORD_LENGTH'])
MIN_SENTENCE_LENGTH = eval(section['MIN_SENTENCE_LENGTH'])
FORBIDDEN_POS = eval(section['FORBIDDEN_POS'])
# FORBIDDEN_TAGS = eval(section['FORBIDDEN_TAGS'])
POSSIBLE_FORBIDDEN_TAGS = eval(section['POSSIBLE_FORBIDDEN_TAGS'])
MUST_TAGS = eval(section['MUST_TAGS'])

PHRASE_LENGTHS = eval(section['PHRASE_LENGTHS'])
MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS = min(PHRASE_LENGTHS), max(PHRASE_LENGTHS)
USE_FILTERED_WIKI_TITLES = eval(section['USE_FILTERED_WIKI_TITLES'])
if USE_FILTERED_WIKI_TITLES:
    _FILTERED = '_onFilteredWikiTitles'
else:
    _FILTERED = ''
logger.info('configs have been read from {}'.format(config_file))

def transform_to_noun_phrases_list(doc):
    np = []
    tokens = nlp(doc, entity=False)
    for i in tokens.noun_chunks:
        tmp = []
        for j in i:
            if not j.is_stop:
                tmp.append(lemmatize(j.lower_))
        np.append('_'.join(tmp))
    return np

def lower_and_remove_non_alphanumeric_wiki_title(title): # title should be a non-empty string
    title = title.lower()
    title2 = title.strip()
    if 'regex' not in lower_and_remove_non_alphanumeric_wiki_title.__dict__:
        lower_and_remove_non_alphanumeric_wiki_title.regex = re.compile(r'^(?!-)([a-z0-9-]+_?)+$')
    if title2 and lower_and_remove_non_alphanumeric_wiki_title.regex.match(title2):
        return title
    else:
        return ''

def lemmatize_and_remove_wiki_title_consisting_of_short_words(title, min_word_length=MIN_WORD_LENGTH):
    title2 = title.strip()
    tmp = [lemmatize(i) for i in title2.split('_')]
    valid = False
    for word in tmp:
        if len(word) >= min_word_length:
            valid = True
            break
    if title2 and valid:
        return '{}\n'.format('_'.join(tmp))
    else:
        return ''

def remove_wiki_title_by_pos_and_tag_rules(title, forbidden_pos=FORBIDDEN_POS, forbidden_tags=POSSIBLE_FORBIDDEN_TAGS, must_tags=MUST_TAGS):
    title2 = title.strip().replace('_', ' ')
    tokens = nlp(title2, tag=True, parse=False, entity=False)
    skip_line = False
    if must_tags:
        exist_one_must_tag = False
    else:
        exist_one_must_tag = True
    for t in tokens:
        if forbidden_pos and t.pos_ in forbidden_pos:
            skip_line = True
            break
        if forbidden_tags and t.tag_ in forbidden_tags:
            skip_line = True
            break
        if must_tags and t.tag_ in must_tags:
            exist_one_must_tag = True
    if title2 and not skip_line and exist_one_must_tag:
        return title
    else:
        return ''

# remove words with forbidden POSs, and make remaining words lowercase and lemmatized
def clean_raw_text_file(infile, outfile, ignored_line_start='<', ignore_existed_file=False, forbidden_pos=FORBIDDEN_POS):
    if ignore_existed_file and os.path.isfile(outfile):
        return
    logger.info('processing file: {}'.format(infile))
    with open(infile) as fin, open(outfile, 'w') as fout:
        for line in fin:
            line = line.strip()
            if not line:
                fout.write('\n')
                continue
            if ignored_line_start != None and line.startswith(ignored_line_start):
                continue
            tokens = nlp(line, tag=True, parse=False, entity=False)
            lemmatized_lower_tokens = []
            for t in tokens:
                if t.is_stop:
                    continue
                if t.pos_ not in forbidden_pos:
                    lemmatized_lower_tokens.append(lemmatize(t.lower_))
            if lemmatized_lower_tokens:
                fout.write('{}\n'.format(' '.join(lemmatized_lower_tokens)))
    logger.info('written into file: {}'.format(outfile))

# remove words with forbidden POSs, and make remaining words lowercase and lemmatized
def clean_raw_text_file_and_add_newline_after_each_document(infile, outfile, ignore_existed_file=False, forbidden_pos=FORBIDDEN_POS):
    if ignore_existed_file and os.path.isfile(outfile):
        return
    logger.info('processing file: {}'.format(infile))
    with open(infile) as fin, open(outfile, 'w') as fout:
        for line in fin:
            line = line.strip()
            if not line:
                # fout.write('\n')
                continue
            if line.startswith('<doc id="'):
                continue
            if line.startswith('</doc>'):
                fout.write('\n')
                continue
            tokens = nlp(line, tag=True, parse=False, entity=False)
            lemmatized_lower_tokens = []
            for t in tokens:
                if t.is_stop:
                    continue
                if t.pos_ not in forbidden_pos:
                    lemmatized_lower_tokens.append(lemmatize(t.lower_))
            if lemmatized_lower_tokens:
                fout.write('{}\n'.format(' '.join(lemmatized_lower_tokens)))
    logger.info('written into file: {}'.format(outfile))

def generate_sentences_of_noun_phrases(infile, outfile):
    logger.info('processing file: {}...'.format(infile))
    with open(infile) as fin, open(outfile, 'w') as fout:
        for line in fin:
            line = line.strip()
            if not line:
                fout.write('\n')
                continue
            if line.startswith('<doc id=') or line.startswith('</doc>'):
                continue
            np_list = transform_to_noun_phrases_list(line)
            if np_list:
                fout.write('{}\n'.format(' '.join(np_list)))
    logger.info('has written into file: {}'.format(outfile))

def generate_sentences_of_given_phrases(given_phrases, infile, outfile, phrase_lengths):
    logger.info('processing file: {}...'.format(infile))
    with open(infile) as fin, open(outfile, 'w') as fout:
        for line in fin:
            word_list = line.split()
            if not word_list:
                fout.write('\n')
                continue
            phrase_list = generate_phrase_list_from_word_list(word_list, given_phrases, phrase_lengths)
            if phrase_list:
                fout.write('{}\n'.format(' '.join(phrase_list)))
    logger.info('has written into file: {}'.format(outfile))

def generate_phrase_list_from_word_list(word_list, given_phrases, phrase_lengths=PHRASE_LENGTHS):
    current_word_index = 0
    phrase_lengths = sorted(phrase_lengths, reverse=True)
    ret = []
    while current_word_index < len(word_list):
        found_new_phrase = False
        for phrase_length in phrase_lengths:
            if current_word_index + phrase_length <= len(word_list):
                new_phrase = '_'.join(word_list[current_word_index: current_word_index + phrase_length])
                if new_phrase in given_phrases:
                    found_new_phrase = True
                    ret.append(new_phrase)
                    current_word_index += phrase_length
                    break
        if not found_new_phrase:
            current_word_index += 1
    return ret

def generate_sentences_of_noun_phrases_at_dir_level(src_dir, dst_dir, chunksize=1, multiprocess=True):
    logger.info('generate_sentences_of_noun_phrases_at_dir_level...')
    multiprocess_dir(src_dir, dst_dir, generate_sentences_of_noun_phrases, chunksize=chunksize, multiprocess=multiprocess)

def generate_sentences_of_given_phrases_at_dir_level(src_dir, dst_dir, given_phrases_file, phrase_lengths=PHRASE_LENGTHS, chunksize=1, multiprocess=True):
    given_phrases = set()
    logger.info('loading {}...'.format(given_phrases_file))
    with open(given_phrases_file) as fin:
        for line in fin:
            given_phrases.add(line.strip())
    logger.info('has loaded {}'.format(given_phrases_file))
    multiprocess_dir(src_dir, dst_dir, generate_sentences_of_given_phrases, phrase_lengths, chunksize=chunksize, multiprocess=multiprocess, necessary_resources=given_phrases)

def train_word2vec_on_text(sentences_dir, model_path, size=SIZE, window=WINDOW, min_count=MIN_COUNT, workers=WORKERS, sample=SAMPLE):
    model = gensim.models.Word2Vec(None, size=size, window=window, min_count=min_count, workers=workers, sample=sample)
    logger.info('building vocabulary from: {}...'.format(sentences_dir))
    model.build_vocab(sentence_generator(sentences_dir))
    model.save(model_path)
    logger.info('training...')
    model.train(sentence_generator(sentences_dir))
    logger.info('saving model to: {}...'.format(model_path))
    model.save(model_path)
    logger.info('done')

def sentence_generator(sentences_dir):
    for dirpath, dirnames, filenames in os.walk(sentences_dir):
        for fname in filenames:
            full_path = os.path.join(dirpath, fname)
            with open(full_path) as fin:
                for line in fin:
                    yield line.split()

def extract_wiki_categories(category_file, outfile, encoding='latin1'):
    categories = set()
    with open(category_file, encoding=encoding) as f:
        lines = f.readlines()
    start_str = 'INSERT INTO `category` VALUES '
    category_regex = re.compile(r"(?<=\(\d+,').*?(?=',\d+,\d+,\d+\))")
    for line in lines:
        if line.startswith(start_str):
            for i in category_regex.findall(line):
                tmp = i.split('_')
                for j in range(len(tmp)):
                    tmp[j] = lemmatize(tmp[j].lower())
                tmp = '_'.join(tmp)
                categories.add(tmp)
    categories = sorted(categories)
    with open(outfile, 'w') as f:
        for i in categories:
            f.write('{}\n'.format(i))

# text rank data in task1
def prepare_wiki_article_for_textrank(infile, outfile_titles, outfile_contents):
    logger.info('infile: {}'.format(infile))
    with open(infile) as fin, open(outfile_titles, 'w') as fout_titles, open(outfile_contents, 'w') as fout_contents:
        current_paper = []
        first_line = True
        valid = None
        for line in fin:
            tmp = line.split()
            if tmp:
                if first_line:
                    if len(tmp) == 1:
                        valid = True
                        fout_titles.write(line)
                    else:
                        valid = False
                    first_line = False
                if valid:
                    for i in tmp:
                        current_paper.append(i)
                else:
                    continue
            else:
                first_line = True
                if current_paper:
                    # tmp = sorted(current_paper.items(), key=lambda x: x[1], reverse=True)
                    # tmp2 = []
                    # for k, v in tmp:
                        # tmp2.append(k)
                        # tmp2.append(str(v))
                    tmp2 = ' '.join(current_paper)
                    fout_contents.write('{}\n'.format(tmp2))
                    current_paper = []
    logger.info('DONE. infile: {}'.format(infile))

def prepare_wiki_article_for_textrank_MP(in_dir, out_dir_titles, out_dir_contents):
    os.makedirs(out_dir_titles, exist_ok=True)
    os.makedirs(out_dir_contents, exist_ok=True)
    params = []
    for i in os.listdir(in_dir):
        in_dir_i = os.path.join(in_dir, i)
        out_dir_i_titles = os.path.join(out_dir_titles, i)
        out_dir_i_contents = os.path.join(out_dir_contents, i)
        os.makedirs(out_dir_i_titles, exist_ok=True)
        os.makedirs(out_dir_i_contents, exist_ok=True)
        for j in os.listdir(in_dir_i):
            infile = os.path.join(in_dir_i, j)
            outfile_titles = os.path.join(out_dir_i_titles, j)
            outfile_contents = os.path.join(out_dir_i_contents, j)
            params.append((infile, outfile_titles, outfile_contents))
    p = mp.Pool()
    p.starmap(prepare_wiki_article_for_textrank, params)

# lda data in task1
def prepare_wiki_article_for_lda(infile, outfile_titles, outfile_contents):
    logger.info('infile: {}'.format(infile))
    with open(infile) as fin, open(outfile_titles, 'w') as fout_titles, open(outfile_contents, 'w') as fout_contents:
        current_paper = dd(int)
        first_line = True
        valid = None
        for line in fin:
            tmp = line.split()
            if tmp:
                if first_line:
                    if len(tmp) == 1:
                        valid = True
                        fout_titles.write(line)
                    else:
                        valid = False
                    first_line = False
                if valid:
                    for i in tmp:
                        current_paper[i] += 1
                else:
                    continue
            else:
                first_line = True
                if current_paper:
                    tmp = sorted(current_paper.items(), key=lambda x: x[1], reverse=True)
                    tmp2 = []
                    for k, v in tmp:
                        tmp2.append(k)
                        tmp2.append(str(v))
                    tmp2 = ' '.join(tmp2)
                    fout_contents.write('{}\n'.format(tmp2))
                    current_paper = dd(int)
    logger.info('DONE. infile: {}'.format(infile))

def prepare_wiki_article_for_lda_MP(in_dir, out_dir_titles, out_dir_contents):
    os.makedirs(out_dir_titles, exist_ok=True)
    os.makedirs(out_dir_contents, exist_ok=True)
    params = []
    for i in os.listdir(in_dir):
        in_dir_i = os.path.join(in_dir, i)
        out_dir_i_titles = os.path.join(out_dir_titles, i)
        out_dir_i_contents = os.path.join(out_dir_contents, i)
        os.makedirs(out_dir_i_titles, exist_ok=True)
        os.makedirs(out_dir_i_contents, exist_ok=True)
        for j in os.listdir(in_dir_i):
            infile = os.path.join(in_dir_i, j)
            outfile_titles = os.path.join(out_dir_i_titles, j)
            outfile_contents = os.path.join(out_dir_i_contents, j)
            params.append((infile, outfile_titles, outfile_contents))
    p = mp.Pool()
    p.starmap(prepare_wiki_article_for_lda, params)

def merge_wiki_article_for_lda(in_dir_titles, in_dir_contents, outfile_titles, outfile_contents):
    infiles = []
    for i in os.listdir(in_dir_titles):
        dir_i_titles = os.path.join(in_dir_titles, i)
        for j in os.listdir(dir_i_titles):
            infile_titles = os.path.join(dir_i_titles, j)
            infile_contents = os.path.join(in_dir_contents, i, j)
            infiles.append((infile_titles, infile_contents))
    titles = dd(list)
    for infile_titles, infile_contents in infiles:
        with open(infile_titles) as ft, open(infile_contents) as fc:
            ft_lines = ft.readlines()
            fc_lines = fc.readlines()
        for i in range(len(ft_lines)):
            ft_line = ft_lines[i].strip()
            fc_line = fc_lines[i].strip()
            titles[ft_line].append(fc_line)
    with open(outfile_titles, 'w') as ot, open(outfile_contents, 'w') as oc:
        for k, v in titles.items():
            v = ' '.join(v)
            ot.write('{}\n'.format(k))
            oc.write('{}\n'.format(v))

def extract_all_articles_of_given_titles(title_file, all_titles_file, all_contents_file, outfile):
    titles = []
    with open(title_file) as fin:
        for line in fin:
            titles.append(line.strip())
    all_articles = {}
    with open(all_titles_file) as f1, open(all_contents_file) as f2:
        lines1 = f1.readlines()
        lines2 = f2.readlines()
    for i in range(len(lines1)):
        all_articles[lines1[i].strip()] = lines2[i]
    with open(outfile, 'w') as fout:
        for t in titles:
            if t in all_articles:
                fout.write(all_articles[t])

def concatenate_files(filenames, outfile):
    with open(outfile, 'w') as fout:
        for fname in filenames:
            with open(fname) as fin:
                for line in fin:
                    fout.write(line)

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # timestamp = '20170901' # the timestamp of wikidump data that you downloaded
    parser.add_argument('timestamp', help='the timestamp of wikipedia dumps that you have downloaded')
    parser.add_argument('wikidata_dir', help='the directory of wikipedia dumps that you have downloaded')
    args = parser.parse_args()

    timestamp = args.timestamp
    document_dir = os.path.join(args.wikidata_dir, 'preprocessed')
    document_cleaned_dir = os.path.join(args.wikidata_dir, 'preprocessed2')
    category_file = os.path.join(args.wikidata_dir, 'enwiki-{}-category.sql'.format(timestamp))
    category_titles_file = os.path.join(args.wikidata_dir, 'enwiki-{}-category-titles'.format(timestamp))
    page_titles_file = os.path.join(args.wikidata_dir, 'enwiki-{}-all-titles-in-ns0'.format(timestamp))
    raw_topics = os.path.join(args.wikidata_dir, 'enwiki-{}-pages-categories-titles'.format(timestamp))
    alphanumeric_topics = os.path.join(args.wikidata_dir, 'enwiki-{}-pages-categories-titles-alphanumeric'.format(timestamp))
    new_sentences_dir = os.path.join(args.wikidata_dir, 'new_sentences')
    model_path = os.path.join(args.wikidata_dir, 'topic_embeddings.model')

    # step 1
    if 1: # process raw text wiki files
        multiprocess_dir(document_dir, document_cleaned_dir, clean_raw_text_file)

    # step 2
    if 1: # extract all category titles
        logger.info('category_file...')
        extract_wiki_categories(category_file, category_titles_file)

    # step 3
    if 1: # concatenate two files of page titles and category titles
        concatenate_files([category_titles_file, page_titles_file], raw_topics)

    # step 4
    if 1: # process wiki titles
        chunksize = 1000
        logger.info('lower_and_remove_non_alphanumeric_wiki_title...')
        multiprocess_file(raw_topics, alphanumeric_topics, lower_and_remove_non_alphanumeric_wiki_title, chunksize, skip_head_line=False)
        logger.info('DONE.')

    # step 5
    if 1: # generate sentences of given phrases
        generate_sentences_of_given_phrases_at_dir_level(wiki_document_cleaned_dir, new_sentences_dir, alphanumeric_topics)

    # step 6
    if 1: # train_word2vec_on_wiki_text
        train_word2vec_on_text(new_sentences_dir, model_path)

# def main():
#     parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     # wiki_titles_raw = '/run/user/1024/enwiki-20160601-all-titles-in-ns0'
#     wiki_titles_raw = '/home/thuzhf/data/wikipedia/20161201/enwiki-20161201-titles-categories'
#     wiki_titles_alphanumeric = '/home/thuzhf/data/wikipedia/20161201/enwiki-20161201-titles-categories_alphanumeric'
#     wiki_titles_nonShort = '/home/thuzhf/data/wikipedia/20161201/enwiki-20161201-titles-categories_alphanumeric_nonShort'
#     wiki_titles_filteredByPosRules = '/home/thuzhf/data/wikipedia/20161201/enwiki-20161201-titles-categories_alphanumeric_nonShort_filteredByPosRules'
#     wiki_titles_filteredByPosAndTagRules = '/home/thuzhf/data/wikipedia/20161201/enwiki-20161201-titles-categories_alphanumeric_nonShort_filteredByPosAndTagRules'
#     if USE_FILTERED_WIKI_TITLES:
#         given_phrases_file = wiki_titles_filteredByPosAndTagRules
#     else:
#         given_phrases_file = wiki_titles_filteredByPosRules

#     wiki_document_dir = '/home/thuzhf/data/wikipedia/wikidump_20160601_document'
#     wiki_document_cleaned_dir = '/home/thuzhf/data/wikipedia/wikidump_20160601_document_cleaned'
#     wiki_document_cleaned_dir_new = '/home/thuzhf/data/wikipedia/wikidump_20160601_document_cleaned_new'
#     wiki_sentences_dir = '/home/thuzhf/data/wikipedia/wikidump_20161201_document_cleaned_{}to{}_words{}'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED)
#     wiki_sentences_dir_new = '/home/thuzhf/data/wikipedia/wikidump_20161201_document_cleaned_new_{}to{}_words{}'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED)
#     wiki_noun_phrases_sentences_dir = '/home/thuzhf/data/wikipedia/wikidump_20160601_document_noun_phrases_sentences'

#     wiki_text_model_path = '/home/thuzhf/work/KEG/FastKATE/model/wiki_titles_on_wikipedia_text/wiki_text_20161201_{}to{}{}_{}d.model'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED, SIZE)
#     wiki_text_noun_phrases_model_path = '/home/thuzhf/work/KEG/FastKATE/model/noun_phrases_on_wiki_text/wiki_text_noun_phrases.model'

#     category_file = '/home/thuzhf/data/wikipedia/20161201/enwiki-20161201-category.sql'
#     category_titles_file = '/home/thuzhf/data/wikipedia/20161201/enwiki-20161201-category_titles'

#     wiki_sentences_dir_new_titles = '/home/thuzhf/data/wikipedia/wikidump_20161201_document_cleaned_new_titles_{}to{}_words{}'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED)
#     wiki_sentences_dir_new_contents = '/home/thuzhf/data/wikipedia/wikidump_20161201_document_cleaned_new_contents_{}to{}_words{}'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED)
#     wiki_sentences_dir_new_titles_for_textrank = '/home/thuzhf/data/wikipedia/wikidump_20161201_document_cleaned_new_titles_for_textrank_{}to{}_words{}'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED)
#     wiki_sentences_dir_new_contents_for_textrank = '/home/thuzhf/data/wikipedia/wikidump_20161201_document_cleaned_new_contents_for_textrank_{}to{}_words{}'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED)

#     # Wikipedia text
#     if 1: # process raw text wiki files
#         multiprocess_dir(wiki_document_dir, wiki_document_cleaned_dir, clean_raw_text_file)

#     if 1: # extract all category titles
#         logger.info('category_file...')
#         extract_wiki_categories(category_file, category_titles_file)

#     if 1: # process wiki titles
#         chunksize = 1000
#         logger.info('lower_and_remove_non_alphanumeric_wiki_title...')
#         multiprocess_file(wiki_titles_raw, wiki_titles_alphanumeric, lower_and_remove_non_alphanumeric_wiki_title, chunksize, skip_head_line=True)
#         # logger.info('lemmatize_and_remove_wiki_title_consisting_of_short_words...')
#         # multiprocess_file(wiki_titles_alphanumeric, wiki_titles_nonShort, lemmatize_and_remove_wiki_title_consisting_of_short_words, chunksize)
#         # logger.info('remove_wiki_title_by_*pos*_rules...')
#         # func = functools.partial(remove_wiki_title_by_pos_and_tag_rules, forbidden_tags=None)
#         # multiprocess_file(wiki_titles_nonShort, wiki_titles_filteredByPosRules, func, chunksize)
#         # logger.info('remove_wiki_title_by_*pos_and_tag*_rules...')
#         # func = functools.partial(remove_wiki_title_by_pos_and_tag_rules, forbidden_pos=None)
#         # multiprocess_file(wiki_titles_filteredByPosRules, wiki_titles_filteredByPosAndTagRules, remove_wiki_title_by_pos_and_tag_rules, chunksize)
#         logger.info('DONE.')
#     if 1: # generate sentences of given phrases
#         generate_sentences_of_given_phrases_at_dir_level(wiki_document_cleaned_dir, wiki_sentences_dir, wiki_titles_alphanumeric)
#     if 1: # train_word2vec_on_wiki_text
#         train_word2vec_on_text(wiki_sentences_dir, wiki_text_model_path)
#     if 0: # generate sentences of noun phrases
#         generate_sentences_of_noun_phrases_at_dir_level(wiki_document_dir, wiki_noun_phrases_sentences_dir)
#     if 0: # train_word2vec_on_wiki_text using noun phrases
#         train_word2vec_on_text(wiki_noun_phrases_sentences_dir, wiki_text_noun_phrases_model_path)

#     if 0: # transform AI papers' titles and abstracts into topics sentences
#         AI_titles_and_abstracts_dir = '/home/thuzhf/work/paper_crawler/data/AI'
#         AI_titles_and_abstracts_dir_cleaned = '/home/thuzhf/work/paper_crawler/data/AI_cleaned'
#         AI_titles_and_abstracts_sentences_dir = '/home/thuzhf/work/paper_crawler/data/AI_processed_sentences'
#         multiprocess_dir(AI_titles_and_abstracts_dir, AI_titles_and_abstracts_dir_cleaned, clean_raw_text_file)
#         generate_sentences_of_given_phrases_at_dir_level(AI_titles_and_abstracts_dir_cleaned, AI_titles_and_abstracts_sentences_dir, wiki_titles_alphanumeric)
#     if 0:
#         infile = '/home/thuzhf/work/paper_crawler/data/selected_papers.txt'
#         outfile = '/home/thuzhf/work/paper_crawler/data/selected_papers_cleaned.txt'
#         clean_raw_text_file(infile, outfile)
#     if 0:
#         in_dir = '/home/thuzhf/work/paper_crawler/data/selected_papers_cleaned'
#         out_dir = '/home/thuzhf/work/paper_crawler/data/selected_papers_topics_sentences'
#         generate_sentences_of_given_phrases_at_dir_level(in_dir, out_dir, wiki_titles_alphanumeric)

#     # # KDD text
#     # if 0:
#     #     kdd_pdf_text_cleaned_dir = '/home/thuzhf/data/papers/kdd/txt_cleaned'
#     #     kdd_pdf_text_cleaned_lemmatized_dir = '/home/thuzhf/data/papers/kdd/txt_cleaned_lemmatized'
#     #     kdd_papers_sentences_dir = '/home/thuzhf/data/papers/kdd/txt_cleaned_lemmatized_{}to{}_words{}'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED)

#     #     kdd_papers_text_model_path = '/home/thuzhf/work/KEG/FastKATE/model/wiki_titles_on_kdd_text/kdd_papers_text_{}to{}{}_{}d.model'.format(MIN_PHRASE_LENGTHS, MAX_PHRASE_LENGTHS, _FILTERED, SIZE)

#     #     if 0: # process raw text paper files (converted from PDFs)
#     #         multiprocess_dir(kdd_pdf_text_cleaned_dir, kdd_pdf_text_cleaned_lemmatized_dir, clean_raw_text_file)
#     #     if 0: # generate sentences of given phrases
#     #         generate_sentences_of_given_phrases_at_dir_level(kdd_pdf_text_cleaned_lemmatized_dir, kdd_papers_sentences_dir, given_phrases_file, multiprocess=False)
#     #     if 0: # train_word2vec_on_kdd_text
#     #         train_word2vec_on_text(kdd_papers_sentences_dir, kdd_papers_text_model_path)

#     if 0:
#         multiprocess_dir(wiki_document_dir, wiki_document_cleaned_dir_new, clean_raw_text_file_and_add_newline_after_each_document)
#     if 0:
#         generate_sentences_of_given_phrases_at_dir_level(wiki_document_cleaned_dir_new, wiki_sentences_dir_new, wiki_titles_alphanumeric)
#     if 0:
#         prepare_wiki_article_for_lda_MP(wiki_sentences_dir_new, wiki_sentences_dir_new_titles, wiki_sentences_dir_new_contents)
#     if 0:
#         prepare_wiki_article_for_textrank_MP(wiki_sentences_dir_new, wiki_sentences_dir_new_titles_for_textrank, wiki_sentences_dir_new_contents_for_textrank)

#     merged_wiki_titles = '/home/thuzhf/data/wikipedia/merged_wiki_titles.txt'
#     merged_wiki_contents = '/home/thuzhf/data/wikipedia/merged_wiki_contents.txt'
#     merged_wiki_titles_for_textrank = '/home/thuzhf/data/wikipedia/merged_wiki_titles_for_textrank.txt'
#     merged_wiki_contents_for_textrank = '/home/thuzhf/data/wikipedia/merged_wiki_contents_for_textrank.txt'
#     if 0:
#         merge_wiki_article_for_lda(wiki_sentences_dir_new_titles, wiki_sentences_dir_new_contents, merged_wiki_titles, merged_wiki_contents)
#     if 0:
#         merge_wiki_article_for_lda(wiki_sentences_dir_new_titles_for_textrank, wiki_sentences_dir_new_contents_for_textrank, merged_wiki_titles_for_textrank, merged_wiki_contents_for_textrank)
#     if 0:
#         parser.add_argument('--title_file')
#         parser.add_argument('--outfile')
#         args = parser.parse_args()
#         extract_all_articles_of_given_titles(args.title_file, merged_wiki_titles_for_textrank, merged_wiki_contents_for_textrank, args.outfile)

if __name__ == '__main__':
    start_t = time.time()
    main()
    end_t = time.time()
    t = end_t - start_t
    print('Time elapsed: {:.4f} minutes'.format(t / 60.))


from enum import Enum
from typing import Optional

from pygame import (
	Color,
	Rect,
	SRCALPHA,
	Surface,
	draw,
	font as fonts,
)

class Center(str, Enum):
	HORIZONTAL = 'top'
	VERTICAL = 'left'

def center(source: Surface, dest: Surface, axes: Optional[Center] = None) -> Rect:
	rect = source.get_rect()
	rect.center = dest.get_rect().center
	if axes is not None:
		exec(f'rect.{axes} = source.get_rect().{axes}')
	return rect

def blit_center(source: Surface, dest: Surface, axes: Optional[Center] = None):
	dest.blit(source, center(source, dest, axes))

def new_surface(size: Optional[int] = 40) -> Surface:
	return Surface((size, size), SRCALPHA)

def render_text(font: str, text: str, size: int, color: Color) -> Surface:
	return fonts.SysFont(font, size).render(text, True, color)

class ModeButton:
	corner_formulae = [
		'(m, m)',
		'(r.w - tr[1].w - m, m)',
		'(m, r.h - tr[2].h - m)',
	]

	@staticmethod
	def __common(
			color: Color,
			texting: Optional[tuple[str, str, int]] = None
	) -> Surface:
		surf = new_surface()
		if texting is not None:
			blit_center(render_text(*texting, color), surf)
		draw.rect(surf, color, (2, 2, 36, 36), 3)
		return surf

	@staticmethod
	def center(font: str, color: Color) -> Surface:
		return ModeButton.__common(color, (font, '12', 25))

	@staticmethod
	def color(font: str, color: Color) -> Surface:
		return ModeButton.__common(color, (font, 'color', 15))

	@staticmethod
	def corner(font: str, color: Color) -> Surface:
		surf = ModeButton.__common(color)
		text = [render_text(font, str(i + 1), 21, color) for i in range(3)]
		tr = [t.get_rect() for t in text]
		m = 6  # m for margin
		r = surf.get_rect()
		coord = [eval(f, loc) for loc in (locals(),) for f in ModeButton.corner_formulae]
		surf.blits(zip(text, coord), False)
		return surf

	@staticmethod
	def digit(font: str, color: Color) -> Surface:
		return ModeButton.__common(color, (font, '9', 55))


import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import scipy.stats as stats

# 可视化的中文处理
plt.rcParams['font.sans-serif'] = 'SimHei'
plt.rcParams['axes.unicode_minus'] = False
plt.style.use('ggplot')
cd = pd.read_csv(r'C:\Users\Lenovo\Desktop\chengduxinfang.csv',encoding='gbk')
cd = cd[cd['states'].isin(['住宅在售','住宅售罄','住宅未开盘','住宅下期待开']) ]
cd.price = cd.price.str.extract('(.*?)元/平')
cd.dropna(subset=['price'],inplace=True)
cd['price'] = cd['price'].astype(int)
# print(cd['price'].unique())
# print(cd.nunique())# 计算DataFrame每一列不重复元素的个数
# print(cd.isnull().sum())# 计算DataFrame每一列缺失值的个数
# print(cd.isnull().sum()/cd.shape[0])
# 计算DataFrame每一列不重复元素的个数
# def func(x):
#     y=len(x.unique())
#     return y
# address_unique=cd.apply(func)
# app_info.comments.describe(percentiles=np.arange(0, 1.2, 0.2))
# print(cd.info())
# print(cd.describe())
# print(cd.describe(include=['object']))
# 新增年龄和工龄两列
# df['age'] = pd.datetime.today().year - df.birthday.dt.year
# df['workage'] = pd.datetime.today().year - df.start_work.dt.year

# loandata['term']=loandata['term'].map(str.strip)
# time.strftime('%Y-%m-%d %H:%M:%S,time.localtime())

# sales['销售时间'], sales['销售星期'] = sales['销售时间'].str.split(' ', 1).str
# sales['销售时间'] = pd.to_datetime(sales['销售时间'],format = '%Y-%m-%d',errors = 'coerce')
# sales = sales[(sales['销售数量'] > 0) & (sales['应收金额'] > 0)]
# month = (sales['销售时间'].max() - sales['销售时间'].min()).days//30

##对去重后的数据按照天进行重新采样
#首先要把索引变成时间
# sales.index = pd.DatetimeIndex(sales['销售时间'])
#然后对其按照每天从新采样
# salesd = sales.resample('D').count()

#画图
# salesd.plot(x = salesd.index, y = '实收金额')
# plt.xlabel('Time')
# plt.ylabel('Money')
# plt.title('xiao shou shu ju')
# plt.show()
#按月采样
# salesm = sales.resample('M').count()

# 数据分列
# grade_split = pd.DataFrame((x.split('-') for x in loandata.grade), index=loandata.index,
#                            columns=['grade', 'sub_grade'])  # 将一列数据拆分开显示为两列
# loandata = pd.merge(loandata, grade_split, how='inner', on="categories", right_index=True,
#                     left_index=True)  # 合并两个pd对象，根据列categories合并
# loandata = pd.merge(loandata, grade_split, right_index=True, left_index=True)  # 直接合并pd对象，两个pd对象顺序一样

sns.set_style("whitegrid")
fig,axes=plt.subplots(1,3) #创建一个一行三列的画布
sns.distplot(cd.price,fit = stats.norm,norm_hist = True,ax=axes[0]) #左图
sns.distplot(cd.price,hist=False,ax=axes[1]) #中图
sns.distplot(cd.price,kde=False,ax=axes[2]) #右图
plt.show()

sns.distplot(cd.price,color='steelblue')
plt.show()

# fig,axes=plt.subplots(1,2)
# sns.distplot(cd.price,norm_hist=True,kde=False,ax=axes[0]) #左图 密度
# sns.distplot(cd.price,kde=False,ax=axes[1]) #右图 计数
# plt.show()

baobao = pd.read_csv(r'C:\Users\Lenovo\Desktop\tianmaotianmaobaobao.csv',encoding='gbk')
baobao=baobao.loc[baobao.sale!='None',:]
baobao.index=range(0,baobao.shape[0])
baobao['new_sale']=baobao.sale.str[:-1].astype('int')
y=lambda x:float(x[:-1]) * 10000 if x.find('万') else float(x)
baobao['new_estimate']=baobao.estimate.apply(y)
baobao=baobao.drop(['sale','estimate'],axis=1)

# data = baobao.corr()
# sns.heatmap(data)
# plt.show()

# print(baobao['new_estimate'].sort_values())

# sns.lmplot(x='price', y='new_estimate',data=baobao,fit_reg=True, scatter=True ) #散点图
# plt.show()
# sns.lmplot(x='price', y='new_estimate',hue='shop',data=baobao,fit_reg=False, scatter=True)
# plt.show()

# train = train[-((train.SalePrice < 200000) &  (train.GrLivArea > 4000))]

# sns.jointplot(x='Area',y='Tprice',data=sh) #散点图
# sns.jointplot(x='Area',y='Tprice',data=sh，kind='hex')
# plt.show()

# lm = sns.lmplot(x = 'Age', y = 'Fare', data = titanic, hue = 'Sex', fit_reg=True)
# lm.set(title = 'Fare x Age')
# axes = lm.axes
# axes[0,0].set_ylim(-5,)
# axes[0,0].set_xlim(-5,85)

# from os import path

# d=path.dirname(__file__)

# df.drop(df[df['职位名称'].str.contains('实习')].index, inplace=True)
# train.drop(train[(train['GrLivArea']>4000) & (train['SalePrice']<300000)].index)

# pattern = '\d+'
# df['work_year'] = df['工作经验'].str.findall(pattern)
# # 数据处理后的工作年限
# avg_work_year = []
# # 工作年限
# for i in df['work_year']:
#    # 如果工作经验为'不限'或'应届毕业生',那么匹配值为空,工作年限为0
#    if len(i) == 0:
#        avg_work_year.append(0)
#    # 如果匹配值为一个数值,那么返回该数值
#    elif len(i) == 1:
#        avg_work_year.append(int(''.join(i)))
#    # 如果匹配值为一个区间,那么取平均值
#    else:
#        num_list = [int(j) for j in i]
#        avg_year = sum(num_list)/2
#        avg_work_year.append(avg_year)
# df['工作经验'] = avg_work_year

# prices_avg = {'0到50/月': 0, '50到100/月': 0, '100到150/月': 0, '150到200/月': 0, '大于200/月': 0}
# 	for d in data:
# 		price = int(float(d.get('price')) / float(d.get('area')))
# 		if price in range(0, 50):
# 			prices_avg['0到50/月'] += 1
# 		elif price in range(50, 100):
# 			prices_avg['50到100/月'] += 1
# 		elif price in range(100, 150):
# 			prices_avg['100到150/月'] += 1
# 		elif price in range(150, 200):
# 			prices_avg['150到200/月'] += 1
# 		else:
# 			prices_avg['大于200/月'] += 1
# 	DrawPie(title='上海租房(单位面积)月租金分布饼图', data=prices_avg, savepath='./results')

# gender_map = {0: 'unknown', 1: 'male', 2: 'female'}
# data['gender'] = data['gender'].apply(lambda x: gender_map[x]) #映射
# data.sample(5) #随机选5行

# newdataset = dataset[-dataset['城区'].isin(['燕郊'])]

# data['year']=data['date'].apply(lambda x:x.year)
# data['month']=data['date'].apply(lambda x:x.month)
# data['day']=data['date'].apply(lambda x:x.day)

# plt.subplot2grid((2,3),(1,0), colspan=2)  # colspan = 2 表示横向跨度是 2
# plt.figure(1, figsize=(40, 60))
plt.subplot(2,2,1)
cd['area'].value_counts().plot(kind='bar')
plt.subplot(2,2,2)
cd['area'].value_counts().plot(kind='hist')
plt.subplot(2,2,3)
cd['area'].value_counts().plot(kind='pie')
plt.subplot(2,2,4)
cd['area'].value_counts().plot(kind='kde')
plt.show()
# plt.subplot(3,2,5)
cd['area'].value_counts().plot(kind='barh')
plt.show()
cd['area'].value_counts().plot(kind='box')
plt.show()
cd['area'].value_counts().plot(kind='area')
plt.show()
cd['area'].value_counts().plot(color='r',marker='D')
plt.show()
baobao.plot(kind='scatter', x='price', y='new_estimate', color='g')
plt.show()
'''Series.plot（kind ='line'，ax = None，figsize = None，use_index = True，title = None，grid = None，legend = False，
  style = None，logx = False，logy = False，loglog = False，xticks = None，yticks = None，xlim = None，ylim = None，rot = None，
  fontsize = None，colormap = None，table = False，yerr = None，xerr = None，label = None，secondary_y = False，** kwds ）'''

import scipy.stats as stats
# 绘制直方图
sns.distplot(a = cd['area'].value_counts(), bins = 10, fit = stats.norm, norm_hist = True,
             hist_kws = {'color':'steelblue', 'edgecolor':'black'},
             kde_kws = {'color':'black', 'linestyle':'--', 'label':'核密度曲线'},
             fit_kws = {'color':'red', 'linestyle':':', 'label':'正态密度曲线'})
# 显示图例
plt.legend()
# 显示图形
plt.show()

# 导入绘图模块
import matplotlib.pyplot as plt
# 设置绘图风格
plt.style.use('ggplot')
# 绘制直方图
sunspots.counts.plot(kind = 'hist', bins = 30, normed = True)
# 绘制核密度图
sunspots.counts.plot(kind = 'kde')
# 图形展现
plt.show()


from tkinter import *
import tkinter.messagebox
import tkinter.filedialog
import os
from word_counter.counter import WordCounter
from filetype_convert.doc2txt import FileTypeConvert

# defining options for opening a directory
class Application(Frame):
    def __init__(self, master=None):
        self.fi_name = ""
        self.out_dir = ""
        self.dir_opt = options = {}
        options['initialdir'] = 'C:\\'
        options['mustexist'] = False
        options['parent'] = root
        options['title'] = 'This is a title'

        Frame.__init__(self, master)
        self.pack()
        self.createWidgets()

    def select_file(self, lb):
        self.fi_name = tkinter.filedialog.askopenfilename(filetypes=[("DOCX",".docx"), ("DOC",".doc"), ("TXT", ".txt")])
        if self.fi_name != '':
            lb.config(text='File chosen: ' + self.fi_name)
        else:
            lb.config(text='You did not choose any file!')

    def save_file(self, lb):
        self.out_dir= tkinter.filedialog.askdirectory(**self.dir_opt)
        if self.out_dir != '':
            lb.config(text='Save to: ' + self.out_dir)
        else:
            lb.config(text='You did not specify any directory!')

    def run(self, fi_name, wds_given, out_dir):

        txt_file = FileTypeConvert(fi_name)
        txt_file.convertDocxToText()

        wc = WordCounter(txt_file.textFilename)
        wc.counter_words_v2(wds_given, out_dir)

        fileExtension = fi_name.split(".")[-1]
        if fileExtension != "txt":
            os.remove(txt_file.textFilename)

        tkinter.messagebox.showinfo('Hi Miss,','Done! \nResult has been saved to '+self.out_dir)

    def createWidgets(self):
        fi_lb = Label(self, text='File Input: ')
        fi_lb.grid(column=0, row=0)
        fi_show_lb = Label(self, text='')
        fi_show_lb.grid(column=1, row=0)
        fi_select_btn = Button(self, text='Select File', command=lambda:self.select_file(fi_lb))
        fi_select_btn.grid(column=2, row=0)

        fo_lb = Label(self, text='Result Save: ')
        fo_lb.grid(column=0, row=1)
        fo_show_lb = Label(self, text='')
        fo_show_lb.grid(column=1, row=1)
        fo_select_btn = Button(self, text='Select File', command=lambda:self.save_file(fo_lb))
        fo_select_btn.grid(column=2, row=1)

        # put given words
        wds_given_entry = Entry(self)
        wds_given_entry.grid(column=0, row=2)

        run_btn = Button(self, text='Run', command=lambda:self.run(self.fi_name, wds_given_entry.get(), self.out_dir))
        run_btn.grid(column=0, row=3)

if __name__ == "__main__":

    root = Tk()
    root.title('TANGTANG(R) Word Counter')
    root.geometry('1240x640')

    app = Application(master=root)

    app.mainloop()


import sys
import random
t = None

def triangle():
    side_length = random.randint(5,50)
    for x in range(1,4):
        t.forward(side_length)
        t.left(360/3)
    t.up()
    t.forward(side_length+10)
    t.down()

def circle():
    radius = random.randint(5,18)
    t.circle(radius)
    t.up()
    t.forward(radius*2+5)
    t.down()

def square():
    side_length = random.randint(5,50)
    for x in range(1,5):
        t.forward(side_length)
        t.left(90)
    t.up()
    t.forward(side_length+10)
    t.down()

def draw_random():
    quantity = random.randint(1,16)
    ans = random.choice([circle,square,triangle])
    if ans == circle():
        radius = random.randint(5,18)
        for x in range (1,quantity):
            circle()
    if ans == square():
        side_length = random.randint(5,50)
        for x in range (1,quantity):
            square()
    if ans == triangle():
        side_length = random.randint(5,50)
        for x in range (1,quantity):
            triangle()


def solution(dartResult):
    # dartResult를 round 별로 나눔
    rounds = []
    round = ''
    for index, ele in enumerate(dartResult):
        if ele.isdigit() and round != '':
            if dartResult[index] == '0' and dartResult[index - 1] == '1':
                round += ele
            else:
                rounds.append(round)
                round = ele
        else:
            round += ele

            if index == len(dartResult) - 1:
                rounds.append(round)

    # 라운드 별 점수 계산
    scores = []
    for index, round in enumerate(rounds):
        score = 0
        option = ''

        if 'S' in round:
            score += int(round.split('S')[0])
        elif 'D' in round:
            score += int(round.split('D')[0]) ** 2
        elif 'T' in round:
            score += int(round.split('T')[0]) ** 3

        if score == 10:
            option = round[3:]
        else:
            option = round[2:]

        if option == '*':
            score *= 2
            if index > 0:
                scores[index - 1] *= 2
        elif option == '#':
            score *= -1

        scores.append(score)

    return sum(scores)


import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt

# Plots two images side by side using matplotlib
def visualize(real_A, real_B, fake_B, fname):
    # 1x3x256x256 to 1x256x256x3
    real_A = real_A.transpose((0, 2, 3, 1))
    real_B = real_B.transpose((0, 2, 3, 1))
    fake_B = fake_B.transpose((0, 2, 3, 1))
    # Pixel values from 0-255
    real_A = np.clip((real_A + 1.0) * (255.0 / 2.0), 0, 255).astype(np.uint8)
    real_B = np.clip((real_B + 1.0) * (255.0 / 2.0), 0, 255).astype(np.uint8)
    fake_B = np.clip((fake_B + 1.0) * (255.0 / 2.0), 0, 255).astype(np.uint8)

    # Create a matplotlib figure with two subplots: one for the real and the other for the fake
    # fill each plot with our buffer array, which creates the image
    fig = plt.figure()
    ax1 = fig.add_subplot(3, 1, 1)
    ax1.set_title('Input')
    ax1.axis('off')
    ax1.imshow(real_A[0])
    ax2 = fig.add_subplot(3, 1, 2)
    ax2.imshow(real_B[0])
    ax2.set_title('Ground Truth')
    ax2.axis('off')
    ax3 = fig.add_subplot(3, 1, 3)
    ax3.imshow(fake_B[0])
    ax3.set_title('pix2pix')
    ax3.axis('off')
    # plt.show()
    plt.savefig(fname)
    plt.close()


# python3.5 buildtagger.py <train_file_absolute_path> <model_file_absolute_path>

import os
import math
import sys

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

import random
import datetime
from collections import defaultdict

## HYPERPARAMETERS ##
DEBUG = False

WORD_VEC_DIM = 128
CHAR_VEC_DIM = 32
CNN_WINDOW_K = 3
CNN_FILTERS_L = 32
LSTM_FEATURES = 64
LSTM_LAYERS = 1
LSTM_DROPOUT = 0

TIME_LIMIT_MIN = 9
TIME_LIMIT_SEC = 0
EPOCH = 2
LEARNING_RATE = 0.001

init_time = datetime.datetime.now()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if DEBUG: print(device)
torch.set_printoptions(threshold=5000)
torch.manual_seed(3940242394)

TAG_TO_IX = {
  '``' : 0,
  '#' : 1,
  '$' : 2,
  "''" : 3,
  ',' : 4,
  '-LRB-' : 5,
  '-RRB-' : 6,
  '.' : 7,
  ':' : 8,
  'CC' : 9,
  'CD' : 10,
  'DT' : 11,
  'EX' : 12,
  'FW' : 13,
  'IN' : 14,
  'JJ' : 15,
  'JJR' : 16,
  'JJS' : 17,
  'LS' : 18,
  'MD' : 19,
  'NN' : 20,
  'NNP' : 21,
  'NNPS' : 22,
  'NNS' : 23,
  'PDT' : 24,
  'POS' : 25,
  'PRP' : 26,
  'PRP$' : 27,
  'RB' : 28,
  'RBR' : 29,
  'RBS' : 30,
  'RP' : 31,
  'SYM' : 32,
  'TO' : 33,
  'UH' : 34,
  'VB' : 35,
  'VBD' : 36,
  'VBG' : 37,
  'VBN' : 38,
  'VBP' : 39,
  'VBZ' : 40,
  'WDT' : 41,
  'WP' : 42,
  'WP$' : 43,
  'WRB' : 44,
}

IX_TO_TAG = {ix:tag for tag, ix in TAG_TO_IX.items()}

## POTENTIAL STRATS ##
# CNN:
#   padding,
#   layers,
#   stride length,
#   combine instead of max pool,
#   activation function
# DROPOUT
# RANDOM INITIALIZATION OF EMBEDDINGS
# UNKNOWN WORDS
# ADAM
# INIT HIDDEN
# FLATTEN PARAMETERS

## MODEL
class POSModel(nn.Module):

  def __init__(self, word_vocab, char_vocab):
    super(POSModel, self).__init__()

    self.word_vocab = word_vocab
    self.char_vocab = char_vocab
    self.word_vocab_length = len(word_vocab)
    self.char_vocab_length = len(char_vocab)
    self.word_embeddings = nn.Embedding(self.word_vocab_length + 1, WORD_VEC_DIM, padding_idx=len(self.word_vocab)).to(device)
    self.char_embeddings = nn.Embedding(self.char_vocab_length + 1, CHAR_VEC_DIM, padding_idx=len(self.char_vocab)).to(device)

    self.conv1d = nn.Conv1d(in_channels=CHAR_VEC_DIM, out_channels=CNN_FILTERS_L, kernel_size=CNN_WINDOW_K, stride=1, padding=(CNN_WINDOW_K-1)//2, bias=True).to(device)
    self.pool = nn.AdaptiveMaxPool1d(1).to(device)

    self.lstm = nn.LSTM(
      input_size=WORD_VEC_DIM+CNN_FILTERS_L,
      hidden_size=LSTM_FEATURES,
      num_layers=LSTM_LAYERS,
      dropout=LSTM_DROPOUT,
      bidirectional=True
    ).to(device)

    self.hidden2tag = nn.Linear(LSTM_FEATURES * 2, len(TAG_TO_IX)).to(device)

  def forward(self, word_sentence):

    word_input = torch.LongTensor([self.word_vocab_length if word not in self.word_vocab else self.word_vocab[word] for word in word_sentence]).to(device)
    word_embeds = self.word_embeddings(word_input).unsqueeze(1).to(device)

    char_input = [torch.LongTensor([len(self.char_vocab) if char not in self.char_vocab else self.char_vocab[char] for char in word]).to(device) for word in word_sentence]

    char_input = nn.utils.rnn.pad_sequence(char_input, batch_first=True, padding_value=self.char_vocab_length)

    char_embeds = self.char_embeddings(char_input).transpose(1,2).to(device)

    char_rep = self.conv1d(char_embeds).to(device)

    char_rep = self.pool(char_rep).transpose(1,2).to(device)

    word_rep = torch.cat((word_embeds, char_rep), dim=2).to(device)

    self.lstm.flatten_parameters()
    output, _ = self.lstm(word_rep)
    out = self.hidden2tag(output.view(len(word_input), -1))
    return out

def train_model(train_file, model_file):
  # use torch library to save model parameters, hyperparameters, etc. to model_file
  reader = open(train_file)
  out_lines = reader.readlines()
  pairs_sentence_lines = []
  reader.close()

  word_vocab = {}
  char_vocab = {}

  for i in range(0, len(out_lines)):
    cur_out_line = out_lines[i].strip()
    cur_out_tokens = cur_out_line.split(' ')
    pairs_sentence = []

    for j in range(0, len(cur_out_tokens)):
      word, _, tag = cur_out_tokens[j].rpartition('/')
      if word not in word_vocab:
        word_vocab[word] = len(word_vocab)
      for char in word:
        if char not in char_vocab:
          char_vocab[char] = len(char_vocab)

      # convert tags to index at this point
      pairs_sentence.append((word, TAG_TO_IX[tag]))

    pairs_sentence_lines.append(pairs_sentence)

  losses = []
  loss_function = nn.CrossEntropyLoss()
  model = POSModel(word_vocab, char_vocab)

  optimizer = optim.Adam(params=model.parameters(), lr=LEARNING_RATE)

  for epoch in range(EPOCH):
    total_loss = 0
    for i, pair_sentence in enumerate(pairs_sentence_lines):

      word_sentence, tag_sentence = zip(*pair_sentence)
      model.zero_grad()

      output = model(word_sentence).to(device)
      loss = loss_function(output, torch.LongTensor(tag_sentence).to(device)).to(device)
      loss.backward()
      optimizer.step()

      if (i+1) % 100 == 0:

        time_diff = datetime.datetime.now() - init_time

        if time_diff > datetime.timedelta(minutes=TIME_LIMIT_MIN, seconds=TIME_LIMIT_SEC):
          torch.save((word_vocab, char_vocab, model.state_dict()), model_file)
          return

        if DEBUG:
          print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Time Elapsed: {}'
                .format(epoch+1, EPOCH, i+1, len(pairs_sentence_lines), loss.item(), time_diff))

      total_loss += loss.item()

    losses.append(total_loss)
    torch.save((word_vocab, char_vocab, model.state_dict()), model_file)

  print('Finished...')

if __name__ == "__main__":
  # make no changes here
  train_file = sys.argv[1]
  model_file = sys.argv[2]
  train_model(train_file, model_file)


#!/usr/bin/env python
#encoding=utf-8
import bottle
from bottle import route, default_app
from google.appengine.ext.webapp.util import run_wsgi_app
from bottle import route, run,redirect,request
t=""
html="""
<html>bottle.run(server='gae')
<title>life is tough,use python~</title>
<body>
<form name="form1" action="/" method="post">
<textarea name="t" cols="80" rows="25">%s</textarea>
<input type="submit" name="sub" value="save" />
</form>
</body>
</html>
"""
from google.appengine.ext import db
class Pastebin(db.Model):
  content   = db.StringProperty(multiline=True)
  date = db.DateTimeProperty(auto_now_add=True)

#@route('/',method=["POST","GET"])
@route('/',method=["POST"])
def index():
    global t,html
    t=request.POST.get("t")
    #print t
    if Pastebin.all().count()==0:
        p=Pastebin(content=t)
    else:
        p=Pastebin.all().order("-date").fetch(1)[0]
        p.content=t
    p.put()
    return html%t

@route('/')
def index():
    global t,html
    #print t
    t=db.GqlQuery("SELECT * FROM Pastebin order by date desc limit 1")
    if t.count()==0:
        p=Pastebin(content="")
        p.put()
        redirect("/")
    #print t
    #print type(t)
    #print t[0].content
    return html%t[0].content

#run(host='localhost', port=8084)
#bottle.run(server='gae')
bottle.debug(True)
app = default_app()
run_wsgi_app(app)


import math

#we define function that will check whether the number is prime or not
def prime(n):
    if n == 1:
        return False
    if n == 2:
        return True
    if n > 2 and n % 2 == 0:
        return False
    max_divisors = math.floor(math.sqrt(n))
    for j in range(3, 1+max_divisors, 2):
        if n % j == 0:
            return False
    return True

number=600851475143

#we go through for loop from 2 to number+1
for i in range(2,number+1):
    #if number can be devided with i, and that quotient is prime
    if number % i == 0 and prime(i):
        #then we print i
        #the last number that is printed is is the largest prime factor of the number
        print(i)


"""Utilities related to data clustering catalogs."""

import logging

import numpy as np
from scipy import interpolate

from cosmopipe.lib.utils import ScatteredBaseClass
from cosmopipe.lib import mpi

def vector_projection(vector, direction):
    r"""
    Vector components of given vectors in a given direction.

    .. math::
       \mathbf{v}_\mathbf{d} &= (\mathbf{v} \cdot \hat{\mathbf{d}}) \hat{\mathbf{d}} \\
       \hat{\mathbf{d}} &= \frac{\mathbf{d}}{\|\mathbf{d}\|}

    Adapted from https://github.com/bccp/nbodykit/blob/master/nbodykit/transform.py

    Parameters
    ----------
    vector : array
        Array of vectors to be projected (along last dimension).

    direction : array
        Projection direction, 1D or 2D (if different direction for each input ``vector``) array.
        It will be normalized.

    Returns
    -------
    projection : array
        Vector components of the given vectors in the given direction.
        Same shape as input ``vector``.
    """
    direction = np.asarray(direction, dtype='f8')
    direction = direction / (direction ** 2).sum(axis=-1)[:, None] ** 0.5
    projection = (vector * direction).sum(axis=-1)
    projection = projection[:, None] * direction

    return projection

class DistanceToRedshift(object):

    """Class that holds a conversion distance -> redshift."""

    def __init__(self, distance, zmax=100., nz=2048, interp_order=3):
        """
        Initialize :class:`DistanceToRedshift`.
        Creates an array of redshift -> distance in log(redshift) and instantiates
        a spline interpolator distance -> redshift.

        Parameters
        ----------
        distance : callable
            Callable that provides distance as a function of redshift (array).

        zmax : float, default=100.
            Maximum redshift for redshift <-> distance mapping.

        nz : int, default=2048
            Number of points for redshift <-> distance mapping.

        interp_order : int, default=3
            Interpolation order, e.g. ``1`` for linear interpolation, ``3`` for cubic splines.
        """
        self.distance = distance
        self.zmax = zmax
        self.nz = nz
        zgrid = np.logspace(-8,np.log10(self.zmax),self.nz)
        self.zgrid = np.concatenate([[0.], zgrid])
        self.rgrid = self.distance(self.zgrid)
        self.interp = interpolate.UnivariateSpline(self.rgrid,self.zgrid,k=interp_order,s=0)

    def __call__(self, distance):
        """Return (interpolated) redshift at distance ``distance`` (scalar or array)."""
        return self.interp(distance)

class RedshiftDensityInterpolator(ScatteredBaseClass):
    """
    Class that computes and interpolates a redshift density histogram :math:`n(z)` from an array of redshift and optionally weights.
    Adapted from: https://github.com/bccp/nbodykit/blob/master/nbodykit/algorithms/zhist.py
    """

    @mpi.MPIInit
    def __init__(self, redshifts, weights=None, bins=None, fsky=1., radial_distance=None, interp_order=1):
        r"""
        Initialize :class:`RedshiftDensityInterpolator`.

        Parameters
        ----------
        redshifts : array
            Array of redshifts.

        weights : array, default=None
            Array of weights, same shape as ``redshifts``. Defaults to 1.

        bins : int, array, string, default=None
            If `bins` is an integer, it defines the number of equal-width
            bins in the given range. If `bins` is a sequence, it defines the bin
            edges, including the rightmost edge, allowing for non-uniform bin widths.
            If 'scott', Scott's rule is used to estimate the optimal bin width
            from the input data. Defaults to 'scott'.

        fsky : float, default=1
            The sky area fraction, which is used in the volume calculation when normalizing :math:`n(z)`.
            ``1`` corresponds to full-sky: :math:`4 \pi` or :math:`\simeq 41253\; \mathrm{deg}^{2}`.

        radial_distance : callable, default=None
            Radial distance to use when converting redshifts into comoving distance.
            If ``None``, ``redshifts`` and optionally ``bins`` are assumed to be in distance units.

        interp_order : int, default=1
            Interpolation order, e.g. ``1`` for linear interpolation, ``3`` for cubic splines.
        """
        def zrange(redshifts):
            if self.is_mpi_scattered():
                return mpi.min_array(redshifts,mpicomm=self.mpicomm),mpi.max_array(redshifts,mpicomm=self.mpicomm)
            else:
                zrange = None
                if self.is_mpi_root():
                    zrange = np.min(redshifts),np.max(redshifts)
                return self.mpicomm.bcast(zrange,root=self.mpiroot)

        if bins is None or bins == 'scott':
            # scott's rule
            if self.is_mpi_scatter():
                var = mpi.var_array(redshifts,aweights=weights,ddof=1,mpicomm=self.mpicomm)
                gsize = mpi.size_array(redshifts)
            else:
                var,gsize = None,None
                if self.is_mpi_root():
                    var = np.cov(redshifts,aweights=weights,ddof=1)
                    gsize = redshifts.size
                var,gsize = self.mpicomm.bcast((var,gsize),root=self.mpiroot)
            sigma = np.sqrt(var)
            dz = sigma * (24. * np.sqrt(np.pi) / gsize) ** (1. / 3)
            zrange = zrange(redshifts)
            nbins = np.ceil((maxval - minval) * 1. / dx)
            nbins = max(1, nbins)
            edges = minval + dx * np.arange(nbins + 1)

        if np.ndim(bins) == 0:
            bins = np.linspace(*zrange(redshifts),num=bins+1,endpoint=True)

        counts = np.histogram(redshifts,weights=weights,bins=bins)
        if self.is_mpi_scattered():
            counts = mpicomm.allreduce(counts,op=mpi.MPI.SUM)

        if radial_distance is not None:
            dbins = radial_distance(bins)
        else:
            dbins = bins
        dvol = fsky*4./3.*np.pi*(dbins[1:]**3 - dbins[:-1]**3)
        self.z = (bins[:-1] + bins[1:])/2.
        self.density = counts/dvol
        self.spline = interpolate.UnivariateSpline(self.z,self.density,k=interp_order,s=0)

    def __call__(self, z):
        """Return density at redshift ``z`` (scalar or array)."""
        return self.spline(z)

def distance(position):
    """Return cartesian distance, taking coordinates along ``position`` last axis."""
    return np.sqrt((position**2).sum(axis=-1))

def cartesian_to_sky(position, wrap=True, degree=True):
    r"""
    Transform cartesian coordinates into distance, RA, Dec.

    Parameters
    ----------
    position : array of shape (N,3)
        Position in cartesian coordinates.

    wrap : bool, default=True
        Whether to wrap RA in :math:`[0, 2 \pi]`.

    degree : bool, default=True
        Whether RA, Dec are in degree (``True``) or radian (``False``).

    Returns
    -------
    dist : array
        Distance.

    ra : array
        Right Ascension.

    dec : array
        Declination.
    """
    dist = distance(position)
    ra = np.arctan2(position[:,1],position[:,0])
    if wrap: ra %= 2.*np.pi
    dec = np.arcsin(position[:,2]/dist)
    conversion = np.pi/180. if degree else 1.
    return dist, ra/conversion, dec/conversion

def sky_to_cartesian(dist, ra, dec, degree=True, dtype=None):
    """
    Transform distance, RA, Dec into cartesian coordinates.

    Parameters
    ----------
    dist : array
        Distance.

    ra : array
        Right Ascension.

    dec : array
        Declination.

    degree : default=True
        Whether RA, Dec are in degree (``True``) or radian (``False``).

    dtype : numpy.dtype, default=None
        :class:`numpy.dtype` for returned array.

    Returns
    -------
    position : array
        position in cartesian coordinates; of shape (len(dist),3).
    """
    conversion = 1.
    if degree: conversion = np.pi/180.
    position = [None]*3
    cos_dec = np.cos(dec*conversion)
    position[0] = cos_dec*np.cos(ra*conversion)
    position[1] = cos_dec*np.sin(ra*conversion)
    position[2] = np.sin(dec*conversion)
    return (dist*np.asarray(position,dtype=dtype)).T


from telebot import TeleBot
from telebot import types
from telebot.types import Message
import config
from modules import crl

bot = TeleBot(config.TOKEN)

@bot.message_handler(commands=['start'])
def welcome(message: Message):
    bot.send_message(message.chat.id, "Старт".format(message.from_user, bot.get_me()), parse_mode="html")

@bot.message_handler(commands=['check'])
def button_message(message : Message):
    markup=types.ReplyKeyboardMarkup(resize_keyboard=True)
    butt_check_crl=types.KeyboardButton("check")
    markup.add(butt_check_crl)
    chaecker1 = crl.CRLBaseChecker()
    bot.send_message(message.chat.id, chaecker1.start().format(message.from_user, bot.get_me()), reply_markup=markup)

bot.polling(none_stop=True)