#!/usr/bin/env python
# coding: utf-8

# In[ ]:

#!/usr/bin/env python
# coding: utf-8

# In[ ]:

# In[ ]:

#!/usr/bin/env python
# coding: utf-8

# In[ ]:

"""
Created on Fri Oct 13 20:37:30 2023

@author: saimo
"""
import streamlit as st
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import altair as alt
from PIL import Image
from sklearn.feature_selection import SelectKBest, chi2
import plotly.express as px
import hiplot as hip
from sklearn.metrics import roc_curve
from sklearn.metrics import accuracy_score, roc_curve, auc
from sklearn.impute import SimpleImputer
#import pandas as pd
from sklearn.utils import resample
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import accuracy_score, roc_curve, auc, f1_score, confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier
from sklearn.neighbors import KNeighborsClassifier
from xgboost import XGBClassifier, XGBRFClassifier
from sklearn.metrics import accuracy_score, confusion_matrix
import matplotlib.pyplot as plt
import numpy as np

st.set_option('deprecation.showPyplotGlobalUse', False)
st.set_page_config(layout="wide")

# Load the Framingham dataset
@st.cache_data
def load_data():
    data = pd.read_csv('Framingham_app/framingham.csv')  # Replace with the actual path to your dataset
    return data

data = load_data()

tab1, tab2, tab3,tab4,tab5,tab6,tab7 = st.tabs(["About Data", "Basic information Plots", "UniVariant Analysis Plots","MultiVariant Analysis Plots","Machine Learning Models","Inference on Inputs","Bio"])

with tab1:
   st.title("Framingham Heart Study Data")
   image = Image.open('Framingham_app/Heart_img.png')
   img = image.resize((image.height, 300))

   st.image(img, caption='Image')

   st.title("About the DATA")
   st.write("The Framingham Heart Study is a long-term, ongoing cardiovascular cohort study that began in 1948 in Framingham, Massachusetts, USA. It's one of the most well-known and influential epidemiological studies of heart disease. The study has provided valuable insights into the risk factors associated with cardiovascular disease and has helped shape our understanding of heart disease and its prevention. The Framingham dataset is a collection of data generated through this study, and it's widely used in epidemiological and public health research.")
   st.write("The dataset contains detailed information about a variety of cardiovascular risk factors and other health-related variables for thousands of individuals. Here's an overview of the key aspects of the Framingham dataset: 1. **Study Participants**: The dataset includes information about thousands of participants from the Framingham area. It has both original and offspring cohorts, meaning it has data from different generations.2. **Data Categories**: The Framingham dataset includes information on a wide range of variables, including:- Demographic information (age, gender, etc.).- Medical history (e.g., diabetes, hypertension).")

   st.title("About this WebAPP")

   st.write("The web app is designed to provide an interactive and visually engaging platform for exploring and visualizing data from the Framingham Heart Study dataset. Users can interact with the app to:")

   st.write("1. Visualize data relationships: Explore relationships between various attributes and Ten-Year Coronary Heart Disease (CHD) status through interactive scatter plots.")

   st.write("2. Understand CHD distribution: View the proportion of CHD cases versus no CHD cases using an interactive pie chart.")

   st.write("3. Interactive 3D visualization: Discover how age, cigarettes per day, and systolic blood pressure relate to Ten-Year CHD using an interactive 3D scatter plot.")

   st.write("4. Missing data analysis: Visualize missing data patterns using heatmaps and bar plots for a comprehensive data overview.")

   st.write("5. Customizable exploration: Users can customize their exploration by selecting attributes and visualizations through drop-down menus and select boxes.")

   st.write("This web app empowers users to gain insights and understand the Framingham dataset visually, enhancing the process of data exploration and analysis.")

   col1,col2 = st.columns(2)
   with col1:
    on = st.toggle('feature list of the dataset')
    if on:
        #st.write('Feature activated!')
        k = list(data.columns)
        st.write(k)
        st.write('Basic Framingham Dataset Information:')
        st.write(f'Total Number of Samples: {data.shape[0]}')
        st.write(f'Number of Features: {data.shape[1]}')
    with col2:
        on1 = st.toggle('summary statistics of the dataset')

        if on1:
            st.write("### Summary Statistics")
            st.write(data.describe())

with tab2:

    chd_counts = data['TenYearCHD'].value_counts()
    chd_proportion = chd_counts / chd_counts.sum()

    st.title("Pie Chart: Proportion of CHD vs. No CHD")

    fig = px.pie(chd_proportion, values=chd_proportion, names=chd_proportion.index,
                 labels={'index': 'CHD Status'}, title="Proportion of CHD vs. No CHD")

    st.plotly_chart(fig)

    st.header('Data Filters')
    age_filter = st.slider('Filter by Age', min_value=int(data['age'].min()), max_value=int(data['age'].max()))
    filtered_data = data[data['age'] <= age_filter]
    # st.write("### Filtered Data")
    # st.write(filtered_data)
    # Show an Altair plot of age distribution
    st.write("### Age Distribution")
    age_chart = alt.Chart(filtered_data).mark_bar().encode(
        x=alt.X('age:Q', bin=True),
        y='count()',
        tooltip=['age:Q', 'count()']
    ).interactive()
    st.altair_chart(age_chart)
    st.write("### Filtered Data")
    st.write(filtered_data)

    # Interactive scatter plot use it in the 3rd tab.
    # st.write("### Interactive Scatter Plot")
    # x_column = st.selectbox("X-axis", filtered_data.columns)
    # y_column = st.selectbox("Y-axis", filtered_data.columns)
    # scatter_chart = alt.Chart(filtered_data).mark_circle().encode(
    #     x=x_column,
    #     y=y_column,
    #     tooltip=[x_column, y_column]
    # ).interactive()
    # st.altair_chart(scatter_chart)

#     missing_data = data.isnull()

# # Use Seaborn to create a heatmap
#     fig = plt.figure(figsize=(10, 6))
#     sns.heatmap(missing_data, cbar=False, cmap='viridis')
#     plt.title('Missing Data in Framingham Dataset')
#     st.pyplot(fig)

    missing_values_count = data.isnull().sum()
    plt.figure(figsize=(10, 6))
    missing_values_count.plot(kind='bar', color='skyblue')
    plt.title("Missing Values by Attribute")
    plt.xlabel("Attributes")
    plt.ylabel("Count of Missing Values")
    plt.xticks(rotation=45)
    st.pyplot()

    numeric_columns = data.select_dtypes(include=['number']).columns
    categorical_columns = data.select_dtypes(exclude=['number']).columns

# Impute missing values based on data type

    st.title ("Data Before Handling Missing Values")
    imputed_data = data.copy()

    col1,col2 = st.columns(2)

    with col1:
     on = st.toggle('feature list of the dataset',['features'])
     if on:
         st.write("Original Data:")
         st.write(data)
     else:
         col1.empty()
        #st.write('Feature activated!')
        # k = list(data.columns)
        # st.write(k)
        # st.write('Basic Framingham Dataset Information:')
        # st.write(f'Total Number of Samples: {data.shape[0]}')
        # st.write(f'Number of Features: {data.shape[1]}')

    with col2:
        on1 = st.toggle('summary statistics of the dataset',['summary'])

        if on1:
            st.write("### Summary Statistics")
            st.write(data.describe())

    for col in numeric_columns:
        imputed_data[col].fillna(imputed_data[col].mean(), inplace=True)

    for col in categorical_columns:
        imputed_data[col].fillna(imputed_data[col].mode().iloc[0], inplace=True)

    st.title("Imputed Data (Mean for Numeric, Mode for Categorical):")
    st.write(imputed_data.describe())
#framingham.csv" with the actual path to your Framingham dataset. This code handles missing values according to the data type of the attribute and provides an imputed dataset for further analysis or visualization.

    # chd_counts = data['TenYearCHD'].value_counts()
    # chd_proportion = chd_counts / chd_counts.sum()

    # st.title("Interactive Pie Chart: Proportion of CHD vs. No CHD")

    # fig = px.pie(chd_proportion, values=chd_proportion, names=chd_proportion.index,
    #              labels={'index': 'CHD Status'}, title="Proportion of CHD vs. No CHD")

    # st.plotly_chart(fig)

    # # Example: Plot a histogram of Age
    # st.subheader("Age Distribution")
    # plt.figure(figsize=(8, 6))
    # sns.histplot(data['age'], bins=20, kde=True)
    # st.pyplot()
    # st.header("A dog")
    # st.image("https://static.streamlit.io/examples/dog.jpg", width=200)
    #st.subheader("Conclusion")
    #st.write(" From the above analysis we can clearly see how the data is distributed and how the missing values look after imputation.")

with tab3:
    data = load_data()

# Basic EDA plots for categorical variables using interactive violin plots
    st.title("Violin Plots for Categorical Variables in Framingham Dataset")

    st.write("Univariate analysis : Here we focus on examining individual variables one at a time. In the context of the Framingham Heart Study dataset, univariate analysis involves studying the relationship between each individual feature (independent variable) and the target variable 'TenYearCHD' (Coronary Heart Disease) to understand how each feature influences the presence of CHD. Univariate analysis helps identify which features have a significant impact on CHD.")

# Define the categorical variables to be visualized
    categorical_columns = ["currentSmoker", "BPMeds", "prevalentStroke", "prevalentHyp","male"]
    selected_variable = st.selectbox("Select a Categorical Variable", categorical_columns)

    fig = px.violin(data, x=selected_variable, y="age", box=True, points="all", title=f"Interactive Violin Plot for {selected_variable} vs Age")
    st.plotly_chart(fig)

    selected_variable = st.radio("Select a Categorical Variable", categorical_columns)

# Create box plots for the selected variable with respect to CHD

    fig = px.box(data, x=selected_variable, y="age", color="TenYearCHD",
              labels={"age": "Age", selected_variable: selected_variable, "TenYearCHD": "CHD"})

    fig.update_layout(
    title=f"Interactive Box Plot for {selected_variable} with Respect to CHD",
    xaxis_title='',
    yaxis_title='Age',
    showlegend=True,
)
    st.plotly_chart(fig)

# Create interactive violin plots for categorical variables
   # Create interactive violin plot for the selected variable

    # for col in categorical_columns:
    #     fig = px.violin(data, x=col, y="age", box=True, points="all", title=f"Interactive Violin Plot for {col} vs Age")
    #     st.plotly_chart(fig)

    st.title("KDE Plots for the Attributes with Respect to CHD")

    # Define the numerical variables to be visualized
    numerical_columns = ["age", "cigsPerDay", "totChol", "sysBP", "diaBP", "BMI", "heartRate", "glucose"]

    # Select box to choose a numerical variable
    # selected_variable = st.selectbox("Select a Numerical Variable", numerical_columns)

# Create KDE plots for the selected numerical variable with respect to CHD

# Filter the data into two subsets based on TenYearCHD
    chd_positive = data[data['TenYearCHD'] == 1]
    chd_negative = data[data['TenYearCHD'] == 0]

    # List of columns to plot
    columns_to_plot = data.columns.drop('TenYearCHD')

    # Plot categorical features (bar plots)
    for column in columns_to_plot:
        if data[column].dtype == 'object':
            plt.figure(figsize=(8, 4))
            sns.countplot(x=column, data=data, hue='TenYearCHD')
            plt.title(f'{column} vs. Ten-Year CHD')
            plt.xticks(rotation=45)
            # plt.show()
            st.pyplot()

    # Plot continuous features (histograms)
    for column in columns_to_plot:
        if data[column].dtype != 'object':
            plt.figure(figsize=(8, 4))
            sns.histplot(chd_negative[column], kde=True, label='No CHD', color='blue', alpha=0.6)
            sns.histplot(chd_positive[column], kde=True, label='CHD', color='red', alpha=0.6)
            plt.title(f'{column} vs. Ten-Year CHD')
            plt.xlabel(column)
            plt.legend()
            # plt.show(
            st.pyplot()
    st.subheader('Conclusion')
    st.write('From the above plots we can see the relation that how each feature is being associate with the output and we also get know about the distribution of the data according to the target class')

with tab4:

    st.title("Multivariant Analyis")

    st.write("Here we examing the relationships between multiple variables simultaneously. In the context of the Framingham Heart Study dataset, you can perform multivariate analysis to understand how combinations of features (independent variables) collectively influence the presence of Ten-Year Coronary Heart Disease (CHD) represented by the 'TenYearCHD' target variable. ")

    selected_variable = st.selectbox("Select a Numerical Variable", data.select_dtypes(include=['number']).columns)

    #st.write("suggested attributes for the scatter plot are")

    non_categorical_columns = data.select_dtypes(exclude=['object']).columns

    # Print the column names that do not have categorical values
    numerical_columns = ["cigsPerDay", "totChol", "sysBP", "diaBP", "BMI", "heartRate", "glucose"]

    st.write ("Suggested Attributes")
    st.write(numerical_columns)

    # Create an interactive scatter plot showing age vs. the selected numerical variable with color-coded CHD
    st.title(f"Interactive Scatter Plot: Age vs. {selected_variable} vs. Ten-Year CHD")

    fig = px.scatter(data, x="age", y=selected_variable, color="TenYearCHD",
                     color_continuous_scale=["blue", "red"],
                     labels={"age": "Age", selected_variable: selected_variable, "TenYearCHD": "CHD"})

    fig.update_layout(
        title=f"Age vs. {selected_variable} vs. Ten-Year CHD",
        xaxis_title="Age",
        yaxis_title=selected_variable
    )

    st.plotly_chart(fig)

    st.title("Interactive 3D Scatter Plot: Age, Cigarettes Per Day, Systolic BP vs. Ten-Year CHD")

    fig = px.scatter_3d(data, x="age", y="cigsPerDay", z="sysBP", color="TenYearCHD",
                        color_continuous_scale=["blue", "red"],
                        labels={"age": "Age", "cigsPerDay": "Cigarettes Per Day", "sysBP": "Systolic BP", "TenYearCHD": "CHD"})

    fig.update_layout(
        scene=dict(xaxis_title="Age", yaxis_title="Cigarettes Per Day", zaxis_title="Systolic BP"),
        title="Age, Cigarettes Per Day, Systolic BP vs. Ten-Year CHD"
    )

    st.plotly_chart(fig)

#visualization with HiPlot
    def save_hiplot_to_html(exp):
        output_file = "hiplot_plot_1.html"
        exp.to_html(output_file)
        return output_file

    st.header("Visualization with HiPlot")
    selected_columns = st.multiselect("Select columns to visualize", imputed_data.columns,default = ['age', 'cigsPerDay', 'totChol', 'sysBP','heartRate','TenYearCHD'])
                                      #color='TenYearCHD', title='Interactive Parallel Coordinates Plot')
    selected_data = imputed_data[selected_columns]
    if not selected_data.empty:
        experiment = hip.Experiment.from_dataframe(selected_data)
        hiplot_html_file = save_hiplot_to_html(experiment)
        st.components.v1.html(open(hiplot_html_file, 'r').read(), height=1500, scrolling=True)
    else:
        st.write("No data selected. Please choose at least one column to visualize.")
    st.subheader('Conclusion')
    st.write('This tab was mainly used to understand how two features of the data played a role in understanding the target disease. From the Hi-plot we can interactively and select the features and data accordingly to see what effect the features have on Heart Disease.')

with tab5:
    st.title("Machine Learning Classifier Performance")
    #data, selected_columns = oad_data()
    st.write('The Predictive Analysis feature within the application utilizes sophisticated machine learning models such as Logistic Regression and Random Forest to unravel the features of the subjects. With a diverse array of models, users can gain varied analytical perspectives on Framingham Data.')
    st.write('This tab serves as a conduit for translating complex data into accessible and interactive insights, enabling users, from decision-makers to the general public, to experiment with data and witness immediate results. This approach not only facilitates the prediction and comprehension of this complex data but also empowers users to engage with and respond to these critical issues proactively. The customization feature allows users to tailor the analysis with respect to the selected feautures and also use top 10 features to dervide insights.')
    st.write('The inclusion of a range of models is pivotal, as it enables users to apply diverse analytical perspectives to the same dataset. This diversity is critical because different models can spotlight distinct facets of the data.')
    st.markdown("""<hr style="height:3px;border:none;color:#333;background-color:#333;" /> """, unsafe_allow_html=True)
    def oad_data():
        data = pd.read_csv('Framingham_app/framingham.csv')

        data = data.sample(frac=0.6, random_state=42)  # Use a fixed random state for reproducibility

        # Preprocessing steps (handle missing values, etc.)
        missing_values_before = data.isnull().sum()
        #st.write("Missing values before imputation:", missing_values_before)

        # Impute missing values for numerical columns
        num_cols = data.select_dtypes(include=['int64', 'float64']).columns
        imputer = SimpleImputer(strategy='median')
        data[num_cols] = imputer.fit_transform(data[num_cols])

        # Check for missing values after imputation
        missing_values_after = data.isnull().sum()

        #st.write("Missing values after imputation:", missing_values_after)

        # Check for and replace infinite values
        data.replace([np.inf, -np.inf], np.nan, inplace=True)

        # Check for missing values after replacing infinite values
        missing_infinite = data.isnull().sum()
        #st.write("Missing/infinite values after replacement:", missing_infinite)

        # Drop any rows that still have NaNs (should be very few if any)
        data.dropna(inplace=True)

        # Preprocessing steps (handle missing values, etc.)
        # ...

        # Splitting the dataset based on class
        target1 = data[data['TenYearCHD'] == 1]
        target0 = data[data['TenYearCHD'] == 0]

        # Resampling to balance the dataset
        target1_resampled = resample(target1, replace=True, n_samples=len(target0), random_state=40)
        data_balanced = pd.concat([target0, target1_resampled])

        # Feature Selection
        X = data_balanced.iloc[:, 0:15]
        y = data_balanced.iloc[:, -1]

        best = SelectKBest(score_func=chi2, k=10)
        best.fit(X, y)

        # Select the top 10 features
        top_features = [X.columns[i] for i in best.get_support(indices=True)]
        data_selected = data_balanced[top_features + ['TenYearCHD']]

        return data_selected

    data = oad_data()

    def train_evaluate_model_cv(model, X, y, cv_folds):
        scores = cross_val_score(model, X, y, cv=cv_folds, scoring='f1')
        mean_score = np.mean(scores)
        accuracy_scores = cross_val_score(model, X, y, cv=cv_folds, scoring='accuracy')
        mean_accuracy_score = np.mean(accuracy_scores)
        st.write(f"Mean Accuracy Score (Cross-Validation): {mean_accuracy_score:.4f}")
        st.write(f"Mean F1 Score (Cross-Validation): {mean_score:.4f}")
        return mean_score, mean_accuracy_score
        #st.write(f"Mean F1 Score (Cross-Validation): {mean_score:.4f}")

        #return mean_score
    # Function to plot top 10 important features
    def plot_top_features(X, y):
        selector = SelectKBest(f_classif, k=10)
        X_new = selector.fit_transform(X, y)
        feature_scores = pd.DataFrame({'Feature': X.columns, 'Score': selector.scores_})
        top_features = feature_scores.nlargest(10, 'Score')

        fig, ax = plt.subplots()
        top_features.plot(x='Feature', y='Score', kind='barh', ax=ax, color='skyblue')
        ax.set_title('Top 10 Important Features')
        st.pyplot(fig)

    # Main app
    #st.title("Machine Learning Classifier Performance")

    #st.title("Machine Learning Classifier Performance")

# Load data
    data = oad_data()

    # Sidebar for feature selection
    all_features = data.drop('TenYearCHD', axis=1).columns.tolist()
    selected_features = st.multiselect("Select Features for Training", all_features, default=all_features[:10])

    # Option to use top 10 features from SelectKBest
    use_top_features = st.checkbox("Use Top 10 Features from SelectKBest", value=True)

    st.markdown("""<hr style="height:3px;border:none;color:#333;background-color:#333;" /> """, unsafe_allow_html=True)

    # Prepare data based on feature selection
    if use_top_features:
        X = data[selected_features]
    else:
        X = data[all_features]
    y = data['TenYearCHD']

    # Load data
    data = oad_data()

    # Sidebar for CV folds and feature selection
    cv_folds = st.slider("Select Number of Cross-Validation Folds", min_value=2, max_value=10, value=5)
    #selected_features = st.sidebar.multiselect("Select Features for Training",
                                           #options=data.columns.drop('TenYearCHD').tolist(),
                                           #default=data.columns.drop('TenYearCHD').tolist()[:10])

    #selected_features = st.sidebar.multiselect("Select Features for Training", data.columns.drop('TenYearCHD'), default=data.columns.drop('TenYearCHD'))
    st.markdown("""<hr style="height:3px;border:none;color:#333;background-color:#333;" /> """, unsafe_allow_html=True)

    # Prepare data
    #X = data[selected_features]
    #y = data['TenYearCHD']

    # Create tabs for each model
    tab_lr, tab_rf, tab_nb, tab_gb = st.tabs(["Logistic Regression", "Random Forest", "Naive Bayes", "Gradient Boosting"])

    with tab_lr:
        C_lr = st.number_input("C (Regularization parameter)", min_value=0.01, max_value=10.0, step=0.01, value=1.0)
        model_lr = LogisticRegression(C=C_lr, max_iter=1000)
        if st.button('Run Logistic Regression'):
            train_evaluate_model_cv(model_lr, X, y, cv_folds)

    with tab_rf:
        n_estimators_rf = st.slider("Number of trees in the forest", min_value=10, max_value=200, value=100)
        model_rf = RandomForestClassifier(n_estimators=n_estimators_rf)
        if st.button('Run Random Forest'):
            train_evaluate_model_cv(model_rf, X, y, cv_folds)

   # with tab_svm:
   #     C_svm = st.number_input("SVM - C (Regularization parameter)", 0.01, 10.0, step=0.01, value=1.0)
   #     kernel_svm = st.selectbox("SVM - Kernel", ("linear", "rbf", "poly"))
   #     model_svm = SVC(C=C_svm, kernel=kernel_svm, probability=True)
   #     if st.button('Run SVM'):
   #         train_evaluate_model_cv(model_svm, X, y, cv_folds)

    with tab_gb:
        n_estimators_gb = st.slider("Number of boosting stages", min_value=10, max_value=200, value=100)
        learning_rate_gb = st.number_input("Gradient Boosting - Learning Rate", 0.01, 1.0, step=0.01, value=0.1)
        model_gb = GradientBoostingClassifier(n_estimators=n_estimators_gb, learning_rate=learning_rate_gb)
        if st.button('Run Gradient Boosting'):
            train_evaluate_model_cv(model_gb, X, y, cv_folds)

    with tab_nb:
    #st.subheader("Naive Bayes Classifier")

    	model_nb = GaussianNB()

    	if st.button('Run Naive Bayes'):
        	train_evaluate_model_cv(model_nb, X, y, cv_folds)

    st.subheader("How each model understands data ?")
    st.write('Logistic Regression: Overview: Logistic Regression is a statistical method used for binary classification problems. It predicts the probability of an instance belonging to a particular class. Application: In the context of Framingham data, Logistic Regression can be used to predict the likelihood of a participant developing a cardiovascular condition based on various input features such as age, cholesterol levels, blood pressure, etc. However becaue of the complex features presnt in the dataset. This models fails to provide accurate predictions')
    st.write('Random Forest: Random Forest is an ensemble learning method that builds multiple decision trees and merges their predictions. It is versatile and can be used for both classification and regression tasks. Application: In the Framingham dataset, Random Forest could be employed to identify important features contributing to cardiovascular risk and provide robust predictions by aggregating outputs from multiple decision trees. This models performs the best and it is able to provide predictions irrespective of the features')
    st.write('Naive Bayes: Naive Bayes is a probabilistic classification algorithm based on Bayes theorem. Despite its simplicity, it often performs well, especially in text classification and simple datasets. Application: Naive Bayes can be applied to predict cardiovascular risk in the Framingham dataset by assuming independence between features, making it suitable for situations where this assumption is reasonable.')
    st.write('Gradient Boosting:Gradient Boosting is an ensemble technique that builds a series of weak learners (usually decision trees) sequentially, with each one correcting errors of its predecessor. Application: In the context of the Framingham data, Gradient Boosting can effectively capture complex relationships between features and the target variable, providing accurate predictions by combining the strengths of multiple weak models.')
    st.write('Experimenting with these models on the Framingham dataset allows for a nuanced understanding of their effectiveness in forecasting and dissecting cardiovascular risk. The choice of model depends on the specific characteristics of the data and the complexity of the scenarios being analyzed. By leveraging this diverse set of models, researchers and analysts can tailor their approach to gain comprehensive insights into the multifaceted nature of cardiovascular health and risk prediction.')
    # Plot top features
    plot_top_features(X, y)
    #st.subheader("How each model understands data ?")

with tab6:

    # Function to load and preprocess data
    def oad_data():
        data = pd.read_csv('Framingham_app/framingham.csv')
        data = data.sample(frac=0.6, random_state=42)

        # Preprocessing steps
        num_cols = data.select_dtypes(include=['int64', 'float64']).columns
        imputer = SimpleImputer(strategy='median')
        data[num_cols] = imputer.fit_transform(data[num_cols])

        data.replace([np.inf, -np.inf], np.nan, inplace=True)
        data.dropna(inplace=True)

        # Resampling to balance the dataset
        target1 = data[data['TenYearCHD'] == 1]
        target0 = data[data['TenYearCHD'] == 0]
        target1_resampled = resample(target1, replace=True, n_samples=len(target0), random_state=40)
        data_balanced = pd.concat([target0, target1_resampled])

        # Feature Selection
        X = data_balanced.iloc[:, :-1]
        y = data_balanced['TenYearCHD']
        best = SelectKBest(score_func=chi2, k=10)
        best.fit(X, y)
        top_features = [X.columns[i] for i in best.get_support(indices=True)]
        data_selected = data_balanced[top_features + ['TenYearCHD']]

        return data_selected, top_features

    data, top_features = oad_data()

    # Function to train the models
    def train_models(X_train, y_train):
        models = {
            'Logistic Regression': LogisticRegression(max_iter=1000),
            'Random Forest': RandomForestClassifier(n_estimators=100),
            'Naive Bayes': GaussianNB(),
            'Gradient Boosting': GradientBoostingClassifier(n_estimators=100)
        }
        for name, model in models.items():
            model.fit(X_train, y_train)
            models[name] = model
        return models

    # Main app
    st.title("Machine Learning Classifier Performance")

    # Split the data
    X = data[top_features]
    y = data['TenYearCHD']
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

    # User input for prediction
    with st.container():
        st.subheader("Make Predictions")
        columns = ['male','age','education','currentSmoker','cigsPerDay','BPMeds','prevalentStroke', 'prevalentHyp','diabetes','totChol','sysBP', 'diaBP', 'BMI', 'heartRate', 'glucose']

        input_data = {}
        for col in top_features:
            if col in ['male', 'currentSmoker', 'prevalentStroke', 'prevalentHyp', 'diabetes','BPMeds']:
                # Binary columns
                input_data[col] = st.selectbox(f"{col.capitalize()}", [0, 1],key=col)
            else:
                # Numerical columns
                # You might need to adjust min_value, max_value, and value based on the actual range of your data
                input_data[col] = st.number_input(f"{col.capitalize()}", min_value=0, max_value=100, value=50,key=col)

        if st.button("Predict"):
            input_df = pd.DataFrame([input_data])
        #input_data = {feature: st.number_input(f"{feature}", value=np.mean(X[feature])) for feature in top_features}
        #if st.button("Predict"):
            #input_df = pd.DataFrame([input_data])
            models = train_models(X_train, y_train)
            predictions = {name: model.predict(input_df)[0] for name, model in models.items()}

        # Displaying the predictions
            for model_name, prediction in predictions.items():
            	if model_name == 'Random Forest':
                	result = "Positive for CHD" if prediction == 1 else "Negative for CHD"
                	st.write(f"{model_name} Prediction: {result}")
            #models = train_models(X_train, y_train)
            #predictions = {name: model.predict(input_df)[0] for name, model in models.items()}
            #for model_name, prediction in predictions.items():
                #st.write(f"{model_name} Prediction: {prediction}")

        st.markdown("""<hr style="height:3px;border:none;color:#333;background-color:#333;" /> """, unsafe_allow_html=True)

        st.subheader("Conclusion")
        st.write("Our innovative web application, built upon the rich Framingham dataset, offers users a powerful and insightful platform for comprehensive data analysis and prediction in cardiovascular health.")
        st.write(" The univariate analysis component provides a meticulous examination of individual variables within the Framingham dataset. Users can delve into the distributions, central tendencies, and variations of key parameters, establishing a foundational understanding of each variable's characteristics. Advancing from univariate analysis, our app seamlessly integrates multivariate analysis capabilities, enabling users to uncover complex relationships and dependencies between various cardiovascular risk factors. This sophisticated exploration facilitates a holistic perspective, empowering users to discern intricate patterns and connections that contribute to a comprehensive understanding of cardiovascular health.")
        st.write("The true value of our application lies in its predictive prowess, driven by machine learning classifiers trained on the Framingham dataset. These models offer users the ability to anticipate potential cardiovascular events, assess risk factors, and make informed decisions for proactive health management. The predictive insights gleaned from our models contribute to a more personalized and preventive approach to cardiovascular care. Our user-friendly interface ensures accessibility for a diverse audience, from healthcare professionals to individuals keen on monitoring their cardiovascular health. By seamlessly integrating analytical tools and machine learning models, our app becomes an indispensable resource for deriving actionable insights from the Framingham dataset, ultimately contributing to enhanced cardiovascular risk assessment and personalized health strategies.")
        st.write("In summary, our web application on the Framingham dataset stands as a comprehensive solution, providing a deep dive into data analysis and predictive modeling specific to cardiovascular health. It is poised to make a meaningful impact on healthcare decision-making and individual well-being, aligning with the broader goals of proactive and personalized healthcare management.")

    # Optional: Add model performance metrics or other analyses here

with tab7:
    st.title("About the Developer ")

    col1, col2 = st.columns(2)

    col1.subheader("Sai Mohan Gajapaka")
    col1.text("Master's in Data Science, MSU")
    col1.write("As a dedicated Python programmer with a robust foundation in mathematical modeling and deep neural networks, I am currently advancing my journey in data science. My academic and research experiences have nurtured a strong proficiency in statistical analysis and machine learning, fueling my drive to tackle challenging problems with innovative solutions. My passion lies in applying my skills to real-world issues, particularly those that can make a positive social impact. I am constantly seeking opportunities that challenge me to grow and refine my abilities in this dynamic field. Beyond my academic pursuits, I have a keen interest in watching anime, which not only serves as a creative outlet but also often inspires my approach to complex problems. I'm deeply curious about the potential of deep learning and its applications, and I am committed to exploring its frontiers to contribute meaningfully to the field of data science.")

    try :
        col2.image("Framingham_app/profile.png")
    except:
     pass


# -*- coding: utf-8 -*-
"""
Created on Wed Jul 31 09:10:52 2019

@author: Acer
"""
#%%
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

data=pd.read_csv(r"D:\projects\zomato.csv")

data.describe()
data.columns

# Transforming rate column

data['rate_new'] = data['rate'].astype(str)
data['rate_new'] = data['rate_new'].apply(lambda x: x.split('/')[0])# Dealing with instanced with 'NEW'
data['rate_new'] = data['rate_new'].apply(lambda x: x.replace('NEW', str(np.nan)))
data['rate_new'] = data['rate_new'].apply(lambda x: x.replace('-', str(np.nan)))
# Changing data type
data['rate_new'] = data['rate_new'].astype(float)

data.drop(['rate'], axis=1, inplace=True)
print(f'{type(data["rate_new"][0])}')

data['approx_cost(for two people)'] = data['approx_cost(for two people)'].str.replace(',','').apply(lambda x:float(x))

#%%
#Dropping unnecessary data
data.drop(['url', 'address', 'dish_liked', 'phone', 'reviews_list', 'menu_item','location'], axis=1, inplace=True)

# Looking for null data
data.isnull().sum()
data = data.dropna(subset=['rate_new', 'approx_cost(for two people)'])
data = data.fillna('Not defined')
data.isnull().sum()

data.reset_index(drop=True)
data.describe()
data
#===========================EDA===========================
#%%
#1.Restaurant Rate Distribution
data['rate_new'].describe()

sns.set(style='darkgrid',palette='muted',color_codes=True)
fig, ax=plt.subplots(figsize=(12,5))
sns.distplot(data['rate_new'],bins=30,color='blue')
ax.set_title('Restaurant Rate Distribution',size=13)
ax.set_xlabel('Rate')
plt.show()
#%%
#2. Approx. cost of 2 people

data['approx_cost(for two people)']
sns.set(style='darkgrid',palette='muted',color_codes=True)
fig, ax=plt.subplots(figsize=(12,5))
sns.distplot(data['approx_cost(for two people)'],bins=10,color='blue')
ax.set_title('Approx cost for two people')
ax.set_xlabel('cost')
plt.show()
#%%
#3.Finding Outliers
#Online_Order
fig, ax=plt.subplots(figsize=(12,7))
sns.boxplot(x='online_order',y='rate_new',data=data)

#BookTable
fig, ax=plt.subplots(figsize=(12,7))
sns.boxplot(x='book_table',y='rate_new',data=data)

#%%
#4.Correlation between rating and cost
fig, ax = plt.subplots(figsize=(10 , 10))
sns.scatterplot(x='rate_new', y='approx_cost(for two people)', data=data, ax=ax)
ax.set_title('Correlation Between Rate and Approx Cost', size=14)
plt.show()

#%%
#5.Correlation between rating and Online Order or Booking Tables
fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(10, 10))
sns.scatterplot(x='rate_new', y='approx_cost(for two people)', hue='online_order',
                data=data, ax=axs[0], palette=['navy', 'crimson'])
sns.scatterplot(x='rate_new', y='approx_cost(for two people)', hue='book_table',
                data=data, ax=axs[1], palette=['navy', 'crimson'])
axs[0].set_title('Cost and Rate Distribution by Online Order', size=14)
axs[1].set_title('Cost and Rate Distribution by Book Table', size=14)
plt.show()
#%%
data.groupby(by='online_order').mean()
#%%
data.groupby(by='book_table').mean()
#%%
#6.Top Rated Restaurant
grouped_rate = data.groupby(by='name', as_index=False).mean()
top_rating = grouped_rate.sort_values(by='rate_new', ascending=False).iloc[:10, np.r_[0, -1]]
top_rating

top_rating.iloc[1, 0] = 'Santa Spa Cuisine'

# Plotting
fig, ax = plt.subplots(figsize=(13, 5))
ax = sns.barplot(y='name', x='rate_new', data=top_rating, palette='Blues_d')
ax.set_xlim([4.7, 4.95])
ax.set_xlabel('Mean Rate')
ax.set_ylabel('')
for p in ax.patches:
    width = p.get_width()
    ax.text(width+0.007, p.get_y() + p.get_height() / 2. + 0.2, '{:1.2f}'.format(width),
            ha="center", color='grey')

ax.set_title('Top 10 Restaurants in Bengaluru by Rate', size=14)
plt.show()
#%%
#Label Encoding

from sklearn.preprocessing import LabelEncoder

lb_en=LabelEncoder()
data['online_order']=lb_en.fit_transform(data['online_order'])
data['online_order'].unique()
data['online_order']
#%%
data['book_table']=lb_en.fit_transform(data['book_table'])
data['book_table'].unique()
data['book_table']
#%%
data['listed_in(type)']=lb_en.fit_transform(data['listed_in(type)'])
data['listed_in(type)']
#%%
data['listed_in(city)'].unique()
data['listed_in(city)']=lb_en.fit_transform(data['listed_in(city)'])
data['listed_in(city)']
#%%
data=pd.read_excel(r"D:\projects\zomato2.xlsx")
data['rest_type'].unique()
data['rest_type']=lb_en.fit_transform(data['rest_type'])
data['rest_type'].unique()
data['rest_type']

data.drop(['cuisines'], axis=1, inplace=True)
data.columns
#data['rate_new'].unique()
data['listed_in(type)'].unique()
#%%
corr = data.corr(method='kendall')
plt.figure(figsize=(15,8))
sns.heatmap(corr, annot=True)

#%%
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import cross_val_score, StratifiedKFold

data.columns
x=data.iloc[:,1:8]
y=data['rate_new']

sc=StandardScaler()
sc.fit(x)
x=sc.transform(x)

x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.10,random_state=20)
#kfold=StratifiedKFold(n_splits=10,random_state=48)
#%%
#LASSO
from sklearn.linear_model import Lasso
from sklearn.metrics import  mean_squared_error
from sklearn.metrics import r2_score
alpha=0.1
lasso=Lasso(alpha=alpha)
lasso.fit(x_train,y_train)
y_train_pred=lasso.predict(x_train)
y_test_pred=lasso.predict(x_test)
print(lasso.coef_)
print('MSE train: %.3f,test: %.3f' % (mean_squared_error(y_train,y_train_pred),mean_squared_error(y_test,y_test_pred)))
print('R^2 train: %.3f,test: %.3f' % (r2_score(y_train,y_train_pred),r2_score(y_test,y_test_pred)))

r2_score_lasso=r2_score(y_test,y_test_pred)
print(lasso)
print("r^2 on test data: %f" % r2_score_lasso)

predictors=data.columns.values[1:8]
coef=pd.Series(lasso.coef_,predictors).sort_values()
coef.plot(kind='bar', title='Modal Coefficients')
#%%
#RandomForest

from sklearn.ensemble import RandomForestRegressor
from sklearn.feature_selection import SelectFromModel
sel = SelectFromModel(RandomForestRegressor(n_estimators = 100))
sel.fit(x_train, y_train)
sel.get_support()
selected_feat= x_train.columns[(sel.get_support())]
print(selected_feat)

#%%
from xgboost import XGBRegressor
from numpy import sort

model = XGBRegressor()
model.fit(x_train, y_train)
# make predictions for test data and evaluate
y_pred = model.predict(x_test)
predictions = [round(value) for value in y_pred]
r2 = r2_score(y_test,y_test_pred)
print("R2: %.2f%%" % (r2 * 100.0))
mse=mean_squared_error(y_test,y_test_pred)
print("MSE: %.2f%%" % (mse * 100.0))
# Fit model using each importance as a threshold
thresholds = sort(model.feature_importances_)
for thresh in thresholds:
	# select features using threshold
	selection = SelectFromModel(model, threshold=thresh, prefit=True)
	select_X_train = selection.transform(x_train)
	# train model
	selection_model = XGBRegressor()
	selection_model.fit(select_X_train, y_train)
	# eval model
	select_X_test = selection.transform(x_test)
	y_pred = selection_model.predict(select_X_test)
	predictions = [round(value) for value in y_pred]
	r2 = r2_score(y_test,y_test_pred)
    #mse=mean_squared_error(y_test,y_test_pred)
	print("Thresh=%.3f, n=%d, r2: %.2f%%" % (thresh, select_X_train.shape[1], r2*100.0))

#%%
#Model Building
#RANDOM FOREST
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression

RForest=RandomForestRegressor(n_estimators=5,random_state=329,min_samples_leaf=.0001)
RForest.fit(x_train,y_train)
y_predict=RForest.predict(x_test)
from sklearn.metrics import r2_score
print("Random forest:", r2_score(y_test,y_predict))
#results=cross_val_score(RForest,x_train,y_train,cv=kfold)
#print("CVS:",results)

#Linear Regression
lm=LinearRegression()
lm.fit(x_train,y_train)
y_pred=lm.predict(x_test)
from sklearn.metrics import r2_score
print("Linear Regression:",r2_score(y_test,y_pred))
#results=cross_val_score(lm,x_train,y_train,cv=kfold)
#print("CVS:",results)

#DecisionTree
from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import export_graphviz
from os import  system

DTree=DecisionTreeRegressor(min_samples_leaf=.0001)
DTree.fit(x_train,y_train)
y_predict=DTree.predict(x_test)
from sklearn.metrics import r2_score
print("Decision Tree:",r2_score(y_test,y_predict))

#results=cross_val_score(DTree,x_train,y_train,cv=kfold)
#print("CVS:",results)

#SVM regressor
from sklearn.svm import SVR
regressor = SVR(kernel = 'rbf')
regressor.fit(x_train,y_train)
y_predict=regressor.predict(x_test)
from sklearn.metrics import r2_score
print("SVM regressor:", r2_score(y_test,y_predict))

#XGBoost Regressor
import xgboost

xgb = xgboost.XGBRegressor(n_estimators=500, learning_rate=0.5, gamma=0, subsample=0.75,colsample_bytree=1, max_depth=7)
xgb.fit(x_train,y_train)
predictions = xgb.predict(x_test)
print("XGB:",r2_score(y_test,predictions))

#KNN
from sklearn import neighbors
r2_val = []
for K in range(20):
    K = K+1
    model = neighbors.KNeighborsRegressor(n_neighbors = K)

    model.fit(x_train, y_train)  #fit the model
    pred=model.predict(x_test) #make prediction on test set
    r2 = r2_score(y_test,pred) #calculate rmse
    r2_val.append(r2) #store rmse values
    print('R2 for k= ' , K , 'is:', r2)

Zomato's Restaurant Rating Prediction.py


import urllib.request, json

from libbottles.utils import connection
from libbottles.exceptions import NoConnection

class Request:

    _headers = {}

    def __init__(self, headers: dict = None):
        self._headers["User-Agent"] = "libbottles client (usebottles.com)"

        if headers is not None:
            self._envs = {**self._headers, **headers}

        if not connection.check():
            raise NoConnection()

    def get(self, url: str):

        req = urllib.request.Request(
            url,
            data=None,
            headers=self._headers
        )

        with urllib.request.urlopen(req) as url:
            data = json.loads(url.read().decode())
            return data


# Add a filter to a palette
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('-i', default="Palette.dmp", help='Input tileset palette file. default is Palette.dmp')
parser.add_argument('-o', default="Palette2.dmp", help='Output tileset palette file. default is Palette2.dmp')
parser.add_argument('-r', default=4, type=int, help='Red value modifier, [-31, 31]. Default is 4')
parser.add_argument('-g', default=-16, type=int, help='Green value modifier, [-31, 31]. Default is -16')
parser.add_argument('-b', default=-16, type=int, help='Blue value modifier, [-31, 31]. Default is -16')
args = parser.parse_args()

input = open(args.i, "rb")
output = open(args.o, "wb")

# Add these values to each colour's R G and B.
redModifier = args.r
greenModifier = args.g
blueModifier = args.b

# Modify colours and concatenate.
for i in range(16):

    inputEntry = ord(input.read(1)) | (ord(input.read(1)) << 8)

    red = (inputEntry & 31) + redModifier
    green = ((inputEntry >> 5) & 31) + greenModifier
    blue = ((inputEntry >> 10) & 31) + blueModifier

    if red < 0:
      red = 0
    elif red > 31:
      red = 31
    if green < 0:
      green = 0
    elif green > 31:
      green = 31
    if blue < 0:
      blue = 0
    elif blue > 31:
      blue = 31

    outputEntry = red | (green << 5) | (blue << 10)

    output.write((outputEntry).to_bytes(2, byteorder='little', signed=False))

input.close()
output.close()


# -*- coding：utf-8 -*-
"""
time:2021/2/24
author:李辰旭
organization: BIT
contact: QQ:316469360
——————————————————————————————
description：
$ 处理二值合成轨迹图的一些函数。
主要包括：
滤波降噪
提取轮廓质心
拟合二次曲线
计算像素高度
——————————————————————————————
note:
python3.7以上版本才可运行
"""
import numpy as np
import cv2 as cv
from scipy.optimize import leastsq
def apply(img_BW,k):
    '''输入二值单通道图像
    返回特定区域被涂黑(k=0)或涂白(k=1)的图像'''
    a=img_BW.copy()
    left=400
    right=1920-left
    mid_left=650
    mid_right=1920-mid_left
    mid_high=500
    a[:,:left]=255*k
    a[:,right:]=255*k
    a[mid_high:,mid_left:mid_right]=255*k
    return a
def find_centroid(img_BW):
    '''输入：二值图像
    返回：质心xy列表'''
    cnts,_ = cv.findContours(img_BW, cv.RETR_EXTERNAL,cv.CHAIN_APPROX_SIMPLE)
    X_Y=[]
    # 遍历轮廓集
    for c in cnts:
        # 计算轮廓区域的图像矩。 在计算机视觉和图像处理中，图像矩通常用于表征图像中对象的形状。这些力矩捕获了形状的基本统计特性，包括对象的面积，质心（即，对象的中心（x，y）坐标），方向以及其他所需的特性。
        M = cv.moments(c)
        cX = int(M["m10"] / M["m00"])
        cY = int(M["m01"] / M["m00"])
        X_Y.append((cX,cY))
    return X_Y

def func(params, x):
    a, b, c = params
    return a * x * x + b * x + c
# 误差函数，即拟合曲线所求的值与实际值的差
def error(params, x, y):
    return func(params, x) - y
# 对参数求解
def slovePara(X,Y):
    '''输入：两个列表
       返回：二次函数的三个参数abc
    '''
    p0 = [10, 10, 10]#abc的初值，还要迭代呢
    X=np.array(X)
    Y=np.array(Y)
    Para = leastsq(error, p0, args=(X, Y))
    a, b, c = Para[0]
    return a,b,c

def track_progress(img_BW,img,grand=950,startline=1250):
    '''输入合成的轨迹图（二值化的单通道图及二值化的三通道图）
       返回发球点的像素高度,返回处理过的图片'''
    #涂抹噪声
    applied=apply(img_BW,0)
    #滤波降噪
    #中值滤波
    mid_filer=cv.medianBlur(applied,3)
    #膨胀
    kernel = np.ones((5, 5), np.uint8)
    frame = cv.dilate(mid_filer, kernel, iterations=2)
    #找质心
    X_Y = find_centroid(frame)
    X=[]
    Y=[]
    for i in X_Y:
        X.append(i[0])
        Y.append(i[1])
        cv.circle(img, (i[0], i[1]), 10, (0, 0, 255), -1)
    #拟合抛物线
    try:
        a,b,c=slovePara(X,Y)
        print('抛物线参数：a=',a,'b=',b,'c=',c)
        #画抛物线
        for w in range(1920):
            h=int(a*(w*w)+b*w+c)
            if h>0&h<1080:
                cv.circle(img, (w, h), 3, (0, 255, 0), -1)
    except:
        print('拟合抛物线时出现错误，好像是因为拟合点少于三个')
    #画地面和起始线
    cv.line(img,(0,grand), (1919,grand), (255,0,0),3)
    cv.line(img,(startline,0), (startline,1079), (255,0,0),3)
    #找发球点，求高度
    try:
        height=a*(startline*startline)+b*startline+c - grand
        if height < 0:
            height*=-1
        print('像素高度=',height)
        return height,frame
    except:
        print('拟合抛物线时出现错误，abc没值')
        return -1,frame

if __name__ == '__main__':
    import os
    cv.namedWindow("obj", cv.WINDOW_NORMAL)
    cv.resizeWindow("obj", int(1920/3), int(1080/3))
    cv.moveWindow("obj", 0, 0)
    cv.namedWindow("applied", cv.WINDOW_NORMAL)
    cv.resizeWindow("applied", int(1920/3), int(1080/3))
    cv.moveWindow("applied", int(1920/3), 0)
    img = cv.imread("D:\\lalala.png")
    img_BW = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
    track_progress(img_BW,img)
    cv.imshow("obj",img)
    cv.imshow("applied",img_BW)
    cv.waitKey(0)


import torch
from torch.nn import functional as F

def compute_mvg(d_latents, latent_name, mean_v, inv_cov_v):
    if latent_name == "W":
        _w = d_latents["W"]
        _v = F.leaky_relu(_w, negative_slope=5.0)
        dv = (_v - mean_v)
        loss = (dv.matmul(inv_cov_v).matmul(dv.T))
        return loss
    elif latent_name == "W+":
        _wp = d_latents["W+"].double()
        _vp = F.leaky_relu(_wp, negative_slope=5.0)
        loss = 0.0
        for idx in range(_vp.shape[1]):
            _v = _vp[:, idx, :]
            dv = (_v - mean_v)
            loss += (dv@inv_cov_v@dv.T)
        return loss.squeeze(0).squeeze(0)

def b_compute_mvg(d_latents, latent_name, mean_v, inv_cov_v):
    if latent_name == "W+":
        _wp = d_latents["W+"].double()
        _vp = F.leaky_relu(_wp, negative_slope=5.0)
        bs = _wp.shape[0]
        inv_cov_v = inv_cov_v.reshape(1, 512, 512).repeat(bs, 1, 1)
        loss = 0.0
        for idx in range(_vp.shape[1]):
            _v = _vp[:, idx, :]
            dv = (_v - mean_v).reshape(-1, 1, 512)
            loss += torch.bmm(torch.bmm(dv, inv_cov_v), torch.transpose(dv, 1, 2)).mean()
        return loss

def delta_loss(latent):
    loss = 0.0
    first_w = latent[:, 0, :]
    for i in range(1, latent.shape[1]):
        delta = latent[:, i, :] - first_w
        delta_loss = torch.norm(delta, 2, dim=1).mean()
        loss += delta_loss
    return loss


import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import time
import random
import os
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from model import *
from user_agent import generate_user_agent
import re

MAIN_URL = 'https://usa.tommy.com/ProductListingView'
db_engine = create_engine("sqlite:///calvin.db", echo=True)
basedir = os.path.abspath(os.path.dirname(__file__))
size_list = []
details_list = []
color_list = []
url_list = []
cat_url_list = []
cookie = {
    'sr_browser_id': '33df22be-572e-4904-ba40-0ad50c8bf097',
    'sr_pik_session_id': 'a93a676-c1cc-79d7-d8d1-e0f2e070f5ea',
    '__utma': '230066073.559163455.1579546275.1583593346.1583599172.13',
    '__utmb': '230066073.9.10.1583599172',
    '__utmc': '230066073',
    '__utmz': '230066073.1581354246.9.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided)',
    '__wid': '496949504',
    '_ga': 'GA1.2.559163455.1579546275',
    '_px2': 'eyJ1IjoiNDdkNjNiNzAtNjA5NC0xMWVhLWE2NDQtZjFhZjM2YWNiMDBkIiwidiI6ImQ2MGQxMGEyLTNiYjUtMTFlYS1hMjE1LTAyNDJhYz'
           'EyMDAwNSIsInQiOjE1ODM2MDA1OTkzNTgsImgiOiJlMTMzMjExNGY0YjE4N2VkZDU0OThlNTY5ZDRkZjIzYjU3NTgwMTVjY2FjNGFlYjk0N'
           'DAyNzYwZWU1Y2ExMzJlIn0=',
    '_pxvid': 'eyJ1IjoiNDdkNjNiNzAtNjA5NC0xMWVhLWE2NDQtZjFhZjM2YWNiMDBkIiwidiI6ImQ2MGQxMGEyLTNiYjUtMTFlYS1hMjE1LTAyNDJh'
              'YzEyMDAwNSIsInQiOjE1ODM2MDA1OTkzNTgsImgiOiJlMTMzMjExNGY0YjE4N2VkZDU0OThlNTY5ZDRkZjIzYjU3NTgwMTVjY2FjNGFl'
              'Yjk0NDAyNzYwZWU1Y2ExMzJlIn0=',
    'sctr': '1|1583532000000',
    'ADRUM': 's=1583600052675&r=https%3A%2F%2Fusa.tommy.com%2Fen%2Fwomen%2Fnewarrivals-women%2Ficon-wide-leg-stripe-pant-ww28013%3F0',
    'JSESSIONID': '0000einWRcx7PVbTVe62TS9mlJv:1crovuh4f'

}

proxy = {'HTTPS': '157.245.138.230:8118'}
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'calvin.db')
HEADERS = {
    'User-Agent': generate_user_agent(device_type="desktop", os=('mac', 'linux')),
    'Accept':'*/*',
    'Cache-Control':'no-cache',
    'Host':'usa.tommy.com',
    'Accept-Encoding':'gzip, deflate, br',
    'Connection':'keep-alive',
    'accept-language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7,uz;q=0.6'}

payload = {
    'catalogId': '10551',
    'isHomeDepartment': 'false',
    'pageSize': '30',
    'disableProductCompare': 'true',
    'langId': '-1',
    'storeId': '10151', #CommerceSearch
    'categoryId': '', #pageId
    'beginIndex': '30',
    'minFacetCount': '1',
    'colorfacetselected': 'false',
    'cache': 'true'
}

engine = create_engine(SQLALCHEMY_DATABASE_URI, echo=True)
Session = sessionmaker(bind=engine)
session = Session()

def read_file_url():
    with open('input.txt', 'r') as file:
        for line in file:
            cat_url_list.append(line.strip('\n'))
    return cat_url_list

def get_html(url, payload=None):
    while True:
        time.sleep(random.randint(random.randint(6, 10), random.randint(12, 27)))
        html = requests.get(url, headers=HEADERS, proxies=proxy, params=payload, cookies=cookie)
        if html.status_code == 200:
            print(html.status_code)
            return html
        elif html.status_code == 403:
            print(html.status_code)
            print('weit to 600 sec')
            time.sleep(random.randint(600,800))
        else:
            time.sleep(random.randint(14, 27))
            print(html.status_code)
            continue

def parser_content(html, image_list):
    # порсит все данные из карточки кроме фото, подумать разбить на несколько функций
    soup = BeautifulSoup(html.text, 'html.parser')
    link = html.url
    try:
        # имя товара
        product_name = soup.find('span', class_='productNameInner').text
    except:
        product_name = None

    try:
        # базовая цена(без скидки)
        price = soup.find('div', id='price_display').find_all('span')[0].text[1:]
    except:
        price = None
    try:
        # акционная цена
        price_sale = soup.find('div', id='price_display').find_all('span')[1].text[1:]
    except (IndexError, ValueError):
        price_sale = None

    try:
        # доступные размеры, на сайте все доступные размеры имеют класс available, поэтому парсим только их
        block_size = soup.find('ul', id='sizes').find_all('li')
        for li in block_size:
            if li['class'] == ['available']:
                size_list.append(li.find('span').text)
    except:
        print(f'Size {None}')

    try:
        # маркированый список Details с доп инфой снизу карточки
        details_group = soup.find('ul', class_='bullets')
        for details in details_group.find_all('li'):
            details_list.append(details.text)
    except:
        details_list.append('')

    try:
        # цветовая схема доступных цветов с сайта
        radiogrup = soup.find('ul', class_='productswatches')
        for color in radiogrup.find_all('li'):
            color_list.append(color['data-color-swatch'])
    except:
        color_list.append('')

    try:
        # парсим 1 цвет
        color = soup.find('ul', class_='productswatches').find('li', class_='active')['data-color-swatch']
    except:
        color = ''

    try:
        # айди обьявления
        universal_id = soup.find('div', class_='universalStyleNumber').find_all('span')[1].text
    except:
        universal_id = ''

    try:
        # парсим категорию товара
        category = soup.find('div', id='breadcrumb').find_all('a')[-2].text + ' ' + \
                   soup.find('div', id='breadcrumb').find_all('a')[-1].text
    except:
        category = ''
    count = 1
    Session = sessionmaker(bind=db_engine)
    session = Session()
    try:
        new_element = Tommy(product_name, price, price_sale, ','.join(size_list), color, ','.join(image_list),
                             ','.join(details_list), category, ','.join(color_list), link)
        session.add(new_element)
        session.commit()
    except:
        pass
    count += 1
    size_list.clear()
    color_list.clear()
    details_list.clear()

def create_dir_name():
    dir_name = 'images'
    try:
        os.mkdir(dir_name)
    except OSError:
        print('Папка существует')
    return dir_name

def chek_images():
    # проверяет номер последней фото в папке, и при запуске парсера след фото будет +1
    num_file = []
    last_image = 0
    try:
        file_list = os.listdir('images')
        for list in file_list:
            num_file.append(int(re.findall(r'\d*', list)[0]))
        num_file.sort()
        print(num_file[-1])
    except(IndexError):
        num_file.append(0)
    last_image = num_file[-1]
    return last_image

def get_photo(html, dir_name):
    count_photo = chek_images()
    image_list = []
    img_name = []
    soup = BeautifulSoup(html.content, 'html.parser')
    image_url = soup.find('div', class_='product_main_image').find('img')['data-src']
    image_list.append(image_url)
    image_list.append(image_url.replace('main', 'alternate1'))
    image_list.append(image_url.replace('main', 'alternate2'))
    image_list.append(image_url.replace('main', 'alternate3'))
    for img in image_list:
        try:
            photo_name = count_photo
            file_obj = requests.get(img, stream=True)
            if file_obj.status_code == 200:
                with open(dir_name+'/'+str(photo_name)+'.JPG', 'bw') as photo:
                    for chunk in file_obj.iter_content(8192):
                        photo.write(chunk)
                count_photo +=1
                img_name.append(str(photo_name))
        except:
            print('Error file_obj')
    return img_name

def get_url_category(html):
    # функция будет парсить в список url всех карточек в список(отсылает отдельные запросы)
    count = 0
    soup = BeautifulSoup(html.content, 'html.parser')
    page_count = soup.find('div', id='filterInfo')['data-total-count']
    all_page = int(page_count) // 30
    prod = soup.find('div', class_='grid').find_all('a', class_='productThumbnail')
    for i in prod:
        url_list.append(i['href'])
    category_id = soup.find('head').find('meta', {'name': 'pageId'})['content']
    payload.update({'categoryId': category_id})
    for page in range(1, all_page + 1):
        count += 30
        payload.update({'beginIndex': count})
        #response = requests.get(MAIN_URL, headers=HEADERS, proxies=proxy, params=payload)
        response = get_html(MAIN_URL, payload=payload)
        print(html.status_code)
        sp = BeautifulSoup(response.content, 'html.parser')
        try:
            prod = sp.find('div', class_='grid').find_all('a', class_='productThumbnail')
            for i in prod:
                url_list.append(i['href'])
        except:
            continue
    return url_list

def main():
    Session = sessionmaker(bind=db_engine)
    session = Session()
    dir_name = create_dir_name()
    cat_url_list = read_file_url()
    for cat_url in cat_url_list:
        html = get_html(cat_url)
        url_list = get_url_category(html)
    for url in url_list:
        card_exist = session.query
        html = get_html(url)
        image_list = get_photo(html, dir_name)
        parser_content(html, image_list)

if __name__ == '__main__':
    main()


from django.db import models
from django.utils.translation import gettext_lazy as _

class VCard(models.Model):
    """
    Vcard model.
    """
    title = models.CharField(
        _("Title"),
        blank=True,
        max_length=150,
        default="",
    )

    def __str__(self):
        return self.title

    class Meta:
        verbose_name = _("VCard")
        verbose_name_plural = _("VCards")


import json
from datetime import timedelta

from db.redis import init_redis_pool

async def add_test_result_to_redis(result_id: int, user_id: int, id_company: int, id_quiz: int, data: dict):
    redis = await init_redis_pool()

    key = f"result_test:{result_id:}:id_user:{user_id}:id_company:{id_company}:id_quiz:{id_quiz}"

    await redis.setex(key, timedelta(hours=48), json.dumps(data))

async def get_value_by_keys(**kwargs):
    values = await get_values_by_keys(**kwargs)

    return None if len(values) == 0 else values[0]

async def get_values_by_keys(**kwargs):
    redis = await init_redis_pool()

    pattern = '*' + '*'.join([f"{key}:{value}" for key, value in kwargs.items()]) + '*'

    keys = await redis.keys(pattern)

    return await get_values(redis, keys)

async def get_values(redis, keys):
    return [json.loads(await redis.get(key)) for key in keys if await redis.get(key)]


import requests
from bs4 import BeautifulSoup
import json

# URL of the web page to scrape
url = 'https://nookipedia.com/wiki/Category:New_Horizons_fish_icons'

# Send an HTTP request to the web page
response = requests.get(url)

# Check if the request was successful
if response.status_code == 200:
    # Parse the HTML content of the page using BeautifulSoup
    soup = BeautifulSoup(response.text, 'html.parser')

    # Find all image tags on the page
    image_tags = soup.find_all('img')

    # Extract the 'src' attribute from each image tag to get the image links
    image_links = [img['src'] for img in image_tags]

    # Create a dictionary with a "urls" key and the image links as its value
    data = {"urls": image_links}

    # Save the data to a JSON file
    with open('image_links.json', 'w') as json_file:
        json.dump(data, json_file, indent=4)

    print("Image links saved to 'image_links.json'")
else:
    print(f"Failed to retrieve the web page. Status code: {response.status_code}")


from trackingsimpy.simulation.revisit_interval import BaseRISimulation
from trackingsimpy.tracking import TrackingComputer
from trackingsimpy.radar import PositionRadar
import filterpy.common
from filterpy.kalman import IMMEstimator, KalmanFilter
from trackingsimpy.common.motion_model import constant_turn_rate_matrix, kinematic_state_transition
from trackingsimpy.target import DefinedTargetProcess
from trackingsimpy.common.measurement_model import position_measurement_matrix

import numpy as np

class DefinedIMM(BaseRISimulation):
    ORDER = 1
    DIM = 2
    DT = 0.1
    MAX_STEPS = 700

    def __init__(self, pfa=1e-6, sn0=50, beamwidth=0.02, n_max=20, x0=np.array([10e3, -200.0, 10e3, 0]),
                 P0=np.eye(4) * 1000, theta_accuracy=0.002):
        var = 0.1*9.81**2
        tr1 = -0.08
        tr2 = 0.1
        probs = np.ones(3) / 3
        p_switch = 1.0 / 100.0
        M = np.array([
            [1 - p_switch, p_switch / 2, p_switch / 2],
            [p_switch / 2, 1 - p_switch, p_switch / 2],
            [p_switch / 2, p_switch / 2, 1 - p_switch]
        ])
        filters = list()

        for i in range(3):
            filters.append(filterpy.common.kinematic_kf(self.DIM, self.ORDER, self.DT))
            filters[i].x = x0
            filters[i].Q = filterpy.common.Q_discrete_white_noise(self.DIM, self.DT, var=var, block_size=self.ORDER+1)

        filters[1].F = constant_turn_rate_matrix(tr1, self.DT)
        filters[2].F = constant_turn_rate_matrix(tr2, self.DT)

        st_models = {
            0: kinematic_state_transition(self.DT, self.ORDER, self.DIM),
            200: constant_turn_rate_matrix(tr1, self.DT),
            300: kinematic_state_transition(self.DT, self.ORDER, self.DIM),
            400: constant_turn_rate_matrix(tr2, self.DT),
            500: kinematic_state_transition(self.DT, self.ORDER, self.DIM)
        }

        p_noises = {
            0: filterpy.common.Q_discrete_white_noise(self.DIM, self.DT, var=var, block_size=self.ORDER + 1)
        }
        target = DefinedTargetProcess(x0, st_models, p_noises, self.MAX_STEPS, self.ORDER, self.DIM)
        tracker = IMMEstimator(filters, probs, M)
        radar = PositionRadar(target, sn0, pfa, beamwidth, self.DIM, self.ORDER, angle_accuracy=theta_accuracy)
        computer = TrackingComputer(tracker, radar, n_max, P0=P0)

        super().__init__(computer)

class DefinedCVCAIMM(BaseRISimulation):
    ORDER = 2
    DIM = 2
    DT = 0.1
    MAX_STEPS = 3000

    def __init__(self, pfa=1e-6, sn0=50, beamwidth=0.02, n_max=20, x0=np.array([30e3, -150, 0, 30e3, 150, 0]),
                 theta_accuracy=0.002):

        # Trackers
        probs = np.ones(2) / 2
        p_switch = 1.0 / 1000.0
        M = np.array([
            [1 - p_switch, p_switch],
            [p_switch, 1 - p_switch]])

        F_ca = kinematic_state_transition(self.DT, self.ORDER, self.DIM)
        F_cv = kinematic_state_transition(self.DT, self.ORDER, self.DIM)
        F_cv[:, 2::3] = 0

        g = 9.81
        var = 4 * g ** 2
        Q = filterpy.common.Q_discrete_white_noise(self.DIM, self.DT, var, block_size=self.ORDER + 1)

        kf_cv = KalmanFilter(dim_x=self.DIM * (self.ORDER + 1), dim_z=self.DIM)
        kf_cv.F = F_cv
        kf_cv.Q = Q
        kf_cv.H = position_measurement_matrix(self.DIM, self.ORDER)

        kf_ca = KalmanFilter(dim_x=self.DIM * (self.ORDER + 1), dim_z=self.DIM)
        kf_ca.F = F_ca
        kf_ca.Q = Q
        kf_ca.H = position_measurement_matrix(self.DIM, self.ORDER)

        filters = [kf_cv, kf_ca]
        tracker = IMMEstimator(filters, probs, M)

        # Target
        st_models = {
            0: F_cv,
            1000: F_ca,
            2000: F_cv
        }
        p_noises = {
            0: Q
        }
        target = DefinedTargetProcess(x0, st_models, p_noises, self.MAX_STEPS, self.ORDER, self.DIM)

        # Radar
        radar = PositionRadar(target, sn0, pfa, beamwidth, self.DIM, self.ORDER, angle_accuracy=theta_accuracy)

        # Computer
        P0 = np.zeros((x0.size,) * 2)
        computer = TrackingComputer(tracker, radar, n_max, P0=P0)

        super().__init__(computer)


import json
import subprocess
import os
import zmq

class CommandLineBarcodeReader():
    def __init__(self, config_path="scandit_commandline",port=5556):
        self.context = zmq.Context()
        self.process = None
        self.config_path = config_path
        self.port = port
        self.start_commandline_zmq_server_if_unstarted()

    def start_commandline_zmq_server_if_unstarted(self):
        socket = self.context.socket(zmq.REQ)
        socket.connect("tcp://localhost:"+str(self.port))
        socket.send(b"Hello")
        message = ""
        try:
            message = socket.recv(flags=zmq.NOBLOCK)
            print(message)
        except Exception as e:
            print("start error")
            print(e)
            f = open(self.config_path,"r")
            commandline=[]
            for line in f.readlines():
                commandline.append(line.strip())
            f.close()
            self.process = subprocess.Popen(commandline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

    def stop_commandline_zmq_server_if_started(self):
        try:
            self.process.kill()
        except:
            print("process not opened")

    def decode_file(self, img_path):
        result_dict = {}
        results = []
        try:
            socket = self.context.socket(zmq.REQ)
            socket.connect("tcp://localhost:"+str(self.port))
            socket.send(bytes(img_path,"utf-8"))
            message = socket.recv()
            json_object = json.loads(message.decode("utf-8"))
            if "results" in json_object:
                results=json_object["results"]
            if "elapsedTime" in json_object:
                result_dict["elapsedTime"]=json_object["elapsedTime"]
        except Exception as e:
            print("decode error")
            print(e)
        result_dict["results"] = results
        return result_dict

if __name__ == '__main__':
    #reader = CommandLineBarcodeReader()
    #reader = CommandLineBarcodeReader(config_path="zxing_commandline",port=5557)
    reader = CommandLineBarcodeReader(config_path="dbr88_commandline",port=6666)

    results = reader.decode_file("D:\\test\\BarcodePerformance\\new\\black_qr_code.png")
    print(results)
    reader.stop_commandline_zmq_server_if_started()


####################################################
##### This is focal loss class for multi class #####
##### University of Tokyo Doi Kento            #####
####################################################

import torch
import torch.nn as nn
import torch.nn.functional as F
# I refered https://github.com/c0nn3r/RetinaNet/blob/master/focal_loss.py

class FocalLoss2d(nn.Module):

    def __init__(self, alpha=1.0, gamma=0, weight=None,ignore_index=-100, size_average=True, with_grad=True):
        super(FocalLoss2d, self).__init__()
        assert gamma>=0.0 and gamma<=5.0,'gamma in [0,5] is okay, but %0.2f'%gamma
        assert alpha>0.0
        self.alpha = alpha
        self.gamma = gamma
        self.weight = weight
        self.size_average = size_average
        self.ignore_index=ignore_index
        self.with_grad=with_grad

    def forward(self, input, target):
        if input.dim()>2:
            input = input.contiguous().view(input.size(0), input.size(1), -1)
            input = input.transpose(1,2)
            input = input.contiguous().view(-1, input.size(2)).squeeze()
        if target.dim()==4:
            target = target.contiguous().view(target.size(0), target.size(1), -1)
            target = target.transpose(1,2)
            target = target.contiguous().view(-1, target.size(2)).squeeze()
        elif target.dim()==3:
            target = target.view(-1)
        else:
            target = target.view(-1, 1)

        # compute the negative likelyhood
        logpt = -F.cross_entropy(input, target,
                                 weight=self.weight,
                                 ignore_index=self.ignore_index,
                                 reduction='none')
        if self.with_grad:
            pt = torch.exp(logpt)
        else:
            with torch.no_grad():
                pt = torch.exp(logpt)
        # compute the loss
        loss = - self.alpha* ((1-pt)**self.gamma) * logpt

        # averaging (or not) loss
        if self.size_average:
            return loss.mean()
        else:
            return loss.sum()

if __name__ == '__main__':
    input=torch.rand(2,10,3,3).float()
    print(input.shape)
    target=torch.rand(2,10,3,3)
    target=torch.argmax(target,dim=1)
    loss_fn=FocalLoss2d()
    loss=loss_fn(input,target)
    print(loss)


import sys
import random
import re
from functools import partial
from tqdm import tqdm

from junkdrawer import generator_looper

def lindenate(liRules, sInput="", lIterations=1):
    """This function iteratively processes a set of find-and-replace rules, liRules, on a given string, sInput.
        By default, it only goes through a single iteration, and processes the rules against an empty string.
            In a traditional Lindenmayer system, rules replace a single character (called the predecessor) with a string
        (called the successor).  In stochastic Lindenmayer systems, this successor can be chosen at random from a
        number of options. This function is modeled after a stochastic L-system, but with a few modifications;
            In this function, predecessors and successors are defined using Regex, which allows for multi-character and
        pattern-based matching, as well as smart replacement via capture groups. This also opens the door for characters
        or character sets which would match to more than one rule, meaning that, unlike L-systems, the order of rule
        application here matters.  With ordered rule application comes the possibility of a later rule affecting the
        successors of the previous rules inside of a single iteration; what I refer as a rule being "protected".
        :param Inputs:
            :param liRules:
                list of rule dictionaries. Each dctRule must have the following pairs:
                        "name":
                            string. Becomes description for progress bar
                        "enabled":
                            boolean. Whether this rule should be applied, or skipped.
                        "protected":
                            boolean. Whether the successor(s) introduced by this rule can be modified by rules
                            later in the list.
                        "predecessor":
                            string (regex). The regex expression searched for in the find-and-replace process
                        "successor":
                            list of tuples; [(number, string (regex)),(number, string (regex))]
                            The successor is stochastically determined via random.random.
                            The first element is a number between 0 and 1 representing the max random.random
                            value for which its corresponding successor, the second element, will be chosen.
                            If the random value is above all options, the successor is an empty string.
                            The tuples MUST be sorted in ascending order by their first element.
            :param sInput:
                String. The text to be mutated by the function
            :param lIterations:
                Number. The number of times to process the string through the rules.
        :return:
            String. The input text, as transformed by (lMaxGen - lCurrGen) iterations through liRules.
    """
    # If we're out of iterations to perform...
    if lIterations <= 0:
        # ...return the input
        return sInput
    sOut = sInput
    # This protection string is a string of equal length to the output.
    # The character sOut[x] is protected if sProtect[x] == "1"
    sProtect = "0" * len(sInput)
    # Loop through each rule
    for dctRule in liRules:
        # if the rule is not enabled...
        if not dctRule["enabled"]:
            # ...skip it
            continue
        # sTempProtect serves the purpose of sProtect within each rule, as a rule is never allowed to overwrite itself
        sTempProtect = sProtect
        objRgx = re.compile(dctRule["predecessor"])
        liReplacements = dctRule["successor"]
        itMatches = objRgx.finditer(sOut)
        lOffset = 0
        # Loop through all matches
        for objMatch in tqdm(itMatches, desc=dctRule["name"], file=sys.stdout):
            lStart = objMatch.span()[0] + lOffset
            lEnd = objMatch.span()[1] + lOffset
            sShieldCheck = sTempProtect[lStart:lEnd]
            # Check whether the match overlaps any protected substrings
            if "1" in sShieldCheck:
                # If there are some zeros in here, this match could be eclipsing another match.
                if "0" not in sShieldCheck:
                    continue
                # Find the next match.  This will either be the eclipsed match, or simply the next math in the iterable
                objMatch = objRgx.search(sOut[lStart+1:])
                # If there aren't any matches left at all in the string, we're done.
                if objMatch is None:
                    break
                # Adjust lStart and lEnd to account for how we sliced the string in line 111
                lStart += objMatch.span()[0] + 1
                lEnd += objMatch.span()[0] + 1
                # sPredecessor = objMatch.group(0)
            # Choose a successor
            fRand = random.random()
            lChoice = -1
            for i in range(len(liReplacements)):
                if fRand > liReplacements[i][0]:
                    continue
                else:
                    lChoice = i
                    break
            if lChoice == -1:
                sSuccessor = ''
            else:
                # The rest of the string is used here in case there are lookahead groups that are referenced by the
                # successor pattern (since they will not be captured in objMatch.group(0))
                sSuccessor = liReplacements[lChoice][1]
                # Manually swap out backreferences, checking for all notation types: \1, \g<1>, \g<name>
                # Step backward so that \20 gets replaced by group 20, not group 2
                for i in reversed(range(len(objMatch.groups())+1)):
                    sSuccessor = sSuccessor.replace("\\" + str(i), objMatch.group(i))
                    sSuccessor = sSuccessor.replace(r"\g<" + str(i) + ">", objMatch.group(i))
                for sGroupName in objMatch.groupdict():
                    sSuccessor = sSuccessor.replace(r"\g<" + sGroupName + ">", objMatch.group(sGroupName))
            # Stitch things back together
            sOut = sOut[:lStart] + sSuccessor + sOut[lEnd:]
            # Protect the affected substring.
            sShield = "1" * len(sSuccessor)
            sTempProtect = sTempProtect[:lStart] + sShield + sTempProtect[lEnd:]
            if dctRule["protected"]:
                sProtect = sProtect[:lStart] + sShield + sProtect[lEnd:]
            else:
                sProtect = sProtect[:lStart] + "0"*len(sShield) + sProtect[lEnd:]
            # The span of the remaining regex matches has already been set, so we need to accommodate for changing
            # string lengths with the lOffset
            lOffset += len(sSuccessor) - (lEnd - lStart)
    # If we're just spinning our wheels and not transforming the string...
    if sInput == sOut:
        # ...there's no need to run through future iterations.
        return sOut
    lIterations -= 1
    sOut = lindenate(liRules, sOut, lIterations)
    return sOut

def lindenator(liRules, sInput="", lIterations=1, lMaxReturns=None):
    """returns a generator object that returns lIterations additional iteration(s) (by default, 1) of lindenate from its
        previous return. First return is simply sInput. if specified, exhausts after lMaxReturns.
    """
    # Are infinite loops better than recursion? I think so
    # yield sInput
    # yield from lindenator(liRules, lindenate(liRules, sInput, lIterations), lIterations)
    if lMaxReturns is None:
        while True:
            yield sInput
            sInput = lindenate(liRules, sInput, lIterations)
    elif lMaxReturns > 0:
        for _i in range(lMaxReturns):
            yield sInput
            sInput = lindenate(liRules, sInput, lIterations)

def main():
    liRules = [
        {
            "name": "Rule1",
            "enabled": False,
            "protected": False,
            "predecessor": r"test",
            "successor": [(1, r"ans")]
        }, {
            "name": "Rule2",
            "enabled": True,
            "protected": True,
            "predecessor": r"1",
            "successor": [(1, r"3")]
        }, {
            "name": "Rule3",
            "enabled": True,
            "protected": True,
            "predecessor": r"(2(?P<middleLetter>[a-z])2)",
            "successor": [(1, r"Z\g<middleLetter>Z")]
        }, {
            "name": "Rule4",
            "enabled": True,
            "protected": True,
            "predecessor": r"[a-zA-Z][\d]",
            "successor": [(.5, r"7"), (1, r"9")]
        },
    ]
    liOverlapRules = [
        {
            "name": "Axiom",
            "enabled": True,
            "protected": True,
            "predecessor": r"^$",
            "successor": [(1, "ABBAAAA")]
        }, {
            "name": "Rule One",
            "enabled": True,
            "protected": True,
            "predecessor": r"(.)(?=AAA)",
            "successor": [(1, "Z")]
        }, {
            "name": "Rule Two",
            "enabled": True,
            "protected": True,
            "predecessor": r"AA",
            "successor": [(1, "CC")]
        }, {
            "name": "Rule Three",
            "enabled": True,
            "protected": True,
            "predecessor": r"A",
            "successor": [(1, "B")]
        }, {
            "name": "Rule Four",
            "enabled": True,
            "protected": True,
            "predecessor": r"B",
            "successor": [(1, "AAAA")]
        },

    ]
    """
    print(lindenate(liRules, "ttest8est82a22b2", 1))
    print(lindenate(liRules, "ttest8est82a22b2", 2))
    print(lindenate(liOverlapRules, "BBAAAA"))
    print(lindenate(liOverlapRules, lIterations=8))
    """
    sInput = input("test?")
    objLReturn = generator_looper(partial(lindenator, liOverlapRules, sInput, lMaxReturns=10))
    bKeepGoing = True
    while bKeepGoing:
        print(next(objLReturn))
        sInput = input("type 'end' to end")
        if sInput == "end":
            bKeepGoing = False
    print("ended")

if __name__ == "__main__":
    main()


import json
import pickle
import random
from os.path import join, dirname

import nltk
from nltk.corpus import treebank
from nltk.tag.sequential import ClassifierBasedPOSTagger

MODEL_META = {
    "corpus": "treebank",
    "lang": "en",
    "model_id": "nltk_treebank_clftagger",
    "tagset": "Penn Treebank",
    "algo": "ClassifierBasedPOSTagger",
    "required_packages": ["nltk"]
}
# initializing training and testing set
nltk.download('treebank')

META = join(dirname(dirname(dirname(__file__))), "JarbasModelZoo", "res")
meta_path = join(META, MODEL_META["model_id"] + ".json")

corpus = treebank.tagged_sents()  # 3914
random.shuffle(corpus)
train_data = corpus[:3000]
test_data = corpus[3000:]

tagger = ClassifierBasedPOSTagger(train=train_data)

a = tagger.evaluate(test_data)
MODEL_META["accuracy"] = a
with open(meta_path, "w") as f:
    json.dump(MODEL_META, f)
print("Accuracy: ", a)  # 0.9309734513274336

# save pickle
path = join(dirname(dirname(dirname(__file__))),
            "models", "postag", MODEL_META["model_id"] + ".pkl")

with open(path, "wb") as f:
    pickle.dump(tagger, f)


import sys
from collections import deque
N, M, V = map(int, sys.stdin.readline().split())

graph = [[0]*(N+1) for i in range(N+1)]

#인접행렬생성
for i in range(M):
    a, b = map(int, sys.stdin.readline().split())
    graph[a][b] = graph[b][a] = 1

visited = [False] * (N + 1)

def dfs(V):
    visited[V] = True
    print(V, end=' ')
    for i in range(1, N+1):
        if not visited[i] and graph[V][i] == 1:
            dfs(i)

def bfs(V):
    print()
    visited = [False] * (N + 1)
    queue = deque([V])
    while queue:
        visited[V] = True
        V = queue.popleft()
        print(V, end=" ")
        for i in range(1, N+1):
            if not visited[i] and graph[V][i] == 1:
                queue.append(i)
                visited[i] = True

dfs(1)
bfs(1)


import logging
from typing import Dict

import grpc

from needlestack.apis import collections_pb2
from needlestack.apis import servicers_pb2
from needlestack.apis import servicers_pb2_grpc
from needlestack.apis import serializers
from needlestack.collections.collection import Collection
from needlestack.collections.shard import Shard
from needlestack.cluster_managers import ClusterManager
from needlestack.servicers.settings import BaseConfig
from needlestack.utilities.rpc import unhandled_exception_rpc

logger = logging.getLogger("needlestack")

class SearcherServicer(servicers_pb2_grpc.SearcherServicer):
    """A gRPC servicer to perform kNN queries on in-memory index structures"""

    collections: Dict[str, Collection]
    collection_protos: Dict[str, collections_pb2.Collection]

    def __init__(self, config: BaseConfig, cluster_manager: ClusterManager):
        self.config = config
        self.cluster_manager = cluster_manager
        self.collections = {}
        self.collection_protos = {}
        self.cluster_manager.register_searcher()
        self.load_collections()

    @unhandled_exception_rpc(servicers_pb2.SearchResponse)
    def Search(self, request, context):
        X = serializers.proto_to_ndarray(request.vector)
        k = request.count
        collection = self.get_collection(request.collection_name)

        if len(X.shape) == 1:
            X = X.reshape(1, -1)

        if collection.dimension == X.shape[1]:
            results = collection.query(X, k, request.shard_names)
            items = [item for i, item in enumerate(results) if i < k]
            return servicers_pb2.SearchResponse(items=items)
        else:
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
            context.set_details(
                f"Collection {collection.name} expected matrix shaped ({collection.dimension}), got {X.shape}"
            )
            return servicers_pb2.SearchResponse()

    @unhandled_exception_rpc(servicers_pb2.RetrieveResponse)
    def Retrieve(self, request, context):
        collection = self.get_collection(request.collection_name)
        item = collection.retrieve(request.id, request.shard_names)
        if item is not None:
            return servicers_pb2.RetrieveResponse(item=item)
        else:
            return servicers_pb2.RetrieveResponse()

    @unhandled_exception_rpc(collections_pb2.CollectionsLoadResponse)
    def CollectionsLoad(self, request, context):
        self.load_collections()
        return collections_pb2.CollectionsLoadResponse()

    def get_collection(self, name: str) -> Collection:
        return self.collections[name]

    def load_collections(self):
        """Load collections from Zookeeper configs

        There are 4 states to handle for each collection:
            - A new collection needs to be loaded
            - An existing collection needs to be dropped
            - An existing collection added/dropped shards
            - No changes
        """
        collection_protos = self.cluster_manager.list_local_collections(
            include_state=False
        )
        current_collections = {name for name in self.collection_protos.keys()}
        new_collections = {proto.name for proto in collection_protos}
        for proto in collection_protos:
            if proto.name in current_collections:
                self._modify_collection(proto)
            else:
                self._add_collection(proto)
        for name in current_collections:
            if name not in new_collections:
                self._drop_collection(name)
        for collection in self.collections.values():
            if collection.update_available():
                logger.debug(f"Update collection {collection.name}")
                self.cluster_manager.set_local_state(
                    collections_pb2.Replica.BOOTING, collection.name
                )
                collection.load()
                self.cluster_manager.set_local_state(
                    collections_pb2.Replica.ACTIVE, collection.name
                )
        self.collection_protos = {proto.name: proto for proto in collection_protos}

    def _add_collection(self, proto: collections_pb2.Collection):
        logger.debug(f"Add collection {proto.name}")
        collection = Collection.from_proto(proto)
        self.cluster_manager.set_local_state(
            collections_pb2.Replica.BOOTING, collection.name
        )
        self.collections[collection.name] = collection
        collection.load()
        self.cluster_manager.set_local_state(
            collections_pb2.Replica.ACTIVE, collection.name
        )

    def _drop_collection(self, name: str):
        logger.debug(f"Drop collection {name}")
        del self.collections[name]

    def _modify_collection(self, proto: collections_pb2.Collection):
        old_proto = self.collection_protos[proto.name]
        if old_proto.SerializeToString() != proto.SerializeToString():
            collection = self.get_collection(proto.name)
            collection.merge_proto(proto)

            old_shards = {shard.name: shard for shard in old_proto.shards}
            new_shards = {shard.name: shard for shard in proto.shards}

            for name, new_shard in new_shards.items():
                if name not in old_shards:
                    logger.debug(f"Add collection shard {proto.name}/{name}")
                    self.cluster_manager.set_local_state(
                        collections_pb2.Replica.BOOTING, collection.name, name
                    )
                    collection.add_shard(Shard.from_proto(new_shard))
                elif (
                    new_shard.SerializeToString()
                    != old_shards[name].SerializeToString()
                ):
                    logger.debug(f"Update collection shard {proto.name}/{name}")
                    self.cluster_manager.set_local_state(
                        collections_pb2.Replica.BOOTING, collection.name, name
                    )
                    collection.drop_shard(name)
                    collection.add_shard(Shard.from_proto(new_shard))

            for name in old_shards.keys():
                if name not in new_shards:
                    logger.debug(f"Drop collection shard {proto.name}/{name}")
                    collection.drop_shard(name)

            collection.load()
            self.cluster_manager.set_local_state(
                collections_pb2.Replica.ACTIVE, collection.name, name
            )


import sys
import time
import os
import gc
import json
import argparse
from pathlib import Path

os.environ["JAX_PLATFORMS"] = "cpu"

import jax
import flax
import numpy as np
import jax.numpy as jnp
import orbax
import orbax.checkpoint
from optax import MaskedNode
from etils import epath
from praxis import base_hyperparams
from praxis import pax_fiddle
from praxis import py_utils
from paxml import checkpoints
from paxml import checkpoint_managers
from paxml import train_states
from paxml import trainer_lib
from flax.traverse_util import flatten_dict, unflatten_dict

from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.generation import GenerationConfig

try:
    import torch
except:
    command = (
        "pip install torch==2.0.0+cpu torchvision==0.15.1+cpu torchaudio==2.0.1 --index-url"
        "https://download.pytorch.org/whl/cpu", "pip install transformers_stream_generator",
        "pip install accelerate"
    )
    subprocess.run(command, stdout=subprocess.PIPE, shell=True)
    import torch

TrainState = train_states.TrainState
CheckpointType = checkpoints.CheckpointType
Checkpointer = checkpoints.Checkpointer
PaxCheckpointHandler = checkpoints.PaxCheckpointHandler
NestedMap = py_utils.NestedMap
checkpoint_type = CheckpointType.GDA
SAVE_INTERVAL_STEPS = 1

LLAMA_STANDARD_CONFIGS = {
    "7B": {
        "dim": 4096,
        "intermediate_size": 11008,
        "n_layers": 32,
        "n_heads": 32,
        "norm_eps": 1e-6,
        "vocab_size": 151936
    },
    "14B": {
        "dim": 5120,
        "intermediate_size": 13696,
        "n_layers": 40,
        "n_heads": 40,
        "norm_eps": 1e-6,
        "vocab_size": 152064
    },
}

step = 0
model_size = '14B'
params = LLAMA_STANDARD_CONFIGS[model_size]
n_layers = params["n_layers"]
n_heads = params["n_heads"]
dim = params["dim"]
intermediate_size = params["intermediate_size"]
head_dim = dim // n_heads
save_opt = False

model = AutoModelForCausalLM.from_pretrained(f"Qwen/Qwen-{model_size}", device_map="auto", trust_remote_code=True).eval()
# pip install tiktoken
# tokenizer = AutoTokenizer.from_pretrained(f"Qwen/Qwen-{model_size}", trust_remote_code=True)

ckpt = {}
for k, v in model.named_parameters():
    ckpt[k] = v
assert len(ckpt) > 0, print(f"ckpt is empty, please model path whether right or error.....")

save_dir = f'gs://llm_base_models/qwen/{model_size}/paxml/checkpoints/'
options = checkpoint_managers.CheckpointManagerOptions(
    max_to_keep=10,
    save_interval_steps=SAVE_INTERVAL_STEPS,
    cleanup_tmp_directories=True,
)
checkpointer = Checkpointer(
    PaxCheckpointHandler(
        enforce_restore_shape_check=False,
        use_ocdbt=False,
    )
)
save_dir = epath.Path(save_dir)
checkpoint_manager = checkpoint_managers.OrbaxCheckpointManager(
    save_dir,
    checkpointer,
    train_input_checkpointer=False,
    options=options,
    checkpoint_type=checkpoint_type,
    tensorstore_use_ocdbt=False,
)

for k, v in model.named_parameters():
    print(k, v.shape)

paxml_to_hf_key_and_shape = {
    "params.lm.embedding_lookup.emb_var": {
        "shape": (vocab_size, dim),
        "map_to_hf": "wte.weight",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.ff_layer.ffn_layer1.linear.w": {
        "shape": (dim, intermediate_size),
        "map_to_hf": "w1.weight",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.ff_layer.ffn_layer1_gate.linear.w": {
        "shape": (dim, intermediate_size),
        "map_to_hf": "w2.weight",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.ff_layer.ffn_layer2.linear.w": {
        "shape": (intermediate_size, dim),
        "map_to_hf": "mlp.c_proj.weight",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.self_attention.query.w": {
        "shape": (dim, n_heads, head_dim),
        "map_to_hf": "q_proj.weight",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.self_attention.query.b": {
        "shape": (dim, n_heads, head_dim),
        "map_to_hf": "q_proj.bias",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.self_attention.key.w": {
        "shape": (dim, n_heads, head_dim),
        "map_to_hf": "k_proj.weight",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.self_attention.key.b": {
        "shape": (dim, n_heads, head_dim),
        "map_to_hf": "k_proj.bias",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.self_attention.value.w": {
        "shape": (dim, n_heads, head_dim),
        "map_to_hf": "v_proj.weight",
    },
     "params.lm.transformer.repeat.sub.x_layers_0.self_attention.value.b": {
        "shape": (dim, n_heads, head_dim),
        "map_to_hf": "v_proj.bias",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.self_attention.post.w": {
        "shape": (dim, n_heads, head_dim),
        "map_to_hf": "attn.c_proj.weight",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.layer_norm.scale": {
        "shape": (dim,),
        "map_to_hf": "ln_1.weight",
    },
    "params.lm.transformer.repeat.sub.x_layers_0.ff_layer.layer_norm.scale": {
        "shape": (dim,),
        "map_to_hf": "ln_2.weight",
    },
    "params.lm.final_ln.scale": {"shape": (dim,), "map_to_hf": "ln_f.weight"},
    "params.lm.softmax.logits_ffn.linear.w": {
        "shape": (dim, vocab_size),
        "map_to_hf": "lm_head",
    },
}

gold_w = ckpt
split_qkv = {}
for k, v in gold_w.items():
    if v.dtype == torch.float32:
        pass
    else:
        v = v.to(torch.float32)
    # o_proj不进行transpose，是个坑
    if len(v.shape) == 2 and "wte.weight" not in k and "attn.c_proj.weight" not in k:
        v = v.transpose(1, 0)
    else:
        print(f"No transpose k: {k}")

    if "c_attn" in k:
        qq = k.replace("c_attn", "q_proj")
        kk = k.replace("c_attn", "k_proj")
        vv = k.replace("c_attn", "v_proj")
        print(f'v.shape')
        if len(v.shape) == 1:
#             v = v.reshape(n_heads, 3 * head_dim)
            split_qkv[qq] = v[..., :dim].detach().numpy().reshape(-1)
            split_qkv[kk] = v[..., dim: 2 * dim].detach().numpy().reshape(-1)
            split_qkv[vv] = v[..., 2 * dim: ].detach().numpy().reshape(-1)
        elif len(v.shape) == 2:
#             v = v.reshape(dim, n_heads, 3 * head_dim)
            split_qkv[qq] = v[..., :dim].detach().numpy().reshape(dim, -1)
            split_qkv[kk] = v[..., dim: 2 * dim].detach().numpy().reshape(dim, -1)
            split_qkv[vv] = v[..., 2 * dim: ].detach().numpy().reshape(dim, -1)
        else:
            raise ValueError(f'qkv shape is error!!!')
    else:
        split_qkv[k] = v.detach().numpy()

for k, v in split_qkv.items():
    print(k, v.shape)

import re

trans_result = {}
flag = 0
with jax.default_device(jax.devices("cpu")[0]):
    for k, v in paxml_to_hf_key_and_shape.items():
        v = v["map_to_hf"]
        k = tuple(k.split("."))
        values = []
        for gold_key, glod_values in split_qkv.items():
            flag = 0
            if v in gold_key:
                flag = 1
                match_res = re.findall("q_proj|k_proj|v_proj|attn.c_proj", v)
                if match_res:
                    if len(glod_values.shape) > 1:
                        glod_values = glod_values.reshape(dim, n_heads, head_dim)
                    else:
                        glod_values = glod_values.reshape(n_heads, head_dim)
                try:
                    layer_index = int(re.findall("\d+", gold_key)[0])
                except:
                    layer_index = 0
                values.append([layer_index, glod_values])
                print(f"match_res: {match_res}|| {len(values)}")

        values = sorted(values, key=lambda x: x[0])
        if len(values) > 1:
            stack_values = np.stack(list(zip(*values))[1])
        else:
            stack_values = values[0][1]
        trans_result[k] = stack_values

print(f"Please simple check model shape and dtype...")
for k, v in trans_result.items():
    k = '.'.join(k)
    print(k, v.shape, v.dtype)

if step is None:
    latest_step = checkpoint_manager.latest_step()
    if save_dir == read_dir:
        step = latest_step + SAVE_INTERVAL_STEPS if latest_step is not None else SAVE_INTERVAL_STEPS
    else:
        step = latest_step

print(f"Model save step is {step}")
start = time.time()

if save_opt:
    with jax.default_device(jax.devices("cpu")[0]):

        opt_state_mv = jax.tree_map(lambda x: jnp.zeros_like(x), trans_result)
        temp_no_prefix, temp_other = {}, {}
        for key_tuple, param in opt_state_mv.items():
            if "repeat" in key_tuple:
                temp_no_prefix[key_tuple] = MaskedNode()
                temp_other[key_tuple] = param
            else:
                temp_no_prefix[key_tuple] = param
                temp_other[key_tuple] = MaskedNode()

        temp_no_prefix = unflatten_dict(temp_no_prefix)
        temp_other = unflatten_dict(temp_other)

        no_prefix = {"count": jnp.array(step), "m": temp_no_prefix, "v": temp_no_prefix}
        other = {"count": jnp.array([step] * n_layers), "m": temp_other, "v": temp_other}
        trans_opt_states = {
            "no_prefix": [{"count": jnp.array(step)}] * 2 + [no_prefix, {"count": jnp.array(step)}],
            f"p#{n_layers}#i-1": [{"count": jnp.array([step] * n_layers)}] * 2
            + [other, {"count": jnp.array([step] * n_layers)}],
        }
        trans_opt_states = [trans_opt_states]
else:
    trans_opt_states = []

new_trainstate = TrainState(
    step=jnp.array(step),
    mdl_vars=unflatten_dict(trans_result),
    opt_states=trans_opt_states,
)
padded_global_shapes = jax.tree_map(
    lambda x: jax.ShapeDtypeStruct(shape=x.shape, dtype=x.dtype) if hasattr(x, "shape") else x,
    new_trainstate,
)
print(f"padded_global_shapes: {padded_global_shapes}")
checkpoint_manager.save(
    step, new_trainstate, padded_global_shapes, train_input_pipeline=None, force=False
)
print(f"Saved model finished. take time: {time.time() - start}s...")