import pickle

import numpy as np
import pyspark.sql.functions as sf
from scalpel.core.cohort import Cohort
from scalpel.core.cohort_collection import CohortCollection
from scalpel.core.io import get_logger, quiet_spark_logger, \
    get_spark_context
from scalpel.core.util import rename_df_columns
from scalpel.drivers.conv_sccs import ConvSccsFeatureDriver

from parameters.experience import (
    get_exposures,
    get_fractures,
    get_subjects,
    get_the_followup,
    read_parameters,
)
from parameters.fall_parameters import STUDY_END, STUDY_START, AGE_REFERENCE_DATE

BUCKET_ROUNDING = "ceil"
RUN_CHECKS = True
AGE_GROUPS = [0, 64, 67, 70, 73, 76, 79, 80, np.Inf]

def pickle_object(obj, path):
    with open(path, "wb") as file:
        pickle.dump(obj, file)

def delete_prevalent(outcomes: Cohort, followup: Cohort) -> Cohort:
    fup_events = rename_df_columns(followup.events, prefix="fup_", keys=("patientID",))
    out_events = outcomes.events.join(fup_events, on="patientID")

    is_valid = (sf.col("start") >= sf.col("fup_start")) & (
        sf.col("start") <= sf.col("fup_end")
    )
    # no condition on outcome end, as it is always null in this study

    prevalent_events = out_events.where(~is_valid)

    prevalent_cases = Cohort(
        "prevalent cases",
        "prevalent_cases",
        subjects=prevalent_events.select("patientID").distinct(),
        events=prevalent_events.select(*outcomes.events.columns),
    )
    return outcomes.difference(prevalent_cases)

def add_pre_exposures(exposures: Cohort, followup: Cohort, delay: int) -> Cohort:
    pre_exposures_events = exposures.events
    pre_exposures_events_cols = pre_exposures_events.columns
    pre_exposures_events = pre_exposures_events.withColumnRenamed('start', 'old_start')
    pre_exposures_events = pre_exposures_events.withColumnRenamed('value', 'old_value')
    pre_exposures_events = pre_exposures_events.withColumn('start',
                                                           sf.date_sub('old_start',
                                                                       int(delay)))
    pre_exposures_events = pre_exposures_events.withColumn('value',
                                                           sf.concat(sf.lit('pre-'),
                                                                     sf.col('old_value')
                                                                     )
                                                           )

    fup_events = rename_df_columns(followup.events, prefix="fup_", keys=("patientID",))
    pre_exposures_events = pre_exposures_events.join(fup_events, on="patientID")

    is_valid = (sf.col("start") >= sf.col("fup_start")) & (
            sf.col("start") <= sf.col("fup_end")
    )
    pre_exposures_events = pre_exposures_events.where(is_valid)\
        .select(*pre_exposures_events_cols)

    events = exposures.events.union(pre_exposures_events)
    return Cohort("Exposures w/ pre-exposures", "", events.select('patientID').drop_duplicates(), events)

def main():
    spark_context = get_spark_context()
    quiet_spark_logger(spark_context.sparkSession)
    spark_context.sparkSession.conf.set("spark.sql.session.timeZone", "UTC")
    logger = get_logger()

    valid_start = sf.col("start").between(STUDY_START, STUDY_END)

    logger.info("Reading parameters")
    parameters = read_parameters()
    logger.info("Reading metadata")
    md = CohortCollection.from_json(parameters["path"])

    clean_outcomes = md.get("fractures").events.where(valid_start)
    md = md.add_cohort(
        "fractures",
        Cohort(
            "fractures",
            "Subjects with fractures",
            clean_outcomes.select("patientID").distinct(),
            clean_outcomes,
        ),
    )

    logger.info("Add age information to patients")

    base_cohort = get_subjects(md, parameters).cache()
    outcomes = get_fractures(md, parameters).cache()
    followup = get_the_followup(md, parameters).cache()
    exposures = add_pre_exposures(get_exposures(md, parameters), followup, 15).cache()  # TODO: HERE, rerun DRUG CONTROL EXPERIMENT

    min_base = base_cohort.intersect_all([followup, exposures, outcomes]).cache()
    min_base.add_subject_information(base_cohort, "omit_all")

    min_exp = exposures.intersection(min_base).cache()
    min_out = outcomes.intersection(min_base).cache()
    min_fup = followup.intersection(min_base).cache()

    logger.debug("Min base subject count {}".format(min_base.subjects.count()))

    min_incident_out = delete_prevalent(min_out, min_fup)
    loader = ConvSccsFeatureDriver(
        min_base,
        min_fup,
        min_exp,
        min_incident_out,
        parameters["bucket"],
        STUDY_START,
        STUDY_END,
        AGE_REFERENCE_DATE,
        AGE_GROUPS,
        BUCKET_ROUNDING,
        RUN_CHECKS,
        outcomes_split_column="category",
    )
    logger.info("Loading features")
    features, labels, censoring = loader.load()
    mapping = loader.mappings[0]
    n_age_groups = loader.n_age_groups

    print("Number of samples {}".format(len(censoring)))
    pickle_object(features, "features")
    pickle_object(labels, "labels")
    pickle_object(censoring, "censoring")
    pickle_object(mapping, "mapping")
    pickle_object(n_age_groups, "age_groups")

if __name__ == "__main__":
    main()


# Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: miha@reciprocitylabs.com
# Maintained By: miha@reciprocitylabs.com

import unittest
from mock import patch

from ggrc import app  # noqa
from ggrc import notification

class TestNotificationsInit(unittest.TestCase):

  @patch("ggrc.notification.get_filter_data")
  def test_get_notification_data(self, get_filter_data):
    """ Test that data does not contain empty emails """

    get_filter_data.return_value = {
        "email@example.com": {},
        "": {},
    }
    notifications = notification.get_notification_data([1, 2])
    self.assertIn("email@example.com", notifications)
    self.assertNotIn("", notifications)


import shutil
import sys
import os
import subprocess
import numpy as np
from scipy.optimize import minimize
import pickle
from pyECP import ecp

def plot_ecps(ecps,rmax):
	import matplotlib.pyplot as plt
	npts=100
	x = np.linspace(0.001,rmax,npts)
	lines=['-','--','-.',':']
	for i,ecp in enumerate(ecps):
		for l in range(len(ecp.n)-1):
			y = [ ecp.eval_channel(l,x[i]) for i in range(npts) ]
			plt.plot(x,y,label='l = '+str(l)+' ECP'+str(i),linestyle = lines[i])
		y = [ ecp.eval_local(x[i]) for i in range(npts) ]
		plt.plot(x,y,label='local ECP'+str(i),linestyle= lines[i])
	plt.legend(frameon=False)
	plt.show()

def get_en_ae():
	states = ['I-s2d6','I-s1d7','I-d8','II-s1d6','II-d7','IX','XV','A-s2d7','III','IV','V','VI','VII','VIII']
	energies = []
	for i,state in enumerate(states):
		statefile = 'states/'+state+'.ip.d'
		shutil.copy(statefile,'runs/ip.d')
		shutil.copy('numHF','runs')
		f = open('runs/tmp.txt','w')
		subprocess.call('./numHF',stdout=f,cwd='runs')
		f.close()
		outfile = open('runs/out','rb')
		energy = float(outfile.readline().split()[0])
		energies.append(energy)
		outfile.close()
	return energies

def get_en_pp():
	states = ['I-s2d6','I-s1d7','I-d8','II-s1d6','II-d7','IX','XV','A-s2d7','III','IV','V','VI','VII','VIII']
	energies = []
	for i,state in enumerate(states):
		statefile = 'pp_states/'+state+'.ip.d'
		shutil.copy(statefile,'runs/ip.d')
		shutil.copy('numHF','runs')
		f=open('runs/tmp.txt','w')
		subprocess.call('./numHF',stdout=f,cwd='runs')
		f.close()
		outfile = open('runs/out','rb')
		energy = float(outfile.readline().split()[0])
		energies.append(energy)
		outfile.close()
	return energies

def generate_objective(opt,obs,show=False):
	opt_vals = []
	[ opt_vals.append(opt[i] - opt[0]) for i in range(1,len(opt)) ]
	opt_vals = np.array(opt_vals)

	obs_vals = []
	[ obs_vals.append(obs[i] - obs[0]) for i in range(1,len(obs)) ]
	obs_vals = np.array(obs_vals)

	f = ((opt_vals-obs_vals)**2).mean()

	if show:
		print(opt_vals)
		print(obs_vals)
		print(obs_vals-opt_vals)
		print(f)

	return f

def optECP(x,*args):
	ae = args[0]
	Zeff = args[1]
	n = args[2]
	log_expt = args[3]
	a = []
	c = []
	k = 0
	for i in range(len(n)):
		a.append([])
		c.append([])
		for j in range(len(n[i])):
			if log_expt:
				a[i].append(np.exp(x[k]))
			else:
				a[i].append(x[k])
			k+=1
			c[i].append(x[k])
			k+=1
	pp = ecp.ECP(Zeff,n,a,c)
	pp.write_ecp('runs/pp.d')
	obs = get_en_pp()
	return generate_objective(ae,obs)

Nfeval = 1
def callbackF(Xi):
	global Nfeval
	f = open('progress.dat','a')
	f.write('{}    '.format(Nfeval))
	for i in range(len(Xi)):
		if i%2 == 0:
			f.write('{}    '.format(Xi[i]))
		else:
			f.write('{}    '.format(Xi[i]))
	f.write('\n')
	f.close()
	Nfeval += 1

def run_optimization(log_expt=True):
	init_pp = ecp.ECP()
	init_pp.read_ecp('initial.pp.d')
	ae = get_en_ae()

	x0 = []
	for i in range(len(init_pp.n)):
		for j in range(len(init_pp.n[i])):
			if log_expt:
				x0.append(np.log(init_pp.alpha[i][j]))
			else:
				x0.append(init_pp.alpha[i][j])
			x0.append(init_pp.coeff[i][j])

	bounds = []
	for x in x0:
		x1=x/10
		x2= x*10
		if x1>x2:
			bounds.append((x2,x1))
		else:
			bounds.append((x1,x2))
	#adding shift and bounds on shift
	#x0.append(0.0)
	#bounds.append((None,None))
	#bounds = tuple(bounds)

	if log_expt:
		cons = ({'type': 'eq',   'fun': lambda x: x[9]*np.exp(x[8]) - x[11]},                                  #bfd local constraint
				{'type': 'eq',   'fun': lambda x: x[9] - init_pp.zeff},                                         # has to be Zeff
				{'type': 'ineq', 'fun': lambda x: np.exp(x[14])*x[15] + np.exp(x[12])*x[13] + np.exp(x[0])*x[1] + np.exp(x[2])*x[3]}, #s smooth constraint
				{'type': 'ineq', 'fun': lambda x: np.exp(x[14])*x[15] + np.exp(x[12])*x[13] + np.exp(x[4])*x[5] + np.exp(x[6])*x[7]}) #p smooth constraint
	else:
		cons = ({'type': 'eq',   'fun': lambda x: x[9]*x[8] - x[11]},                  #bfd local constraint
				{'type': 'eq',   'fun': lambda x: x[9] - init_pp.zeff},                 # has to be Zeff
				{'type': 'ineq', 'fun': lambda x: x[14]*x[15] + x[12]*x[13] + x[0]*x[1] + x[2]*x[3]}, #s smooth constraint
				{'type': 'ineq', 'fun': lambda x: x[14]*x[15] + x[12]*x[13] + x[4]*x[5] + x[6]*x[7]}) #p smooth constraint

	print('Starting optimizations')
	for eps in [1.0,0.1,0.01,0.001,0.0001]:
		try:
			with open('results/res_eps'+str(eps)+'.p','rb') as f:
				print('Already completed optimization with eps {}'.format(eps))
				prev_result = pickle.load(f)
				x0 = prev_result.x
		except IOError:
			f = open('progress.dat','w')
			f.write('EPS: {}   \n'.format(eps))
			f.write('{}    '.format('It'))
			for i in range(len(x0)):
				f.write('X{}    '.format(i))
			f.write('\n')
			f.close()
			result = minimize(optECP, x0, args=(ae,init_pp.zeff,init_pp.n,log_expt),
					 method='SLSQP',
					 constraints = cons,
					 bounds = bounds,
					 callback=callbackF,
					 options={'eps': eps,
						      'ftol': 1e-07,
							  'maxiter': 200})
			x0 = result.x
			pickle.dump(result,open('results/res_eps'+str(eps)+'.p','wb'))

def test_results(log_expt=True):
	opts = []
	for eps in [1.0,0.1,0.01,0.001,0.0001]:
		print('eps: {}'.format(eps))
		try:
			with open('results/res_eps'+str(eps)+'.p','rb') as f:
				opt = pickle.load(f)
				print(opt)
				opts.append(opt)
		except:
			continue
		print()
		print()

	init_pp = ecp.ECP()
	init_pp.read_ecp('initial.pp.d')
	init_pp.write_ecp('runs/pp.d')
	init = get_en_pp()

	pps = []
	for opt in opts:
		a = []
		c = []
		k = 0
		for i in range(len(init_pp.n)):
			a.append([])
			c.append([])
			for j in range(len(init_pp.n[i])):
				if log_expt:
					a[i].append(np.exp(opt.x[k]))
				else:
					a[i].append(opt.x[k])
				k+=1
				c[i].append(opt.x[k])
				k+=1
		opt_pp = ecp.ECP(init_pp.zeff,init_pp.n,a,c)
		pps.append(opt_pp)
	#plot_ecps([init_pp]+pps,1.5)
	pps[-1].write_ecp('pp.d')
	pps[-1].write_ecp('runs/pp.d')
	en = get_en_pp()
	ae = get_en_ae()
	print('INITIAL')
	generate_objective(ae,init,show=True)
	print('FINAL')
	generate_objective(ae,en,show=True)

if __name__ == '__main__':
	pp = ecp.ECP()
	pp.read_ecp('initial.pp.d')
	pp.write_ecp('runs/pp.d')
	print(get_en_pp())


import os
import sys
import unittest

import pytorch_lightning as pl
import pytorch_lightning.loggers
from pytorch_lightning.utilities.parsing import AttributeDict

from deep_depth_transfer import PoseNetResNet, DepthNetResNet, UnsupervisedCriterion
from deep_depth_transfer.data import KittiDataModuleFactory
from deep_depth_transfer.models import ScaledUnsupervisedDepthModel
from test.data_module_mock import DataModuleMock

if sys.platform == "win32":
    WORKERS_COUNT = 0
else:
    WORKERS_COUNT = 4

class TestUnsupervisedDepthModel(unittest.TestCase):
    def setUp(self) -> None:
        current_folder = os.path.dirname(os.path.abspath(__file__))
        dataset_folder = os.path.join(os.path.dirname(current_folder), "datasets", "kitti")
        data_module_factory = KittiDataModuleFactory(range(0, 301, 1), directory=dataset_folder)
        self._data_module = data_module_factory.make_dataset_manager(
            final_image_size=(128, 384),
            transform_manager_parameters={"filters": True},
            batch_size=1,
            num_workers=WORKERS_COUNT,
            split=(0.8, 0.1, 0.1)
        )
        self._data_module = DataModuleMock(self._data_module)

        pose_net = PoseNetResNet()
        depth_net = DepthNetResNet()
        criterion = UnsupervisedCriterion(self._data_module.get_cameras_calibration(), 1, 1)

        params = AttributeDict(
            lr=1e-3,
            beta1=0.99,
            beta2=0.9,
            scale_lr=1e-3,
            initial_log_scale=0.,
            initial_log_min_depth=0.,
            initial_log_pose_scale=0.,
        )
        self._model = ScaledUnsupervisedDepthModel(params, pose_net, depth_net, criterion).cuda()

    def test_unsupervised_depth_model(self):
        tb_logger = pl.loggers.TensorBoardLogger('logs/')
        trainer = pl.Trainer(logger=tb_logger, max_epochs=1, gpus=1, progress_bar_refresh_rate=20)
        trainer.fit(self._model, self._data_module)


import shutil
from pathlib import Path

import httplib2
import sys
import csv
import re
from datetime import datetime, timedelta

from googleapiclient.discovery import build_from_document
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import argparser, run_flow

SITUATION_TXT = "situation.txt"

SOURCES = [
    #{"type": "channel", "id": "UCJ0dhh1UFNjzdCHa12IT-4A", "name": "Silverman", "cat": "lateshow"},
    #{"type": "channel", "id": "UCarEovlrD9QY-fy-Z6apIDQ", "name": "Minhaj", "cat": "lateshow"},
    {"type": "channel", "id": "UC3XTzVzaHQEd30rQbuvCtTQ", "name": "Oliver", "cat": "lateshow"},
    {"type": "channel", "id": "UCwWhs_6x42TyRM4Wstoq8HA", "name": "Noah", "cat": "lateshow"},
    {"type": "channel", "id": "UC18vz5hUUqxbGvym9ghtX_w", "name": "Bee", "cat": "lateshow"},
    {"type": "channel", "id": "UCVTyTA7-g9nopHeHbeuvpRA", "name": "Meyers", "cat": "lateshow"},
    {"type": "channel", "id": "UCMtFAi84ehTSYSE9XoHefig", "name": "Colbert", "cat": "lateshow"},
    {"type": "channel", "id": "UCy6kyFxaMqGtpE3pQTflK8A", "name": "Maher", "cat": "lateshow"},
    {"type": "channel", "id": "UC8-Th83bH_thdKZDJCrn88g", "name": "Fallon", "cat": "lateshow"},
    {"type": "channel", "id": "UCi7GJNg51C3jgmYTUwqoUXA", "name": "Conan", "cat": "lateshow"},
    {"type": "channel", "id": "UCJ0uqCI0Vqr2Rrt1HseGirg", "name": "Corden", "cat": "lateshow"},
    {"type": "channel", "id": "UCa6vGFO9ty8v5KZJXQxdhaw", "name": "Kimmel", "cat": "lateshow"},
    {"type": "channel", "id": "UCqFzWxSCi39LnW1JKFR3efg", "name": "SNL", "cat": "lateshow"},
    {"type": "channel", "id": "UC2gzy_aI-luPtEpL-GzQP6w", "name": "Singh", "cat": "lateshow"},
    #{"type": "channel", "id": "UCXIJgqnII2ZOINSWNOGFThA", "name": "FoxNews", "cat": "news"},
    #{"type": "channel", "id": "UCaXkIU1QidjPwiAYu6GcHjg", "name": "MSNBC", "cat": "news"},
    #{"type": "channel", "id": "UCupvZG-5ko_eiXAupbDfxWw", "name": "CNN", "cat": "news"},
    #{"type": "channel", "id": "UC8p1vwvWtl6T73JiExfWs1g", "name": "CBSNews", "cat": "news"},
    #{"type": "channel", "id": "UCeY0bbntWzzVIaj2z3QigXg", "name": "NBCNews", "cat": "news"},
    #{"type": "channel", "id": "UCBi2mrWuNuyYy4gbM6fU18Q", "name": "ABCNews", "cat": "news"},
    #{"type": "channel", "id": "UCzuqE7-t13O4NIDYJfakrhw", "name": "DemocracyNow", "cat": "onlineNewsComment"},
    #{"type": "channel", "id": "UC1yBKRuGpC1tSM73A0ZjYjQ", "name": "TheYoungTurks", "cat": "onlineNewsComment"},
    #{"type": "channel", "id": "UCLXo7UDZvByw2ixzpQCufnA", "name": "Vox", "cat": "onlineNewsComment"},
    #{"type": "channel", "id": "UCvsye7V9psc-APX6wV1twLg", "name": "AlexJones", "cat": "pundit"},
]

DAY_INCR = 2
DT_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
CLIENT_SECRETS_FILE = "client_secrets10.json"
YOUTUBE_READ_WRITE_SSL_SCOPE = "https://www.googleapis.com/auth/youtube.force-ssl"
YOUTUBE_API_SERVICE_NAME = "youtube"
YOUTUBE_API_VERSION = "v3"
MISSING_CLIENT_SECRETS_MESSAGE = "WARNING: Please configure OAuth 2.0"
CSV_PATH = "/Users/paulineziserman/GoogleDrive/these_db/csv/2012/2012_raw/"

def add_one_month(dt0):
    dt1 = dt0.replace(day=1)
    dt2 = dt1 + timedelta(days=32)
    dt3 = dt2.replace(day=1)
    return dt3

def get_authenticated_service():
    args = argparser.parse_args()
    flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE, scope=YOUTUBE_READ_WRITE_SSL_SCOPE,
                                   message=MISSING_CLIENT_SECRETS_MESSAGE)
    storage = Storage("%s-oauth2.json" % sys.argv[0])
    credentials = storage.get()
    if credentials is None or credentials.invalid:
        credentials = run_flow(flow, storage, args)
    with open("youtube-v3-discoverydocument.json", "r") as f:
        doc = f.read()
        return build_from_document(doc, http=credentials.authorize(httplib2.Http()))

def get_videos(youtube_api, source, d_from, d_to):
    result = youtube_api.search().list(
        part="snippet",
        channelId=source["id"],
        publishedAfter=d_from.strftime(DT_FORMAT),
        publishedBefore=d_to.strftime(DT_FORMAT),
        safeSearch="none",
        type="video",
        maxResults=50
    ).execute()
    items = result["items"]
    print("get_videos. source: " + source["name"] + " from:" + d_from.strftime(DT_FORMAT) + " to:" + d_to.strftime(
        DT_FORMAT) + " : " + str(len(items)))
    return items

def get_stats(youtube_api, video_id):
    result = youtube_api.videos().list(
        part="contentDetails,statistics",
        id=video_id
    ).execute()
    return result["items"][0]

def get_comments(youtube_api, video_id, page_token):
    try:
        result = youtube_api.commentThreads().list(
            part="snippet",
            videoId=video_id,
            pageToken=page_token,
            # order="relevance",
            textFormat="plainText",
            maxResults=100
        ).execute()
        return result, result["items"], result.get("nextPageToken", False)
    except:
        print("get_comments: API error. videoId:" + video_id)
        return None, None, None

def clean_str(dirty_string):
    coding_string = str(dirty_string.encode("ascii", "ignore"))
    clean1 = re.sub(r'\s+', ' ', coding_string).strip()
    clean2 = re.sub(r',+', ';', clean1)
    res = clean2[2:-1]
    return res

def read_video(youtube_api, video_writer, source, video):
    res_stat = get_stats(youtube_api, video["id"]["videoId"])
    details = res_stat["contentDetails"]
    stats = res_stat["statistics"]

    if "commentCount" not in stats:
        comment_count = 0
        comment_lock = True
    else:
        comment_count = stats["commentCount"]
        comment_lock = False
    if "likeCount" not in stats:
        like_count = 0
    else:
        like_count = stats["likeCount"]
    if "dislikeCount" not in stats:
        dislike_count = 0
    else:
        dislike_count = stats["dislikeCount"]

    video_writer.writerow([source["cat"],
                           source["name"],
                           video["id"]["videoId"],
                           clean_str(video["snippet"]["title"]),
                           clean_str(video["snippet"]["description"]),
                           clean_str(video["snippet"]["publishedAt"]),
                           details["duration"],
                           stats["viewCount"],
                           like_count,
                           dislike_count,
                           comment_count,
                           comment_lock
                           ])

def read_comments(youtube_api, comment_writer, video):
    print("    read_comments START. video id:" + video["id"]["videoId"])
    page_token = None
    i = 0
    for _ in range(150):
        if page_token != False:
            res, comments, page_token = get_comments(youtube_api, video["id"]["videoId"], page_token)
            if res is None:
                break
            for comment in comments:
                clc = comment["snippet"]["topLevelComment"]["snippet"]
                if "authorChannelId" in clc:
                    author_id = clc["authorChannelId"]["value"]
                else:
                    author_id = "UNKNOWN"
                comment_writer.writerows([[comment["snippet"]["videoId"],
                                           comment["snippet"]["topLevelComment"]["id"], # comment id
                                           author_id,
                                           clean_str(clc["authorDisplayName"]),
                                           clean_str(clc["textDisplay"]),
                                           comment["snippet"]["totalReplyCount"],
                                           clc["likeCount"],
                                           clean_str(clc["publishedAt"])
                                           ]])
                i += 1
                if i % 1000 == 0:
                    print("        read_comments. Nbr : " + str(i))

def compute_left_and_right(d_current, d_to):
    left = d_current
    dif = d_to - left
    dif_days = dif.days
    if dif_days < DAY_INCR:
        right = d_to
    else:
        right = d_current + timedelta(days=DAY_INCR, seconds=-1)
    d_current = d_current + timedelta(days=DAY_INCR)
    return d_current, left, right

def get_file_name(file_type, source, period):
    return CSV_PATH + file_type + source["name"] + "_" + period["period"] + ".tmp"

def get_csv_video(video_file):
    video_writer = csv.writer(video_file, delimiter=',', quoting=csv.QUOTE_MINIMAL)
    video_writer.writerow(["cat", "source", "videoId", "videoTitle", "videoDesc",
                           "videoPublishedAt", "duration", "viewCount", "likeCount", "dislikeCount",
                           "commentCount", "commentLock"])
    return video_writer

def get_csv_comment(comment_file):
    comment_writer = csv.writer(comment_file, delimiter=',', quoting=csv.QUOTE_MINIMAL)
    comment_writer.writerow(["videoId", "commentId", "authorId", "author", "text", "replies", "likes", "publishedAt"])
    return comment_writer

def is_done(last_situation, source, period):
    if last_situation is None:
        return False
    period_current = period["period"]
    source_current = source["name"]
    for s in last_situation:
        if s[0] == period_current and s[1] == source_current:
            return True
    return False

def move_tmp_to_csv(file_type, source, period):
    tmp_name = CSV_PATH + file_type + source["name"] + "_" + period["period"] + ".tmp"
    csv_name = CSV_PATH + file_type + source["name"] + "_" + period["period"] + ".csv"
    shutil.move(tmp_name, csv_name)

def read_source(youtube_api, last_situation, source, period):
    print("read_source START. Period: " + period["period"] + " source: " + source["name"])
    with open(get_file_name("videos_", source, period), "w") as video_file:
        video_writer = get_csv_video(video_file)
        with open(get_file_name("comments_", source, period), "w") as comment_file:
            comment_writer = get_csv_comment(comment_file)
            d_current = period["from"]
            while d_current < period["to"]:
                d_current, left, right = compute_left_and_right(d_current, period["to"])
                videos = get_videos(youtube_api, source, left, right)
                for video in videos:
                    read_video(youtube_api, video_writer, source, video)
                    read_comments(youtube_api, comment_writer, video)
                    if video["id"]["videoId"] == "gcSwBR6l8QE":
                        import pdb
                        pdb.set_trace()

    move_tmp_to_csv("videos_", source, period)
    move_tmp_to_csv("comments_", source, period)
    write_situation(last_situation, period, source)

def get_last_situation():
    last_situation_file = Path(SITUATION_TXT)
    last = None
    situations = []
    if last_situation_file.is_file():
        with open(SITUATION_TXT, "r") as last_situation_file:
            csv_reader_situation = csv.reader(last_situation_file, delimiter=',')
            for line in csv_reader_situation:
                if len(line) >= 2:
                    situations.append(line)
                    last = line
    print("Last : " + str(last))
    return situations

def write_situation(last_situation, period, source):
    with open(SITUATION_TXT, "a") as situation_file:
        situation_writer = csv.writer(situation_file, delimiter=',', quoting=csv.QUOTE_MINIMAL)
        situation_writer.writerow([period["period"], source["name"], str(datetime.now())])
    last_situation.append([period["period"], source["name"], str(datetime.now())])

def read_sources():
    youtube_api = get_authenticated_service()
    last_situation = get_last_situation()
    d_current = FROM
    while d_current < TO:
        period = {"period": str(d_current.year) + "-" + str(d_current.month).zfill(2),
                  "from": d_current,
                  "to": add_one_month(d_current) + timedelta(seconds=-1)}
        for source in SOURCES:
            if is_done(last_situation, source, period):
                print("read_source DONE. Period: " + period["period"] + " source: " + source["name"])
            else:
                read_source(youtube_api, last_situation, source, period)
        d_current = add_one_month(d_current)

FROM = datetime(2020, 11, 1)
TO = datetime(2020, 12, 1) + timedelta(seconds=-1)

read_sources()


import pandas as pd
import os
import matplotlib.pyplot as plt
import datetime

class Picture(object):

    def __init__(self, path):
        self.id_list=self.getFileNames(path)
        self.draw(self.id_list)

    def getFileNames(self,path):
        list = os.listdir(path)
        return list

    def draw(self, id_list):
        for id in id_list:
            df = self.getDataframe(id)
            start=df.index[0]
            start = start + datetime.timedelta(days=10)
            end= start + datetime.timedelta(days = 10)
            plt.figure()
            df.plot(title=id.split(".")[0],xlim=[start,end])
            plt.savefig("pictures/" + id.split(".")[0])
            plt.close()

    def getDataframe(self, id):
        df = pd.read_csv("parts/" + id)
        df["timestamp"]=pd.to_datetime(df["timestamp"]*1000000000)
        df.set_index("timestamp", inplace=True)
        df1 = df["value"]
        return df1

if __name__=='__main__':

    interval=Picture("parts")


from rest_framework import serializers

from departments.models import Employee, Department

class DepartmentSerializer(serializers.ModelSerializer):
    class Meta:
        model = Department
        fields = (
            'name',
        )

class EmployeeSerializer(serializers.ModelSerializer):
    department = DepartmentSerializer(many=False, read_only=True)

    class Meta:
        model = Employee
        fields = (
            'id',
            'department',
            'full_name',
            'email',
        )


import time
import random
import os
import requests
from colorama import *

init(autoreset=True)

class jg:
	names = ["Миша", "Аимбет", "Хакеры из ананимус", "Дова", "Клайд", "Влад гей","когда миксед", "я люблю сосать член","пуська", "бля", "твоя мама гей", "сука", "бля"]
	def helloWorld():
		print(Fore.CYAN + 'WEBHOOK FLOOD')
		print("github.com/jackglow/webhookspam")
	def run(urmsg):
		hf = open("hooks.txt", "r")
		hooks = hf.read()
		hf.close()
		totalhooks = 0
		hooks = hooks.split('\n')
		for hook in hooks:
			totalhooks += 1
			print(Fore.YELLOW+"Loaded "+str(totalhooks)+" webhook in total...")
		print(Fore.GREEN + "Webhooks loaded.")
		print(Fore.YELLOW + "Running message (30) send...")
		for x in range(0,30):
			for hook in hooks:
				r = requests.post(hook, data={'username': random.choice(jg.names), 'content': urmsg, "avatar_url": "https://avatars3.githubusercontent.com/u/13962537?s=460&v=4"})

jg.helloWorld()
print(Fore.YELLOW + "Enter message> ", end="")
f = input()
jg.run(f)


from detect_text import parse_screenshot, compress
from google_search import run_query_all
import os
import glob
import time
from watchdog.events import FileSystemEventHandler

compressed = True

class ParseSearchHandler(FileSystemEventHandler):

    def on_created(self, event):
        start = time.time()
        # by default, mac saves ss on desktop
        all_img = glob.glob('/Users/jpchen/Desktop/*.png')
        # get the last ss
        latest_img = max(all_img, key=os.path.getctime)
        if compressed:
            latest_img = compress(latest_img)
        try:
            q_and_a = parse_screenshot(latest_img, should_launch=True, compressed=compressed)
            question, answer = q_and_a['question'], q_and_a['answers']
            negative_q = ' not ' in question.lower() or "isn ' t " in question.lower() or ' never ' in question.lower()
            answer = run_query_all(question, answer, is_negative=negative_q)
        except:
            pass
        end = time.time()
        try:
            # read aloud the answer
            os.system('say "{} is the most likely answer."'.format(answer))
            print('Elapsed wall time: {} seconds', end - start)
        except:
            # something went wrong
            os.system('say fuck if i know')


#-------------------------------------------------------------------------------
# Name:        POXLSParser
# Purpose:     JS Wright Purchase Order Parser for Tradex Update
#
# Author:      Raju.Surapuraju
#
# Created:     04/01/2019
# Copyright:   (c) Raju.Surapuraju 2018
# Licence:     Causeway Technologies Limited
#-------------------------------------------------------------------------------

import re
import os
import configparser

settings = configparser.ConfigParser()
settings._interpolation = configparser.ExtendedInterpolation()
settings.read('C:\\config\\config.ini')
varciteXMLInput = settings.get('InputData', 'citeXMLInput')
varciteCSV = settings.get('InputData', 'citeCSV')

citeXML = varciteXMLInput
citeCSV = varciteCSV

if os.path.exists(citeCSV):
  os.remove(citeCSV)
else:
  print("citeCSV file does not exist")

def parseXML():
    with open(citeXML) as cxFH:
        while True:
            line = cxFH.readline()
            line = str(line)
            invoiceNumStr = re.search('DocNumber="(.+?)" DocType=', line)
            catalogueNumberStr = re.search('E.7140.ItemNumber.D="(.+?)"/', line)
            catalogueLineNumberStr = re.search('E.1082.LineItemNumber.D="(.+?)">', line)
            catalogueLineQuantityStr = re.search('E.6060.Quantity.D="(.+?)"', line)
            if invoiceNumStr:
                invoiceNum = invoiceNumStr.group(1)
                print ("Invoice Number:" + invoiceNum)

            if catalogueNumberStr:
                catalogueNumber = catalogueNumberStr.group(1)
                print (" Catelogue Number:" + catalogueNumber)

            if catalogueLineNumberStr:
                catalogueLineNumber = catalogueLineNumberStr.group(1)
                print (" Catalogue Line Number:" + catalogueLineNumber)

            if catalogueLineQuantityStr:
                catalogueLineQuantity = catalogueLineQuantityStr.group(1)
                print (" Catalogue Line Quantity:" + catalogueLineQuantity)
                print ()

                csvString =  (invoiceNum + ',' +  catalogueNumber + ','  + catalogueLineNumber + ',' + catalogueLineQuantity)
                csvFH = open(citeCSV, "a")
                csvFH.write(csvString + '\n')
                csvFH.close()

            if not line:
                break
        cxFH.close()

def main():
    parseXML()

if __name__ == '__main__':
    main()


import pandas as pd
import numpy as np
import os, sys
import matplotlib.pyplot as plt

from parse import *

sys.path.append("../utils/")
from utils import *

subset_noise = False
noise_level = '1-2en01'

subset_difficulty = True
diff_level = '1en01'

data_dir = '../../out/'
games = []
games += get_games(data_dir, 'experiment')
#games += ['tmp']

data = get_data(data_dir, games)
#data = data[data['n_players'] < 6]
#data = data[data['score'] > 0.7]

if subset_noise:
    data = data[data['noise'] == noise_level]
if subset_difficulty:
    data = data[data['difficulty'] == diff_level]

def get_shape(i):
    shapes = ['$a$',
              '$b$',
              '$c$',
              '$d$',
              '$e$',
              '$f$',
              '$g$',
              '$h$',
              '$i$',
              '$j$',
              '$k$',
              '$l$',
              '$m$',
              '$n$',
              '$o$',
              '$p$',
              '$q$',
              '$r$',
              '$s$',
              '$t$',
              '$u$',
              '$v$',
              '$w$',
              '$x$',
              '$y$',
              '$z$',
        (4,0,0), (3,0,0), (4,0,45), (0,3,0), (3,0,90),
              (3,0,180), (3,0,270),
              (5,0,0), (6,0,0), (6,0,90), (4,2,0), (4,3,0),
              (5,2,0), (5,3,0),
              "d", "*", "$a$"]
    return shapes[i % len(shapes)]

plt.rcParams.update(pd.tools.plotting.mpl_stylesheet)

fig, ax = plt.subplots()
ax.margins(0.05)

colors = ['#348ABD',  '#467821', '#E4C334', '#A60628']

counts = dict([(n,0) for n in set(data['n_players'])])

data['nn_players'] = data['n_players'] + np.random.randn(len(data))*0.1

i = 0
for noise in set(data['noise']):
    if not subset_noise or noise == noise_level:
        noise_sub = data[data['noise'] == noise]
        x = sorted(set(noise_sub['n_players']))
        y = []
        for n in x:
            y += [np.mean(noise_sub[noise_sub['n_players'] == n]['score'])]
        ax.plot(x, y, c = colors[i], lw = 10, alpha = 0.5)
    i += 1

i = 0
j = 0
for noise in set(data['noise']):
    if not subset_noise or noise == noise_level:
        noise_sub = data[data['noise'] == noise]
        for game in set(noise_sub['game']):
            sub = noise_sub[noise_sub['game'] == game]
            n = list(set(sub['n_players']))[0]
            ax.plot(sub['nn_players'], sub['score'], marker='o', linestyle='', ms = 20, c = colors[i])
            ax.plot(sub['nn_players'], sub['score'], marker=get_shape(counts[n]), linestyle='', ms = 10, c = 'black')
            counts[n] += 1
    i += 1

#legend = ax.legend(loc='center left', bbox_to_anchor=(1, 0.5), title = 'Background', numpoints=1)
#legend.get_title().set_fontsize('30')
#plt.setp(plt.gca().get_legend().get_texts(), fontsize='20')

plt.xlabel('Number of Players', fontsize=50)
plt.ylabel('Individual Score', fontsize=50)
ax.tick_params(axis='x', labelsize=30)
ax.tick_params(axis='y', labelsize=30)

fig = plt.gcf()
fig.set_size_inches(18.5,10.5)
if subset_difficulty:
    fig.savefig('../../plots/performance-' + diff_level + '.png',dpi=100)
elif subset_noise:
    fig.savefig('../../plots/performance-' + noise_level + '.png',dpi=100)
else:
    fig.savefig('../../plots/performance-all.png',dpi=100)

#plt.show()


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
#  handler.py
#
#  Copyright 2017 Andrei Tumbar <atuser@Kronos>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#

from http import server
import json, os
from shutil import rmtree
from .r_handler import MainRHandler
from .yt import YTSearchParser
import urllib.request

class postHandler (server.SimpleHTTPRequestHandler):
    def parse_post (self, _str):
        try:
            itms = _str.split ("&")
            ret = {}
            for x in itms:
                name, val = x.split ('=')
                ret[name] = val
        except:
            return None
        return ret

    def do_GET(self):
        if self.path.find('.cache/') != -1:
            if (not os.path.isfile(self.path[1:])):
                self.send_response(301)
                self.send_header('Location','http://' + self.path[self.path.find('.cache/') + len('.cache/'):])
                self.end_headers()
                return

        if self.path[0:7] == "/?code=":
            try:
                self.__OAUTHLOGIN (self.path[7:])
            except:
                pass
            self.send_response(301)
            self.send_header('Location','http://localhost:8000')
            self.end_headers()
            return

        f = self.send_head()
        if f:
            try:
                self.copyfile(f, self.wfile)
            finally:
                f.close()

    #def do_PLAY (self): # Song played
    #

    def do_CHECKOAUTH (self):
        if MainRHandler.is_logged_in:
            self.send_response(200)
        else:
            self.send_response(403)
        self.end_headers()

    def do_INC (self): # Increment song playcount
        MainRHandler.gmusic.gm_library.increment_song (self.path[1:])
        MainRHandler.gmusic.write_data ()
        self.send_response(200)
        self.end_headers()

    def do_DELETE (self):
        _id = self.path[1:]
        MainRHandler.gmusic.gm_library.rm_song (_id)
        MainRHandler.gmusic.refresh()
        MainRHandler.gmusic.write_data ()
        self.send_response(200)
        self.end_headers()

    def do_ADDTPL (self): # Add song to playlist
        index, playlist = self.path[1:].split ('/')
        MainRHandler.gmusic.gm_library.add_song_to_playlist (index, playlist)
        MainRHandler.gmusic.refresh()
        MainRHandler.gmusic.write_data ()
        self.send_response(200)
        self.end_headers()

    def do_RMFPL (self): # Remove plid from playlist
        plid = self.path[1:]
        MainRHandler.gmusic.gm_library.rm_song_from_playlist (plid)
        MainRHandler.gmusic.refresh()
        MainRHandler.gmusic.write_data ()
        self.send_response(200)
        self.end_headers()

    def do_STREAM (self):
        i = self.path[1:]
        try:
            out = str.encode(MainRHandler.gmusic.gm_library.get_stream(i))
        except:
            self.send_response (502)
            self.end_headers()
            return
        else:
            self.send_response(200)
        self.end_headers()
        self.wfile.write (out)

    def do_GETOAUTHURL (self):
        self.send_response(200)
        self.end_headers()
        self.wfile.write (MainRHandler.gmusic.gm_api_man.get_login_url().encode ())

    def __OAUTHLOGIN (self, code):
        sys.stdout.flush()
        s = MainRHandler.gmusic.gm_api_man.get_auth_token(code)
        status = MainRHandler.gmusic.gm_api_man.login(s)
        MainRHandler.is_logged_in = status

    def do_POST (self):
        self.data_string = self.rfile.read(int(self.headers['Content-Length']))

        data = self.parse_post(self.data_string.decode("utf-8"))
        res = MainRHandler.r_get (self.path, data)
        self.send_response(res)
        self.end_headers()

    def do_RELOAD (self):
        MainRHandler.gmusic.write_data ()
        self.send_response(200)
        self.end_headers()

    def do_UPLOAD (self):
        stat = MainRHandler.gmusic.gm_api_man.upload (self.path, enable_matching=True)
        MainRHandler.gmusic.write_data ()
        if stat:
            self.send_response (200)
        else:
            self.send_response (504)
        self.end_headers

    def do_SEARCHYT (self):
        buf = YTSearchParser ()
        if self.path[0] == '/':
            self.path = self.path[1:]
        buf.feed (urllib.request.urlopen ("https://www.youtube.com/results?search_query=" + self.path).read().decode("utf-8"))
        bufs = buf.search_finds
        buf.search_finds = []
        self.send_response(200)
        self.end_headers()
        for u in bufs:
            self.wfile.write ((u + '\n').encode())

    def do_UPLOADALL (self): #Uploads all songs in .temp then deletes them
        stat = MainRHandler.gmusic.gm_api_man.upload (prepend_to_items(os.listdir('.temp'), '.temp/'), enable_matching=True)
        rmtree ('.temp')
        MainRHandler.gmusic.write_data ()
        self.send_response (200)
        self.end_headers()

    def do_YTDL (self):
        self.send_response (MainRHandler._ytdl ('http://www.youtube.com/watch?v=' + self.path[1:]))
        self.end_headers()

    def do_SHUTDOWN (self):
        self.send_response(200)
        self.end_headers()
        exit(0)

def prepend_to_items (arr, prepend):
    ret = []
    for i in arr:
        ret.append (prepend + i)
    return ret

class postHandlerDebug (postHandler):
    def do_DEBUG (self):
        try:
            exec ('t = ' + self.path)
        except:
            self.send_response(400)
            self.end_headers()
            return
        else:
            self.send_response(200)

        t = eval(self.path)
        self.wfile.write (str(t).encode())


from pymongo import MongoClient
from faker import Factory
fake = Factory.create('ru_RU')

dataset1 = []
dataset2 = []

for _ in range(0,150):
  dataset1.append(
    {"name": fake.color_name(),
     "description": fake.text(max_nb_chars=50)}
  )
  dataset2.append(
    {"name": fake.name(),
     "description": fake.text(max_nb_chars=100),
     "tags": []}
  )

client = MongoClient('localhost', 27017)

db = client['boojom']

db.tags.remove()
db.objects.remove()

db.tags.insert(dataset1)
db.objects.insert(dataset2)


from django.conf.urls import url

from . import views
from .views import export as export_views

urlpatterns = [
    url(r'^$', views.ReportIndexView.as_view(), name='index'),
    url(r'^hour_meter$', views.HourmeterReportView.as_view(), name='hour_meter'),
    url(r'^biaya_per_alat$', views.BiayaabPerAlatReportView.as_view(), name='biaya_per_alat'),
    url(r'^biaya_per_tanggal$', views. BiayaabPerTanggalReportView.as_view(), name='biaya_per_tanggal'),
    url(r'^bbm_alat_berat$', views.BBMabReportView.as_view(), name='bbm_ab'),
    url(r'^transaksi_tangki_induk$', views.TransaksiTangkiIndukReportView.as_view(), name='transaksi_tangki_induk'),
    url(r'^transaksi_mobil_tangki$', views.TransaksiMobilTangkiReportView.as_view(), name='transaksi_mobil_tangki'),
    url(r'^hasil_produksi$', views.ProduksiReportView.as_view(), name='hasil_produksi'),
    url(r'^export_hour_meter$', export_views.HourMeterExportView.as_view(), name='export_hour_meter'),
    url(r'^export_biaya_per_alat$', export_views.BiayaabPerAlatExportView.as_view(), name='export_biaya_per_alat'),
    url(r'^export_biaya_per_tanggal$', export_views.BiayaabPerTanggalExportView.as_view(), name='export_biaya_per_tanggal'),
    url(r'^export_bbm_ab$', export_views.BBMabExportView.as_view(), name='export_bbm_ab'),
    url(r'^export_transaksi_tangki_induk', export_views.TransaksiTangkiIndukExportView.as_view(), name='export_transaksi_tangki_induk'),
    url(r'^export_transaksi_mobil_tangki', export_views.TransaksiMobilTangkiExportView.as_view(), name='export_transaksi_mobil_tangki'),
    url(r'^export_produksi', export_views.ProduksiExportView.as_view(), name='export_produksi'),
]


# -*- coding: utf-8 -*-
# @File : getExcelData.py
# _author_=feng
# date: 2021/1/13
# openpyxl(xlsx表格复制)
from openpyxl import load_workbook
import openpyxl
import xlrd
from xlutils.copy import copy

# 1- 打开excel表
def get_excelData(sheetName, startRow, endRow):
    dataList = []
    # 1-excel表路径
    excelDir = 'D:/Auto_test_python/TP_Api_Test/data/TP_接口自动化测试用例V1.1.xls'
    # 2- 打开excel对象--formatting_info=True  保持样式
    workBook = xlrd.open_workbook(excelDir, formatting_info=True)
    # workBook = openpyxl.load_workbook(excelDir)
    # workSheetNames = workBook.sheet_names()#获取所有的表名
    # print(workSheetNames)
    # 3- 获取某一个指定的表
    workSheet = workBook.sheet_by_name(sheetName)
    # 4- 读取单元格---返回是字符串---cell(行号，列号)  从0开始
    for one in range(startRow - 1, endRow):
        reqBodyData = workSheet.cell(one, 9).value  # 请求body
        respData = workSheet.cell(one, 11).value  # 响应数据
        dataList.append((reqBodyData, respData))  # 封装一个列表里嵌套元组

    return dataList

# #可以自动识别用例数
# def get_excelData2(sheetName,caseName):
#     '''
#     :param sheetName: 表名
#     :param caseName: 某一个接口的用例名称
#     :return:
#     '''
#     resList = []
#     # 1-excel表路径
#     excelDir = r'D:\Auto_test_python\TP_Api_Test\data\TP_接口自动化测试用例V1.0.xls'
#     # 2- 打开excel对象--formatting_info=True  保持样式
#     workBook = xlrd.open_workbook(excelDir,formatting_info=True)
#     # workSheetNames = workBook.sheet_names()#获取所有的表名
#     # print(workSheetNames)
#     # 3- 获取某一个指定的表
#     workSheet = workBook.sheet_by_name(sheetName)
#     # 4- 读取一列数据
#     # print(workSheet.col_values(0))
#     idx=0 # 开始的下标
#     for one in workSheet.col_values(0):
#         if caseName in one:
#             reqBodyData = workSheet.cell(idx,9).value #请求body
#             respData =  workSheet.cell(idx,11).value #响应数据
#             resList.append((json.loads(reqBodyData),json.loads(respData))) # 封装一个列表里嵌套元组
#         idx += 1#
#     return resList
#
#     # for one in range(startRow-1,endRow):
#     #     reqBodyData = workSheet.cell(one,9).value#请求body
#     #     respData =  workSheet.cell(one,11).value#响应数据
#     #     resList.append((reqBodyData,respData))#封装一个列表里嵌套元组
#     # return resList
#
#
def set_excelData():
    # 1-excel表路径
    excelDir = '../data/TP_接口自动化测试用例V1.1.xls'
    # 2- 打开excel对象--formatting_info=True  保持样式
    workBook = xlrd.open_workbook(excelDir, formatting_info=True)
    workBookNew = copy(workBook)  # 复制一个新excel文件对象
    # workSheetNew = workBookNew.get_sheet(n)
    # 取复制出来的新excel文件对象的第一个子表
    workBookNew.save(r'../report/res.xls')
    return workBookNew
    # 复制出来的excel对象，复制出来excel对象的第一个子表

if __name__ == '__main__':
    a = get_excelData('1登录模块', 3, 3)
    # b=json.loads(a)
    print(a)
    print(type(a))


# 14.01.2013
# @hugo_dc

# Download code from SAP Server

import wx
import Toolbars
import Utils
import sap
import Messages
import db

def Show(self):
    f = DownloadCode(self, id=-1, title="Download ABAP Sourcecode")
    f.Show()

class DownloadCode(wx.Frame):
    fields = []
    def __init__(self, parent, id, title):
        self.parent = parent
        wx.Frame.__init__(self, parent, id, title, style = wx.DEFAULT_FRAME_STYLE & ~ (  wx.RESIZE_BORDER | wx.RESIZE_BOX | wx.MINIMIZE_BOX ) , size=(250, 190) )
        panel = wx.Panel(self, -1)

        self.CreateStatusBar()
        toolbar = self.CreateToolBar()
        toolbar = Toolbars.getToolbar(self, toolbar, 'downloadwindow')
        toolbar.Realize()
        self.fields = []
        self.servers = db.getServers()

        servers = []
        for server in self.servers:
            servers.append(server[0])

        wx.StaticText(panel, -1, "SAP Server: ", pos=(20, 55))
        self.choice = wx.Choice(panel, -1, pos=(120, 50), choices=servers )

        y = 20

        y = Utils.createField(self, panel, 'Program name:', 100, y)

    def OnDownload(self, event):
        message = Utils.setParams(self)
        if message != None:
            Messages.messageError(message, 'Error downloading code')
        else:
            ix = self.choice.GetCurrentSelection()

            if ix < 0:
                Messages.messageError('Select a server', 'Download ABAP Source code')
            else:
                abap = sap.getCode(self.servers[ix], self.params[0])
                if abap != "":
                    self.parent.code = abap
                    self.parent.codetext.SetValue(abap)
                    self.parent.ReloadHighlight()
                    Messages.messageInformation('Source Code downloaded successfully!', 'Download')
                    self.parent.local = False
                    self.parent.ChangeRootName(self.params[0].upper(), True)
                    self.parent.sap_program = self.params[0].upper()
                    self.parent.sap_server  = self.servers[ix]
                    self.parent.tree.SetBackgroundColour('blue violet')
                    self.parent.SetTitle('Zorse - SAP:\\' + self.params[0].upper())
                    self.Destroy()
                else:
                    Messages.messageError('Error downloading program', 'Error downloading code')


#!/usr/bin/env python
# coding=utf-8
from time import sleep
from test.common.aems_case import AeMSCase
from test.page.SmallCellMgtPage.file_mgt_page import FileManagementPage
from utils.common.utx import Tag, tag

driver = AeMSCase().driver
file_management_page = FileManagementPage(driver=driver)

class TestFileManagement(AeMSCase):

    def setUp(self):
        AeMSCase.setUp(self)
        file_management_page.act_open_tab("Small Cell Management", "File Management")
        if not AeMSCase.column_names:
            AeMSCase.column_names = file_management_page.get_column_names()

    @tag(Tag.HIGH)
    def test_upload(self):
        files = file_management_page.get_upload_files(self.version)
        # to upload all profile which format is .csv (end with csv)
        for file in files:
            if file.endswith(".csv"):
                file_management_page.act_upload_file(file)
                sleep(0.25)
                file_management_page.find_xpath_by_text('button', 'Confirm').click()
                file_management_page.get_alert_text_and_dismiss()
                sleep(0.25)

    def tearDown(self):
        AeMSCase.tearDown(self)


import re
from os.path import join

import click
import untangle
import pypandoc
from slugify import slugify

@click.command()
@click.argument('xmlfile', type=click.File('rb'))
@click.option('--output-dir', type=click.Path(exists=True,
              dir_okay=True, file_okay=False))
def main(xmlfile, output_dir):
  obj = untangle.parse(xmlfile)
  for page in obj.mediawiki.page:
    title = page.title.cdata
    filename = slugify(title) + '.md'
    permalink = '/' + wikimedia_slugify(title)
    author = page.revision.contributor.username.cdata
    date = page.revision.timestamp.cdata
    body = page.revision.text.cdata
    body_without_category = re.sub(r"\[\[Category:[^\]]+\]\]", '', body)
    category_match = re.search(r"\[\[Category:([^\]]+)\]\]", body)
    category = '' if category_match is None else category_match[1]

    try:
      body_md = pypandoc.convert_text(body_without_category, 'gfm',
                 format='mediawiki')

      contents = """---
title: {}
permalink: {}
layout: page
author: {}
date: {}
category: {}
---
{}
""".format(title, permalink, author, date, category, body_md)

      click.echo(title)

      filepath = join(output_dir, filename)
      with open(filepath, 'w') as file:
        file.write(contents)

    except Exception as e:
      click.echo(e, err=True)

def wikimedia_slugify(input):
  return input.replace(' ', '_')

if __name__ == '__main__':
  main()


from django.shortcuts import render
from blog.models import BlogPost,Blogger,Category

def index(request):
    blogposts = BlogPost.objects.order_by('-blog_date')
    featured_blogposts = BlogPost.objects.order_by('-blog_date')
    categories = Category.objects.all()
    context = {
        'blogposts':blogposts,
        'featured_blogposts':featured_blogposts,
        'categories':categories
    }
    return render(request,'main/index.html',context)

def about(request):
    return render(request,'main/about.html')

def search(request):
    query = request.GET['query']
    queryset_title = BlogPost.objects.order_by('-blog_date').filter(title__icontains=query)
    context = {
        'blogposts':queryset_title,
    }

    return render(request,'main/search.html',context)


import tensorflow as tf
import logging
tf.get_logger().setLevel(logging.ERROR)
import numpy as np
from model_utils import biGRUCell, initial_point_interpolation

class mrnn ():
  """MRNN class with core functions.

  Attributes:
    - x: incomplete data
    - model_parameters:
      - h_dim: hidden state dimensions
      - batch_size: the number of samples in mini-batch
      - iteration: the number of iteration
      - learning_rate: learning rate of model training
  """

  def __init__(self, x, model_parameters):

    # Set Parameters
    self.no, self.seq_len, self.dim = x.shape

    self.h_dim = model_parameters['h_dim']
    self.batch_size = model_parameters['batch_size']
    self.iteration = model_parameters['iteration']
    self.learning_rate = model_parameters['learning_rate']

  def rnn_train (self, x, m, t, f):
    """Train RNN for each feature.

    Args:
      - x: incomplete data
      - m: mask matrix
      - t: time matrix
      - f: feature index
    """
    tf.compat.v1.reset_default_graph()

    with tf.compat.v1.Session() as sess:

      # input place holders
      target = tf.placeholder(tf.float32, [self.seq_len, None, 1])
      mask = tf.placeholder(tf.float32, [self.seq_len, None, 1])

      # Build rnn object
      rnn = biGRUCell(3, self.h_dim, 1)
      outputs = rnn.get_outputs()
      loss = tf.sqrt(tf.reduce_mean(tf.square(mask*outputs - mask*target)))
      optimizer = tf.compat.v1.train.AdamOptimizer(self.learning_rate)
      train = optimizer.minimize(loss)

      sess.run(tf.compat.v1.global_variables_initializer())

      # Training
      for i in range(self.iteration):
        # Batch selection
        batch_idx = np.random.permutation(x.shape[0])[:self.batch_size]

        temp_input = np.dstack((x[:,:,f], m[:,:,f], t[:,:,f]))
        temp_input_reverse = np.flip(temp_input, 1)

        forward_input = np.zeros([self.batch_size, self.seq_len, 3])
        forward_input[:,1:,:] = temp_input[batch_idx,
                                           :(self.seq_len-1), :]

        backward_input = np.zeros([self.batch_size, self.seq_len, 3])
        backward_input[:,1:,:] = temp_input_reverse[batch_idx,
                                                    :(self.seq_len-1),:]

        _, step_loss = \
        sess.run([train, loss],
                 feed_dict=
                 {mask: np.transpose(np.dstack(m[batch_idx,:,f]),[1, 2, 0]),
                  target: np.transpose(np.dstack(x[batch_idx,:,f]),[1, 2, 0]),
                  rnn._inputs: forward_input,
                  rnn._inputs_rev: backward_input})

      # Save model
      inputs = {'forward_input': rnn._inputs,
                'backward_input': rnn._inputs_rev}
      outputs = {'imputation': outputs}

      save_file_name = 'tmp/mrnn_imputation/rnn_feature_' + str(f+1) + '/'
      tf.compat.v1.saved_model.simple_save(sess, save_file_name,
                                           inputs, outputs)

  def rnn_predict (self, x, m, t):
    """Impute missing data using RNN block.

    Args:
      - x: incomplete data
      - m: mask matrix
      - t: time matrix

    Returns:
      - imputed_x: imputed data by rnn block
    """
    # Output Initialization
    imputed_x = np.zeros([self.no, self.seq_len, self.dim])

    # For each feature
    for f in range(self.dim):

      temp_input = np.dstack((x[:,:,f], m[:,:,f], t[:,:,f]))
      temp_input_reverse = np.flip(temp_input, 1)

      forward_input = np.zeros([self.no, self.seq_len, 3])
      forward_input[:,1:,:] = temp_input[:,:(self.seq_len-1), :]

      backward_input = np.zeros([self.no, self.seq_len, 3])
      backward_input[:,1:,:] = temp_input_reverse[:,:(self.seq_len-1),:]

      save_file_name = 'tmp/mrnn_imputation/rnn_feature_' + str(f+1)  + '/'

      # Load saved model
      graph = tf.Graph()
      with graph.as_default():
        with tf.compat.v1.Session() as sess:
          sess.run(tf.compat.v1.global_variables_initializer())
          tf.compat.v1.saved_model.loader.load(sess,
                                               [tf.compat.v1.saved_model.SERVING],
                                               save_file_name)
          fw_input = graph.get_tensor_by_name('inputs:0')
          bw_input = graph.get_tensor_by_name('inputs_rev:0')
          output = graph.get_tensor_by_name('map/TensorArrayStack/TensorArrayGatherV3:0')

          imputed_data = sess.run(output,
                                  feed_dict={fw_input: forward_input,
                                             bw_input: backward_input})

          imputed_x[:, :, f] = (1-m[:,:,f]) * np.transpose(np.squeeze(imputed_data)) + \
                               m[:,:,f] * x[:,:,f]

    # Initial poitn interpolation for better performance
    imputed_x = initial_point_interpolation (x, m, t, imputed_x)

    return imputed_x

  def fc_train(self, x, m, t):
    """Train Fully Connected Networks after RNN block.

    Args:
      - x: incomplete data
      - m: mask matrix
      - t: time matrix
    """
    tf.compat.v1.reset_default_graph()

    # rnn imputation results
    rnn_imputed_x = self.rnn_predict(x, m, t)

    # Reshape the data for FC train
    x = np.reshape(x, [self.no * self.seq_len, self.dim])
    rnn_imputed_x = np.reshape(rnn_imputed_x, [self.no * self.seq_len, self.dim])
    m = np.reshape(m, [self.no * self.seq_len, self.dim])

    # input place holders
    x_input = tf.placeholder(tf.float32, [None, self.dim])
    target = tf.placeholder(tf.float32, [None, self.dim])
    mask = tf.placeholder(tf.float32, [None, self.dim])

    # build a FC network
    U = tf.compat.v1.get_variable("U", shape=[self.dim, self.dim],
                                  initializer=tf.contrib.layers.xavier_initializer())
    V1 = tf.compat.v1.get_variable("V1", shape=[self.dim, self.dim],
                                   initializer=tf.contrib.layers.xavier_initializer())
    V2 = tf.compat.v1.get_variable("V2", shape=[self.dim, self.dim],
                                   initializer=tf.contrib.layers.xavier_initializer())
    b = tf.Variable(tf.random.normal([self.dim]))

    L1 = tf.nn.sigmoid((tf.matmul(x_input, tf.linalg.set_diag(U, np.zeros([self.dim,]))) + \
                        tf.matmul(target, tf.linalg.set_diag(V1, np.zeros([self.dim,]))) + \
                        tf.matmul(mask, V2) + b))

    W = tf.Variable(tf.random.normal([self.dim]))
    a = tf.Variable(tf.random.normal([self.dim]))
    hypothesis = W * L1 + a

    outputs = tf.nn.sigmoid(hypothesis)

    # reshape out for sequence_loss
    loss = tf.sqrt(tf.reduce_mean(tf.square(outputs - target)) )

    # Optimizer
    optimizer = tf.compat.v1.train.AdamOptimizer(self.learning_rate)
    train = optimizer.minimize(loss)

    # Sessions
    sess = tf.compat.v1.Session()
    sess.run(tf.compat.v1.global_variables_initializer())

    # Training step
    for i in range(self.iteration * 20):
      batch_idx = np.random.permutation(x.shape[0])[:self.batch_size]
      _, step_loss = sess.run([train, loss],
                              feed_dict={x_input: x[batch_idx, :],
                                         target: rnn_imputed_x[batch_idx, :],
                                         mask: m[batch_idx, :]})

    # Save model
    inputs = {'x_input': x_input,
              'target': target,
              'mask': mask}
    outputs = {'imputation': outputs}

    save_file_name = 'tmp/mrnn_imputation/fc_feature/'
    tf.compat.v1.saved_model.simple_save(sess, save_file_name,
                                         inputs, outputs)

  def rnn_fc_predict(self, x, m, t):
    """Impute missing data using RNN and FC.

    Args:
      - x: incomplete data
      - m: mask matrix
      - t: time matrix

    Returns:
      - fc_imputed_x: imputed data using RNN and FC
    """
    # rnn imputation results
    rnn_imputed_x = self.rnn_predict(x, m, t)

    # Reshape the data for FC predict
    x = np.reshape(x, [self.no * self.seq_len, self.dim])
    rnn_imputed_x = np.reshape(rnn_imputed_x, [self.no * self.seq_len, self.dim])
    m = np.reshape(m, [self.no * self.seq_len, self.dim])

    save_file_name = 'tmp/mrnn_imputation/fc_feature/'

    # Load saved data
    graph = tf.Graph()
    with graph.as_default():
      with tf.compat.v1.Session() as sess:

        sess.run(tf.compat.v1.global_variables_initializer())
        tf.compat.v1.saved_model.loader.load(sess, [tf.compat.v1.saved_model.SERVING],
                                             save_file_name)
        x_input = graph.get_tensor_by_name('Placeholder:0')
        target = graph.get_tensor_by_name('Placeholder_1:0')
        mask = graph.get_tensor_by_name('Placeholder_2:0')
        outputs = graph.get_tensor_by_name('Sigmoid_1:0')

        fc_imputed_x = sess.run(outputs, feed_dict={x_input: x,
                                                    target: rnn_imputed_x,
                                                    mask: m})

    # Reshape imputed data to 3d array
    fc_imputed_x = np.reshape(fc_imputed_x, [self.no, self.seq_len, self.dim])
    m = np.reshape(m, [self.no, self.seq_len, self.dim])
    x = np.reshape(x, [self.no, self.seq_len, self.dim])

    fc_imputed_x = fc_imputed_x * (1-m) + x * m
    fc_imputed_x = initial_point_interpolation (x, m, t, fc_imputed_x)

    return fc_imputed_x

  def fit(self, x, m, t):
    """Train the entire MRNN.

    Args:
      - x: incomplete data
      - m: mask matrix
      - t: time matrix
    """
    # Train RNN part
    for f in range(self.dim):
      self.rnn_train(x, m, t, f)
      print('Finish ' + str(f+1) + '-th feature training with RNN for imputation')
    # Train FC part
    self.fc_train(x, m, t)
    print('Finish M-RNN training with both RNN and FC for imputation')

  def transform(self, x, m, t):
    """Impute missing data using the entire MRNN.

    Args:
      - x: incomplete data
      - m: mask matrix
      - t: time matrix

    Returns:
      - imputed_x: imputed data
    """
    # Impute with both RNN and FC part
    imputed_x = self.rnn_fc_predict(x, m, t)

    return imputed_x


import os
from hydra.experimental import compose, initialize
initialize(config_path="./conf")
import util
util.config._config = compose("config.yaml")

import torch
from torchvision.utils import save_image
from evolution.generator import Generator, GeneratorDataset
from tqdm import tqdm
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter

def store_from_dataset(dataset, path, batch_size=50):
    dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, drop_last=False)
    i = 0
    for batch, _ in tqdm(dataloader):
        for image in batch:
            save_image((image + 1) / 2, os.path.join(path, f"image_{i}.png"))
            i += 1

def main(samples=100, output="output"):
    path = os.path.join(os.path.dirname(__file__), "generator.pkl")
    generator = Generator.load(path)
    generator_dataset = GeneratorDataset(generator, size=samples)
    os.makedirs(output, exist_ok=True)
    store_from_dataset(generator_dataset, output)

if __name__ == "__main__":
    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("-o", "--output", type=str, default="output", help='Output Path')
    parser.add_argument("-s", "--samples", default=100, type=int, help='Number of samples to generate')
    args = parser.parse_args()
    print(args)
    main(samples=args.samples, output=args.output)


#!/usr/bin/python3

import sys
import glob
import os
import natsort
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import re
import numpy

data = pd.DataFrame(dtype='float64', columns=["Omega", "Implementation", "Time"])
# cublas_data =
# cucosma_data = pd.DataFrame(dtype='float64')
# cutlass_data = pd.DataFrame(dtype='float64')
# omega_data = pd.DataFrame(dtype='float64')
# time_unit_data = pd.DataFrame(dtype='float64')

configs = "/home/neville/git/cucosma/Benchmarks/128x128x8_Benchmarks/square*/config*"
nvprofs = "/home/neville/git/cucosma/Benchmarks/cosmaBenchmarks/square*/nvprof*"
configs = "/home/neville/git/cucosma/Benchmarks/cosmaBenchmarks/square*/config*"

configs_array = natsort.natsorted(glob.glob(configs))
nvprofs_array = natsort.natsorted(glob.glob(nvprofs))
# print(configs_array)

for i in range(len(configs_array)):
    # print(configs_array[i])
    # print(nvprofs_array[i])
    # print(nvprofs_cosma_array[i])
    # print()
    config_df = pd.read_csv(configs_array[i])
    nvporf_df = pd.read_csv(nvprofs_array[i], skiprows=3)

    M = int(re.findall(r'\d+', config_df.iloc[10, 0])[0])
    N = int(re.findall(r'\d+', config_df.iloc[11, 0])[0])
    K = int(re.findall(r'\d+', config_df.iloc[12, 0])[0])
    THREADBLOCK_TILE_M = re.findall(r'\d+', config_df.iloc[13, 0])[0]
    THREADBLOCK_TILE_N = re.findall(r'\d+', config_df.iloc[14, 0])[0]
    THREADBLOCK_TILE_K = re.findall(r'\d+', config_df.iloc[15, 0])[0]
    LOAD_K = re.findall(r'\d+', config_df.iloc[16, 0])[0]
    WARP_TILE_M = re.findall(r'\d+', config_df.iloc[17, 0])[0]
    WARP_TILE_N = re.findall(r'\d+', config_df.iloc[18, 0])[0]
    THREAD_TILE_M = re.findall(r'\d+', config_df.iloc[19, 0])[0]
    THREAD_TILE_N = re.findall(r'\d+', config_df.iloc[20, 0])[0]

    # print(config_df)
    # print(nvporf_df)
    # omega_data = omega_data.append({'Omega': M}, ignore_index=True)
    # time_unit_data = time_unit_data.append({'Unit': nvporf_df['Duration'][0]}, ignore_index=True)

    col_cosma = nvporf_df[nvporf_df["Name"].str.contains("cosmaSgemm", na=False)]['Duration'].astype(float)
    col_cosma_reduction = nvporf_df[nvporf_df["Name"].str.contains("cosmaSplitK", na=False)]['Duration'].astype(float)
    col_cosma.reset_index(drop=True, inplace=True)
    col_cosma_reduction.reset_index(drop=True, inplace=True)

    if len(col_cosma_reduction) != 0:
        col_cosma = col_cosma + col_cosma_reduction

    col_cosma.reset_index(drop=True, inplace=True)
    col_cutlass = nvporf_df[nvporf_df["Name"].str.contains("cutlass", na=False)]['Duration'].astype(float)
    col_cutlass.reset_index(drop=True, inplace=True)

    #  col_cutlass = nvporf_df[nvporf_df["Name"].str.contains("cutlass", na=False)]['Duration'].astype(float)
    # col_cutlass.reset_index(drop=True, inplace=True)
    # cutlass_data[i] = col_cutlass

    cublas_kernel = nvporf_df["Name"][1]
    cublas_reduction = nvporf_df["Name"][2]

    if cublas_kernel == "[CUDA memset]":
        cublas_kernel = nvporf_df["Name"][2]
        cublas_reduction = nvporf_df["Name"][3]

    if cublas_reduction == "[CUDA memcpy HtoD]":
        cublas_reduction = ""

    cublas_real_name = cublas_kernel

    col_cublas_kernel = nvporf_df[nvporf_df["Name"] == cublas_kernel]['Duration'].astype(float)
    col_cublas_reduction = nvporf_df[nvporf_df["Name"] == cublas_reduction]['Duration'].astype(float)

    col_cublas_kernel.reset_index(drop=True, inplace=True)
    col_cublas_reduction.reset_index(drop=True, inplace=True)

    # print(col_cublas_kernel)
    # print(col_cublas_reduction)

    if len(col_cublas_reduction) != 0:
        col_cublas_kernel = col_cublas_kernel + col_cublas_reduction
        cublas_real_name = cublas_kernel + "+" + cublas_reduction

    # cublas_data[i] = col_cublas_kernel
    cublas_real_name = cublas_real_name.replace('volta_sgemm_', '')
    cublas_real_name = cublas_real_name.replace('_tt', '')
    cublas_real_name = cublas_real_name.replace(
        '<int, int, float, float, float, int=128, int=16, int=4, int=4, bool=0, cublasGemvParams<cublasGemvTensorStridedBatched<float const >, cublasGemvTensorStridedBatched<float>, float>>(float const , float, float)',
        '')
    cublas_real_name = cublas_real_name.replace('void', '')
    cublas_real_name = cublas_real_name.replace(' ', '')
    cublas_real_name = cublas_real_name.replace('_kernel<float,float,float>(cublasSplitKParams<float>,floatconst*,floatconst*,float*,floatconst*,floatconst*)', '')

    print(M, ": ", cublas_real_name)

    if nvporf_df['Duration'][0] == "ms":
        col_cosma = col_cosma * 1000
        col_cutlass = col_cutlass * 1000
        col_cublas_kernel = col_cublas_kernel * 1000

    curr_df_cublas = pd.DataFrame(dtype='float64', columns=["Omega", "Implementation", "Time"])
    curr_df_cutlass = pd.DataFrame(dtype='float64', columns=["Omega", "Implementation", "Time"])
    curr_df_cosma = pd.DataFrame(dtype='float64', columns=["Omega", "Implementation", "Time"])

    cublas_mean = numpy.mean(col_cublas_kernel)

    #print((M * N * (2 * K + 3)) / (col_cublas_kernel * 10e3))

    curr_df_cublas['Omega'] = numpy.full(100, M)
    curr_df_cublas['Implementation'] = numpy.full(100, cublas_real_name)
    curr_df_cublas['Time'] = (M * N * (2 * K + 3)) / (col_cublas_kernel * 10e5)

    curr_df_cutlass['Omega'] = numpy.full(100, M)
    curr_df_cutlass['Implementation'] = numpy.full(100, "CUTLASS")
    curr_df_cutlass['Time'] = (M * N * (2 * K + 3)) / (col_cutlass * 10e5)

    curr_df_cosma['Omega'] = numpy.full(100, M)
    curr_df_cosma['Implementation'] = numpy.full(100, "cuCOSMA")
    curr_df_cosma['Time'] = (M * N * (2 * K + 3)) / (col_cosma * 10e5)

    data = data.append(curr_df_cutlass, ignore_index=True).append(curr_df_cublas, ignore_index=True).append(curr_df_cosma, ignore_index=True)

# cuBLAS_mean_df = cuBLAS_mean_df.append({'Omega': M, 'Mean': cublas_mean}, ignore_index=True)

# print(cuBLAS_mean_df)

print(data)
# print(time_unit_data)

data.to_csv("data_square_flops.csv", index=False)
# cuBLAS_mean_df.to_csv("cuBLAS_mean_square.csv", index=False)
# print(cuBLAS_mean_df)

# time_unit_data.to_csv("time_unit_data_square.csv", index=False)

exit(1)


import os
import tkinter as tk
from datetime import datetime, date
from tkinter import ttk

from tkcalendar import DateEntry

from Database.Sheet import Sheet
from Generator.Module import Module

class MediaSpace(Module):
    def __init__(self, root):
        super(MediaSpace, self).__init__(root)
        self.root.title("Invoice Generator - Media Space")
        self.module_name = 'MediaSpace'
        self.root.geometry('460x635')

    def load_database(self):
        self.SPACES = Sheet('mediaspace_spaces')
        self.ITEMS = Sheet('mediaspace_items')
        self.EXTRAS = Sheet('mediaspace_extras')

    def build_window(self):
        super(MediaSpace, self).place_admin_fields()
        super(MediaSpace, self).place_customer_fields()
        self.place_space_reservation_details()
        super(MediaSpace, self).place_item_reservation_fields()
        super(MediaSpace, self).place_notes()
        self.clear()

    def place_space_reservation_details(self):
        frame = tk.LabelFrame(self.master, text='Reservation Details', font=('Arial', 14), padx=10, pady=10,
                              borderwidth=0, highlightthickness=0)

        for i, text in enumerate(['Selection', 'Day', 'Timings']):
            tk.Label(frame, text=text, font=('Arial', 12), padx=10, width=6).grid(row=0, column=i, padx=10)

        options = []
        for i in range(12):
            str_num = str(i)

            if len(str_num) == 1:
                str_num = '0' + str_num

            options.append(str_num + ':' + '00 am')
            options.append(str_num + ':' + '30 am')

        options.append('12' + ':' + '00 pm')
        options.append('12' + ':' + '30 pm')

        for i in range(1, 12):
            str_num = str(i)

            if len(str_num) == 1:
                str_num = '0' + str_num

            options.append(str_num + ':' + '00 pm')
            options.append(str_num + ':' + '30 pm')

        self.reservation_details = dict()
        for i in range(3):
            item_field = ttk.Combobox(frame, values=self.SPACES.list_items(), width=22)
            item_field.grid(row=i + 1, column=0)

            date_field = DateEntry(frame, width=8, background='black', foreground='white', borderwidth=2)
            date_field.grid(row=i + 1, column=1)

            timing_frame = tk.LabelFrame(frame, borderwidth=0, highlightthickness=0)

            start_time = ttk.Combobox(timing_frame, values=options, width=8)
            end_time = ttk.Combobox(timing_frame, values=options, width=8)

            start_time.grid(row=0, column=1)
            end_time.grid(row=0, column=2)
            timing_frame.grid(row=i + 1, column=2)
            timings = (start_time, end_time)
            self.reservation_details[item_field] = (date_field, timings)

        discount_frame = tk.LabelFrame(frame, borderwidth=0, highlightthickness=0)
        tk.Label(discount_frame, text="Discount", font=('Arial', 10), pady=10, width=6).pack(side=tk.LEFT)

        self.discount = tk.Spinbox(discount_frame, from_=0, to=25, increment=5, wrap=True, width=3)
        self.discount.pack(side=tk.RIGHT, padx=20)

        discount_frame.grid(row=i + 2, column=0, sticky=tk.W)

        frame.pack(padx=20, pady=2, anchor=tk.W)

    def read_inputs(self, *args):

        def convert_to_num(str, isFloat=True):
            test_str = str
            if isFloat:
                test_str = str.replace('.', '', 1)

            if test_str == '':
                return 0
            elif test_str.isnumeric():
                return float(str)
            else:
                return -1

        def reformat(string_date, char='/'):
            date = string_date.split('/')
            for i, entry in enumerate(date):
                if len(entry) == 1:
                    date[i] = '0' + entry

            return char.join(date)

        def generate_col_ids(chars):
            res = []

            for num in range(25, 36):
                entry = []
                for char in chars:
                    entry.append(char + str(num))
                res.append(entry)

            return res

        def separate(string_date, split_char='/'):
            string_date = string_date.split(split_char)
            m, d, y = [int(num) for num in string_date]
            return y, m, d

        admin, customer, company = self.admin.get(), self.customer.get(), self.company.get()
        credit = convert_to_num(self.credit.get())

        discount = convert_to_num(self.discount.get()) / 100

        current_file_count = str(len(os.listdir('Invoices/MediaSpace/')) + 1)
        zeros = ''.join(['0'] * (5 - len(current_file_count.split())))
        invoice_number = zeros + current_file_count

        invoice_date = datetime.date(datetime.now()).strftime("%m/%d/%y")
        days = {'mon': 0, 'tue': 1, 'wed': 2, 'thu': 3, 'fri': 4, 'sat': 5, 'sun': 6}

        billables, reservation_descriptions = [], []
        for space, info in self.reservation_details.items():
            if space.get():
                name = space.get()
                _date, timings = info

                start, end = [timing.get() for timing in timings]

                # seperate time from am/pm
                start_time, start_meridiem = start.split(' ')
                end_time, end_meridiem = end.split(' ')

                start_hr, start_min = [int(num) for num in start_time.split(':')]
                end_hr, end_min = [int(num) for num in end_time.split(':')]

                # reformat timing to 24-hr format for calculations
                if start_meridiem == 'pm' and start_hr != 12:
                    start_hr += 12

                if end_meridiem == 'pm' and end_hr != 12:
                    end_hr += 12

                duration = end_hr - start_hr + end_min / 60 - start_min / 60
                time = start + ' - ' + end

                # Extract quantities
                _, weekday_rate, weekend_rate, unit = self.SPACES[name]

                # Choose rate based on weekday
                display_name = name
                dt = date(*separate(_date.get()))
                if dt.weekday() == days['sun'] or dt.weekday() == days['sat']:
                    rate = weekend_rate
                    display_name = name + ' - Weekend'
                else:
                    rate = weekday_rate

                booking_date = reformat(_date.get())

                # Save info
                billables += [(display_name, duration, unit, float(rate))]
                reservation_descriptions += [(booking_date, time)]

        reserved_items = []
        for item, info in self.table_fields.items():
            if item.get() != '':

                _rate, _qty = info

                rate = convert_to_num(_rate.get())
                qty = convert_to_num(_qty.get(), isFloat=False)

                if rate <= 0 or qty <= 0:
                    raise Exception("Invalid rate or quanitity used")

                item_name = item.get()
                if item_name in self.ITEMS:
                    _, _, unit = self.ITEMS[item_name]
                else:
                    unit = "Item"

                reserved_items += [(item_name, qty, unit, rate)]

        billables += reserved_items

        try:
            if admin == "":
                raise Exception("Insert Admin Name")

            if customer == "":
                raise Exception("Insert Customer Name")

            if company == "":
                raise Exception("Insert Company Name")

            if credit < 0:
                raise Exception("Credit is a positive number")

            if discount < 0 or discount > 1:
                raise Exception("Discount is a number between 0 and 100")

            for name, qty, unit, rate in reserved_items:
                if qty <= 0:
                    raise Exception("Invalid quantity used in item reservation")
                if rate <= 0:
                    raise Exception("Invalid rate used in item reservation")

        except Exception as e:
            return e

        target = 'Invoices/MediaSpace/' + invoice_number + "_" + customer

        cells = {
            'admin': 'A6',
            'name': 'A13',
            'company': 'A14',
            'invoice_date': 'G12',
            'invoice_number': 'G11',
            'due_date': 'G13',
            'reservation_details': [['A18', 'B18'], ['A19', 'B19'], ['A20', 'B20']],
            'table': (generate_col_ids(['A', 'C', 'D', 'F'])),
            'discount': 'F40',
            'credit': 'G43',
            'notes': 'A48'
        }

        # place info
        entries = [(cells['admin'], admin),
                   (cells['name'], customer),
                   (cells['company'], company),
                   (cells['invoice_date'], invoice_date),
                   (cells['invoice_number'], invoice_number),
                   (cells['due_date'], reservation_descriptions[0][0]),
                   (cells['notes'], self.notes.get("1.0", 'end-1c'))]

        for placement, entry in zip(cells['table'], billables):
            for cell, info in zip(placement, entry):
                entries.append((cell, info))

        for i, entry in enumerate(zip(cells['reservation_details'], reservation_descriptions)):
            placement, description = entry
            for cell, info in zip(placement, description):
                entries.append((cell, info))

        hidden_entries = []
        for cell1, cell2 in cells['reservation_details'][i + 1:]:
            hidden_entries.append((cell1, 0))

        hidden_entries.append((cells['credit'], credit))
        hidden_entries.append((cells['discount'], discount))

        return target, entries, hidden_entries

    def clear(self, *args):
        self.admin.delete(0, 'end')
        self.admin.insert(0, 'Omar Eddin')

        self.customer.delete(0, 'end')

        self.company.delete(0, 'end')
        self.company.insert(0, 'Individual')

        self.credit.delete(0, 'end')

        for selection, info_fields in self.reservation_details.items():
            # day, start, end =
            for field in [selection, info_fields[0], *info_fields[1]]:
                field.delete(0, 'end')

        # self.discount.delete(0, 'end')

        for item_field, info_fields in self.table_fields.items():
            for field in [item_field, *info_fields]:
                field.delete(0, 'end')

        self.notes.delete(1.0, 'end')
        self.notes.insert(1.0, 'Please make your payment through Paypal to info@makeplus.us')

if __name__ == '__main__':
    window = tk.Tk()
    module = MediaSpace(window)
    module.start()


# encoding: utf-8
"""
@version: ??
@author: chenyitao
@license: Apache Licence
@software: PyCharm
@file: kline_protocol.py
@time: 2018/4/16 10:23
"""
import logging
from string import lower

from .binance_protocol_base import BinanceWSProtocolBase

log = logging.getLogger(__name__)

class BinanceWSKLineProtocol(BinanceWSProtocolBase):

    platform = 'binance'

    feature = 'binance.kline'

    def start(self):
        log.info('Binance KLine Started...')

    def message(self, data):
        if data.get('data'):
            self.update_market(data)

    def update_market(self, data):
        data = data.get('data')
        tick = data.get('k')
        date = data.get('E')
        currency_pair = lower(data.get('s'))
        index = self.req_currency_pairs.index(currency_pair)
        currency_pair = self.src_currency_pairs[index]
        market_info = {'platform': self.platform,
                       'currencyPair': currency_pair,
                       'high': tick.get('h'),
                       'low': tick.get('l'),
                       'open': tick.get('o'),
                       'last': tick.get('c'),
                       'volume': tick.get('v'),
                       'date': date}
        self.storage(market_info)


"""
Created on Wed Feb  5 13:04:17 2020

@author: matias
"""

import numpy as np
from matplotlib import pyplot as plt
from scipy.optimize import minimize
import  matplotlib.pyplot as mp
from scipy import optimize
import scipy
import math

import sys
import os
from os.path import join as osjoin
from pc_path import definir_path
path_git, path_datos_global = definir_path()
os.chdir(path_git)
sys.path.append('./Software/Funcionales/')
from funciones_data import leer_data_pantheon
from funciones_LambdaCDM_1 import params_to_chi2
#%%

min_z = 0
max_z = 3

os.chdir(path_git+'/Software/Estadística/Datos/Datos_pantheon/')
zcmb,zhel, Cinv, mb = leer_data_pantheon('lcparam_full_long_zhel.txt'
                        ,min_z=min_z,max_z=max_z)
sn = len(zcmb)
Cinv.shape

#%%
#Parametros a ajustar
H0_true =  73.5 #Unidades de (km/seg)/Mpc
M_true = -19.22

# Calculo el chi 2
omegas = np.linspace(0,1,100)
chies = np.zeros((len(omegas)))
for i,omega_m in enumerate(omegas):
    theta = omega_m
    chies[i] = params_to_chi2(theta,[M_true,H0_true], zcmb, zhel,
                Cinv, mb) #Chis normalizados

#%%
mins = np.where(chies==np.min(chies))
print(chies[mins[0][0]], omegas[mins[0][0]])

chis=chies
#%% Para ver cuanto es un sigma, me fijo cuales valores de chi estan a +- 1.07
delta_chi = chies-min(chies)
index_min = np.where(chies==min(chies))[0][0]
index_sigma_Om = np.where(delta_chi<1.07)[0][0] #1 corresponde a 1 sigma
index_sigma_om = np.where(delta_chi<1.07)[0][-1]
omega_posta = omegas[index_min]

#%%
# OJOTA, chequear que este bien esto que estoy reportando
print('Omega es {} +{} -{}'.format(omega_posta,abs(omega_posta-omegas[index_sigma_Om]),abs(omega_posta-omegas[index_sigma_om])))

#%% ACA EMPIEZA LO DE CLAUDIA
#graficos
plt.close()
plt.figure()
plt.plot(omegas,chies,'r.')
#mp.savefig('Mabsvschi27348.pdf')
#%%
Likelihood = np.exp(-0.5*(chies-chies.min()))
#print(Likelihood)
Maximo_Like = Likelihood.max()   #max de la likelihood (normalizada a 1)

#Pruebo distintos niveles (cortes en altura para calcular los errores):
Level = Maximo_Like - 0.318
#Level_1 = Maximo_Like - 0.1
#Level_2 = Maximo_Like - 0.2
#Level_3 = Maximo_Like - 0.3
#Level_4 = Maximo_Like - 0.4
#Level_412 = Maximo_Like - 0.412

#Pasos:
# elijo un nivel en el eje y (Likelihood)
# busco los valores de x que correspondan a ese likelihood.
# Integro entre esos dos valores
# Hago el cociente con la integral total
# Cuando llego al 68% del valor total, ya encontré el intervalo a 1sigma.

integral_total= scipy.integrate.trapz(Likelihood,omegas)
y = Level*np.ones(100)    #linea horizontal con un nivel de Likelihood

#Calculo los puntos del eje x donde la Likelihood corta la horizontal:
idx = np.argwhere(np.diff(np.sign(y - Likelihood))).flatten()

#Los indices son:
indice_1= idx[0]
indice_2= idx[1]

print(indice_1, indice_2)

#Calculo las integrales acumulativas hasta los puntos x_1 y x_2:
integral_1 = scipy.integrate.cumtrapz(Likelihood,omegas)[indice_1]
integral_2 = scipy.integrate.cumtrapz(Likelihood,omegas)[indice_2]

#Restando las integrales acumulativas, calculo la integral en el rango:
integral_rango = integral_2 - integral_1

#Hago el cociente con la integral total:
cociente= integral_rango/integral_total

#Me fijo para que Level obtengo que el area sea el 68% del total:
print(cociente*100.)

#Me fijo cuales son los valores de Mabs:
print(omegas[indice_1])
print(omegas[indice_2])

#Me fijo cual es el indice del maximo Likelihood:
# Get the indices of maximum element in numpy array
idx_max = np.where(Likelihood == np.amax(Likelihood))

print(omegas[idx_max])

#Calculo los errores inferiores y superiores:
error_1 = omegas[idx_max] - omegas[indice_1] #error inferior
error_2 = omegas[indice_2] - omegas[idx_max] #error superior

print(error_1)
print(error_2)
print(omegas[indice_2] , omegas[idx_max] , omegas[indice_1])

#plt.close()
plt.figure()
plt.plot(omegas,Likelihood,'r.')
#plt.savefig('Lik#e_Mabs.pdf')
#%%

#efectivamente da una parabola (el grafico lo hace mas abajo con el ajuste ya hecho)
#ajustamos un polinomio de grado dos a la parabola y buscamos el minimo
fit=np.polyfit(omegas,chies,2) #esto me devuelve un array con a,b,c
pol=np.poly1d(fit) #esto me lo convierte para poder evaluar el pol
Mmin = optimize.fmin(pol, -19)

#Mmin[0] es el minimo en Mabs. y pol(Mmin[0]) el valor del chi2 en el minimo
#ahora hay que encontrar el error. hay que sumarle 1.00 al chi2 de acuerdo a la tabla de la pg 815 del numerical recipies.
chi2sigma=pol(Mmin[0])+1.00
#ahora hay que encontrar el Mabs asociado a este valor de chi2. depejo la cuadratica considerando c=c-chi2sigma
Mabssigma=(-fit[1]+math.sqrt((fit[1]**2.-4.0*fit[0]*(fit[2]-chi2sigma))))/(2.0*fit[0])
print(Mmin[0],abs(Mabssigma-Mmin[0]),pol(Mmin[0])) #escribo el minimo, la desviacion a 1 sigma  y el chi2 minimo

#ploteo el el chi2 y el fit
#Claudia: xp=np.linspace(-19.1,-19.7,100)
#xp=np.linspace(-19.28,-19.22,100)
#xp=np.linspace(-19.1,-19.7,100)
xp=omegas
#plt.close()
plt.figure()
plt.plot(omegas,chies,'r.',xp,pol(xp),'g.')
#plt.savefig('MabsvschiLCDM.pdf')