import numpy as np
from scipy.integrate import solve_ivp

from syssim.core import NodeDifferential, InputPort, OutputPort
from syssim.core.port import InputPort, OutputPort

import matplotlib.pyplot as plt

class NodeSimpleBattery(NodeDifferential):
    def __init__(self, x0, **kwargs):
        in_current = InputPort("in_current", self)
        in_draw = InputPort("in_current_draw", self)

        out_soc = OutputPort("out_soc", self)

        ports = {
            in_current.name: in_current,
            in_draw.name: in_draw,
            out_soc.name: out_soc,
        }

        super().__init__(x0, ports, **kwargs)

    def initialize(self):
        self._eff = self._config["efficiency"]
        self._ts = []
        self._inp = []
        self._t = 0

    # def finalize(self):
    #     plt.plot(self._ts, self._inp)
    #     plt.show()

    def update(self, sim_time: float):
        i_inp = self._ports["in_current"].read()
        i_out = self._ports["in_current_draw"].read()

        if i_inp == None:
            i_inp = 0
        if i_out == None:
            i_out = 0

        self._ts.append(sim_time)
        self._inp.append(i_inp)

        dt = sim_time - self._t
        # print(
        #     f"{(i_inp - i_out) * dt / 3600 / self._config['capacity_ah']} | {self._x}"
        # )
        # print(f'{i_inp}')
        self._x += (
            (self._eff * i_inp - (1 / self._eff) * i_out)
            * dt
            / 3600
            / self._config["capacity_ah"]
        )
        self._x = np.clip(self._x, 0.0, 1.0)

        self._t = sim_time

        self._ports["out_soc"].shift_out(self._x)


#!/usr/bin/env python
import os
import sys
import numpy as np
from keras.models import Sequential, load_model
from keras.layers import Dense, Activation, Dropout, TimeDistributed, LSTM
from keras.optimizers import Adam
from sklearn.preprocessing import MinMaxScaler
from keras.callbacks import Callback, ModelCheckpoint
import codecs

def load_cfg(cfg):
    # TO
    # We should use argparser instead hard code

    with codecs.open(cfg, "r", encoding = "utf8") as f:
        data = f.readlines()
        params = {}
        for line in data:
            line = line.strip().split("=")
            params[line[0]] = int(line[1])

        return params["batch_size"], params["num_steps"], params["input_dim"]

class KerasBatchGenerator(object):

    def __init__(self, data, batch_size, num_steps, input_dim):
        self.data = data
        self.batch_size = batch_size
        self.num_steps = num_steps
        self.input_dim = input_dim
        self.cur_idx = 0

    def generate(self):

        scaler = MinMaxScaler(feature_range=(0,1))

        xx = np.zeros((self.batch_size, self.num_steps, self.input_dim))
        yy = np.zeros((self.batch_size))

        while True:
            for i in range(self.batch_size):
                if self.cur_idx >= len(self.data):
                    self.cur_idx = 0
                xt = self.data[self.cur_idx: self.cur_idx+self.num_steps]
                xt = scaler.fit_transform(xt)
                xx[i,:,:] = xt
                self.cur_idx += self.num_steps
                yy[i] = self.data[self.cur_idx][0]
                self.cur_idx += 1
            yield xx, yy

def init_lstm(num_steps, input_dim, hidden_size = 50):

    model = Sequential()
    model.add(LSTM(hidden_size, input_shape = (num_steps, input_dim), return_sequences = True))
    model.add(LSTM(hidden_size, input_shape = (num_steps, input_dim), return_sequences = False))
    model.add(Dense(1))
    model.add(Activation("linear"))
    optimizer = Adam
    model.compile(loss="mean_absolute_error", optimizer = "rmsprop")

    print(model.summary())

    return model

def analyze_data(data, num_steps):

    length = len(data)

    return length//(num_steps+1)

def train(fp, batch_size, num_steps, input_dim):

    data = np.loadtxt(fp, "float")
    print(len(data))
    samples_tot = analyze_data(data, num_steps)
    print(samples_tot)
    spe = samples_tot//batch_size
    print(spe)
    train_data_generator = KerasBatchGenerator(data, batch_size, num_steps, input_dim)
    md = init_lstm(num_steps, input_dim, 50)
    #checkpoint = ModelCheckpoint(filepath = "./", monitor = "val_acc", verbose = 1, save_best_only = True, mode = "max")
    #callback_list = [checkpoint]

    md.fit_generator(train_data_generator.generate(), steps_per_epoch = spe, epochs = 30)
    #md.fit_generator(train_data_generator.generate(), steps_per_epoch = spe, callbacks = callback_list, epochs = 30)
    md.save_weights("final_model_weights.h5")
def test(fp, batch_size, num_steps, input_dim):
    md = init_lstm(num_steps, input_dim, 50)
    md.load_weights("final_model_weights")

    return

def main(feats, stage):

    batch_size, num_steps, input_dim = load_cfg("../conf/nn.conf")
    if stage == "0":
        print("Training starts...")
        fp = feats+"/"+"train"
        train(fp, batch_size, num_steps, input_dim)
    elif stage == "1":
        print("Testing starts...")
        fp = feats+"/"+"test"
        test(fp, batch_size, num_steps, input_dim)
    else:
        print("Invalid command.")
        exit(0)

if __name__ == "__main__":

    if len(sys.argv) != 3:
        print("Usage: {} input stage<0: training, 1: test>".format(sys.argv[0]))
        exit(0)
    main(sys.argv[1], sys.argv[2])


import os
import sublime
import re
import sublime_plugin

import FuzzyFilePath.expression as Context
from FuzzyFilePath.project.ProjectManager import ProjectManager
import FuzzyFilePath.common.path as Path
from FuzzyFilePath.common.verbose import log
import FuzzyFilePath.common.selection as Selection

ID = "goto file"

class FfpGotoFileCommand(sublime_plugin.TextCommand):
    """ go to file """
    def run(self, edit):
        current_directory = os.path.dirname(self.view.file_name())

        context = Context.get_context(self.view)
        if context.get("valid") is False:
            return log(ID, "abort, no valid path given:", context.get("needle"))

        path = context.get("needle")
        project = ProjectManager.get_current_project()

        if not (path and project):
            return log(ID, "path or project invalid", path, project)

        is_relative = Path.is_relative(path)
        if is_relative:
            path = Path.get_absolute_path(current_directory, path)
            path = re.sub(project.get_directory(), "", path)

        path = re.sub("^[\\\\/\.]", "", path)
        files = project.find_file(path)

        if len(files) == 0:
            return log(ID, "failed finding file", path)

        if len(files) == 1:
            self.open_file(project.get_directory(), files[0])
        else:
            # if javascript, search for index.js
            current_scope = self.view.scope_name(Selection.get_position(self.view))
            if re.search("\.js ", current_scope):
                for file in files:
                    if "index.js" in file:
                        return self.open_file(project.get_directory(), file)

            self.files = files
            self.project_folder = project.get_directory()
            self.view.show_popup_menu(files, self.select_file)

    def select_file(self, index):
        self.open_file(self.project_folder, self.files[index])
        self.files = None
        self.project_folder = None

    def open_file(self, project_folder, filepath):
        path = os.path.join(project_folder, filepath)
        sublime.active_window().open_file(path)


from . import _wrap
from ..arrow import vector as _av
import polars as pl

cells_to_coordinates = _wrap(_av.cells_to_coordinates, ret_type=pl.DataFrame)
cells_bounds = _av.cells_bounds
cells_bounds_arrays = _wrap(_av.cells_bounds_arrays, ret_type=pl.DataFrame)
cells_to_wkb_polygons = _wrap(_av.cells_to_wkb_polygons, ret_type=pl.Series)
cells_to_wkb_points = _wrap(_av.cells_to_wkb_points, ret_type=pl.Series)
vertexes_to_wkb_points = _wrap(_av.vertexes_to_wkb_points, ret_type=pl.Series)
directededges_to_wkb_lines = _wrap(_av.directededges_to_wkb_lines, ret_type=pl.Series)
directededges_to_wkb_linestrings = _wrap(_av.directededges_to_wkb_linestrings, ret_type=pl.Series)
wkb_to_cells = _wrap(_av.wkb_to_cells, ret_type=pl.Series)
geometry_to_cells = _wrap(_av.geometry_to_cells, ret_type=pl.Series)

__all__ = [
    cells_to_coordinates.__name__,
    cells_bounds.__name__,
    cells_bounds_arrays.__name__,
    cells_to_wkb_polygons.__name__,
    cells_to_wkb_points.__name__,
    vertexes_to_wkb_points.__name__,
    directededges_to_wkb_lines.__name__,
    directededges_to_wkb_linestrings.__name__,
    wkb_to_cells.__name__,
    geometry_to_cells.__name__,
]


import os
def rename_files():
    file_list=os.listdir(r"\prank")
    path=os.getcwd()
    os.chdir(r"\prank")
    #print file_list
    for file_name in file_list:
        new_name=file_name.translate(None,"1234567890")
        os.rename(file_name,new_name)
    os.chdir(path)
rename_files()

New folder/read this -extract files open prank folder then run file then run this program.py


#!/usr/bin/env python
from __future__ import division
import sys, os
from scripts.RunQC.Utilities import FileUtils as fileUtils
import xml.etree.ElementTree as ET
from scripts.RunQC.Metrics.TileMetrics import TileMetrics
from scripts.RunQC.Metrics.ErrorMetrics import ErrorMetrics

class MiSeqRunQualityInfo :

	# Checks phix alignment error rate
	def EvaluateErrorMetrics( self , config ) :

		path_element_list = config.basecalls_dir.split( 'Data' + os.path.sep + 'Intensities' + os.path.sep + 'BaseCalls')
		interop_path      = os.path.join( path_element_list[0] , 'InterOp' )
		error_metrics_file = os.path.join( interop_path , 'ErrorMetricsOut.bin' )

		if os.path.exists( error_metrics_file ) :

			metrics = ErrorMetrics( error_metrics_file )
			self.setAveragePhixErrorRate( metrics.average_error_rate )

	# calculates average percent failure amongst all tiles
	def CalculatePercentFailedDemux( self , config ) :

		alignment_dir = fileUtils.AlignmentDirectoryFromBaseCalls( config.basecalls_dir )

		demux_report  = os.path.join( alignment_dir , "DemultiplexSummaryF1L1.txt" )

		exists , message = fileUtils.exists( demux_report )

		if exists :

			percent_failed = list()

			foundHeader = False
			finished    = False
			for line in open( demux_report , 'rb' ) :
				if not foundHeader :
					if line.strip().startswith("SampleName") :
						foundHeader = True
						continue
					else : continue
				if foundHeader and not finished :
					tokens = line.strip().split('\t')
					if len( tokens ) > 2 :
						percent_failed.append( float( tokens[1] ) )
					else : finished = True

			if len( percent_failed ) != 0 :
				average_percent_failed = reduce( lambda x, y: x + y, percent_failed ) / len( percent_failed )
			else : average_percent_failed = 0

			self.setPercentFailedDemux( average_percent_failed )

	# Checks for percentage of tiles where zero reads demux.
	# Can indicate a problem with the flowcell - large spots, dust inhibiting image analysis or
	# it's also been seen when data is moved while it's being analyzed
	def CalculatePercentTileDemuxFail( self , config ) :

		alignment_dir = fileUtils.AlignmentDirectoryFromBaseCalls( config.basecalls_dir )

		demux_report  = os.path.join( alignment_dir , "DemultiplexSummaryF1L1.txt" )

		exists , message = fileUtils.exists( demux_report )

		if exists :

			zero_tiles  = 0
			total_tiles = 0

			foundHeader = False
			finished    = False
			for line in open( demux_report , 'rb' ) :
				if not foundHeader :
					if line.strip().startswith("SampleName") :
						foundHeader = True
						continue
					else : continue
				if foundHeader and not finished :
					tokens = line.strip().split('\t')
					if len( tokens ) > 2 :
						column_count = 0
						for token in tokens :
							if column_count > 1 :
								if float( token ) <= 0 :
									zero_tiles += 1
								total_tiles += 1
							column_count += 1
					else : finished = True

			if total_tiles > 0 :
				average_percent_failed = ( zero_tiles / total_tiles ) * 100
			else : average_percent_failed = 100

			self.setPercentTilesFailedDemux( average_percent_failed )

	# Checks for things like flow cell density etc.
	def EvaluateTileMetrics( self , config ) :

		path_element_list = config.basecalls_dir.split( 'Data' + os.path.sep + 'Intensities' + os.path.sep + 'BaseCalls')
		interop_path      = os.path.join( path_element_list[0] , 'InterOp' )

		tile_metrics_file = os.path.join( interop_path , 'TileMetricsOut.bin' )

		if os.path.exists( tile_metrics_file ) :

			metrics = TileMetrics( tile_metrics_file )

			self.setAverageDensity( metrics.average_density )
			self.setAverageDensityPF( metrics.average_density_pf )
			self.setPhasingApplied( metrics.average_phasing )
			self.setPrephasingApplied( metrics.average_prephasing )
			self.setPercentPhiXAligned( metrics.average_phix_aligned )
			self.setClusterCountPF( metrics.total_pf_clusters )
			self.setPercentPF( metrics.Get_Percent_PF() )

	# Sanity Check Fastq Files
	def VerifyFastqFilesExist( self , config ) :

		fastq_files_exist = True
		message = "File not found"

		for sampleNumber , sampleInfo in config.sampleInfoDictionary.iteritems() :
			fastq_files_exist , message = fileUtils.exists( sampleInfo.read_1_fastq )
			if not fastq_files_exist : break
			if config.is_paired_end :
				fastq_files_exist , message = fileUtils.exists( sampleInfo.read_2_fastq )
				if not fastq_files_exist : break

		self.setFastqFilesExist( fastq_files_exist , message )

	# Checks that the instrument completed
	# demux & fastq file generation
	def VerifyPrimaryAnalysisComplete( self , config ) :

		illumina_alignment_dir = fileUtils.AlignmentDirectoryFromBaseCalls( config.basecalls_dir )
		checkpoint_file        = os.path.join( illumina_alignment_dir, "Checkpoint.txt" )

		if os.path.exists( checkpoint_file ) :

			f = open( checkpoint_file , 'rb')
			line = f.readline().strip()
			f.close()

			step = int( line )

			if step >= 3 : self.setPrimaryAnalysisComplete( True , step )
			else : self.setPrimaryAnalysisComplete( False , step )

	# Warn if unknown illumina SW is detected.
	# sometimes they ship products that are not
	# ready for prime time - we should detect that
	def VerifyIlluminaSoftwareVersions( self , config ) :

		versions_file = os.path.join( os.path.dirname( __file__ ) , 'instrument_software_versions.xml' )

		if os.path.exists( versions_file  ) :

			known_versions_used = True

			tree = ET.parse( versions_file )
			root = tree.getroot()

			FPGA_versions = list()
			MCS_versions  = list()
			RTA_versions  = list()
			MSR_versions  = list()

			for version in root.findall( "./FPGA_versions/version" ) : # hardware
				FPGA_versions.append( version.text )
			for version in root.findall( "./MCS_versions/version" ) :  # firmware
				MCS_versions.append( version.text )
			for version in root.findall( "./RTA_versions/version" ) :  # real time anlaysis sw ( primary analysis )
				RTA_versions.append( version.text )
			for version in root.findall( "./MSR_versions/version" ) :  # miseq reporter ( secondary analysis )
				MSR_versions.append( version.text )

			versions = dict()

			versions['fpga_version'] = "unknown"
			versions['mcs_version']  = "unknown"
			versions['rta_version']  = "unknown"
			versions['msr_version']  = "unknown"

			# first 3 are read from run params file
			path_element_list = config.basecalls_dir.split( 'Data' + os.path.sep + 'Intensities' + os.path.sep + 'BaseCalls' )
			runParametersFile = os.path.join( path_element_list[0], 'RunParameters.xml' )
			if not os.path.exists( runParametersFile ) :
				runParametersFile = os.path.join( path_element_list[0], 'runParameters.xml' ) # sometimes it's lowercase!

			if os.path.exists( runParametersFile ) :

				tree = ET.parse( runParametersFile )
				root = tree.getroot()

				for child in root:
					if child.tag == "FPGAVersion" :
						versions['fpga_version'] = child.text
					elif child.tag == "MCSVersion" :
						versions['mcs_version'] = child.text
					elif child.tag == "RTAVersion" :
						versions['rta_version'] = child.text

			# Miseq reporter writes to generate fastq stats file
			generateFastqStatsFile = os.path.join( fileUtils.AlignmentDirectoryFromBaseCalls( config.basecalls_dir ) , 'GenerateFASTQRunStatistics.xml' )

			if os.path.exists( generateFastqStatsFile ) :

				tree = ET.parse( generateFastqStatsFile )
				root = tree.getroot()

				for child in root:
					if child.tag == "RunStats" :
						msr_element = child.find('AnalysisSoftwareVersion')
						versions['msr_version'] = msr_element.text

				if versions['fpga_version'] not in FPGA_versions :
					known_versions_used = False
				if versions['mcs_version'] not in MCS_versions :
					known_versions_used = False
				if versions['rta_version'] not in RTA_versions :
					known_versions_used = False
				if versions['msr_version'] not in MSR_versions :
					known_versions_used = False

				self.setMiSeqSWVersions( known_versions_used , versions )

	# Given a base call dir, write the report
	def writeReport( self , config , version, working_dir ) :

		#file = open( os.path.join( config.run_root , "RunQuality_" + config.flowcellID + ".csv" ) , 'wb' )
		if not os.path.exists( os.path.join( working_dir , config.runID ) ):
			os.makedirs( os.path.join( working_dir , config.runID ) )
		file = open( os.path.join( working_dir , config.runID, config.runID + "_stats.csv" ) , 'wb' )

		file.write( "FLAG,METRIC,QUALITY_SCORE,MESSAGE,VERSION,RUN_ID\n" )
		#file.write( "Instrument Software Check" +
				#',' + 'N/A' +
				#',' + str( self.known_sw_versions_used_score ) +
				#',' + self.known_sw_versions_used_msg +
				#',' + version +
				#',' + config.flowcellID + '\n' )
		#file.write( "Fastq File Sanity Check" +
				#',' + 'N/A' +
				#',' + str( self.fastq_files_score ) +
				#',' + self.fastq_files_exist_msg +
				#',' + version +
				#',' + config.flowcellID + '\n' )
		#file.write( "Instrument Run Complete" +
				#',' + str( self.last_primary_analysis_step ) +
				#',' + str( self.primary_analysis_complete_score ) +
				#',' + self.primary_analysis_complete_msg +
				#',' + version +
				#',' + config.flowcellID + '\n' )
		file.write( "Phasing" +
				',' + str( "%0.2f" % self.average_phasing ) +
				',' + str( self.phasing_score )  +
				',' + self.phasing_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "Prephasing" +
				',' + str( "%0.2f" % self.average_prephasing ) +
				',' + str( self.prephasing_score )  +
				',' + self.prephasing_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "PhiX Control Error Rate" +
				',' + str( "%0.2f" % self.average_phix_error_rate ) +
				',' + str( self.average_phix_error_rate_score )  +
				',' + self.average_phix_error_rate_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "Percent PF"
				',' + str( "%0.2f" % self.percent_pf ) +
				',' + str( self.percent_pf_score ) +
				',' + self.percent_pf_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "Cluster Density" +
				',' + str( int( self.cluster_density ) ) +
				',' + str( self.cluster_density_score ) +
				',' + self.cluster_density_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "Cluster Density PF" +
				',' + str( int( self.cluster_density_pf ) ) +
				',' + str( self.cluster_density_pf_score ) +
				',' + self.cluster_density_pf_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "Reads PF" +
				',' + str( self.total_pf_clusters ) +
				',' + str( self.reads_pf_score ) +
				',' + self.reads_pf_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "Percent PhiX Aligned" +
				',' + str( "%0.2f" % self.average_phix_aligned ) +
				',' + str( self.percent_phix_aligned_score ) +
				',' + self.percent_phix_aligned_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "Percent Failed Demux" +
				',' + str( "%0.2f" % self.average_demux_failed ) +
				',' + str( self.percent_demux_failed_score ) +
				',' + self.percent_demux_failed_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.write( "Percent Tiles Failed Demux" +
				',' + str( "%0.2f" % self.percent_tile_demux_fail ) +
				',' + str( self.percent_tile_demux_fail_score ) +
				',' + self.percent_tile_demux_fail_msg +
				',' + version +
				',' + config.irunID + '\n' )
		file.close()

	# Given a base call dir, write the report
	def writeReport2( self , config , version, working_dir ) :

		#file = open( os.path.join( config.run_root , "RunQuality_" + config.flowcellID + ".csv" ) , 'wb' )
		if not os.path.exists( os.path.join( working_dir , config.runID ) ):
			os.makedirs( os.path.join( working_dir , config.runID ) )
		file = open( os.path.join( working_dir , config.runID, config.runID + "_stats.csv" ) , 'ab' )

		file.write( "RunId,Tool,FLAG,StandardGroup,Type,Metric,Comments\n" )
		wantedMetricsNames = ['Phasing','Prephasing','PhiX Control Error Rate','Percent PF','Cluster Density','Cluster Density PF','Reads PF','Percent PhiX Aligned','Percent Failed Demux','Percent Tiles Failed Demux']
		wantedMetricsVariables = ['average_phasing','average_prephasing','average_phix_error_rate','percent_pf','cluster_density','cluster_density_pf','total_pf_clusters','average_phix_aligned','average_demux_failed','percent_tile_demux_fail']
		wantedMetricsMsgs = ['phasing_msg','prephasing_msg','average_phix_error_rate_msg','percent_pf_msg','cluster_density_msg','cluster_density_pf_msg','reads_pf_msg','percent_phix_aligned_msg','percent_demux_failed_msg','percent_tile_demux_fail_msg']

		if len(wantedMetricsNames) == len(wantedMetricsVariables) == len(wantedMetricsMsgs)  :
			for index,MetricName in enumerate(wantedMetricsNames):
				MetricVariable = wantedMetricsVariables[index]
				MetricMsg = wantedMetricsMsgs[index]

				file.write( config.irunID +
					',' + version +
					',' + MetricName +
					',' + 'ALL' +
					',' + 'ALL' +
					',' + str( "%0.2f" % self.__dict__[MetricVariable] ) +
					',' + self.__dict__[MetricMsg] + '\n'
				)

		file.close()

	# Average error rate from RTA SeqAn alignment of phix
	def setAveragePhixErrorRate( self , average_error_rate ) :

		self.average_phix_error_rate = average_error_rate

		if average_error_rate <= self.max_error_rate :
			self.average_phix_error_rate_msg = "PhiX Error Rate [spec: <= " + str( "%.2f" % self.max_error_rate ) + "]: " + str( "%.2f" % average_error_rate )
			self.average_phix_error_rate_ok = True
			self.average_phix_error_rate_score = 10

		else :
			self.average_phix_error_rate_ok = True
			self.average_phix_error_rate_msg = "PhiX Error Rate is greater than spec [spec: <= " + str( "%.2f" % self.max_error_rate ) + "]: " + str( "%.2f" % average_error_rate )

			if average_error_rate > self.max_error_rate and average_error_rate <= self.max_error_rate + 0.1 : self.average_phix_error_rate_score = 9
			elif average_error_rate > self.max_error_rate + 0.1 and average_error_rate <= self.max_error_rate + 0.2 : self.average_phix_error_rate_score = 8
			elif average_error_rate > self.max_error_rate + 0.2 and average_error_rate <= self.max_error_rate + 0.3 : self.average_phix_error_rate_score = 7
			elif average_error_rate > self.max_error_rate + 0.3 and average_error_rate <= self.max_error_rate + 0.4 : self.average_phix_error_rate_score = 6
			elif average_error_rate > self.max_error_rate + 0.4 and average_error_rate <= self.max_error_rate + 0.5 : self.average_phix_error_rate_score = 5
			elif average_error_rate > self.max_error_rate + 0.5 and average_error_rate <= self.max_error_rate + 0.6 : self.average_phix_error_rate_score = 4
			elif average_error_rate > self.max_error_rate + 0.6 and average_error_rate <= self.max_error_rate + 0.7 : self.average_phix_error_rate_score = 3
			elif average_error_rate > self.max_error_rate + 0.7 and average_error_rate <= self.max_error_rate + 0.8 : self.average_phix_error_rate_score = 2
			elif average_error_rate > self.max_error_rate + 0.8 : self.average_phix_error_rate_score = 1 ; self.average_phix_error_rate_ok = False

	# Average demux failure percent amongst all tiles
	def setPercentFailedDemux( self , average_percent_failed ) :

		self.average_demux_failed = average_percent_failed

		if average_percent_failed <= self.max_mean_demux_failure :

			self.percent_demux_failed_msg = "Average percent demux failure is within spec [spec: <= " + str( self.max_mean_demux_failure ) + "]: " + str( "%.2f" % average_percent_failed )
			self.percent_demux_failed_ok = True
			self.percent_demux_failed_score = 10

		else :
			self.percent_demux_failed_ok = True
			self.percent_demux_failed_msg = "Average percent demux failure is greater than spec [spec: <= " + str( self.max_mean_demux_failure ) + "]: " + str( "%.2f" % average_percent_failed)
			self.percent_demux_failed_score = 10

			if average_percent_failed > self.max_mean_demux_failure         and average_percent_failed < self.max_mean_demux_failure + 3  : self.percent_demux_failed_score = 9
			elif average_percent_failed >= self.max_mean_demux_failure + 3  and average_percent_failed < self.max_mean_demux_failure + 7  : self.percent_demux_failed_score = 8
			elif average_percent_failed >= self.max_mean_demux_failure + 7  and average_percent_failed < self.max_mean_demux_failure  + 11 : self.percent_demux_failed_score = 7
			elif average_percent_failed >= self.max_mean_demux_failure + 11 and average_percent_failed < self.max_mean_demux_failure + 14 : self.percent_demux_failed_score = 6
			elif average_percent_failed >= self.max_mean_demux_failure + 14 and average_percent_failed < self.max_mean_demux_failure + 16 : self.percent_demux_failed_score = 5
			elif average_percent_failed >= self.max_mean_demux_failure + 16 and average_percent_failed < self.max_mean_demux_failure + 18 : self.percent_demux_failed_score = 4
			elif average_percent_failed >= self.max_mean_demux_failure + 18 and average_percent_failed < self.max_mean_demux_failure + 22 : self.percent_demux_failed_score = 3
			elif average_percent_failed >= self.max_mean_demux_failure + 22 and average_percent_failed < self.max_mean_demux_failure + 25 : self.percent_demux_failed_score = 2
			elif average_percent_failed >= self.max_mean_demux_failure + 25 : self.percent_demux_failed_score = 1 ; self.percent_demux_failed_ok = False

	# Percentage of tiles where demux completely failed.  Indicates problem
	# with the flowcell like dust.  Can also be caused by moving the data while
	# it's being analyzed - the threads die, the fastq 'appears' valid ( not truncated )
	# but it's missing reads..
	def setPercentTilesFailedDemux( self , percent_failed ) :

		self.percent_tile_demux_fail_msg = percent_failed

		if percent_failed <= self.max_percent_tile_demux_fail :

			self.percent_tile_demux_fail_msg   = "Average percent demux tile failure is within spec [spec: <= " + str( self.max_percent_tile_demux_fail ) + "]: " + str( "%.2f" % percent_failed )
			self.percent_tile_demux_fail_ok    = True
			self.percent_tile_demux_fail_score = 10
			self.percent_tile_demux_fail       = percent_failed

		else :

			self.percent_tile_demux_fail_msg   = "Average percent demux tile failure is greater than spec [spec: <= " + str( self.max_percent_tile_demux_fail ) + "]: " + str( "%.2f" % percent_failed )
			self.percent_tile_demux_fail_ok    = False
			self.percent_tile_demux_fail_score = 10
			self.percent_tile_demux_fail       = percent_failed

			if percent_failed > self.max_percent_tile_demux_fail         and percent_failed < self.max_percent_tile_demux_fail + 3  : self.percent_tile_demux_fail_score = 9
			elif percent_failed >= self.max_percent_tile_demux_fail + 3  and percent_failed < self.max_percent_tile_demux_fail + 7  : self.percent_tile_demux_fail_score = 8
			elif percent_failed >= self.max_percent_tile_demux_fail + 7  and percent_failed < self.max_percent_tile_demux_fail + 11 : self.percent_tile_demux_fail_score = 7
			elif percent_failed >= self.max_percent_tile_demux_fail + 11 and percent_failed < self.max_percent_tile_demux_fail + 14 : self.percent_tile_demux_fail_score = 6
			elif percent_failed >= self.max_percent_tile_demux_fail + 14 and percent_failed < self.max_percent_tile_demux_fail + 16 : self.percent_tile_demux_fail_score = 5
			elif percent_failed >= self.max_percent_tile_demux_fail + 16 and percent_failed < self.max_percent_tile_demux_fail + 18 : self.percent_tile_demux_fail_score = 4
			elif percent_failed >= self.max_percent_tile_demux_fail + 18 and percent_failed < self.max_percent_tile_demux_fail + 22 : self.percent_tile_demux_fail_score = 3
			elif percent_failed >= self.max_percent_tile_demux_fail + 22 and percent_failed < self.max_percent_tile_demux_fail + 25 : self.percent_tile_demux_fail_score = 2
			elif percent_failed >= self.max_percent_tile_demux_fail + 25 : self.percent_tile_demux_fail_score = 1

	# Sometimes illumina ships half-baked and if we encounter
	# an unknown we want to warn at least
	def setMiSeqSWVersions( self , known_sw_versions_used , versions ) :

		if known_sw_versions_used :
			self.known_sw_versions_used_msg = "Known & tested instrument SW was used to generate fastq files"
			self.known_sw_versions_used_ok = True
			self.known_sw_versions_used_score = 10
		else :
			self.known_sw_versions_used_msg = "WARNING: Unknown & untested (by Trovagene) instrument SW was used to generate data!"
			self.known_sw_versions_used_ok = False
			self.known_sw_versions_used_score = 1

	# Checks that the instrument dropped the flag
	# indicating step-3 completed ( finished making fastqs )
	def setPrimaryAnalysisComplete( self , complete , step ) :

		if complete == True :
			self.primary_analysis_complete_msg = "Min required step: 3 achieved: 3"
			self.primary_analysis_complete_ok = True
			self.primary_analysis_complete_score = 10
			self.last_primary_analysis_step = step
		else :
			self.primary_analysis_complete_msg = "Min required step (3) not achieved: " + str( complete )
			self.primary_analysis_complete_ok = False
			self.primary_analysis_complete_score = -1
			self.last_primary_analysis_step = step

	# Sanity check to verify fastq files exist
	def setFastqFilesExist( self , exist , msg ) :

		if exist :
			self.fastq_files_exist_ok = True
			self.fastq_files_exist_msg = "All Fastq files exist"
			self.fastq_files_score = 10
		else :
			self.fastq_files_exist_ok = False
			self.fastq_files_exist_msg = msg
			self.fastq_files_score = -1

	# The minimum required number of clusters passed the chastity filter:
	# highest_sig_int / (highest_sig_int + 2nd_highest_sig_int)
	# Clusters pass filter if no more than one base call in the first 25 cycles has a chastity of less than 0.6
	def setPercentPF( self , percent_pf ) :

		self.percent_pf = percent_pf

		if percent_pf >= self.min_percent_PF :
			self.percent_pf_msg = "Percent PF is within spec [spec: >= " + str( self.min_percent_PF ) + "]: " + str( "%.2f" % percent_pf )
			self.percent_pf_ok = True
			self.percent_pf_score = 10

		else : # somewhat sigmoidal drop is qualites as pf decreases

			self.percent_pf_msg = "Percent PF is less than spec [spec: >= " + str( self.min_percent_PF ) + "]: " + str( "%.2f" % percent_pf )

			if percent_pf   < self.min_percent_PF - 0.05 : self.percent_pf_score = 9
			elif percent_pf < self.min_percent_PF - 0.09 : self.percent_pf_score = 8
			elif percent_pf < self.min_percent_PF - 0.13 : self.percent_pf_score = 7
			elif percent_pf < self.min_percent_PF - 0.15 : self.percent_pf_score = 6
			elif percent_pf < self.min_percent_PF - 0.17 : self.percent_pf_score = 5
			elif percent_pf < self.min_percent_PF - 0.19 : self.percent_pf_score = 4
			elif percent_pf < self.min_percent_PF - 0.20 : self.percent_pf_score = 3
			elif percent_pf < self.min_percent_PF - 0.21 : self.percent_pf_score = 2
			elif percent_pf < self.min_percent_PF - 0.22 : self.percent_pf_score = 1

	# The PF cluster count should be within the density specification set by illumina for the V3 chemistry
    # Single Reads: 22-25 M
    # Paired End Reads: 44-50 M
    # For now while we are sequencing mono-template we may actually get better performance with lower density.
    # Perhaps even to the point of obtaining a very low quality score using the scale implemented.
    # A test we should add is a measure of the distance between Raw clusters and PF clusters.  If things went
    # well they should be close together - but how close?
	def setClusterCountPF( self , total_pf_clusters ) :

		self.total_pf_clusters = total_pf_clusters

		if total_pf_clusters > self.min_reads_PF and total_pf_clusters < self.max_reads_PF :

			self.reads_pf_msg = "The expected number of reads are available: " + str( "%.0f" % total_pf_clusters )
			self.reads_pf_ok = True
			self.reads_pf_score = 10

		elif total_pf_clusters > self.max_reads_PF :

				self.reads_pf_msg = "The number of reads passing filter exceeds the spec (" + str( "%.0f" % self.max_reads_PF ) + "): " + str( "%.0f" % total_pf_clusters )
				self.reads_pf_ok = True

				if total_pf_clusters    > self.max_reads_PF + 1000000 and total_pf_clusters < self.max_reads_PF + 1500000 : self.reads_pf_score = 9
				elif total_pf_clusters >= self.max_reads_PF + 1500000 and total_pf_clusters < self.max_reads_PF + 2000000 : self.reads_pf_score = 8
				elif total_pf_clusters >= self.max_reads_PF + 2000000 and total_pf_clusters < self.max_reads_PF + 2500000 : self.reads_pf_score = 7
				elif total_pf_clusters >= self.max_reads_PF + 2500000 and total_pf_clusters < self.max_reads_PF + 3000000 : self.reads_pf_score = 6
				elif total_pf_clusters >= self.max_reads_PF + 3000000 and total_pf_clusters < self.max_reads_PF + 3500000 : self.reads_pf_score = 5
				elif total_pf_clusters >= self.max_reads_PF + 3500000 and total_pf_clusters < self.max_reads_PF + 4000000 : self.reads_pf_score = 4
				elif total_pf_clusters >= self.max_reads_PF + 4000000 and total_pf_clusters < self.max_reads_PF + 4500000 : self.reads_pf_score = 3
				elif total_pf_clusters >= self.max_reads_PF + 4500000 and total_pf_clusters < self.max_reads_PF + 5000000 : self.reads_pf_score = 2
				elif total_pf_clusters >= self.max_reads_PF + 5000000                                                     : self.reads_pf_score = 1

		elif total_pf_clusters < self.min_reads_PF :

				self.reads_pf_msg = "The number of reads passing filter does not meet spec (" + str( "%.0f" % self.min_reads_PF ) + "): " + str( "%.0f" % total_pf_clusters )

				if   total_pf_clusters > self.min_reads_PF - 2000000  : self.reads_pf_score = 9 ; self.reads_pf_ok = True
				elif total_pf_clusters > self.min_reads_PF - 4000000  : self.reads_pf_score = 8 ; self.reads_pf_ok = True
				elif total_pf_clusters > self.min_reads_PF - 8000000  : self.reads_pf_score = 7 ; self.reads_pf_ok = True
				elif total_pf_clusters > self.min_reads_PF - 10000000 : self.reads_pf_score = 6 ; self.reads_pf_ok = True
				elif total_pf_clusters > self.min_reads_PF - 12000000 : self.reads_pf_score = 5
				elif total_pf_clusters > self.min_reads_PF - 14000000 : self.reads_pf_score = 4
				elif total_pf_clusters > self.min_reads_PF - 16000000 : self.reads_pf_score = 3
				elif total_pf_clusters > self.min_reads_PF - 17000000 : self.reads_pf_score = 2
				elif total_pf_clusters > 9000000                      : self.reads_pf_score = 1

	# Sets the percentage of PhiX which aligned.  Currently we are spiking in 5% and so we'd expect to see 5% aligned.
	# If not it may be due to:
	# 1. PhiX sample degradation - a tube freeze/thawed multiple times over a week of use may degrade so that even though
	#    the lab thinks they are spiking in 5% it is now 2%. Recommend aliquot and store single use tubes.
	# 2. Can also indicate overall poor sequencing quality if the error (1) is avoided.
	def setPercentPhiXAligned( self , average_phix_aligned ) :

		self.average_phix_aligned = average_phix_aligned

		if average_phix_aligned >= self.min_percent_phix_aligned :

			self.percent_phix_aligned_msg = "Percent PhiX Aligned is within spec [spec: >= " + \
			str( "%.2f" % self.min_percent_phix_aligned ) + "]: " + str( "%.2f" % average_phix_aligned )

			# Too much PHIX indicates low to no human
			if average_phix_aligned < self.max_percent_phix_aligned :
				self.percent_phix_aligned_ok = True
				self.percent_phix_aligned_score = 10
			else :
				self.percent_phix_aligned_ok = False
				self.percent_phix_aligned_score = 1
				self.percent_phix_aligned_msg = "Percent PhiX Aligned indicates overloading PhiX or underloading Human: " + str( "%.2f" % average_phix_aligned )
		else :

			self.percent_phix_aligned_msg = "Percent PhiX Aligned is less than spec [spec: " + \
			str( "%.2f" %  self.min_percent_phix_aligned ) + "]: " + str( "%.2f" % average_phix_aligned )

			if   average_phix_aligned < self.min_percent_phix_aligned and average_phix_aligned >= self.min_percent_phix_aligned - 0.5 :       self.percent_phix_aligned_score = 9 ; self.percent_phix_aligned_ok = True
			elif average_phix_aligned < self.min_percent_phix_aligned - 0.5 and average_phix_aligned >= self.min_percent_phix_aligned - 1 :   self.percent_phix_aligned_score = 8 ; self.percent_phix_aligned_ok = True
			elif average_phix_aligned < self.min_percent_phix_aligned - 1   and average_phix_aligned >= self.min_percent_phix_aligned - 1.5 : self.percent_phix_aligned_score = 7 ; self.percent_phix_aligned_ok = True
			elif average_phix_aligned < self.min_percent_phix_aligned - 1.5 and average_phix_aligned >= self.min_percent_phix_aligned - 2 :   self.percent_phix_aligned_score = 6 ; self.percent_phix_aligned_ok = True
			elif average_phix_aligned < self.min_percent_phix_aligned - 2   and average_phix_aligned >= self.min_percent_phix_aligned - 2.5 : self.percent_phix_aligned_score = 5 ; self.percent_phix_aligned_ok = True
			elif average_phix_aligned < self.min_percent_phix_aligned - 2.5 and average_phix_aligned >= self.min_percent_phix_aligned - 3 :   self.percent_phix_aligned_score = 4 ; self.percent_phix_aligned_ok = True
			elif average_phix_aligned < self.min_percent_phix_aligned - 3   and average_phix_aligned >= self.min_percent_phix_aligned - 3.5 : self.percent_phix_aligned_score = 3 ; self.percent_phix_aligned_ok = True
			elif average_phix_aligned < self.min_percent_phix_aligned - 3.5 and average_phix_aligned >= self.min_percent_phix_aligned - 4 :   self.percent_phix_aligned_score = 2 ; self.percent_phix_aligned_ok = True
			elif average_phix_aligned < self.min_percent_phix_aligned - 4 :                                                                   self.percent_phix_aligned_score = 1 ; self.percent_phix_aligned_ok = True

	# Sets the Prephasing applied status: Correction for % of DNA strands per cluster which
	# have erroneously fallen ahead of the current cycle by one or more bases
	def setPrephasingApplied( self , average_prephasing ) :

		self.average_prephasing = average_prephasing

		if average_prephasing <= self.max_prephasing :

			self.prephasing_score = 10
			self.prephasing_msg = "Prephasing applied is within spec [spec: <= " + str( "%.2f" % self.max_prephasing ) + "]: " + str( "%.2f" % average_prephasing )
			self.prephasing_ok = True

		else : # primitive quality score for now...

			difference = average_prephasing - self.max_prephasing

			if difference < 0.1 : self.prephasing_score = 9
			elif difference >= 0.1 and difference <= 0.2 : self.prephasing_score = 8
			elif difference > 0.2 and difference <= 0.3 : self.prephasing_score = 7
			elif difference > 0.3 and difference <= 0.4 : self.prephasing_score = 6
			elif difference > 0.4 and difference <= 0.5 : self.prephasing_score = 5
			elif difference > 0.5 and difference <= 0.6 : self.prephasing_score = 4
			elif difference > 0.6 and difference <= 0.7 : self.prephasing_score = 3
			elif difference > 0.7 and difference <= 0.8 : self.prephasing_score = 2
			elif difference > 0.8 : self.prephasing_score = 1

			self.prephasing_msg = "Prephasing applied is greater than spec [spec: <= " + str( "%.2f" % self.max_prephasing ) + "]: " + \
			str( "%.2f" % average_prephasing ) + ". Difference is: " + str( "%.2f" % difference )

	# Sets the Phasing applied status: Correction for % of DNA strands per cluster which
	# have erroneously fallen behind the current cycle by one or more bases
	def setPhasingApplied( self , average_phasing ) :

		self.average_phasing = average_phasing

		if average_phasing <= self.max_phasing :

			self.phasing_score = 10
			self.phasing_msg = "Phasing applied is within spec [spec: <= " + str( "%.2f" % self.max_phasing ) + "]: " + str( "%.2f" % average_phasing )
			self.phasing_ok = True

		else : # primitive quality score for now...

			difference = average_phasing - self.max_phasing

			if difference < 0.1 : self.phasing_score = 9
			elif difference >= 0.1 and difference <= 0.2 : self.phasing_score = 8
			elif difference > 0.2 and difference <= 0.3 : self.phasing_score = 7
			elif difference > 0.3 and difference <= 0.4 : self.phasing_score = 6
			elif difference > 0.4 and difference <= 0.5 : self.phasing_score = 5
			elif difference > 0.5 and difference <= 0.6 : self.phasing_score = 4
			elif difference > 0.6 and difference <= 0.7 : self.phasing_score = 3
			elif difference > 0.7 and difference <= 0.8 : self.phasing_score = 2
			elif difference > 0.8 : self.phasing_score = 1

			self.phasing_msg = "Phasing applied is greater than spec [spec: <= " + str( "%.2f" % self.max_phasing ) + "]: " + \
			str( "%.2f" % average_phasing ) + ". Difference is: " + str( "%.2f" % difference )

	# One of the most important metrics relating run quality.
	# More important that it's not too dense (over-cooked) rather than not dense enough (undercooked)
	# For now while we are sequencing mono-template we may actually get better performance with lower density.
	# Perhaps even to the point of obtaining a very low quality score using the scale implemented.
    # A test we should add is a measure of the distance between Raw clusters and PF clusters.  If things went
    # well they should be close together - but how close?
	def setAverageDensity( self , average_density ) :

		self.cluster_density = average_density

		if average_density <= self.max_cluster_density and average_density >= self.min_cluster_density :

			self.cluster_density_msg = "Raw cluster density is within spec [spec: >= " + str( int( self.min_cluster_density / 1000 ) ) + "K/mm^2 and <= " + \
			str( int( self.max_cluster_density / 1000 ) ) + "K/mm^2]: " + str( int( average_density / 1000 ) ) + "K/mm^2"

			self.cluster_density_ok = True
			self.cluster_density_score = 10

		else :

			if average_density < self.min_cluster_density : # undercooked, better than overcooked
				percent_difference = abs( self.min_cluster_density - average_density ) / abs( ( self.min_cluster_density + average_density ) / 2 ) * 100
				if percent_difference < 15 : self.cluster_density_score = 9
				elif percent_difference >= 15 and percent_difference < 35   : self.cluster_density_score = 8
				elif percent_difference >= 35 and percent_difference < 50   : self.cluster_density_score = 7
				elif percent_difference >= 50 and percent_difference < 65   : self.cluster_density_score = 6
				elif percent_difference >= 65 and percent_difference < 80   : self.cluster_density_score = 5
				elif percent_difference >= 80 and percent_difference < 95   : self.cluster_density_score = 4
				elif percent_difference >= 95 and percent_difference < 110  : self.cluster_density_score = 3
				elif percent_difference >= 110 and percent_difference < 125 : self.cluster_density_score = 2
				elif percent_difference >= 125 : self.cluster_density_score = 1

			elif average_density > self.max_cluster_density :
				percent_difference = abs( self.max_cluster_density - average_density ) / abs( ( self.max_cluster_density + average_density ) / 2 ) * 100
				if percent_difference < 1 : self.cluster_density_score = 9
				elif percent_difference >= 1 and percent_difference < 5   : self.cluster_density_score = 8
				elif percent_difference >= 5 and percent_difference < 10  : self.cluster_density_score = 7
				elif percent_difference >= 10 and percent_difference < 15 : self.cluster_density_score = 6
				elif percent_difference >= 15 and percent_difference < 20 : self.cluster_density_score = 5
				elif percent_difference >= 20 and percent_difference < 25 : self.cluster_density_score = 4
				elif percent_difference >= 25 and percent_difference < 30 : self.cluster_density_score = 3
				elif percent_difference >= 30 and percent_difference < 35 : self.cluster_density_score = 2
				elif percent_difference >= 35 : self.cluster_density_score = 1

			self.cluster_density_msg = "Raw cluster density is not within spec [spec: >= " + str( "%.0f" % ( self.min_cluster_density / 1000 ) ) + "K/mm^2 and <= " + \
                str( "%.0f" % ( self.max_cluster_density  / 1000 ) ) + "K/mm^2]: " + str( "%.0f" % ( average_density / 1000 ) )  + \
				" K/mm^2. Percent Difference is: " + str( "%.0f" % ( percent_difference ) )

		# This is density for data which has passed the chastity filter.  The raw density
		# is more informative but it's not always estimated correctly by the illumina
		# software.  PF density should be lower but relatively close if the true
		# density is in the sweet spot of the illumina image analysis algorithms.
		# For now while we are sequencing mono-template we may actually get better performance with lower density.
		# Perhaps even to the point of obtaining a very low quality score using the scale implemented.
		# A test we should add is a measure of the distance between Raw clusters and PF clusters.  If things went
		# well they should be close together - but how close?
	def setAverageDensityPF( self , average_density_pf ) :

		self.cluster_density_pf = average_density_pf

		if average_density_pf <= self.max_cluster_density and average_density_pf >= self.min_cluster_density :

			self.cluster_density_pf_msg = "PF cluster density is within spec [spec: >= " + str( int( self.min_cluster_density / 1000 ) ) + "K/mm^2 and <= " + \
			str( int( self.max_cluster_density / 1000 ) ) + "K/mm^2]: " + str( int( average_density_pf / 1000 ) ) + "K/mm^2"

			self.cluster_density_pf_ok = True
			self.cluster_density_pf_score = 10

		else :

			if average_density_pf < self.min_cluster_density : # undercooked, better than overcooked
				percent_difference = abs( self.min_cluster_density - average_density_pf ) / abs( ( self.min_cluster_density + average_density_pf ) / 2 ) * 100
				if percent_difference < 15 : self.cluster_density_pf_score = 9
				elif percent_difference >= 15 and percent_difference < 35   : self.cluster_density_pf_score = 8
				elif percent_difference >= 35 and percent_difference < 50   : self.cluster_density_pf_score = 7
				elif percent_difference >= 50 and percent_difference < 65   : self.cluster_density_pf_score = 6
				elif percent_difference >= 65 and percent_difference < 80   : self.cluster_density_pf_score = 5
				elif percent_difference >= 80 and percent_difference < 95   : self.cluster_density_pf_score = 4
				elif percent_difference >= 95 and percent_difference < 110  : self.cluster_density_pf_score = 3
				elif percent_difference >= 110 and percent_difference < 125 : self.cluster_density_pf_score = 2
				elif percent_difference >= 125 : self.cluster_density_pf_score = 1

			elif average_density_pf > self.max_cluster_density :
				percent_difference = abs( self.max_cluster_density - average_density_pf ) / abs( ( self.max_cluster_density + average_density_pf ) / 2 ) * 100
				if percent_difference < 1 : self.cluster_density_pf_score = 9
				elif percent_difference >= 1 and percent_difference < 5   : self.cluster_density_pf_score = 8
				elif percent_difference >= 5 and percent_difference < 10  : self.cluster_density_pf_score = 7
				elif percent_difference >= 10 and percent_difference < 15 : self.cluster_density_pf_score = 6
				elif percent_difference >= 15 and percent_difference < 20 : self.cluster_density_pf_score = 5
				elif percent_difference >= 20 and percent_difference < 25 : self.cluster_density_pf_score = 4
				elif percent_difference >= 25 and percent_difference < 30 : self.cluster_density_pf_score = 3
				elif percent_difference >= 30 and percent_difference < 35 : self.cluster_density_pf_score = 2
				elif percent_difference >= 35 : self.cluster_density_pf_score = 1

			self.cluster_density_pf_msg = "PF cluster density is not within spec [spec: >= " + str( "%.0f" % ( self.min_cluster_density / 1000 ) ) + "K/mm^2 and <= " + \
                str( "%.0f" % ( self.max_cluster_density / 1000 ) ) + "K/mm^2]: " + str( "%.0f" % ( average_density_pf / 1000 ) )  + \
				" K/mm^2. Percent Difference is: " + str( "%.0f" % ( percent_difference ) )

	# Failure of any of these results in a "No" answer to the
	# question: "Is this data good enough for teriary analysis?"
	# Secondary analysis being fastq file generation via MiSeq Reporter
	def OK( self ) :
		if self.fastq_files_exist_ok and self.primary_analysis_complete_ok and self.average_phix_error_rate_ok and \
		   self.known_sw_versions_used_ok and self.reads_pf_ok and self.percent_phix_aligned_ok and \
		   self.percent_demux_failed_ok and self.percent_tile_demux_fail_ok : return True , "AOK"
		elif not self.fastq_files_exist_ok :
			return False , self.fastq_files_exist_msg
		elif not self.primary_analysis_complete_ok :
			return False , self.primary_analysis_complete_msg
		elif not self.average_phix_error_rate_ok :
			return False , self.average_phix_error_rate_msg
		elif not self.known_sw_versions_used_ok :
			return False , self.known_sw_versions_used_msg
		elif not self.reads_pf_ok :
			return False , self.reads_pf_msg
		elif not self.percent_phix_aligned_ok :
			return False , self.percent_phix_aligned_msg
		elif not self.percent_demux_failed_ok :
			return False , self.percent_demux_failed_msg
		elif not self.percent_tile_demux_fail_ok :
			return False , self.percent_tile_demux_fail_msg

	####################################
	# MAIN INITIALIZATION

	def __init__( self , config ):

		self.average_phix_aligned    = 0
		self.total_pf_clusters       = 0
		self.percent_pf              = 0
		self.average_prephasing      = 0
		self.average_phasing         = 0
		self.cluster_density         = 0
		self.cluster_density_pf      = 0
		self.average_demux_failed    = 0
		self.average_phix_error_rate = 0
		self.percent_tile_demux_fail = 0

		self.cluster_density_msg           = "File not found"
		self.cluster_density_pf_msg        = "File not found"
		self.phasing_msg                   = "File not found"
		self.prephasing_msg                = "File not found"
		self.percent_phix_aligned_msg      = "File not found"
		self.cluster_count_pf_msg          = "File not found"
		self.percent_pf_msg                = "File not found"
		self.reads_pf_msg                  = "File not found"
		self.fastq_files_exist_msg         = "File not found"
		self.primary_analysis_complete_msg = "File not found"
		self.known_sw_versions_used_msg    = "File not found"
		self.percent_demux_failed_msg      = "File not found"
		self.average_phix_error_rate_msg   = "File not found"
		self.percent_tile_demux_fail_msg   = "File not found"

		self.cluster_density_ok           = False
		self.cluster_density_pf_ok        = False
		self.phasing_ok                   = False
		self.prephasing_ok                = False
		self.percent_phix_aligned_ok      = False
		self.reads_pf_ok                  = False
		self.percent_pf_ok                = False
		self.fastq_files_exist_ok         = False
		self.primary_analysis_complete_ok = False
		self.known_sw_versions_used_ok    = False
		self.percent_demux_failed_ok      = False
		self.average_phix_error_rate_ok   = False
		self.percent_tile_demux_fail_ok   = False

		self.cluster_density_score           = 0
		self.cluster_density_pf_score        = 0
		self.phasing_score                   = 0
		self.prephasing_score                = 0
		self.percent_phix_aligned_score      = 0
		self.reads_pf_score                  = 0
		self.percent_pf_score                = 0
		self.fastq_files_score               = 0
		self.primary_analysis_complete_score = 0
		self.known_sw_versions_used_score    = 0
		self.percent_demux_failed_score      = 0
		self.average_phix_error_rate_score   = 0
		self.percent_tile_demux_fail_score   = 0

		self.last_primary_analysis_step = 3    # an illumina workflow step, last step in GenerateFASTQ workflow
		self.min_reads_PF = 17500000           # based on cluster density for v3 chemistry:(1200-1400 K/m) diverse or 500-1200 K/m monotemplate PF.  For diverse: 22000000
		self.max_reads_PF = 22000000           # based on cluster density for v3 chemsitry: (1200-1400 K/m diverse or 500-1200 K/m monotemplate PF ) For diverse: 25000000
		self.min_percent_phix_aligned = 5      # currently spiking in 5% but more is better for mono-template. Lower score if degrades.
		self.max_percent_phix_aligned = 60     # indicates at most only 40% of the flowcell is human, which is not good
		self.min_percent_PF = 75               # not too conservative considering monotemplate
		self.max_error_rate = 0.50             # Control PhiX RTA/SeqAn alignment error rate
		self.max_phasing = 0.5                 # percent of clusters falling behind (ideal is < 0.25)
		self.max_prephasing = 0.3              # percent clusters getting ahead (ideal is < 0.15)
		self.max_cluster_density = 1200000     # MiSeq v3 reagents accommodate an optimal raw cluster density of ( 1200-1400 K/m diverse or 500-1200 K/m monotemplate )
		self.min_cluster_density = 500000      # MiSeq v3 reagents accommodate an optimal raw cluster density of ( 1200-1400 K/m diverse or 500-1200 K/m monotemplate )
		self.max_mean_demux_failure = 20       # 5-10% is considered good, this is liberal because we're sequencing mono-template
		self.max_percent_tile_demux_fail = 10  # Percentage of tiles were zero reads demultiplex

		self.VerifyFastqFilesExist( config )
		self.VerifyPrimaryAnalysisComplete( config )
		self.VerifyIlluminaSoftwareVersions( config )
		self.EvaluateTileMetrics( config )
		self.CalculatePercentFailedDemux( config )
		self.CalculatePercentTileDemuxFail( config )
		self.EvaluateErrorMetrics( config )


import sys
sys.stdout = open("output.txt","w")

import re
import requests

url = "http://books.toscrape.com/catalogue/category/books/mystery_3/index.html"
response = requests.get(url)
print(response.ok)

string = response.text
string = string.replace("\n", " ")

compiler = re.compile(r'
',)

result = re.findall(compiler, string)
print(len(result))
print(result[0])

Web Crawling/find_specific category of book.py


from pathlib import Path
import json
from copy import deepcopy

import pytest
from mongomock import MongoClient
from requests import HTTPError

from analyst.web_api import (
    GetStockDataTask,
    get,
    get_ticker_symbols,
    get_daily_prices,
    get_financial_statements,
    NoDataError,
)
from analyst.task_base import AnalystTaskBase

snapshot_file_path = Path.cwd() / "tests/fixtures/stock_snapshot.json"
with open(snapshot_file_path, "r", encoding="utf-8") as f:
    stock_snapshot = json.load(f)

mock_financials_quarter = stock_snapshot["data"]["financial_statements"]["quarter"]
mock_daily_prices = stock_snapshot["data"]["prices"]
mock_ticker_symbols = [
    {
        "symbol": "AMEX_PASS_1",
        "type": "stock",
        "exchange": "American Stock Exchange",
        "price": 20.1,
    },
    {"symbol": "TRUST_FILTERED_1", "type": "trust"},
    {
        "symbol": "NASDAQ_PASS_1",
        "type": "stock",
        "exchange": "NASDAQ Capital Market",
        "price": 100.0,
    },
    {
        "symbol": "NASDAQ_PASS_2",
        "type": "stock",
        "exchange": "NASDAQ Global Market",
        "price": 20.0,
    },
    {
        "symbol": "NYSE_PASS_1",
        "type": "stock",
        "exchange": "New York Stock Exchange Arca",
        "price": 31.0,
    },
    {
        "symbol": "NYSE_FILTERED_1",
        "type": "stock",
        "exchange": "New York Stock Exchange Arca",
        "price": 19.0,
    },
    {
        "symbol": "LSE_FILTERED_1",
        "type": "stock",
        "exchange": "London Stock Exchange",
        "price": 100.0,
    },
    {"symbol": "ETF_FILTERED_1", "type": "etf"},
]
mock_filtered_ticker_symbols = [
    {"symbol": "AMEX", "type": "stock", "exchange": "American Stock Exchange"},
    {"symbol": "NASDAQ", "type": "stock", "exchange": "NASDAQ Global Market"},
    {"symbol": "NASDAQ2", "type": "stock", "exchange": "Nasdaq capital Market"},
    {"symbol": "NYSE", "type": "stock", "exchange": "New York Stock Exchange Arca"},
]

def mock_response(data: dict | list, status_code: int):
    class MockResponse:
        def __init__(self, data_: dict | list, status_code_: int):
            self.data = data_
            self.status_code = status_code_

        def json(self):
            return self.data

        def raise_for_status(self):
            if 200 <= self.status_code < 300:
                return
            raise HTTPError(f"TEST ERROR: {self.status_code}")

    return MockResponse(data, status_code)

class TestGetStockDataTask:
    def setup_method(self):
        client = MongoClient()
        self.mock_db_client = client
        db_name = AnalystTaskBase.DB_NAME
        task_collection_name = AnalystTaskBase.TASK_COLLECTION_NAME
        stock_data_collection_name = AnalystTaskBase.STOCK_DATA_COLLECTION_NAME
        screener_collection_name = AnalystTaskBase.SCREENER_COLLECTION_NAME
        self.task_collection = client[db_name][task_collection_name]
        self.stock_data_collection = client[db_name][stock_data_collection_name]
        self.screener_collection = client[db_name][screener_collection_name]

    def teardown_method(self):
        self.mock_db_client.close()

    def test_get_single_stock_data_and_save_success(self, mocker):
        mocker.patch(
            "analyst.web_api.get_financial_statements",
            return_value=mock_financials_quarter,
        )
        mocker.patch("analyst.web_api.get_daily_prices", return_value=mock_daily_prices)
        task = GetStockDataTask("TEST", self.mock_db_client)
        task.get_single_stock_data_and_save({"symbol": "TEST"})
        filter_ = {"symbol.symbol": "TEST"}
        count = self.stock_data_collection.count_documents(filter_)
        assert count == 1

    def test_get_single_stock_data_and_save_no_financials(self, mocker):
        mocker.patch(
            "analyst.web_api.get_financial_statements",
            return_value=None,
        )
        mocker.patch("analyst.web_api.get_daily_prices", return_value=mock_daily_prices)
        task = GetStockDataTask("TEST", self.mock_db_client)
        task.get_single_stock_data_and_save({"symbol": "TEST"})
        filter_ = {"symbol.symbol": "TEST"}
        count = self.stock_data_collection.count_documents(filter_)
        assert count == 0

    def test_get_single_stock_data_and_save_no_prices(self, mocker):
        mocker.patch(
            "analyst.web_api.get_financial_statements",
            return_value=mock_financials_quarter,
        )
        mocker.patch("analyst.web_api.get_daily_prices", return_value=None)
        task = GetStockDataTask("TEST", self.mock_db_client)
        task.get_single_stock_data_and_save({"symbol": "TEST"})
        filter_ = {"symbol.symbol": "TEST"}
        count = self.stock_data_collection.count_documents(filter_)
        assert count == 0

    def test_save_to_screener_collection(self):
        task = GetStockDataTask("TEST", self.mock_db_client)
        symbol_names = ["A", "B"]
        docs = []
        for s in symbol_names:
            copy_symbol = deepcopy(stock_snapshot["symbol"])
            copy_symbol["symbol"] = s
            docs.append(
                {
                    "taskId": task.task_id,
                    "symbol": copy_symbol,
                    "data": stock_snapshot["data"],
                }
            )
        self.stock_data_collection.insert_many(docs)
        task.save_to_screener_collection()
        filter_ = {"taskId": task.task_id}
        count = self.screener_collection.count_documents(filter_)
        doc = self.screener_collection.find_one(filter_)
        assert count == 1
        assert doc["tickerSymbols"] == symbol_names

    def test_run(self, mocker):
        mocker.patch(
            "analyst.web_api.get_ticker_symbols",
            return_value=mock_filtered_ticker_symbols,
        )
        mocker.patch(
            "analyst.web_api.get_financial_statements",
            return_value=mock_financials_quarter,
        )
        mocker.patch("analyst.web_api.get_daily_prices", return_value=mock_daily_prices)
        task = GetStockDataTask("TEST", self.mock_db_client)
        task.run(20.0)
        filter_ = {"taskId": task.task_id}
        count_stock_data = self.stock_data_collection.count_documents(filter_)
        count_screener = self.screener_collection.count_documents(filter_)
        screener_result = self.screener_collection.find_one(filter_)
        doc_task = self.task_collection.find_one(filter_)
        assert count_stock_data == 4
        assert len(screener_result["tickerSymbols"]) == 4
        assert count_screener == 1
        assert doc_task["taskType"] == "get_stock_data"

class TestGet:
    def test_get_success(self, mocker):
        m = mocker.patch(
            "requests.Session.get", return_value=mock_response(mock_daily_prices, 200)
        )
        data = get(
            "https://financialmodelingprep.com/api/v3/historical-price-full/TEST",
            {"test": "test"},
        )
        args, kwargs = m.call_args
        assert args == (
            "https://financialmodelingprep.com/api/v3/historical-price-full/TEST",
        )
        assert "apikey" in kwargs["params"]
        assert kwargs["params"]["test"] == "test"
        assert type(data) is dict

    def test_det_daily_prices_no_data_error(self, mocker):
        mocker.patch("requests.Session.get", return_value=mock_response([], 200))
        url = "https://financialmodelingprep.com/api/v3/historical-price-full/TEST"
        with pytest.raises(NoDataError) as e:
            get("https://financialmodelingprep.com/api/v3/historical-price-full/TEST")
        assert str(e.value) == f"No data returned: {url}"

class TestGetDailyPrices:
    def test_det_daily_prices_success_default(self, mocker):
        m = mocker.patch("analyst.web_api.get", return_value=mock_daily_prices)
        prices = get_daily_prices("TEST")
        m.assert_called_once_with(
            "https://financialmodelingprep.com/api/v3/historical-price-full/TEST",
            {},
        )
        assert type(prices) is dict

    def test_det_daily_prices_success(self, mocker):
        m = mocker.patch("analyst.web_api.get", return_value=mock_daily_prices)
        prices = get_daily_prices("TEST", "2023-01-01", "2023-01-02")
        m.assert_called_once_with(
            "https://financialmodelingprep.com/api/v3/historical-price-full/TEST",
            {"from": "2023-01-01", "to": "2023-01-02"},
        )
        assert type(prices) is dict

    def test_det_daily_prices_http_error(self, mocker):
        mocker.patch("analyst.web_api.get", side_effect=HTTPError("TEST"))
        prices = get_daily_prices("TEST", "2023-01-01", "2023-01-02")
        assert prices is None

    def test_det_daily_prices_no_data_error(self, mocker):
        mocker.patch("requests.Session.get", return_value=mock_response([], 200))
        prices = get_daily_prices("TEST", "2023-01-01", "2023-01-02")
        assert prices is None

class TestGetFinancialStatements:
    def test_det_daily_prices_success_default(self, mocker):
        m = mocker.patch("analyst.web_api.get", return_value=mock_financials_quarter)
        financials = get_financial_statements("TEST")
        m.assert_called_once_with(
            "https://financialmodelingprep.com/api/v3/income-statement/TEST",
            {"limit": 10, "period": "quarter"},
        )
        assert type(financials) is list

    def test_det_daily_prices_success(self, mocker):
        m = mocker.patch("analyst.web_api.get", return_value=mock_financials_quarter)
        financials = get_financial_statements("TEST", 20, "annual")
        m.assert_called_once_with(
            "https://financialmodelingprep.com/api/v3/income-statement/TEST",
            {"limit": 20, "period": "annual"},
        )
        assert type(financials) is list

    def test_det_daily_prices_http_error(self, mocker):
        mocker.patch("analyst.web_api.get", side_effect=HTTPError("TEST"))
        financials = get_financial_statements("TEST")
        assert financials is None

    def test_det_daily_prices_no_data_error(self, mocker):
        mocker.patch("requests.Session.get", return_value=mock_response([], 200))
        financials = get_financial_statements("TEST")
        assert financials is None

def test_get_ticker_symbols(mocker):
    mocker.patch("requests.get", return_value=mock_response(mock_ticker_symbols, 200))
    sym = get_ticker_symbols(20.0)
    assert len(sym) == 4
    assert [s["symbol"] for s in sym] == [
        "AMEX_PASS_1",
        "NASDAQ_PASS_1",
        "NASDAQ_PASS_2",
        "NYSE_PASS_1",
    ]


import random
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
a=[]
b=[]
e=[]
fig, ax = plt.subplots(1)
for i in range(1,6):
    files=pd.read_csv("../../result4/Combine fuzzy and deep q "+str(i+1)+"/ketqua_oneday.csv")
    x=files["mean_reward"].to_numpy()[0:100]
    x = x[1:100]
    #print(x.shape)
    a.append(x)
    files=pd.read_csv("../../result4/deep q learning "+str(i)+"/ketqua_oneday.csv")
    xx=files["mean_reward"].to_numpy()[0:100]
    xx = xx[1:100]
    #print(xx.shape)
    b.append(xx)
    files=pd.read_csv("../DDQL_5phut"+str(i)+".csv")
    xxx=files["mean_reward"].to_numpy()[0:99]
    e.append(xxx)
    print(xxx.shape)
    #print(a)
m=[i for i in range(1,100)]

c=pd.read_csv("../../result4/fuzzy/fuzzy_150.csv")
d=pd.read_csv("../../result4/random/fuzzy_150.csv")

ax.plot(m,np.average(a,axis=0),marker='^', markevery=10,label='FDQO',color="orange")
ax.plot(m,np.average(b,axis=0),marker='o', markevery=10,label='DQL',color="blue")
ax.plot(m,np.average(e,axis=0),marker='x', markevery=10,label='DDQL',color="green")
# ax.fill_between(m, np.max(b,axis=0),np.min(b,axis=0), facecolor='#b9deff', alpha=0.5)

# # ax.plot(m,c["Fuzzy Controller"][0:100],marker='P', markevery=10,label='Fuzzy')
# # ax.plot(m,d["Random"][0:100],marker='x', markevery=10,label='Random')
# ax.fill_between(m, np.max(a,axis=0),np.min(a,axis=0), facecolor='#FFA107', alpha=0.5)
# ax.fill_between(m, np.max(c,axis=0),np.min(c,axis=0), facecolor='#b9deff', alpha=0.5)
ax.legend(loc='lower left', bbox_to_anchor=(0., 1.02, 1., .102), ncol=2)

plt.ylim(0,1)
#ax.set_xticks([0,0.2,0.4,0.6,0.8,1.0])
ax.set_yticks([0,0.2,0.4,0.6,0.8,1.0])
ax.set_xlabel('Time slots',fontsize=15)
ax.set_ylabel('Average QoE',fontsize=15)
plt.grid(alpha=0.5)
#plt.show()

plt.savefig("variation.pdf")


import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import os, json
import pdb
import matplotlib.ticker as ticker
import glob2, argparse
from os.path import expanduser
import visualization_module as viz

if __name__ == '__main__' :

    parser = argparse.ArgumentParser(description='data viz', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # reference directories
    parser.add_argument('--sd', default='/home/mattiacarletti/Desktop/plots/', help='Save directory')
    parser.add_argument('--base', default='/mnt/DATA/Mattia_data/results/', help='Base directory')
    # the following 3 parameters define the sub-directories from which we load data
    parser.add_argument('--exp', required=True, help='Experiment to be considered')
    parser.add_argument('--arch', required=True, nargs='*', help='List of architectures to be considered ("resnet18", "resnet34")')
    parser.add_argument('--datasets', required=True, nargs='*', help='List of datasets to be considered ("cifar10", "cifar100", "imagenet_lt")')
    # quantity to plot on the y-axis (plot) and phase
    parser.add_argument('--plot', required=True, help='Plot to be created ("loss", "top1", "class_count")')
    parser.add_argument('--phase', help='Phase to plot ("train", "val", "train_val")')
    parser.add_argument('--epochs', nargs='*', help='List of epochs to be considered (for class count hist)')
    # comparison at plot level (grouping variables)
    parser.add_argument('--hue', default=None, type=json.loads, help='Hue grouping variable for the seaborn plot')
    parser.add_argument('--style', default=None, type=json.loads, help='Style grouping variable for the seaborn plot (ignored when using phase="train_val")')
    parser.add_argument('--size', default=None, type=json.loads, help='Size grouping variable for the seaborn plot')
    # comparison at file level
    parser.add_argument('--cfl', default=None, type=json.loads, help='List of variables for comparisons at file level')
    opt = vars(parser.parse_args())

    # when phase='train_val', we use plain line for val and dashed line for train
    if opt['phase'] == 'train_val':
        opt['style'] = None
        print('\n"style" argument ignored since "phase"=="train_val"\n')
    # when plot='class_count', opt['style'] and opt['size'] cannot be used
    if opt['plot'] == 'class_count' and opt['style'] is not None and opt['size'] is not None:
        opt['style'] = None
        opt['size'] = None
        print('\n"style" and "size" arguments cannot be used since "plot"=="class_count"\n')
    # if 'arch' and 'dataset' are not grouping variables,
    # information about the architecture and dataset must be included in 'cfl'
    options = [list(opt[name].keys())[0] for name in ['hue','style','size'] if opt[name]]
    if 'arch' not in options:
        assert opt['cfl'] is not None, "'arch' must be added to the variables for comparison at file level"
        assert 'arch' in opt['cfl'].keys(), "'arch' must be added to the variables for comparison at file level"
    if 'dataset' not in options:
        assert opt['cfl'] is not None, "'dataset' must be added to the variables for comparison at file level"
        assert 'dataset' in opt['cfl'].keys(), "'dataset' must be added to the variables for comparison at file level"
    # PLOTS
    if opt['plot'] in ['loss', 'top1']:
        if opt['phase'] in ['train','val']:
            viz.compared_plot(opt)
        elif opt['phase'] == 'train_val':
            viz.compared_plot_train_val(opt)
    elif opt['plot'] == 'class_count':
        viz.compared_class_count_hist(opt)
    else:
        NotImplementedError


from typing import List, Optional

from pydantic import Field, root_validator

from pydatalab.models.items import Item
from pydatalab.models.utils import Constituent, InlineSubstance

class Sample(Item):
    """A model for representing an experimental sample."""

    type: str = Field("samples", const="samples", pattern="^samples$")

    chemform: Optional[str] = Field(example=["Na3P", "LiNiO2@C"])
    """A string representation of the chemical formula or composition associated with this sample."""

    synthesis_constituents: List[Constituent] = Field([])
    """A list of references to constituent materials giving the amount and relevant inlined details of consituent items."""

    synthesis_description: Optional[str]
    """Free-text details of the procedure applied to synthesise the sample"""

    @root_validator
    def add_missing_synthesis_relationships(cls, values):
        """Add any missing sample synthesis constituents to parent relationships"""
        from pydatalab.models.relationships import RelationshipType, TypedRelationship

        constituents_set = set()
        if values.get("synthesis_constituents") is not None:
            existing_parent_relationship_ids = set()
            if values.get("relationships") is not None:
                existing_parent_relationship_ids = set(
                    relationship.item_id or relationship.refcode
                    for relationship in values["relationships"]
                    if relationship.relation == RelationshipType.PARENT
                )
            else:
                values["relationships"] = []

            for constituent in values.get("synthesis_constituents", []):
                # If this is an inline relationship, just skip it
                if isinstance(constituent.item, InlineSubstance):
                    continue
                if (
                    constituent.item.item_id not in existing_parent_relationship_ids
                    and constituent.item.refcode not in existing_parent_relationship_ids
                ):
                    relationship = TypedRelationship(
                        relation=RelationshipType.PARENT,
                        item_id=constituent.item.item_id,
                        type=constituent.item.type,
                        description="Is a constituent of",
                    )
                    values["relationships"].append(relationship)

                # Accumulate all constituent IDs in a set to filter those that have been deleted
                constituents_set.add(constituent.item.item_id)

        # Finally, filter out any parent relationships with item that were removed
        # from the synthesis constituents
        values["relationships"] = [
            rel
            for rel in values["relationships"]
            if not (
                rel.item_id not in constituents_set
                and rel.relation == RelationshipType.PARENT
                and rel.type in ("samples", "starting_materials")
            )
        ]

        return values


from absl.testing import absltest
from jaxonnxruntime import call_onnx
from jaxonnxruntime.experimental import custom_ops  # pylint: disable=unused-import
import numpy as np

import onnx
from onnx import helper as onnx_helper

class ZerosLikeTest(absltest.TestCase):

  def test_basic(self):
    # Create input and output tensors.
    input_tensor = onnx_helper.make_tensor_value_info(
        "X", onnx.TensorProto.FLOAT, [1, 3, 224, 224]
    )
    output_tensor = onnx_helper.make_tensor_value_info(
        "Y", onnx.TensorProto.FLOAT, [1, 3, 224, 224]
    )

    # Create the ZerosLike node
    node = onnx_helper.make_node(
        "ZerosLike",
        inputs=["X"],
        outputs=["Y"],
        dtype=onnx.TensorProto.FLOAT,
        domain="jaxonnxruntime",
    )

    # Create the graph with the node
    graph_def = onnx_helper.make_graph(
        [node],
        "ZerosLike_Model",
        [input_tensor],
        [output_tensor],
    )

    # Create the model
    onnx_model = onnx_helper.make_model(
        graph_def,
        producer_name="JAX-ONNX",
        opset_imports=[
            onnx_helper.make_opsetid(
                onnx.defs.ONNX_DOMAIN, onnx.defs.onnx_opset_version()
            ),
            onnx_helper.make_opsetid("jaxonnxruntime", 1),
        ],
    )
    x = np.random.randn(3, 4, 5).astype(np.float32)
    y = np.zeros_like(x).astype(np.float32)
    inputs = [x]
    jax_model_func, jax_model_params = call_onnx.call_onnx_model(
        onnx_model, inputs
    )
    outputs = jax_model_func(jax_model_params, inputs)
    expect_outputs = [y]
    self.assertLen(outputs, 1, f"output is {outputs}")
    np.testing.assert_allclose(outputs[0], expect_outputs[0], atol=1e-6)

if __name__ == "__main__":
  absltest.main()


class Solution:
    """
    @param A: an integer array
    @return: nothing
    """
    def sortIntegers2(self, A):
        # write your code here
        self.sortHelper(A, 0, len(A) - 1)

    def sortHelper(self, A, first, last):
        if first < last:
            splitIdx = self.partition(A, first, last)

            self.sortHelper(A, first, splitIdx - 1)
            self.sortHelper(A, splitIdx + 1, last)

    def partition(self, nums, first, last):
        pivot = nums[first]

        leftIdx = first + 1
        rightIdx = last

        done = False
        while not done:
            while leftIdx <= rightIdx and nums[leftIdx] <= pivot:
                leftIdx += 1

            while leftIdx <= rightIdx and nums[rightIdx] >= pivot:
                rightIdx -= 1

            if rightIdx < leftIdx:
                done = True
            else:
                nums[leftIdx], nums[rightIdx] = nums[leftIdx], nums[rightIdx]

        nums[first] = nums[rightIdx]
        nums[rightIdx] = pivot

        return rightIdx

if __name__ == "__main__":
    sol = Solution()
    nums = [3,2,1,4,5]
    sol.sortIntegers2(nums)


"""module that contains helper function to load models

Models in separate packages should make themselves available to vak by including
'vak.models' in the entry_points of their setup.py file.

For example, if you had a package `grunet` containing a model
that was instantiated by a function `GRUnet`,
then that package would include the following in its setup.py file:

setup(
    ...
    entry_points={'vak.models': 'GRUnet = grunet:GRUnet'},
    ...
)

For more detail on entry points in Python, see:
https://packaging.python.org/guides/creating-and-discovering-plugins/#using-package-metadata
https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins
https://amir.rachum.com/blog/2017/07/28/python-entry-points/
"""
from .. import entry_points

MODELS_ENTRY_POINT = 'vak.models'

def find():
    """find installed vak.models

    returns generator that yields model name and function for loading
    """
    for entrypoint in entry_points._iter(MODELS_ENTRY_POINT):
        yield entrypoint.name, entrypoint.load()

def from_model_config_map(model_config_map, num_classes, input_shape, logger=None):
    """get models that are ready to train, given their names and configurations.

    Given a dictionary that maps model names to configurations,
    along with the number of classes they should be trained to discriminate and their input shape,
    return a dictionary that maps model names to instances of the model

    Parameters
    ----------
    model_config_map : dict
        where each key-value pair is model name : dict of config parameters
    num_classes : int
        number of classes model will be trained to classify
    input_shape : tuple, list
        e.g. (channels, height, width).
        Batch size is not required for input shape.
    logger : logging.Logger
        instance created by vak.logging.get_logger. Default is None.

    Returns
    -------
    models_map : dict
        where keys are model names and values are instances of the models, ready for training
    """
    MODELS = {model_name: model_builder for model_name, model_builder in find()}

    models_map = {}
    for model_name, model_config in model_config_map.items():
        # pass section dict as kwargs to config parser function
        model_config['network'].update(
            num_classes=num_classes,
            input_shape=input_shape,
        )
        try:
            model = MODELS[model_name].from_config(config=model_config)
        except KeyError:
            model = MODELS[f'{model_name}Model'].from_config(config=model_config, logger=logger)
        models_map[model_name] = model
    return models_map


import sys
from collections import deque

input = sys.stdin.readline

M, N = map(int, input().split())
board = [list(input().rstrip()) for _ in range(M)]

visited = []
dx = [1, -1, 0, 0]
dy = [0, 0, 1, -1]
result_v = 0
result_o = 0

for m in range(M):
    for n in range(N):
        queue = deque()
        temp = []
        if board[m][n] in ["v", "o", "."] and [m, n] not in visited:
            queue.append([m, n])
            temp.append(board[m][n])
            visited.append([m, n])
            while queue:
                m_, n_ = queue.popleft()
                for i in range(4):
                    nx = dx[i] + n_
                    ny = dy[i] + m_
                    if nx < 0 or N <= nx or ny < 0 or M <= ny:
                        continue
                    if board[ny][nx] == '#':
                        continue
                    if [ny, nx] not in visited:
                        queue.append([ny, nx])
                        temp.append(board[ny][nx])
                        visited.append([ny, nx])
        if temp:
            if temp.count('v') < temp.count('o'):
                result_o += temp.count('o')
            else:
                result_v += temp.count('v')
print(result_o, result_v)


#!/usr/bin/env python

##################################################
## DEPENDENCIES
import sys
import os
import os.path
try:
    import builtins as builtin
except ImportError:
    import builtins as builtin
from os.path import getmtime, exists
import time
import types
from Cheetah.Version import MinCompatibleVersion as RequiredCheetahVersion
from Cheetah.Version import MinCompatibleVersionTuple as RequiredCheetahVersionTuple
from Cheetah.Template import Template
from Cheetah.DummyTransaction import *
from Cheetah.NameMapper import NotFound, valueForName, valueFromSearchList, valueFromFrameOrSearchList
from Cheetah.CacheRegion import CacheRegion
import Cheetah.Filters as Filters
import Cheetah.ErrorCatchers as ErrorCatchers

##################################################
## MODULE CONSTANTS
VFFSL=valueFromFrameOrSearchList
VFSL=valueFromSearchList
VFN=valueForName
currentTime=time.time
__CHEETAH_version__ = '2.4.4'
__CHEETAH_versionTuple__ = (2, 4, 4, 'development', 0)
__CHEETAH_genTime__ = 1484975073.056371
__CHEETAH_genTimestamp__ = 'Sat Jan 21 16:04:33 2017'
__CHEETAH_src__ = '/home/mattias/xmds-2.2.3/admin/staging/xmds-2.2.3/xpdeint/xsil2graphics2/PythonImport.tmpl'
__CHEETAH_srcLastModified__ = 'Sun Jul 28 14:51:41 2013'
__CHEETAH_docstring__ = 'Autogenerated by Cheetah: The Python-Powered Template Engine'

if __CHEETAH_versionTuple__ < RequiredCheetahVersionTuple:
    raise AssertionError(
      'This template was compiled with Cheetah version'
      ' %s. Templates compiled before version %s must be recompiled.'%(
         __CHEETAH_version__, RequiredCheetahVersion))

##################################################
## CLASSES

class PythonImport(Template):

    ##################################################
    ## CHEETAH GENERATED METHODS

    def __init__(self, *args, **KWs):

        super(PythonImport, self).__init__(*args, **KWs)
        if not self._CHEETAH__instanceInitialized:
            cheetahKWArgs = {}
            allowedKWs = 'searchList namespaces filter filtersLib errorCatcher'.split()
            for k,v in list(KWs.items()):
                if k in allowedKWs: cheetahKWArgs[k] = v
            self._initCheetahInstance(**cheetahKWArgs)

    def description(self, **KWS):

        ## Generated from @def description: Creates text to import data from XSIL files into PyLab. at line 25, col 1.
        trans = KWS.get("trans")
        if (not trans and not self._CHEETAH__isBuffering and not callable(self.transaction)):
            trans = self.transaction # is None unless self.awake() was called
        if not trans:
            trans = DummyTransaction()
            _dummyTrans = True
        else: _dummyTrans = False
        write = trans.response().write
        SL = self._CHEETAH__searchList
        _filter = self._CHEETAH__currentFilter

        ########################################
        ## START - generated method body

        write('''Creates text to import data from XSIL files into PyLab.''')

        ########################################
        ## END - generated method body

        return _dummyTrans and trans.response().getvalue() or ""

    def loadXSILFile(self, xsilFile, **KWS):

        ## CHEETAH: generated from @def loadXSILFile($xsilFile) at line 31, col 1.
        trans = KWS.get("trans")
        if (not trans and not self._CHEETAH__isBuffering and not callable(self.transaction)):
            trans = self.transaction # is None unless self.awake() was called
        if not trans:
            trans = DummyTransaction()
            _dummyTrans = True
        else: _dummyTrans = False
        write = trans.response().write
        SL = self._CHEETAH__searchList
        _filter = self._CHEETAH__currentFilter

        ########################################
        ## START - generated method body

        write('''#!/usr/bin/env python
from xpdeint.XSILFile import XSILFile

xsilFile = XSILFile("''')
        _v = VFFSL(SL,"xsilFile.filename",True) # u'$xsilFile.filename' on line 35, col 22
        if _v is not None: write(_filter(_v, rawExpr='$xsilFile.filename')) # from line 35, col 22.
        write('''")

def firstElementOrNone(enumerable):
  for element in enumerable:
    return element
  return None

''')
        for objectNum, xsilObject in enumerate(xsilFile.xsilObjects): # generated from line 42, col 3
            for var in xsilObject.independentVariables: # generated from line 43, col 5
                varName = ''.join([str(VFFSL(SL,"var.name",True)),'_',str(VFFSL(SL,"objectNum",True)+1)])
                _v = VFFSL(SL,"varName",True) # u'$varName' on line 45, col 1
                if _v is not None: write(_filter(_v, rawExpr='$varName')) # from line 45, col 1.
                write(''' = firstElementOrNone(_["array"] for _ in xsilFile.xsilObjects[''')
                _v = VFFSL(SL,"objectNum",True) # u'$objectNum' on line 45, col 72
                if _v is not None: write(_filter(_v, rawExpr='$objectNum')) # from line 45, col 72.
                write('''].independentVariables if _["name"] == "''')
                _v = VFFSL(SL,"var.name",True) # u'${var.name}' on line 45, col 122
                if _v is not None: write(_filter(_v, rawExpr='${var.name}')) # from line 45, col 122.
                write('''")
''')
            for var in xsilObject.dependentVariables: # generated from line 47, col 5
                varName = ''.join([str(VFFSL(SL,"var.name",True)),'_',str(VFFSL(SL,"objectNum",True)+1)])
                _v = VFFSL(SL,"varName",True) # u'$varName' on line 49, col 1
                if _v is not None: write(_filter(_v, rawExpr='$varName')) # from line 49, col 1.
                write(''' = firstElementOrNone(_["array"] for _ in xsilFile.xsilObjects[''')
                _v = VFFSL(SL,"objectNum",True) # u'$objectNum' on line 49, col 72
                if _v is not None: write(_filter(_v, rawExpr='$objectNum')) # from line 49, col 72.
                write('''].dependentVariables if _["name"] == "''')
                _v = VFFSL(SL,"var.name",True) # u'${var.name}' on line 49, col 120
                if _v is not None: write(_filter(_v, rawExpr='${var.name}')) # from line 49, col 120.
                write('''")
''')
        write('''
# Write your plotting commands here.
# You may want to import pylab (from pylab import *) or matplotlib (from matplotlib import *)

''')

        ########################################
        ## END - generated method body

        return _dummyTrans and trans.response().getvalue() or ""

    def respond(self, trans=None):

        ## CHEETAH: main method generated for this template
        if (not trans and not self._CHEETAH__isBuffering and not callable(self.transaction)):
            trans = self.transaction # is None unless self.awake() was called
        if not trans:
            trans = DummyTransaction()
            _dummyTrans = True
        else: _dummyTrans = False
        write = trans.response().write
        SL = self._CHEETAH__searchList
        _filter = self._CHEETAH__currentFilter

        ########################################
        ## START - generated method body

        #
        # PythonImport.tmpl
        #
        # Created by Thomas Antioch on 2013-05-31.
        # Modified by Thomas Antioch on 2013-07-18.
        # Modified by Graham Dennis on 2013-07-28.
        #
        # Copyright (c) 2009-2013, Joe Hope and Graham Dennis
        #
        # This program is free software: you can redistribute it and/or modify
        # it under the terms of the GNU General Public License as published by
        # the Free Software Foundation, either version 2 of the License, or
        # (at your option) any later version.
        #
        # This program is distributed in the hope that it will be useful,
        # but WITHOUT ANY WARRANTY; without even the implied warranty of
        # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
        # GNU General Public License for more details.
        #
        # You should have received a copy of the GNU General Public License
        # along with this program.  If not, see <http://www.gnu.org/licenses/>.
        #
        write('''

''')

        ########################################
        ## END - generated method body

        return _dummyTrans and trans.response().getvalue() or ""

    ##################################################
    ## CHEETAH GENERATED ATTRIBUTES

    _CHEETAH__instanceInitialized = False

    _CHEETAH_version = __CHEETAH_version__

    _CHEETAH_versionTuple = __CHEETAH_versionTuple__

    _CHEETAH_genTime = __CHEETAH_genTime__

    _CHEETAH_genTimestamp = __CHEETAH_genTimestamp__

    _CHEETAH_src = __CHEETAH_src__

    _CHEETAH_srcLastModified = __CHEETAH_srcLastModified__

    name = 'Python'

    defaultExtension = 'py'

    _mainCheetahMethod_for_PythonImport= 'respond'

## END CLASS DEFINITION

if not hasattr(PythonImport, '_initCheetahAttributes'):
    templateAPIClass = getattr(PythonImport, '_CHEETAH_templateClass', Template)
    templateAPIClass._addCheetahPlumbingCodeToClass(PythonImport)

# CHEETAH was developed by Tavis Rudd and Mike Orr
# with code, advice and input from many other volunteers.
# For more information visit http://www.CheetahTemplate.org/

##################################################
## if run from command line:
if __name__ == '__main__':
    from Cheetah.TemplateCmdLineIface import CmdLineIface
    CmdLineIface(templateObj=PythonImport()).run()


from django.views.decorators.csrf import csrf_exempt
from django.contrib import messages
from django.conf import settings
from django.shortcuts import render, redirect
from django.urls import resolve, reverse
from django.views.generic import View
from ..forms import *
import json
import requests

class AddressLogic:
  """
  Logic class to Address requests
  """
  def status(self, request, *args, **kwargs):
    """
    Yields the status of a given eth address
    :param request:
    :param args:
    :param kwargs:
    :return:
    """
    try:
      page = request.GET.get('page', 1)
      items_per_page = request.GET.get('items', 7)
      transactions_filter = request.GET.get('transactions_type', 'external')
      ethaddr = kwargs['ethaddr'] if 'ethaddr' in kwargs else None
      address = Address.get_by_ethaddr(ethaddr)

      if transactions_filter == "internal":
        transactions_filter = "internal"
        transactions_type = "txlistinternal"
      else:
        transactions_filter = "external"
        transactions_type = "txlist"

      url = f"{settings.ETHSCAN_HOST}?" \
            f"module=account&" \
            f"action={transactions_type}&" \
            f"address=0x{address.address}&" \
            f"startblock=0&" \
            f"endblock=99999999&" \
            f"page={page}&" \
            f"offset={items_per_page}&" \
            f"sort=asc&" \
            f"apikey={settings.ETHSCAN_API_KEY}"

      response = requests.get(url)
      response_data = {}
      if response.status_code == 200:
        response_data = response.json()

      context = {'address': address,
                 'transactions': response_data['result'],
                 'page': page,
                 'next_page': int(page)+1,
                 'previous_page': int(page)-1 if int(page) > 1 else 1,
                 'transactions_type': transactions_filter}

      return render(request, 'details.html', context)
    except Exception as e:
      print(e)
      return render(request, '500.html', {})

  def track_get(self, request, *args, **kwargs):
    """
    Yields an empty form for registering an ethereum address
    :param request:
    :param args:
    :param kwargs:
    :return:
    """
    try:
      context = {}
      form = TrackForm()
      context['form'] = form
      return render(request, 'form.html', context)
    except Exception as e:
      print(e)
      return render(request, '500.html', {})

  @csrf_exempt
  def track_post(self, request, *args, **kwrags):
    """
    Register an ethereum address to start tracking
    :param request:
    :param args:
    :param kwrags:
    :return:
    """
    try:
      context = {}
      form = TrackForm(request.POST)
      if form.is_valid():
        # add message
        a = form.save(commit=True)
        messages.add_message(request, messages.SUCCESS,
                             f"New ETH Address {a.address} added")
        return redirect(reverse('home'))
      context['form'] = form
      return render(request, 'form.html', context)
    except Exception as e:
      print(e)
      return render(request, '500.html', {})

class AddressController(View, AddressLogic):
  """
  Address Page controller
  """
  def get(self, request, *args, **kwargs):
    url_name = resolve(request.path_info).url_name

    if url_name == 'address_status':
      return self.status(request, *args, **kwargs)
    elif url_name == 'address_track':
      return self.track_get(request, *args, **kwargs)
    else:
      # todo: unknown
      return None

  def post(self, request, *args, **kwargs):
    url_name = resolve(request.path_info).url_name

    if url_name == 'address_track':
      return self.track_post(request, *args, **kwargs)
    else:
      # todo: unknown
      return None