#
# Author: John Wieczorek
#
# Build script for XML-based extension files for use with the Integrated Publishing
# Toolkit (IPT). XML files produced from this script need to go into the appropriate
# directories in https://github.com/gbif/rs.gbif.org/tree/master/core or
# https://github.com/gbif/rs.gbif.org/tree/master/extension/dwc with file names reflecting
# version dates (e.g., dwc_taxon_2023-07-07.xml).
# Extensions are defined first in the CSVtoXMLConverter class' __init__ member function
# by populating the dictionary extensions_configuration.
# One extension can be generated per run of the script, with the extension's name and
# destination file as parameters (see main() for syntax).
#
__version__ = '2023-07-09T20:21-03:00'

import csv
import sys
import argparse
from xml.sax.saxutils import escape

class CSVtoXMLConverter:
    '''
    Class to manage the configurations for Darwin Core extension definitions and build
    XML output from the term history document (../vocabulary/term_versions.csv).
    '''
    def __init__(self, csv_file_path):
        self.csv_file_path = csv_file_path
        self.root_element = None
        self.row_element = None
        self.extensions_configuration = {
            "Occurrence": {
                "title":"Darwin Core Occurrence",
                "namespace":"http://rs.tdwg.org/dwc/terms/",
                "rowType":"http://rs.tdwg.org/dwc/terms/Occurrence",
                "dc:issued":"2023-07-07",
                "dc:relation":"http://rs.tdwg.org/dwc/terms/index.htm#Occurrence",
                "dc:description":"An existence of a dwc:Organism at a particular place at a particular time.",
                "included_terms":[
                    "type",
                    "modified",
                    "language",
                    "license",
                    "rightsHolder",
                    "accessRights",
                    "bibliographicCitation",
                    "references",
                    "institutionID",
                    "collectionID",
                    "datasetID",
                    "institutionCode",
                    "collectionCode",
                    "datasetName",
                    "ownerInstitutionCode",
                    "basisOfRecord",
                    "informationWithheld",
                    "dataGeneralizations",
                    "dynamicProperties",
                    "occurrenceID",
                    "catalogNumber",
                    "recordNumber",
                    "recordedBy",
                    "recordedByID",
                    "individualCount",
                    "organismQuantity",
                    "organismQuantityType",
                    "sex",
                    "lifeStage",
                    "reproductiveCondition",
                    "caste",
                    "behavior",
                    "vitality",
                    "establishmentMeans",
                    "degreeOfEstablishment",
                    "pathway",
                    "georeferenceVerificationStatus",
                    "occurrenceStatus",
                    "preparations",
                    "disposition",
                    "associatedMedia",
                    "associatedOccurrences",
                    "associatedReferences",
                    "associatedSequences",
                    "associatedTaxa",
                    "otherCatalogNumbers",
                    "occurrenceRemarks",
                    "organismID",
                    "organismName",
                    "organismScope",
                    "associatedOrganisms",
                    "previousIdentifications",
                    "organismRemarks",
                    "materialSampleID",
                    "verbatimLabel",
                    "eventID",
                    "parentEventID",
                    "eventType",
                    "fieldNumber",
                    "eventDate",
                    "eventTime",
                    "startDayOfYear",
                    "endDayOfYear",
                    "year",
                    "month",
                    "day",
                    "verbatimEventDate",
                    "habitat",
                    "samplingProtocol",
                    "sampleSizeValue",
                    "sampleSizeUnit",
                    "samplingEffort",
                    "fieldNotes",
                    "eventRemarks",
                    "locationID",
                    "higherGeographyID",
                    "higherGeography",
                    "continent",
                    "waterBody",
                    "islandGroup",
                    "island",
                    "country",
                    "countryCode",
                    "stateProvince",
                    "county",
                    "municipality",
                    "locality",
                    "verbatimLocality",
                    "minimumElevationInMeters",
                    "maximumElevationInMeters",
                    "verbatimElevation",
                    "verticalDatum",
                    "minimumDepthInMeters",
                    "maximumDepthInMeters",
                    "verbatimDepth",
                    "minimumDistanceAboveSurfaceInMeters",
                    "maximumDistanceAboveSurfaceInMeters",
                    "locationAccordingTo",
                    "locationRemarks",
                    "decimalLatitude",
                    "decimalLongitude",
                    "geodeticDatum",
                    "coordinateUncertaintyInMeters",
                    "coordinatePrecision",
                    "pointRadiusSpatialFit",
                    "verbatimCoordinates",
                    "verbatimLatitude",
                    "verbatimLongitude",
                    "verbatimCoordinateSystem",
                    "verbatimSRS",
                    "footprintWKT",
                    "footprintSRS",
                    "footprintSpatialFit",
                    "georeferencedBy",
                    "georeferencedDate",
                    "georeferenceProtocol",
                    "georeferenceSources",
                    "georeferenceRemarks",
                    "geologicalContextID",
                    "earliestEonOrLowestEonothem",
                    "latestEonOrHighestEonothem",
                    "earliestEraOrLowestErathem",
                    "latestEraOrHighestErathem",
                    "earliestPeriodOrLowestSystem",
                    "latestPeriodOrHighestSystem",
                    "earliestEpochOrLowestSeries",
                    "latestEpochOrHighestSeries",
                    "earliestAgeOrLowestStage",
                    "latestAgeOrHighestStage",
                    "lowestBiostratigraphicZone",
                    "highestBiostratigraphicZone",
                    "lithostratigraphicTerms",
                    "group",
                    "formation",
                    "member",
                    "bed",
                    "identificationID",
                    "verbatimIdentification",
                    "identificationQualifier",
                    "typeStatus",
                    "identifiedBy",
                    "identifiedByID",
                    "dateIdentified",
                    "identificationReferences",
                    "identificationVerificationStatus",
                    "identificationRemarks",
                    "taxonID",
                    "scientificNameID",
                    "acceptedNameUsageID",
                    "parentNameUsageID",
                    "originalNameUsageID",
                    "nameAccordingToID",
                    "namePublishedInID",
                    "taxonConceptID",
                    "scientificName",
                    "acceptedNameUsage",
                    "parentNameUsage",
                    "originalNameUsage",
                    "nameAccordingTo",
                    "namePublishedIn",
                    "namePublishedInYear",
                    "higherClassification",
                    "kingdom",
                    "phylum",
                    "class",
                    "order",
                    "superfamily",
                    "family",
                    "subfamily",
                    "tribe",
                    "subtribe",
                    "genus",
                    "genericName",
                    "subgenus",
                    "infragenericEpithet",
                    "specificEpithet",
                    "infraspecificEpithet",
                    "cultivarEpithet",
                    "taxonRank",
                    "verbatimTaxonRank",
                    "scientificNameAuthorship",
                    "vernacularName",
                    "nomenclaturalCode",
                    "taxonomicStatus",
                    "nomenclaturalStatus",
                    "taxonRemarks"
                ],
                "required":["basisOfRecord"],
                "thesauri":{
                    "type":"http://rs.gbif.org/vocabulary/dcterms/type.xml",
                    "basisOfRecord":"http://rs.gbif.org/vocabulary/dwc/basis_of_record_2022-02-02.xml",
                    "organismQuantityType":"http://rs.gbif.org/vocabulary/gbif/quantity_type_2015-07-10.xml",
                    "establishmentMeans":"http://rs.gbif.org/vocabulary/dwc/establishment_means_2022-02-02.xml",
                    "degreeOfEstablishment":"http://rs.gbif.org/vocabulary/dwc/degree_of_establishment_2022-02-02.xml",
                    "pathway":"http://rs.gbif.org/vocabulary/dwc/pathway_2022-02-02.xml",
                    "occurrenceStatus":"http://rs.gbif.org/vocabulary/gbif/occurrence_status_2020-07-15.xml",
                    "sampleSizeUnit":"http://rs.gbif.org/vocabulary/gbif/unit_of_measurement_2015-07-10.xml",
                    "taxonRank":"http://rs.gbif.org/vocabulary/gbif/rank_2015-04-24.xml",
                    "nomenclaturalCode":"http://rs.gbif.org/vocabulary/gbif/nomenclatural_code.xml"
                },
                "gbif_additions":[
#                        "<property group='Record Level' name='source' type='uri' namespace='http://purl.org/dc/terms/' qualName='http://purl.org/dc/terms/source' dc:relation='' dc:description='Deprecated in favor over new dcterm:references! A URI link or reference to the source of this record. A link to a webpage or RESTful webservice is recommended.  URI is mandatory format.' examples='http://www.catalogueoflife.org/annual-checklist/show_species_details.php?record_id=6197868' required='false'/>"
                ]
            },
            "Event": {
                "title":"Darwin Core Event",
                "namespace":"http://rs.tdwg.org/dwc/terms/",
                "rowType":"http://rs.tdwg.org/dwc/terms/Event",
                "dc:issued":"2023-07-07",
                "dc:relation":"http://rs.tdwg.org/dwc/terms/index.htm#Event",
                "dc:description":"An action that occurs at some location during some time.",
                "included_terms":[
                    "type",
                    "modified",
                    "language",
                    "license",
                    "rightsHolder",
                    "accessRights",
                    "bibliographicCitation",
                    "references",
                    "institutionID",
                    "datasetID",
                    "institutionCode",
                    "datasetName",
                    "ownerInstitutionCode",
                    "informationWithheld",
                    "dataGeneralizations",
                    "dynamicProperties",
                    "eventID",
                    "parentEventID",
                    "eventType",
                    "fieldNumber",
                    "eventDate",
                    "eventTime",
                    "startDayOfYear",
                    "endDayOfYear",
                    "year",
                    "month",
                    "day",
                    "verbatimEventDate",
                    "habitat",
                    "samplingProtocol",
                    "sampleSizeValue",
                    "sampleSizeUnit",
                    "samplingEffort",
                    "fieldNotes",
                    "eventRemarks",
                    "locationID",
                    "higherGeographyID",
                    "higherGeography",
                    "continent",
                    "waterBody",
                    "islandGroup",
                    "island",
                    "country",
                    "countryCode",
                    "stateProvince",
                    "county",
                    "municipality",
                    "locality",
                    "verbatimLocality",
                    "minimumElevationInMeters",
                    "maximumElevationInMeters",
                    "verbatimElevation",
                    "verticalDatum",
                    "minimumDepthInMeters",
                    "maximumDepthInMeters",
                    "verbatimDepth",
                    "minimumDistanceAboveSurfaceInMeters",
                    "maximumDistanceAboveSurfaceInMeters",
                    "locationAccordingTo",
                    "locationRemarks",
                    "decimalLatitude",
                    "decimalLongitude",
                    "geodeticDatum",
                    "coordinateUncertaintyInMeters",
                    "coordinatePrecision",
                    "pointRadiusSpatialFit",
                    "verbatimCoordinates",
                    "verbatimLatitude",
                    "verbatimLongitude",
                    "verbatimCoordinateSystem",
                    "verbatimSRS",
                    "footprintWKT",
                    "footprintSRS",
                    "footprintSpatialFit",
                    "georeferencedBy",
                    "georeferencedDate",
                    "georeferenceProtocol",
                    "georeferenceSources",
                    "georeferenceRemarks",
                    "geologicalContextID",
                    "earliestEonOrLowestEonothem",
                    "latestEonOrHighestEonothem",
                    "earliestEraOrLowestErathem",
                    "latestEraOrHighestErathem",
                    "earliestPeriodOrLowestSystem",
                    "latestPeriodOrHighestSystem",
                    "earliestEpochOrLowestSeries",
                    "latestEpochOrHighestSeries",
                    "earliestAgeOrLowestStage",
                    "latestAgeOrHighestStage",
                    "lowestBiostratigraphicZone",
                    "highestBiostratigraphicZone",
                    "lithostratigraphicTerms",
                    "group",
                    "formation",
                    "member",
                    "bed"
                ],
                "required":[],
                "thesauri":{
                    "type":"http://rs.gbif.org/vocabulary/dcterms/type.xml",
                    "sampleSizeUnit":"http://rs.gbif.org/vocabulary/gbif/unit_of_measurement_2015-07-10.xml"
                },
                "gbif_additions":[
#                        "<property group='Record Level' name='source' type='uri' namespace='http://purl.org/dc/terms/' qualName='http://purl.org/dc/terms/source' dc:relation='' dc:description='Deprecated in favor over new dcterm:references! A URI link or reference to the source of this record. A link to a webpage or RESTful webservice is recommended.  URI is mandatory format.' examples='http://www.catalogueoflife.org/annual-checklist/show_species_details.php?record_id=6197868' required='false'/>"
                ]
            },
            "Taxon": {
                "title":"Darwin Core Taxon",
                "namespace":"http://rs.tdwg.org/dwc/terms/",
                "rowType":"http://rs.tdwg.org/dwc/terms/Taxon",
                "dc:issued":"2023-07-07",
                "dc:relation":"http://rs.tdwg.org/dwc/terms/index.htm#Taxon",
                "dc:description":"A group of organisms (sensu http://purl.obolibrary.org/obo/OBI_0100026) considered by taxonomists to form a homogeneous unit.",
                "included_terms":[
                    "taxonID",
                    "scientificNameID",
                    "acceptedNameUsageID",
                    "parentNameUsageID",
                    "originalNameUsageID",
                    "nameAccordingToID",
                    "namePublishedInID",
                    "taxonConceptID",
                    "scientificName",
                    "acceptedNameUsage",
                    "parentNameUsage",
                    "originalNameUsage",
                    "nameAccordingTo",
                    "namePublishedIn",
                    "namePublishedInYear",
                    "higherClassification",
                    "kingdom",
                    "phylum",
                    "class",
                    "order",
                    "superfamily",
                    "family",
                    "subfamily",
                    "tribe",
                    "subtribe",
                    "genus",
                    "genericName",
                    "subgenus",
                    "infragenericEpithet",
                    "specificEpithet",
                    "infraspecificEpithet",
                    "cultivarEpithet",
                    "taxonRank",
                    "verbatimTaxonRank",
                    "scientificNameAuthorship",
                    "vernacularName",
                    "nomenclaturalCode",
                    "taxonomicStatus",
                    "nomenclaturalStatus",
                    "taxonRemarks",
                    "modified",
                    "language",
                    "license",
                    "rightsHolder",
                    "accessRights",
                    "bibliographicCitation",
                    "references",
                    "institutionCode",
                    "institutionID",
                    "datasetID",
                    "datasetName",
                    "informationWithheld"
                ],
                "required":[],
                "thesauri":{
                    "taxonRank":"http://rs.gbif.org/vocabulary/gbif/rank_2015-04-24.xml",
                    "nomenclaturalCode":"http://rs.gbif.org/vocabulary/gbif/nomenclatural_code.xml"
                },
                "gbif_additions":[
#                        "<property group='Record Level' name='source' type='uri' namespace='http://purl.org/dc/terms/' qualName='http://purl.org/dc/terms/source' dc:relation='' dc:description='Deprecated in favor over new dcterm:references! A URI link or reference to the source of this record. A link to a webpage or RESTful webservice is recommended.  URI is mandatory format.' examples='http://www.catalogueoflife.org/annual-checklist/show_species_details.php?record_id=6197868' required='false'/>"
                ]
            },
            "Identification": {
                "title":"Darwin Core Identification History",
                "namespace":"http://rs.tdwg.org/dwc/terms/",
                "rowType":"http://rs.tdwg.org/dwc/terms/Identification",
                "dc:issued":"2023-07-07",
                "dc:subject":"dwc:Occurrence",
                "dc:relation":"http://rs.tdwg.org/dwc/terms/index.htm#Identification",
                "dc:description":"Support for multiple identifications (determinations) of dwc:Occurrences. All identifications including the most current one should be listed, while the current one should also be repeated in the Occurrence Core.",
                "included_terms":[
                    "identificationID",
                    "verbatimIdentification",
                    "identificationQualifier",
                    "typeStatus",
                    "identifiedBy",
                    "identifiedByID",
                    "dateIdentified",
                    "identificationReferences",
                    "identificationVerificationStatus",
                    "identificationRemarks",
                    "taxonID",
                    "scientificNameID",
                    "acceptedNameUsageID",
                    "parentNameUsageID",
                    "originalNameUsageID",
                    "nameAccordingToID",
                    "namePublishedInID",
                    "taxonConceptID",
                    "scientificName",
                    "acceptedNameUsage",
                    "parentNameUsage",
                    "originalNameUsage",
                    "nameAccordingTo",
                    "namePublishedIn",
                    "namePublishedInYear",
                    "higherClassification",
                    "kingdom",
                    "phylum",
                    "class",
                    "order",
                    "superfamily",
                    "family",
                    "subfamily",
                    "tribe",
                    "subtribe",
                    "genus",
                    "genericName",
                    "subgenus",
                    "infragenericEpithet",
                    "specificEpithet",
                    "infraspecificEpithet",
                    "cultivarEpithet",
                    "taxonRank",
                    "verbatimTaxonRank",
                    "scientificNameAuthorship",
                    "vernacularName",
                    "nomenclaturalCode",
                    "taxonomicStatus",
                    "nomenclaturalStatus",
                    "taxonRemarks"
                ],
                "required":[],
                "thesauri":{
                    "taxonRank":"http://rs.gbif.org/vocabulary/gbif/rank_2015-04-24.xml",
                    "nomenclaturalCode":"http://rs.gbif.org/vocabulary/gbif/nomenclatural_code.xml"
                },
                "gbif_additions":[
#                        "<property group='Record Level' name='source' type='uri' namespace='http://purl.org/dc/terms/' qualName='http://purl.org/dc/terms/source' dc:relation='' dc:description='Deprecated in favor over new dcterm:references! A URI link or reference to the source of this record. A link to a webpage or RESTful webservice is recommended.  URI is mandatory format.' examples='http://www.catalogueoflife.org/annual-checklist/show_species_details.php?record_id=6197868' required='false'/>"
                ]
            },
            "MeasurementOrFacts": {
                "title":"Darwin Core Measurement or Facts",
                "namespace":"http://rs.tdwg.org/dwc/terms/",
                "rowType":"http://rs.tdwg.org/dwc/terms/MeasurementOrFact",
                "dc:issued":"2023-07-07",
                "dc:subject":"dwc:Occurrence dwc:Event dwc:Taxon",
                "dc:relation":"http://rs.tdwg.org/dwc/terms/index.htm#MeasurementOrFact",
                "dc:description":"Support for multiple Darwin Core MeasurementOrFacts per core record. Allows links to any core.",
                "included_terms":[
                    "measurementID",
                    "parentMeasurementID",
                    "measurementType",
                    "measurementValue",
                    "measurementAccuracy",
                    "measurementUnit",
                    "measurementDeterminedBy",
                    "measurementDeterminedDate",
                    "measurementMethod",
                    "measurementRemarks"
                ],
                "required":["measurementType"],
                "thesauri":{
                },
                "gbif_additions":[
#                        "<property group='Record Level' name='source' type='uri' namespace='http://purl.org/dc/terms/' qualName='http://purl.org/dc/terms/source' dc:relation='' dc:description='Deprecated in favor over new dcterm:references! A URI link or reference to the source of this record. A link to a webpage or RESTful webservice is recommended.  URI is mandatory format.' examples='http://www.catalogueoflife.org/annual-checklist/show_species_details.php?record_id=6197868' required='false'/>"
                ]
            },
            "ResourceRelationships": {
                "title":"Darwin Core Resource Relationships",
                "namespace":"http://rs.tdwg.org/dwc/terms/",
                "rowType":"http://rs.tdwg.org/dwc/terms/ResourceRelationship",
                "dc:issued":"2023-07-07",
                "dc:subject":"dwc:Occurrence dwc:Event dwc:Taxon",
                "dc:relation":"http://rs.tdwg.org/dwc/terms/index.htm#ResourceRelationship",
                "dc:description":"Support for multiple Darwin Core ResourceRelationships between records in the Core, in an extension, or resources external to the data set. The identifiers for subject (resourceID) and object (relatedResourceID) may exist in the dataset or be accessible via an externally resolvable identifier.",
                "included_terms":[
                    "resourceRelationshipID",
                    "resourceID",
                    "relationshipOfResourceID",
                    "relatedResourceID",
                    "relationshipOfResource",
                    "relationshipAccordingTo",
                    "relationshipEstablishedDate",
                    "relationshipRemarks"
                ],
                "required":[
                    "resourceID",
                    "relatedResourceID",
                    "relationshipOfResource"
                ],
                "thesauri":{
                },
                "gbif_additions":[
#                        "<property group='Record Level' name='source' type='uri' namespace='http://purl.org/dc/terms/' qualName='http://purl.org/dc/terms/source' dc:relation='' dc:description='Deprecated in favor over new dcterm:references! A URI link or reference to the source of this record. A link to a webpage or RESTful webservice is recommended.  URI is mandatory format.' examples='http://www.catalogueoflife.org/annual-checklist/show_species_details.php?record_id=6197868' required='false'/>"
                ]
            }
        }

    def get_xml(self, extension_name):
        '''
        Use extension_name to read the XML configuration and build the XML.
        '''
        extension = self.extensions_configuration.get(extension_name)
        if extension is None:
            print(f'Extension {extension_name} not found in extensions configuration.')
            return
        xml =  '<?xml version="1.0" encoding="UTF-8"?>\n'
        xml += '<?xml-stylesheet type="text/xsl" href="/style/human.xsl"?>\n'
        xml += '<extension xmlns="http://rs.gbif.org/extension/"\n'
        xml += '    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n'
        xml += '    xmlns:dc="http://purl.org/dc/terms/"\n'
        xml += '    xsi:schemaLocation="http://rs.gbif.org/extension/ http://rs.gbif.org/schema/extension.xsd"\n'
        xml += f'    dc:title="{extension.get("title")}"\n'
        xml += f'    name="{extension_name}"\n'
        xml += f'    namespace="{extension.get("namespace")}"\n'
        xml += f'    rowType="{extension.get("rowType")}"\n'
        xml += f'    dc:issued="{extension.get("dc:issued")}"\n'
        subject = extension.get("dc:subject")
        if subject is not None:
            xml += f'    dc:subject="{extension.get("dc:subject")}"\n'
        xml += f'    dc:relation="{extension.get("dc:relation")}"\n'
        description = extension.get("dc:description")
        description = escape(description, {'"':'"'})
        xml += f'    dc:description="{description}">\n'
        xml += '\n'
        with open(self.csv_file_path, 'r') as csv_file:
            reader = csv.reader(csv_file)
            header = next(reader)
            for row in reader:
                row_dict = dict(zip(header, row))
                if row_dict["status"] != "recommended":
                    break
                if row_dict['rdf_type'] == "http://www.w3.org/2000/01/rdf-schema#Class":
                    continue
                if row_dict["term_localName"] not in extension.get("included_terms"):
                    continue
                term_xml =  "<property "
                group = row_dict["organized_in"].rsplit("/", 1)[1]
                if group == "":
                    group="Record-level"
                term_xml += f'group="{group}" '
                term_xml += f'name="{row_dict["term_localName"]}" '
                namespace = row_dict["term_iri"].rsplit('/', 1)[0]
                term_xml += f'namespace="{namespace}/" '
                term_xml += f'qualName="{row_dict["term_iri"]}" '
                relation_base = "http://rs.tdwg.org/dwc/terms/index.htm#"
                if namespace == "http://purl.org/dc/elements/1.1/":
                    relation_base = "https://dwc.tdwg.org/terms/#dc:"
                elif namespace == "http://purl.org/dc/terms/":
                    relation_base = "https://dwc.tdwg.org/terms/#dcterms:"
                term_xml += f'dc:relation="{relation_base}{row_dict["term_localName"]}" '
                description = row_dict["definition"]
                if row_dict.get("comments") is not None and len(row_dict.get("comments"))>0:
                    description += f' {row_dict["comments"]}'
                description = escape(description, {'"':'"'})
                term_xml += f'dc:description="{description}" '
                examples = row_dict.get("examples") or ""
                examples = escape(examples, {'"':'"'})
                term_xml += f'examples="{examples}" '
                if row_dict["term_localName"] in extension.get("required"):
                    term_xml += f'required="true"/>'
                else:
                    term_xml += f'required="false"/>'
                xml += f'    {term_xml}\n'
        for addition in extension.get("gbif_additions"):
            addition = escape(addition,{'"':'"'})
            xml += f'    {addition}'
        xml += "</extension>"
        return xml

    def write_xml(self, extension_name, filename):
        '''
        Use extension_name to read the XML configuration, build the XML, and write the
        result to the output file.
        '''
        with open(filename, 'w') as xml_file:
            xml_file.write(self.get_xml(extension_name))

def _getoptions():
  ''' Parse command line options and return them.'''
  parser = argparse.ArgumentParser()

  help = 'extension name (required)'
  parser.add_argument("-e", "--extension_name", required=True, help=help)

  help = 'output file name with full path (required)'
  parser.add_argument("-o", "--outputfile", required=True, help=help)

  return parser.parse_args()

def main():
    ''' Get the commend-line options for extension to build and output file for it.'''
    options = _getoptions()

    '''If the required parameters are not given provide a script syntax message and exit.'''
    if options.extension_name is None or len(options.extension_name)==0 \
        or options.outputfile is None or len(options.outputfile)==0:
      s =  'syntax:\n'
      s += 'python buildxml.py'
      s += ' -e Taxon'
      s += ' -o dwc_taxon_2023-07-07.xml'
      print(f'{s}')
      exit(1)

    '''Build the selected extension XML file from the normative term history document.'''
    term_versions_file = "../vocabulary/term_versions.csv"
    print(f"Building the {options.extension_name} extension XML file:")

    '''Create the class that reads the term versions file and constructs the XML output.'''
    converter = CSVtoXMLConverter(term_versions_file)

    '''
    Build the XML extension file based on the choice of extension and write the result
    to the output file.
    '''
    converter.write_xml(options.extension_name, options.outputfile)
    print(f"Extension {options.extension_name} written to {options.outputfile}.")

if __name__ == "__main__":
    sys.exit(main())


import torch
from torchtext import data
SEED = 1234
torch.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
import jieba
import re
def x_tokenize(x):
    str = re.sub('[^\u4e00-\u9fa5]', "", x)
    return jieba.lcut(str)

TEXT = data.Field(sequential=True, tokenize=x_tokenize,fix_length=100,
            use_vocab=True)
LABEL = data.Field(sequential=False,
            use_vocab=False)

train, dev, test = data.TabularDataset.splits(path='dataset',
                                              train='csv_train.csv',
                                              validation='csv_dev.csv',
                                              test='csv_test.csv',
                                              format='csv',
                                              skip_header=True,
                                              csv_reader_params={'delimiter':','},
                                              fields=[('text',TEXT),('label',LABEL)])

TEXT.build_vocab(train)

train_iter, val_iter, test_iter = data.BucketIterator.splits((train,dev,test),
                                                             batch_size = 256,
                                                             shuffle=True,
                                                             sort=False,
                                                             sort_within_batch=False,
                                                             repeat=False)

def getTEXT():
    return TEXT
def getLabel():
    return LABEL
def getIter():
    return train_iter, val_iter, test_iter


# -*- coding: utf-8 -*-
from django.db import models

PATTERN_TYPE = (('A', u'Дуга'),
                ('At', u'Тентовая Дуга'),
                ('L', u'Петля'),
                ('Lp', u'Павлиний глаз'),
                ('Ws', u'Завиток спиралькой'),
                ('W', u'Завиток кружками'),
                ('2L', u'Двойная петля'),
                ('C', u'Сложный узор'))

ORIENT_TYPE = (('U', u'Ульнарная'), # Завиток: Прав рука - против, Левая - по; Петля: хвостик от Бол пал
               ('R', u'Радиальная')) # Завиток: Прав рука - по, Левая - против;  Петля: хвостик к Бол пал

class FingerPrints(models.Model):
    pattern = models.CharField(max_length=2, choices=PATTERN_TYPE, verbose_name=u'Тип узора')
    orient = models.CharField(max_length=1, choices=ORIENT_TYPE, null=True, default=None, verbose_name=u'Ориентация узора')
    rc_u = models.IntegerField(default=0, blank=True, verbose_name = u'Ульнарный ГС')
    rc_r = models.IntegerField(default=0, blank=True, verbose_name = u'Радиальный ГС')

    class Meta:
        verbose_name = u'палец'
        verbose_name_plural = u'пальцы'

    def __unicode__(self):
        return "%s %s (%s, %s)" % (self.pattern, self.orient, self.rc_u, self.rc_r)

    @property
    def is_complicated(self):
        return self.pattern == 'Ws' or self.pattern == 'W' or self.pattern == '2L' or self.pattern == 'C'

class Human(models.Model):
    name = models.CharField(max_length=256, unique=True, verbose_name=u'имя')
    desc = models.TextField(blank=True, verbose_name=u'описание')
    # далее - вектора и вероятность от 0 до 10 присутствия этого вектора
    ur = models.NullBooleanField(default=None, verbose_name='У')
    an = models.NullBooleanField(default=None, verbose_name='А')
    ko = models.NullBooleanField(default=None, verbose_name='К')
    my = models.NullBooleanField(default=None, verbose_name='М')
    zr = models.NullBooleanField(default=None, verbose_name='Зр')
    zv = models.NullBooleanField(default=None, verbose_name='Зв')
    o  = models.NullBooleanField(default=None, verbose_name='Ор')
    ob = models.NullBooleanField(default=None, verbose_name='Об')

    # далее - руки от 1(Бол) до 5(Миз) пальца
    r1 = models.ForeignKey(FingerPrints, verbose_name=u'П Бол', related_name = 'r1')
    r2 = models.ForeignKey(FingerPrints, verbose_name=u'П Ук', related_name = 'r2')
    r3 = models.ForeignKey(FingerPrints, verbose_name=u'П Ср', related_name = 'r3')
    r4 = models.ForeignKey(FingerPrints, verbose_name=u'П Без', related_name = 'r4')
    r5 = models.ForeignKey(FingerPrints, verbose_name=u'П Миз', related_name = 'r5')
    l1 = models.ForeignKey(FingerPrints, verbose_name=u'Л Бол', related_name = 'l1')
    l2 = models.ForeignKey(FingerPrints, verbose_name=u'Л Ук', related_name = 'l2')
    l3 = models.ForeignKey(FingerPrints, verbose_name=u'Л Ср', related_name = 'l3')
    l4 = models.ForeignKey(FingerPrints, verbose_name=u'Л Без', related_name = 'l4')
    l5 = models.ForeignKey(FingerPrints, verbose_name=u'Л Миз', related_name = 'l5')

    class Meta:
        verbose_name = u'человек'
        verbose_name_plural = u'люди'

    def __unicode__(self):
        return self.name

    @property
    def sum_rc(self):
        s = 0
        s += self.r1.rc_u + self.r1.rc_r
        s += self.r2.rc_u + self.r2.rc_r
        s += self.r3.rc_u + self.r3.rc_r
        s += self.r4.rc_u + self.r4.rc_r
        s += self.r5.rc_u + self.r5.rc_r

        s += self.l1.rc_u + self.l1.rc_r
        s += self.l2.rc_u + self.l2.rc_r
        s += self.l3.rc_u + self.l3.rc_r
        s += self.l4.rc_u + self.l4.rc_r
        s += self.l5.rc_u + self.l5.rc_r
        return s

    @property
    def complicated_count(self):
        s = 0
        if self.r1.is_complicated: s += 1
        if self.r2.is_complicated: s += 1
        if self.r3.is_complicated: s += 1
        if self.r4.is_complicated: s += 1
        if self.r5.is_complicated: s += 1
        if self.l1.is_complicated: s += 1
        if self.l2.is_complicated: s += 1
        if self.l3.is_complicated: s += 1
        if self.l4.is_complicated: s += 1
        if self.l5.is_complicated: s += 1

        return s


from ._animado import Animado
from ._suertudo import Suertudo
from ._equipado import Equipado
from engine.globs.event_dispatcher import EventDispatcher

class Combativo(Animado, Suertudo, Equipado):
    is_damageable = True

    def hurt(self, damage):
        damage -= self.dureza
        self['Salud'] -= damage
        EventDispatcher.trigger('MobWounded', self.tipo,
                                {'mob': self,
                                 "value": self['Salud'],
                                 "stat": "Salud",
                                 "factor": -damage}
                                )

        if self['Salud'] <= 0:
            if self.death_img is not None:
                self.image = self.death_img
                # esto queda hasta que haga sprites 'muertos' de los npcs
                # pero necesito más resolución para hacerlos...
            self.dead = True
            EventDispatcher.trigger('MobDeath', self.tipo, {'obj': self, 'mapa': self.parent})

    def atacar(self, sprite):
        if super().atacar(sprite) and sprite.is_damageable:
            x, y = self.direcciones[self.direccion]

            x, y = x * self['Retroceso'], y * self['Retroceso']
            a, b = self.luck(), sprite.luck()

            if self['Ataque']+a >= sprite['Evasión']+b:
                EventDispatcher.trigger('AttackSuccess', self.tipo, {'victim': sprite, 'attaker': self})
                sprite.reubicar(x, y)
                sprite.hurt(self['DañoCC'])
            else:
                EventDispatcher.trigger('MissedAttack', self.tipo, {'mob': sprite})


#  F28PL 2015/2016
#  Labs H- (http://www.macs.hw.ac.uk/~gabbay/F28PL/labs.html)
#  Christian Gregg
#  Lab Helpers:
#    Helen Rankin - hr80@hw.ac.uk
#    Pierre - pl96@hw.ac.uk
#    Andrea - att31@hw.ac.uk

###################################################################
# Question H (Python lists, polymorphism, recursion, and reduce). #
###################################################################

#   Make sure to test all your programs below, including on [].

#   Using iteration (a for-next or a while loop or similar), write a program mult1 that inputs a list of integers or reals, and returns their product (the result of multiplying all the numbers together). mult1 should return 1 for the empty list, because 1 is the multiplicative unit. Thus
#       mult1([]) calculates 1, and
#       mult1([1,2,3,10]) calculates 60.
def mult1(list):
    sum = 1
    for x in list:
        sum *= x
    return sum
#   Using recursion, write a program mult2 that inputs a list of integers and
#   returns their product (warning: this question is very slightly harder than
#   it seems; make sure to test your answer).
def mult2(list):
    if list:
        return list[0]*(mult2(list[1:]))
    else:
        return 1
#   Using range or otherwise, write a one-line piece of code that breaks mult2
#   (i.e. make it crash when it “should” return a value). Why did this happen?
mult2(range(5)) # range creates an iterator, not a list, iterators do not have
                # the random access property ie you cannot perform i[j] on
                # an iterator i to access its jth element, because that 'element'
                # has not yet been computed, iterators do not actually have 'elements'.

#   Using Python lambda and reduce (see specifically the example on line 2 of
#   this link (https://docs.python.org/3/library/functools.html?highlight=reduce#functools.reduce))
#   , write a program mult3 that inputs a list of integers and returns their product.

import functools
def mult3(l):
    if l == []:
        return 0
    return functools.reduce(lambda x, y: x*y, l)
#  What do mult1, mult2, and mult3 return if applied to [1.0,2,3.0,10]?
#  What would corresponding programs do if they were written in ML and applied to such an input?
#  What does this tell us about ML vs Python typing?

#       All three return 60.0.
#       ML would return a type error when trying to create a list of ints and reals to input.
#       Python typing is less strict, and python allows lists containing different types

#   Order mult1, mult2, and mult3 from slowest to fastest and explain your order.
# TODO
#   An issue with the programs above is that they only work for numbers, however,
#   there is no reason in principle they should not also work with strings or
#   lists (by concatenation). Note that in Python you can check whether a
#   variable x has type str (for example) with if type(x) is str:
#   print("x is a string!"), and similarly for lists if type(x) is
#   list: print("x is a list!").
#   Write a program multpoly that outputs the product of a list of numbers,
#   and the concatenation (i.e. chaining) of a list of strings or lists. So
#       multpoly([1,2,3,10]) calculates 60, and
#       multpoly(["1","2","3","10"]) calculates "12310", and
#       multpoly([[1],[2,3],[10]]) calculates [1,2,3,10].
#   We do not care what multpoly calculates on the empty list. We do care about
#   a clear and elegant implementation.
def multpoly(l):
    if type(l[0]) is int:
        product = 1
        for x in l:
            product *= x
        return product
    elif type(l[0]) is str:
        concat = ""
        for x in l:
            concat += x
        return concat
    elif type(l[0]) is list:
        concat =[]
        for x in l:
            concat += x
        return concat
    else:
        return l

#   Using if type(x) is list or otherwise, write a recursive program flatten
#   that inputs an arbitrarily nested list structure and outputs a “flattened”
#   list consisting of the list of all the data in the list, but with any nested
#   list structure removed. So for instance,
#       flatten([]) returns [], and
#       flatten([["hi"],5]) returns ["hi",5], and
#       flatten([[[],[["hello"],[" "]],[1],[[["world"]],[]]],"!"]) returns ["hello"," ",1,"world!"].
def flatten(arg):
    if arg == []:
        return arg
    elif type(arg[0]) == list:
        return flatten(arg[0]) + flatten(arg[1:])
    else:
        return [arg[0]] + flatten(arg[1:])

############################
# Question I (More Python) #
############################

#   Write a program factors that inputs a number n, which you may assume is
#   greater than 2, and outputs the list of its prime factors (prime numbers
#   are 2, 3, 5, 7, 11, 13, 17, and so on). You may assume basic arithmetic
#   operations, including modulus n % k and divisibility. However, if you find a
#   library call “prime_factors_of”, you can’t use that, sorry.
def factors(n):
    current_n = n
    factor = 0
    for div in range(2, current_n+1):
        if current_n % div == 0:
            factor = div
            current_n //= div
            break
    if n == 1:
        return []
    elif factor == n:
        return [n]
    else:
        return [factor]+factors(current_n)

#   Write a program largest that inputs a nonempty list of numbers and outputs
#   its greatest element. You do not need to worry about error-handling,
#   e.g. if the input is the empty list.
def largest(arg):
    largest = arg[0]
    for i in arg:
        if i > largest:
            largest = i
    return largest

#   Write a program largest_factor that inputs a number n and outputs its
#   largest prime factor.
def largest_factor(n):
    return largest(factors(n))

#   Write a program firstbigfib that inputs a number n and outputs the index of
#   the first Fibonacci number to contain n digits.
def fib():
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

def firstbigfib(n):
    index = 0
    fibgen = fib()
    current = next(fibgen)
    while n > len(str(abs(current))):
        current = next(fibgen)
        index += 1
    return index

#   Write a program firstbigf that inputs a number n and a function f mapping
#   numbers to numbers, and outputs the least strictly positive number i
#   (so i is in [1,2,3,...]) such that f(i) contains n digits. So for example
#   if f=lambda x:10**x then firstbigf f 10 should calculate 9 (because 10**9,
#   which is equal to 1000000000, has ten digits).
def firstbigf(f,n):
    i = 1
    current = f(1)
    while n > len(str(abs(current))):
        i += 1
        current = f(i)
    return i
#   Suggest, in very general terms, how we might optimise firstbigf to run on
#   an m-core architecture where m>1, with a roughly m-fold speedup.

#   A Pythagorean triple is a 3-tuple of strictly positive numbers (x,y,z)
#   such that z*z=x*x+y*y. Using Python generators (page 97 of these slides or
#   the generator that yields items example here), write a Python generator
#   function triples() to generate all Pythagorean triples. So:
#       x=triples() and then next(x) should return a first triple, and then
#       next(x) should return a second triple, and then
#       next(x) should return a third one, and so on.
# TODO: Need to find an algorithm that created triples in order
def triples():
    n, m = 1, 2
    while True:
        a = (m*m) - (n*n)
        b = 2 * n * m
        c = (m*m) + (n*n)
        yield (a, b, c)
        m = m+1

#   Test your program further by running it on list(itertools.islice(triples(), 1, 300))
#   (using the Internet or otherwise, make sure you understand what this does;
#   you may need to import itertools).

#####################################################
# Question J (Python lists, sets, and dictionaries) #
#####################################################

# State what the following Python programs calculate, and explain why:
#   1. { 0:"zero", 1:"one", 2:"two" }[1]
#       Calculates the value that is associsted with the key 1
#   2. range(1000)[0]
#       Calculates the first (0th) integer calculated by range()
#   3. for x in range(1,1000): print(list(enumerate(range(x)))[0][0])
#       the program prints out the print out the first element of the first element
#       in the enumerated list of the range 0 to x-1, 999 times. (just prints 0 999 times).
#
#       >>> list(enumerate(range(10)))
#           [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9)]
#       >>> list(enumerate(range(10)))[0]
#           (0, 0)
#       >>> list(enumerate(range(10)))[0][0]
#
#   4. "hello world"[:-2]
#       returns the string with all the characters bar the last 2, ie each characters
#       is indexed, it returns all the characters from index 0 to index -3 (index -2 is not inclusive)

# Explain the difference in behaviour between the following two code snippets:
#   x=[""] # Snippet 1
#   for i in x: x.append("")
#   x=[""] # Snippet 2
#   for i in x[:]: x.append("")
#
#       Snippet 1 will append "" to the list x infinetly, as it is iterating over the list x
#       which is being updated to add a new object to it on every iteration, ie the for loop has no end.
#
#       Snippet 2 on the other hand will only append "" to the list once, as at the beginning there is
#       only one object in list x, this is because the for loop is iterating over a copy of x that is
#       created at the the beginning of the iteration (it iterates over that value returned by x[:])
#       Snippet 2 will double the number of objects in the list.

# Explain the behaviour of the following code snippet:
#   x=[[]] ; x[0].extend(x)
#   while True:
#       print(x)
#       x = x[0]
#
#       By extending x[0] by x, x[0] is extended by itself, this means x[0] contains a reference to
#       itself. (it is recursive, x[0] is equal to x[0] which is equal to x[0] and so on... )
#       Because x[0] contains a refernce to itself, assigning x to x[0] is eqivalent to assigning x to itself.
#       This can be seen by adding a call to id(x) underneath print(x), which shows that
#       after the first iteration, the element being printed is the same (held at the same point in memory).

# The following Python program returns an error.
#   {[0]:0}[[0]]
# Explain what the error is and why it arises.
#       The error is caused due to the fact that the keys in a dictionary must be imutable,
#       lists are mutable objects.
# Suggest how to fix it.
#       Use tuples, which are immutable instead of lists. ie {(0):0}[(0)]

# Using list comprehensions and lambda, write a program that if given a number n, calculates the first n even numbers (starting from 0).
[x for x in list(range(n*2)) if x % 2 == 0]

# Using list comprehensions and lambda, write a program that if given a list of numbers l, will return the sublist of those numbers in l that are divisible by 3.
[x for x in l if x % 3 == 0 ]
# The zip function inputs two lists (which you may assume have the same length) and outputs the list of pairs of the lists ‘zipped’ together. For instance, zip([1,2,3],["one","two","three"]) returns [(1,"one"),(2,"two"),(3,"three")]. Implement zip yourself, from first principles, using list comprehension (so not using a for-next loop and the append method; too easy).
def my_zip(list1,list2):
    return [(x,y) for x in list1 for y in list2 if list1.index(x) == list2.index(y)]

# The following Python programs calculates the first ten squares and the first ten pentagonal numbers:
#   squares = [x*x for x in range(10)]
#   pentagonals = [x*(3*x-1)//2 for x in range(10)]
#   (For this question, 0 is a square and a pentagonal number.)
# Based on those examples, and using list comprehensions if possible, write a Python program that calculates a list of numbers that are both square and pentagonal (up to some computational bound, i.e. the program is allowed to stop after a while). Test your program by calculating the first four such numbers.

# TODO make this better ffs
squares = [x*x for x in range(100000)]
pentagonals = [x*(3*x-1)//2 for x in range(100000)]
square_pent = [x for x in squares if (x in pentagonals)]

# Nonnegative integers can be implemented in Python by nest(0)=[] and nest(n+1)=[nest(n)]. Write programs nest and unnest which translate between nonnegative integers and the nesting implemenation.
def nest(n):
    if n == 0:
        return []
    else:
        return [nest(n-1)]

def unnest(n):
    if n == []:
        return 0
    else:
        return 1 + unnest(n[0])
# We do not care what happens at incorrect input. Implement addition and multiplication as direct functions on this implementation (i.e. not by untranslating, adding the numbers, and retranslating).
def add(x, y):
    if y == []:                 # if y is 0 return x (nothing added)
        return x
    else:
        return add([x], y[0])  # call add with x+1 and y-1

def multiply_help(n1,n2,n3):
    if n1 == [] or n2 == []:
        return []
    elif n2 == [[]]:
        return n1
    else:
        return multiply_help(add(n1,n3), n2[0],n3)

def multiply(n1,n2):
    return multiply_help(n1,n2,n1)
#(The student who finds this kind of thing amusing, might be interested in the surreal numbers.)


# Universidade Federal de Campina Grande - UFCG
# Programação 1 - 2019.2
# Guilherme Aureliano

def soma_intervalo(a, b):
    soma = 0
    cont = a
    lim = b
    while True:
        soma += cont
        cont += 1
        if cont == (lim + 1): break
    return soma


import tensorflow as tf
import pickle
import numpy as np
import matplotlib.pyplot as plt

"""
You have to implement an image classification neural network for the same dataset as before. Feel free to reduce
the number of examples for each class in your dataset, if it takes too long to train on your hardware.

For this task, you should make at least two neural networks: the fully connected one that works on the same extracted
features as before and another one convolutional with class prediction at the end.

You can do it either in Keras or Pytorch. Better to do both.

As an output, you should provide your code, trained model files (2 pcs. at least), your dataset, and the same precision
metrics [calculated on test images] as you did before.

Your code should provide 3 execution modes/functions: train (for training new model), test (for testing the trained
and saved model on the test dataset), and infer (for inference on a particular folder with images or a single image).
"""

def save_the_model(model, fname):
    import json
    print('Saving the model')
    json_string = model.to_json()
    with open(fname + '.txt', 'w', encoding='utf-8') as f:
        json.dump(json_string, f, ensure_ascii=False)
    print('Saving the weights')
    model.save_weights(fname + '.hdf5')
    print('Done')

def get_model_from_files(fnm):
    import json
    from tensorflow.keras.models import model_from_json
    with open(fnm + '.txt') as infile:
        json_string = json.load(infile)
    model = model_from_json(json_string)
    model.load_weights(fnm + '.hdf5', by_name=False)
    model.compile(loss='binary_crossentropy', optimizer='Adagrad', metrics=['accuracy'])
    model.summary()
    return model

def fully_connected_model():

    # Uploaded features from the last homework. The number of features reduced from 546 to 40 with L1 regularization.

    with open('train_dataset.pickle', 'rb') as f:
        train_dataset_df = pickle.load(f)

    target = train_dataset_df.pop('y')
    Y = np.asarray(target).astype(np.float32)
    X = np.asarray(train_dataset_df).astype(np.float32)
    dataset = tf.data.Dataset.from_tensor_slices((X, Y)).batch(16)
    features, labels = next(iter(dataset))
    train_dataset = dataset.shuffle(len(train_dataset_df)).batch(1)

    # The code for a moder training

    # model2 = tf.keras.Sequential([
    #     tf.keras.layers.Dense(30, activation=tf.nn.relu, input_shape=(40,)),  # input shape required
    #     tf.keras.layers.Dense(20, activation=tf.nn.relu),
    #     tf.keras.layers.Dense(17)
    # ])
    #
    # loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    #
    # def loss(model, x, y, training):
    #     # training=training is needed only if there are layers with different
    #     # behavior during training versus inference (e.g. Dropout).
    #     y_ = model(x, training=training)
    #
    #     return loss_object(y_true=y, y_pred=y_)
    #
    # def grad(model, inputs, targets):
    #     with tf.GradientTape() as tape:
    #         loss_value = loss(model, inputs, targets, training=True)
    #     return loss_value, tape.gradient(loss_value, model.trainable_variables)
    #
    # optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
    #
    # loss_value, grads = grad(model2, features, labels)
    # optimizer.apply_gradients(zip(grads, model2.trainable_variables))
    #
    # # Keep results for plotting
    # train_loss_results = []
    # train_accuracy_results = []
    #
    # num_epochs = 201
    #
    # for epoch in range(num_epochs):
    #     epoch_loss_avg = tf.keras.metrics.Mean()
    #     epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()
    #
    #     # Training loop - using batches of 32
    #     for x, y in train_dataset:
    #         # Optimize the model
    #         loss_value, grads = grad(model2, x, y)
    #         optimizer.apply_gradients(zip(grads, model2.trainable_variables))
    #
    #         # Track progress
    #         epoch_loss_avg.update_state(loss_value)  # Add current batch loss
    #         # Compare predicted label to actual label
    #         # training=True is needed only if there are layers with different
    #         # behavior during training versus inference (e.g. Dropout).
    #         epoch_accuracy.update_state(y, model2(x, training=True))
    #
    #     # End epoch
    #     train_loss_results.append(epoch_loss_avg.result())
    #     train_accuracy_results.append(epoch_accuracy.result())
    #
    #     if epoch % 50 == 0:
    #         print("Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}".format(epoch,
    #                                                                     epoch_loss_avg.result(),
    #                                                                     epoch_accuracy.result()))

    # Training log:

    """ Epoch 000: Loss: 2.813, Accuracy: 12.292%
        Epoch 050: Loss: 2.133, Accuracy: 51.979%
        Epoch 100: Loss: 1.946, Accuracy: 61.667%
        Epoch 150: Loss: 1.785, Accuracy: 68.750%
        Epoch 200: Loss: 1.673, Accuracy: 73.125%"""

    with open('train_accuracy_results.pickle', 'rb') as f2:
        train_accuracy_results = pickle.load(f2)

    with open('train_loss_results.pickle', 'rb') as f3:
        train_loss_results = pickle.load(f3)

    fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))
    fig.suptitle('Training Metrics')

    axes[0].set_ylabel("Loss", fontsize=14)
    axes[0].plot(train_loss_results)

    axes[1].set_ylabel("Accuracy", fontsize=14)
    axes[1].set_xlabel("Epoch", fontsize=14)
    axes[1].plot(train_accuracy_results)
    plt.show()

    # save_the_model(model2, 'model_fully_connected')
    #-------------------------------------------------------------------------------------

    with open('test_dataset.pickle', 'rb') as f2:
        test_dataset_df = pickle.load(f2)

    target_test = test_dataset_df.pop('y')
    Y_test = np.asarray(target_test).astype(np.float32)
    X_test = np.asarray(test_dataset_df).astype(np.float32)
    dataset_t = tf.data.Dataset.from_tensor_slices((X_test, Y_test))
    test_dataset = dataset_t.shuffle(len(test_dataset_df)).batch(1)

    model3 = get_model_from_files('model_fully_connected')

    test_accuracy = tf.keras.metrics.Accuracy()

    for (x, y) in test_dataset:
        # training=False is needed only if there are layers with different
        # behavior during training versus inference (e.g. Dropout).
        logits = model3(x, training=False)
        prediction = tf.argmax(logits, axis=1, output_type=tf.int32)
        test_accuracy(prediction, y)

    print("Test set accuracy: {:.3%}".format(test_accuracy.result())) # Test set accuracy: 15.625%

if __name__ == '__main__':
    fully_connected_model()


# Return a collection of numbers from user.
def getNums():
    nums = []
    while True:
        num = int(input("Number? "))
        if num == -1 :
            break
        nums.append(num)
    return nums

# Client code.
nums = getNums()
for n in nums:
    print("  %d" % n)


"""
Script for building specific layers needed by GAN architecture.
"""
import torch.nn as nn

from torch_mimicry.modules import spectral_norm

def SNConv2d(*args, **kwargs):
    r"""
    Wrapper for applying spectral norm on conv2d layer.
    """
    if kwargs.get('default', True):
        return nn.utils.spectral_norm(nn.Conv2d(*args, **kwargs))

    else:
        return spectral_norm.SNConv2d(*args, **kwargs)

def SNLinear(*args, **kwargs):
    r"""
    Wrapper for applying spectral norm on linear layer.
    """
    if kwargs.get('default', True):
        return nn.utils.spectral_norm(nn.Linear(*args, **kwargs))

    else:
        return spectral_norm.SNLinear(*args, **kwargs)

def SNEmbedding(*args, **kwargs):
    r"""
    Wrapper for applying spectral norm on embedding layer.
    """
    if kwargs.get('default', True):
        return nn.utils.spectral_norm(nn.Embedding(*args, **kwargs))

    else:
        return spectral_norm.SNEmbedding(*args, **kwargs)

class ConditionalBatchNorm2d(nn.Module):
    r"""
    Conditional Batch Norm as implemented in
    https://github.com/pytorch/pytorch/issues/8985

    Attributes:
        num_features (int): Size of feature map for batch norm.
        num_classes (int): Determines size of embedding layer to condition BN.
    """
    def __init__(self, num_features, num_classes):
        super().__init__()
        self.num_features = num_features
        self.bn = nn.BatchNorm2d(num_features, affine=False)
        self.embed = nn.Embedding(num_classes, num_features * 2)
        self.embed.weight.data[:, :num_features].normal_(
            1, 0.02)  # Initialise scale at N(1, 0.02)
        self.embed.weight.data[:,
                               num_features:].zero_()  # Initialise bias at 0

    def forward(self, x, y):
        r"""
        Feedforwards for conditional batch norm.

        Args:
            x (Tensor): Input feature map.
            y (Tensor): Input class labels for embedding.

        Returns:
            Tensor: Output feature map.
        """
        out = self.bn(x)
        gamma, beta = self.embed(y).chunk(
            2, 1)  # divide into 2 chunks, split from dim 1.
        out = gamma.view(-1, self.num_features, 1, 1) * out + beta.view(
            -1, self.num_features, 1, 1)

        return out


#!/usr/bin/env python3

import boto3
from os import environ
from tabulate import tabulate
from datetime import datetime

ec2_res = boto3.resource('ec2', region_name=environ['AWS_REGION_NAME'])
ec2_cli = boto3.client('ec2', region_name=environ['AWS_REGION_NAME'])

def get_owner_id():
    return boto3.client('sts').get_caller_identity().get('Account')

def get_images_info(owner_id):
    for ami in ec2_res.images.filter(Owners=owner_id):

        cdate = datetime.strptime(ami.creation_date, '%Y-%m-%dT%H:%M:%S.%fZ')
        yield {
            'creation_date': cdate,
            'id': ami.id,
            'name': ami.name,
        }

def group_by_name(amis):
    groups = {}

    for ami in amis:
        last_sep_index = [
            pos for pos, char in enumerate(ami['name']) if char == '_'
        ][-1]

        name = ami['name'][0:last_sep_index]
        if not groups.get(name):
            groups[name] = []

        groups[name].append(ami)

    return groups

def sort_by_age(group):
    amis = []

    def to_dict():
        dict_ = {}

        for g in group:
            cdate = g['creation_date']
            dict_[cdate] = g

        return dict_

    for ami in sorted(to_dict().items(), reverse=True):
        _, info = ami
        amis.append(info)

    return amis

def filter_for_deletion(amis):
    min_available = int(environ['MINIMUM_AVAILABLE_AMIS'])

    if len(amis) <= min_available:
        return amis, None

    to_keep = amis[0:min_available]
    to_delete = amis[min_available:]

    return to_keep, to_delete

def output(group, keep_list, del_list):
    def fl(amis):
        L = []

        for ami in amis:
            d = ami['creation_date']
            L.append([
                '{0}-{1}-{2} {3}:{4}:{5}'.format(
                    d.month, d.day, d.year, d.hour, d.minute, d.second
                ),
                ami['name'],
                ami['id']
            ])

        return tabulate(L, headers=['Creation Date', 'AMI Name', 'AMI Id'])

    #
    charseq = '*' * len(group)
    print('\n{0}\n{1}\n{2}\n'.format(charseq, group, charseq))

    #
    print('\n\nAMIS TO KEEP:\n\n{0}'
          '\n\nAMIS TO DELETE:\n\n{1}\n'.format(
              (fl(keep_list) if keep_list else 'No AMIs to keep.'),
              (fl(del_list) if del_list else 'No AMIs to delete.')
          ))

def lambda_handler(event, context):
    owner  = get_owner_id()
    amis   = get_images_info([owner])
    groups = group_by_name(amis)

    for title, group in groups.items():
        sorted_list = sort_by_age(group)
        keep_list, del_list = filter_for_deletion(sorted_list)

        output(title, keep_list, del_list)

        if del_list:
            for ami in del_list:
                ec2_cli.deregister_image(ImageId=ami['id'])


HEADER_LENGTH = 2048
HEADER_LENGTH = 2048
message = input(f"{my_username} > ")
# Ana Júlia Costa Santana
# Matrícula : 11811ETE003

# Semana05 - Programa 2 - Servidor#

import socket
import sys
import select
import threading
from threading import *

server = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR, 1)

if len(sys.argv) != 3:
    print ("Correct usage: script, IP address, port number")

HEADER_LENGTH = 10
IP = "127.0.0.1"
Port = int(input('Escolha a porta -->'))

print("[STARTING...] O servidor está sendo inicializado")
print('Rodando no IP: '+IP)
print('Rodando na porta: '+ str(Port))

server.bind((IP,Port))
server.listen(100)

clients_list = []

def clientthread(client_socket,addr):
	conn.send("Bem vindo(a) ao chatroom!") #Mensagem de confirmação

	while True:
		try:
			message = client_socket.recv(HEADER_LENGTH)
			if message:
				print("<" + addr[0] + ">" + message)
				message_to_send = "<" + addr[0] + "> " + message
				broadcast(message_to_send, client_socket)

			else:
				remove(client_socket)
		except:
			continue

def broadcast(message, connection):
    for clients in clients_list:
        if clients!= connection:
            try:
                clients.send(message)
            except:
                clients.close()
                remove(clients)

def remove(connection):
    if connection in clients_list:
        clients_list.remove(connection)

while True:
	client_socket, addr = server.accept()
	clients_list.append(client_socket)
	print("Nova conexão" + addr[0] + "aceita")
	start_thread(clientthread,(client_socket,addr))

client_socket.close()
server.close()


from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect

from core import cards
from core import menu
from core.authentication import AuthenticationManager
from health.models.activity import Activity
from health.models.consumption import Consumption
from health.models.weight import WeightHistory

def index(request):
    return render(request, 'index.html')

@login_required
def dashboard(request):
    user = AuthenticationManager.get_session_user(request=request)

    if not user.is_authenticated:
        return redirect('core:register')

    page_cards = cards.get_cards(request)
    menu_items = menu.get_menu_items('Dashboard')

    context = {
        'username': user.username,
        'menu_items': menu_items,
        'cards': page_cards
    }

    return render(request, 'dashboard.html', context)

@login_required
def timeline(request):
    user = AuthenticationManager.get_session_user(request=request)
    menu_items = menu.get_menu_items('Timeline')

    consumptions = Consumption.objects.filter(user=user, is_active='1')
    activities = Activity.objects.filter(user=user, is_active='1')
    weights = WeightHistory.objects.filter(user=user, is_active='1')

    context = {
        'username': user.username,
        'menu_items': menu_items,
        'consumptions': consumptions,
        'activities': activities,
        'weights': weights,
    }

    return render(request, 'timeline.html', context)

@login_required
def profile(request):
    user = AuthenticationManager.get_session_user(request=request)
    menu_items = menu.get_menu_items('Profile')

    data = {
        'username': user.username,
        'first_name': user.first_name,
        'last_name': user.last_name,
        'fullname': user.get_full_name(),
        'email': user.email,
        'gender': user.profile.gender if user.profile.gender is not None else "",
        'age': user.profile.get_age() if user.profile.birth_date is not None else "",
        'birth_date': user.profile.birth_date if user.profile.birth_date is not None else "",
        'bmi': user.profile.bmi if user.profile.bmi is not None else "",
        'weight': user.profile.weight if user.profile.weight is not None else "",
        'height': user.profile.height if user.profile.height is not None else ""
    }

    context = {
        'profile': data,
        'menu_items': menu_items
    }

    return render(request, 'profile.html', context)


# -*- coding: utf-8 -*-
"""
WSPIA_eu spider created on the top of ATSSpider

scrapy crawl wspia_eu -a iteration=1 -a extract=1 -a mining_job_id=9999 -a url="http://www.biurokarier.wspia.eu/lista-ogloszen?q="

Seed URL:
    http://www.biurokarier.wspia.eu/lista-ogloszen?q=
"""

from re import compile
from scrapy.http import Request
from scrapy.selector import Selector
from urlparse import urljoin

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import ConvertDateString, NormalizedJoin, Prefix

pattern = {
    'ref_number': compile(r'id\/(\d+)'),
}

class WSPIA_eu(ATSSpider):

    name = 'wspia_eu'

    def parse(self, response):
        sel = Selector(response)

        for href in sel.xpath(
            '//div[@class="pageItemContent"]/div/div[@class="careerItem"]//div[@class="careerItemPosition"]/a/@href'
        ).extract():
            yield Request(
                callback=self.parse_job_callback(),
                url=urljoin(response.url, href)
            )
        # pagination
        next_page = sel.xpath(
            '//div[@class="pager"]/div/ul/li[@class="pagerNext"]/span/a/@href'
        ).extract()
        if next_page:
            yield Request(
                callback=self.parse,
                url=urljoin(response.url, next_page[0])
            )

    def parse_job(self, response):
        sel = Selector(response)

        loader = BrightcorpItemLoader(selector=sel)

        loader.add_xpath(
            'title',
            '//div[contains(@class, "pageItem")]/h1[@class="pageItemName"]/text()'
        )
        loader.add_xpath(
            'company',
            '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Nazwa firmy:', 'utf-8')
        )
        loader.add_xpath(
            'industry',
            '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Branża:', 'utf-8')
        )
        loader.add_xpath(
            'location',
            [
                '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Miasto:', 'utf-8'),
                '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Region:', 'utf-8'),
            ],
            NormalizedJoin(', ')
        )
        loader.add_xpath(
            'date',
            '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Data publikacji:', 'utf-8'),
            ConvertDateString('%d-%m-%Y')
        )
        loader.add_xpath(
            'description',
            '//div[@class="pageItemContent"]/div/div[@class="careerContent"]'
        )
        loader.add_value(
            'referencenumber',
            response.url,
            Prefix('%s-' % self.name),
            re=pattern['ref_number']
        )
        loader.add_value('url', response.url)
        loader.add_value('apply_url', response.url)

        yield loader.load_item()


from PyQt5 import QtWidgets, QtGui, QtCore
import sys
import os
import matplotlib.pyplot as plt
import shutil

class Window(QtWidgets.QWidget):

    def __init__(self):
        super().__init__()

        self.init_ui()

        self.setWindowTitle("Graph OP. Application")

    def init_ui(self):

        self.pixmap = QtGui.QPixmap("default_graph.png")  ## grafiğin geleceği konumda boş bir fotoğraf
        self.default_graph_label = QtWidgets.QLabel()
        self.default_graph_label.setPixmap(self.pixmap)
        self.default_graph_label.setAlignment(QtCore.Qt.AlignCenter) ## boş fotoğraf eklendi.

        ## Label'lar tanımlanıyor.
        self.input1 = QtWidgets.QLabel("Input 1:")
        self.input2 = QtWidgets.QLabel("Input 2:")
        self.input3 = QtWidgets.QLabel("Input 3:")
        self.input4 = QtWidgets.QLabel("Input 4:")
        self.input5 = QtWidgets.QLabel("Input 5:")
        self.list_label = QtWidgets.QLabel("LIST OF GRAPHS")
        self.labelGN = QtWidgets.QLabel("Give a name to your graph:")
        self.saveOperations_label = QtWidgets.QLabel("SAVE OPERATIONS")
        self.del_and_clear_op_label = QtWidgets.QLabel("DELETE & CLEAR OPERATIONS")

        self.result1_tag = QtWidgets.QLabel("# Result 1 #")
        self.result2_tag = QtWidgets.QLabel("# Result 2 #")
        self.result3_tag = QtWidgets.QLabel("# Result 3 #")
        self.result4_tag = QtWidgets.QLabel("# Result 4 #")
        self.result5_tag = QtWidgets.QLabel("# Result 5 #")

        self.result1_label = QtWidgets.QLabel("not calculated")
        self.result2_label = QtWidgets.QLabel("not calculated")
        self.result3_label = QtWidgets.QLabel("not calculated")
        self.result4_label = QtWidgets.QLabel("not calculated")
        self.result5_label = QtWidgets.QLabel("not calculated")

        ## Girdilerin alınacağı line editler.
        self.number1 = QtWidgets.QLineEdit()
        self.number2 = QtWidgets.QLineEdit()
        self.number3 = QtWidgets.QLineEdit()
        self.number4 = QtWidgets.QLineEdit()
        self.number5 = QtWidgets.QLineEdit()
        self.giveName = QtWidgets.QLineEdit()
        self.labelGN.setStyleSheet("QLabel {color: '#FFD700'}")
        self.ListW = QtWidgets.QListWidget()
        self.graphTitle = QtWidgets.QLineEdit()

        ## BUTTONS.
        self.saveGraph = QtWidgets.QPushButton("Save Mode (Save One By One)")
        self.clearALL = QtWidgets.QPushButton("Clear The Whole List")
        self.deleteALL = QtWidgets.QPushButton("Delete All Graphs")
        self.saveALL = QtWidgets.QPushButton("Save All Graphs As Archive")
        self.button = QtWidgets.QPushButton("Create Graphs and Calculate")

        ## MSG BOX.
        self.errorMSG = QtWidgets.QMessageBox()

        ## FONT ADJUSTMENT.
        self.buttonFonts = QtGui.QFont("Trebuchet MS", 10, QtGui.QFont.Bold)
        self.labelFonts = QtGui.QFont("Trebuchet MS", 10, QtGui.QFont.Bold)
        self.operationFonts = QtGui.QFont("Trebuchet MS", 11, QtGui.QFont.Bold)

        ## Buttons colors & fonts.
        self.button.setStyleSheet("QPushButton {background: #FFD700; color: 'black';}")
        self.saveGraph.setStyleSheet("QPushButton {background: #FFD700; color: 'black';}")
        self.clearALL.setStyleSheet("QPushButton {background: #FFD700; color: 'black';}")
        self.deleteALL.setStyleSheet("QPushButton {background: #FFD700; color: 'black';}")
        self.saveALL.setStyleSheet("QPushButton {background: #FFD700; color: 'black';}")
        self.button.setFont(self.buttonFonts)
        self.saveGraph.setFont(self.buttonFonts)
        self.clearALL.setFont(self.buttonFonts)
        self.deleteALL.setFont(self.buttonFonts)
        self.saveALL.setFont(self.buttonFonts)

        ## Labels colors & fonts.
        self.input1.setStyleSheet("QLabel {color: #FFD700;}")
        self.input2.setStyleSheet("QLabel {color: #FFD700;}")
        self.input3.setStyleSheet("QLabel {color: #FFD700;}")
        self.input4.setStyleSheet("QLabel {color: #FFD700;}")
        self.input5.setStyleSheet("QLabel {color: #FFD700;}")
        self.list_label.setStyleSheet("QLabel {color: #FFD700;}")
        self.labelGN.setStyleSheet("QLabel {color: #FFD700;}")
        self.input1.setFont(self.labelFonts)
        self.input2.setFont(self.labelFonts)
        self.input3.setFont(self.labelFonts)
        self.input4.setFont(self.labelFonts)
        self.input5.setFont(self.labelFonts)
        self.list_label.setFont(self.labelFonts)
        self.labelGN.setFont(self.labelFonts)

        self.result1_label.setStyleSheet("QLabel {color: #FF0000;}")
        self.result2_label.setStyleSheet("QLabel {color: #FF0000;}")
        self.result3_label.setStyleSheet("QLabel {color: #FF0000;}")
        self.result4_label.setStyleSheet("QLabel {color: #FF0000;}")
        self.result5_label.setStyleSheet("QLabel {color: #FF0000;}")

        self.result1_label.setFont(self.labelFonts)
        self.result2_label.setFont(self.labelFonts)
        self.result3_label.setFont(self.labelFonts)
        self.result4_label.setFont(self.labelFonts)
        self.result5_label.setFont(self.labelFonts)

        self.result1_tag.setStyleSheet("QLabel {color: #FFD700;}")
        self.result2_tag.setStyleSheet("QLabel {color: #FFD700;}")
        self.result3_tag.setStyleSheet("QLabel {color: #FFD700;}")
        self.result4_tag.setStyleSheet("QLabel {color: #FFD700;}")
        self.result5_tag.setStyleSheet("QLabel {color: #FFD700;}")

        self.result1_tag.setFont(self.labelFonts)
        self.result2_tag.setFont(self.labelFonts)
        self.result3_tag.setFont(self.labelFonts)
        self.result4_tag.setFont(self.labelFonts)
        self.result5_tag.setFont(self.labelFonts)

        self.del_and_clear_op_label.setStyleSheet("QLabel {color: #3FCCFF;}")
        self.saveOperations_label.setStyleSheet("QLabel {color: #3FCCFF;}")

        self.del_and_clear_op_label.setFont(self.labelFonts)
        self.saveOperations_label.setFont(self.labelFonts)

        ## ortalıyoruz bazı labelları.
        self.del_and_clear_op_label.setAlignment(QtCore.Qt.AlignCenter)
        self.saveOperations_label.setAlignment(QtCore.Qt.AlignCenter)
        self.list_label.setAlignment(QtCore.Qt.AlignCenter)

        ## Adding widgets.
        v_box = QtWidgets.QVBoxLayout()
        v_box2 = QtWidgets.QVBoxLayout()

        v_box.addWidget(self.input1)
        v_box.addWidget(self.number1)
        v_box.addWidget(self.input2)
        v_box.addWidget(self.number2)
        v_box.addWidget(self.input3)
        v_box.addWidget(self.number3)
        v_box.addWidget(self.input4)
        v_box.addWidget(self.number4)
        v_box.addWidget(self.input5)
        v_box.addWidget(self.number5)
        v_box.addWidget(self.labelGN)
        v_box.addWidget(self.giveName)
        v_box.addWidget(self.button)

        v_box.addWidget(self.result1_tag)
        v_box.addWidget(self.result1_label)
        v_box.addWidget(self.result2_tag)
        v_box.addWidget(self.result2_label)
        v_box.addWidget(self.result3_tag)
        v_box.addWidget(self.result3_label)
        v_box.addWidget(self.result4_tag)
        v_box.addWidget(self.result4_label)
        v_box.addWidget(self.result5_tag)
        v_box.addWidget(self.result5_label)

        v_box2.addWidget(self.default_graph_label)
        v_box2.addWidget(self.list_label)
        v_box2.addWidget(self.ListW)
        v_box2.addWidget(self.saveOperations_label)
        v_box2.addWidget(self.saveALL)
        v_box2.addWidget(self.saveGraph)
        v_box2.addWidget(self.del_and_clear_op_label)
        v_box2.addWidget(self.deleteALL)
        v_box2.addWidget(self.clearALL)

        h_box = QtWidgets.QHBoxLayout()

        h_box.addLayout(v_box)
        h_box.addLayout(v_box2)

        self.all_graphs_list = []  ## a list for graph pictures.
        self.setLayout(h_box)
        self.show()

        self.button.clicked.connect(lambda: self.showGraph(self.giveName.text()))
        self.saveGraph.clicked.connect(self.bridge2)
        self.clearALL.clicked.connect(self.clearALL_list)
        self.deleteALL.clicked.connect(self.delete_all_item)
        self.saveALL.clicked.connect(self.saveALL_of_them)

    def showGraph(self, file_name):

        self.all_graphs_list.append(file_name + ".png")  ## putting the picture into the list.
        self.all_graphs_list = list(dict.fromkeys(self.all_graphs_list))  ## controlling the same file name
        print(self.all_graphs_list)

        plt.close()

        ## girilen herhangi bir boş veya string değer olursa diye try-except içerisine aldık.
        ## ValueError alırsak hatası mesajı veriyoruz.
        try:
            ## rastgele hesaplamalar.
            self.result1 = (int(self.number1.text()) ** 2) / 6
            self.result2 = (int(self.number2.text())) - self.result1
            self.result3 = self.result1 + self.result2
            self.result4 = (int(self.number3.text()) + int(self.number4.text())) / int(self.number5.text())
            self.result5 = self.result1 / self.result4

            ## Hesaplamaları -not calculated- kısmındaki labellara setText diyerek güncelliyoruz.
            self.result1_label.setText(str(self.result1))
            self.result2_label.setText(str(self.result2))
            self.result3_label.setText(str(self.result3))
            self.result4_label.setText(str(self.result4))
            self.result5_label.setText(str(self.result5))

            ## Kırmızıdan yeşile çeviriyoruz rengi.
            self.result1_label.setStyleSheet("QLabel {color: #00FF00;}")
            self.result2_label.setStyleSheet("QLabel {color: #00FF00;}")
            self.result3_label.setStyleSheet("QLabel {color: #00FF00;}")
            self.result4_label.setStyleSheet("QLabel {color: #00FF00;}")
            self.result5_label.setStyleSheet("QLabel {color: #00FF00;}")

            ## Grafik için gelen sonuçları liste içreisine aldık.
            result_List = [self.result1, self.result2,
                           self.result3, self.result4,
                           self.result5]

            # use the plot function
            ## grafiik oluşturuluyor.
            plt.figure(figsize=(3.5,3))
            plt.plot([-100,-50, 0, 50, 100], result_List, linestyle ='-', linewidth = 3, color = 'orange', marker = 'o')

            ## basılan butonda aynı zamanda grafiğin ismini alıp (parametre olarak aldık) ona göre grafik oluşturduğu için
            ## kullanıcının boş dosya ismi girme olasılığı mevcut ve dolayısıyla burada dosya isminin length'ini alıp ona göre
            ## basit bir if bloğu ile sorguluyoruz. en az 3 karakterli dosya ismi istiyoruz.
            if len(file_name) < 2:
                self.errorMSG.setIcon(QtWidgets.QMessageBox.Warning)
                self.errorMSG.setText("Please enter at least 3 character file name.")
                self.errorMSG.setWindowTitle("WARNING")
                self.errorMSG.exec_()
            else:
                plt.title(file_name) #grafik başlığını alıyoruz.
                plt.savefig(file_name + ".png")  ## grafiği png olarak kaydediyoruz.
                self.newGraph = QtGui.QPixmap(file_name+".png") ## grafiğimiz tanımlanıyor (arayüzde göstermek için)
                self.default_graph_label.setPixmap(self.newGraph)  ## grafik yenisiyle değiştiriliyor.
                self.default_graph_label.setAlignment(QtCore.Qt.AlignCenter)  ## grafik ortalanıyor.
                self.ListW.addItem(file_name + ".png")  ## grafik listemize ekleniyor.
                self.ListW.itemClicked.connect(self.listUpload) ## burada listUpload function'ına bağlanıyoruz (kullanıcı listedeki her grafiğe tıkladığında)
            #print(os.path.splitext("")[0])  ## bunu niye yapmışım bilmiyorum, dursun :*)
        except ValueError:
            self.errorMSG.setIcon(QtWidgets.QMessageBox.Critical)
            self.errorMSG.setText("Please enter a number to every input!")
            self.errorMSG.setWindowTitle("ERROR")
            self.errorMSG.exec_()

    ## 247.satırda tanımlanmış yeni grafik fotoğrafımızı, kullanıcı listwidget'da her tıkladığında değiştirmek için
    ## bir function.
    def listUpload(self,q):

        self.newGraph = QtGui.QPixmap(str(q.text()))  ## burada q bizim listemizde tıkladığımız item'ları tutar.
        self.default_graph_label.setPixmap(self.newGraph)  ## bu yeni atanmış grafiği tekrar grafik kısmına yansıtıyoruz.

    ## save mode açılıyor.
    ## buradda bridge dememin sebebi, iki fonksiyon arası köprü işlevi görüyor
    ## burada kullanıcı save mode butonuna bastıktan sonra artık listwidget'da tıkladığı her grafik
    ## kaydediliyor. çünkü createRAR fonksiyonuna gidiyoruz orada q parametresini veriyoruz ve
    ## bu parametre listwidget içerisindeki graifğin ismini tutuyor.
    def bridge2(self):

        self.ListW.itemClicked.connect(self.createRAR)  ## listwidget'da bir item'a basıldığında createRAR fonksiyonuna git

    ## grafiklerin kaydedilmesi
    def createRAR(self, q):

        ## bir dosya ismi belirttik.
        save_path = 'save_archive'
        ## dosya ismi bize listwidget'dan q olarak geliyor
        dir_name = str(q.text()) # dosya ismini dir_name'e tanımlıyoruz.
        shutil.make_archive(str(q.text()) + "_saveFile", 'zip', ".", dir_name)  ## arşivleme işlemi yapılıyor.
        shutil.move(str(q.text()) + "_saveFile.zip", save_path)  ## arşivleri daha önce 279.satırda belirttiğimiz dosyanın içerisine move ile atıyoruz. (bu kopyalama işlemi değildir.)

        ## işlemin yapıldığına dair mesaj gönderiyoruz user'a.
        self.errorMSG.setIcon(QtWidgets.QMessageBox.Information)
        self.errorMSG.setText("Operation has done successfully. Saved as archive at ~/save_archive")
        self.errorMSG.setWindowTitle("Successed")
        self.errorMSG.exec_()

    ## listwidget'ımız temizleniyor.
    def clearALL_list(self):
        self.ListW.clear()
        self.all_graphs_list.clear() ## ayrıca dosya isimlerinin alındığı listeyi de temizliyoruz ki sonradan sıkıntı oluşmasın.

    def delete_all_item(self):

        ## 182.satırda tanımladığımız listenin içerisine gelen grafiklerin silinme işlemi (işletim sisteminden silinme işlemi.)
        ## singleGraph all_graphs_list içerisindeki her bir grafiği geziyor, o listenin içerisindeki grafiğin ismini kullanarak
        ## os.remove(singleGraph) satırıyla siliyoruz sistemden.
        for singleGraph in self.all_graphs_list:
            os.remove(singleGraph)

        self.ListW.clear()  ## listwdiget'ımız temizleniyor.
        self.all_graphs_list.clear() ## grafik listemiz temizleniyor.
        self.errorMSG.setIcon(QtWidgets.QMessageBox.Information) ## kullancııya mesaj veriyoruz.
        self.errorMSG.setText("Operation has done successfully. All of them deleted from folder.")
        self.errorMSG.setWindowTitle("Successed")
        self.errorMSG.exec_()

    ## tüm grafiklrei kaydetme işlemi
    def saveALL_of_them(self):

        myListRange = len(self.all_graphs_list)  ## kaç tane grafiğimiz varsa o kadar işlem yapılmasını istediğimiz için
                                                    ## grafik listemizin uzunluğunu alıyoruz.

        save_path = 'save_archive'  ## grafiklerin kaydedileceği dosya ismi tanımlanıyor.

        for singleGraph in range(0,myListRange):  ## burada uzunluğu kullanıyoruz.
            ## Şimdi burada olay şu, bize singleGraph bir sayı veriyor
            ## biz bu sayıyı grafik listemizin indexi olarak kullanıyoruz her index arttığında
            ## grafiğimizin ismiyle rar kaydediyoruz.
            shutil.make_archive(self.all_graphs_list[singleGraph] + "_saveFile", 'zip', ".", self.all_graphs_list[singleGraph])
            shutil.move(self.all_graphs_list[singleGraph]+ "_saveFile.zip", save_path) ## grafikleri 317.satırda belirtilen dosyanın içerisine atıyoruz (kopyalama işlemi değildir.)

        self.errorMSG.setIcon(QtWidgets.QMessageBox.Information)
        self.errorMSG.setText("Operation has done successfully. Saved as archive at ~/save_archive")
        self.errorMSG.setWindowTitle("Successed")
        self.errorMSG.exec_()

app = QtWidgets.QApplication(sys.argv)
window = Window()
window.move(650, 120)
app.setStyle("Fusion")
window.setFixedSize(500, 800)
window.setStyleSheet("Window {background : #434343;}")
sys.exit(app.exec_())


import pytest
from datetime import date, datetime
from analyzers import AnalyzeTokens
import os

pytestmark = pytest.mark.usefixtures("spark")

def test_token_stat(spark):
    path = os.getcwd() + "/tests/fixtures/token_stat"
    start_date = date(2017, 1, 15)
    end_date = date(2017, 1, 16)

    analyzer = AnalyzeTokens(storage_path=path, start_date=start_date, end_date=end_date, spark_session=spark,
                             database=None, current_datetime=datetime(2017, 2, 15, 15, 10))

    files = analyzer.get_files_to_analyze()

    expected_files = [path + '/2017/01/15/token_stat.json.log', path + '/2017/01/16/token_stat.json.log']

    assert len(files) == len(expected_files)
    assert len(set(files) - set(expected_files)) == 0

    results = analyzer.get_data()
    expected_results = [(u'token:2', date(2017, 1, 15), 1),
                        (u'token:2', date(2017, 1, 16), 1),
                        (u'token:3', date(2017, 1, 15), 6),
                        (u'token:1', date(2017, 1, 15), 2),
                        (u'token:1', date(2017, 1, 16), 2)]
    assert len(results) == len(expected_results)
    assert results == expected_results
    assert analyzer.get_log_analyzer_stats(datetime(2017, 2, 15, 15, 12)) == \
           "[OK] [2017-02-15 15:12:00] [2017-02-15 15:10:00] [TokenStatsUpdater] [120]"

def test_token_stat_empty_file(spark):
    path = os.getcwd() + "/tests/fixtures/token_stat"
    start_date = date(2017, 1, 17)
    end_date = date(2017, 1, 17)
    analyzer = AnalyzeTokens(storage_path=path, start_date=start_date, end_date=end_date, spark_session=spark,
                             database=None)

    files = analyzer.get_files_to_analyze()
    expected_files = [path + '/2017/01/17/token_stat.json.log']
    assert len(files) == len(expected_files)
    results = analyzer.get_data()
    assert len(results) == 0


# Year - has many months
# Month - has many days
# Day - name, hours

days_in_may = {
    1 : 'Saturday',
    2 : 'Sunday',
    3: 'Monday',
    4: 'Tuesday',
    5: 'Wednesday',
    6: 'Thursday',
    7: 'Friday'
}

class Day:
    def __init__(self, name, date, month):
        self.name = name
        self.date = date
        self.month = month

    def __str__(self):
        return f'{self.name} the {self.date} of {self.month.name}'

class Month:
    def __init__(self, name, placement):
        self.name = name
        self.placement = placement
        self.days = []

    def __str__(self):
        return f'{self.name}({self.placement})'

may = Month("May", 5)
# yesterday = Day("Wednesday", 5, may)
# today = Day("Thursday", 6, may)
# tomorrow = Day("Friday", 7, may)

for date, name in days_in_may.items():
    day = Day(name, date, may)
    may.days.append(day)

for day in may.days:
    print(day)