# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

"""
See documentation for usage
https://github.com/CGCookie/blender-addon-updater

"""

import urllib.request
import urllib
import os
import json
import zipfile
import shutil
import asyncio
import threading
import time
import fnmatch
from datetime import datetime, timedelta

# blender imports, used in limited cases
import bpy
import addon_utils

# -----------------------------------------------------------------------------
# Define error messages/notices & hard coded globals
# -----------------------------------------------------------------------------

# currently not used
DEFAULT_TIMEOUT = 10
DEFAULT_PER_PAGE = 30

# -----------------------------------------------------------------------------
# The main class
# -----------------------------------------------------------------------------

class Singleton_updater(object):
    """
	This is the singleton class to reference a copy from,
	it is the shared module level class
	"""

    def __init__(self):

        self._engine = GithubEngine()
        self._user = None
        self._repo = None
        self._website = None
        self._current_version = None
        self._tags = []
        self._tag_latest = None
        self._tag_names = []
        self._latest_release = None
        self._include_branches = False
        self._include_branch_list = ['master']
        self._include_branch_autocheck = False
        self._manual_only = False
        self._version_min_update = None
        self._version_max_update = None

        # by default, backup current addon if new is being loaded
        self._backup_current = True
        self._backup_ignore_patterns = None

        # set patterns for what files to overwrite on update
        self._overwrite_patterns = ["*.py", "*.pyc"]
        self._remove_pre_update_patterns = []

        # by default, don't auto enable/disable the addon on update
        # as it is slightly less stable/won't always fully reload module
        self._auto_reload_post_update = False

        # settings relating to frequency and whether to enable auto background check
        self._check_interval_enable = False
        self._check_interval_months = 0
        self._check_interval_days = 7
        self._check_interval_hours = 0
        self._check_interval_minutes = 0

        # runtime variables, initial conditions
        self._verbose = False
        self._fake_install = False
        self._async_checking = False  # only true when async daemon started
        self._update_ready = None
        self._update_link = None
        self._update_version = None
        self._source_zip = None
        self._check_thread = None
        self._skip_tag = None

        # get from module data
        self._addon = __package__.lower()
        self._addon_package = __package__  # must not change
        self._updater_path = os.path.join(os.path.dirname(__file__),
                                          self._addon + "_updater")
        self._addon_root = os.path.dirname(__file__)
        self._json = {}
        self._error = None
        self._error_msg = None
        self._prefiltered_tag_count = 0

        # to verify a valid import, in place of placeholder import
        self.invalidupdater = False

    # -------------------------------------------------------------------------
    # Getters and setters
    # -------------------------------------------------------------------------

    @property
    def engine(self):
        return self._engine.name

    @engine.setter
    def engine(self, value):
        if value.lower() == "github":
            self._engine = GithubEngine()
        elif value.lower() == "gitlab":
            self._engine = GitlabEngine()
        elif value.lower() == "bitbucket":
            self._engine = BitbucketEngine()
        else:
            raise ValueError("Invalid engine selection")

    @property
    def private_token(self):
        return self._engine.token

    @private_token.setter
    def private_token(self, value):
        if value == None:
            self._engine.token = None
        else:
            self._engine.token = str(value)

    @property
    def addon(self):
        return self._addon

    @addon.setter
    def addon(self, value):
        self._addon = str(value)

    @property
    def verbose(self):
        return self._verbose

    @verbose.setter
    def verbose(self, value):
        try:
            self._verbose = bool(value)
            if self._verbose == True:
                print(self._addon + " updater verbose is enabled")
        except:
            raise ValueError("Verbose must be a boolean value")

    @property
    def include_branches(self):
        return self._include_branches

    @include_branches.setter
    def include_branches(self, value):
        try:
            self._include_branches = bool(value)
        except:
            raise ValueError("include_branches must be a boolean value")

    @property
    def include_branch_list(self):
        return self._include_branch_list

    @include_branch_list.setter
    def include_branch_list(self, value):
        try:
            if value == None:
                self._include_branch_list = ['master']
            elif type(value) != type(['master']) or value == []:
                raise ValueError("include_branch_list should be a list of valid branches")
            else:
                self._include_branch_list = value
        except:
            raise ValueError("include_branch_list should be a list of valid branches")

    @property
    def overwrite_patterns(self):
        return self._overwrite_patterns

    @overwrite_patterns.setter
    def overwrite_patterns(self, value):
        if value == None:
            self._overwrite_patterns = ["*.py", "*.pyc"]
        elif type(value) != type(['']):
            raise ValueError("overwrite_patterns needs to be in a list format")
        else:
            self._overwrite_patterns = value

    @property
    def remove_pre_update_patterns(self):
        return self._remove_pre_update_patterns

    @remove_pre_update_patterns.setter
    def remove_pre_update_patterns(self, value):
        if value == None:
            self._remove_pre_update_patterns = []
        elif type(value) != type(['']):
            raise ValueError("remove_pre_update_patterns needs to be in a list format")
        else:
            self._remove_pre_update_patterns = value

    # not currently used
    @property
    def include_branch_autocheck(self):
        return self._include_branch_autocheck

    @include_branch_autocheck.setter
    def include_branch_autocheck(self, value):
        try:
            self._include_branch_autocheck = bool(value)
        except:
            raise ValueError("include_branch_autocheck must be a boolean value")

    @property
    def manual_only(self):
        return self._manual_only

    @manual_only.setter
    def manual_only(self, value):
        try:
            self._manual_only = bool(value)
        except:
            raise ValueError("manual_only must be a boolean value")

    @property
    def auto_reload_post_update(self):
        return self._auto_reload_post_update

    @auto_reload_post_update.setter
    def auto_reload_post_update(self, value):
        try:
            self._auto_reload_post_update = bool(value)
        except:
            raise ValueError("Must be a boolean value")

    @property
    def fake_install(self):
        return self._fake_install

    @fake_install.setter
    def fake_install(self, value):
        if type(value) != type(False):
            raise ValueError("fake_install must be a boolean value")
        self._fake_install = bool(value)

    @property
    def user(self):
        return self._user

    @user.setter
    def user(self, value):
        try:
            self._user = str(value)
        except:
            raise ValueError("User must be a string value")

    @property
    def json(self):
        if self._json == {}:
            self.set_updater_json()
        return self._json

    @property
    def repo(self):
        return self._repo

    @repo.setter
    def repo(self, value):
        try:
            self._repo = str(value)
        except:
            raise ValueError("User must be a string")

    @property
    def website(self):
        return self._website

    @website.setter
    def website(self, value):
        if self.check_is_url(value) == False:
            raise ValueError("Not a valid URL: " + value)
        self._website = value

    @property
    def async_checking(self):
        return self._async_checking

    @property
    def api_url(self):
        return self._engine.api_url

    @api_url.setter
    def api_url(self, value):
        if self.check_is_url(value) == False:
            raise ValueError("Not a valid URL: " + value)
        self._engine.api_url = value

    @property
    def stage_path(self):
        return self._updater_path

    @stage_path.setter
    def stage_path(self, value):
        if value == None:
            if self._verbose: print("Aborting assigning stage_path, it's null")
            return
        elif value != None and not os.path.exists(value):
            try:
                os.makedirs(value)
            except:
                if self._verbose: print("Error trying to staging path")
                return
        self._updater_path = value

    @property
    def tags(self):
        if self._tags == []:
            return []
        tag_names = []
        for tag in self._tags:
            tag_names.append(tag["name"])

        return tag_names

    @property
    def tag_latest(self):
        if self._tag_latest == None:
            return None
        return self._tag_latest["name"]

    @property
    def latest_release(self):
        if self._releases_latest == None:
            return None
        return self._latest_release

    @property
    def current_version(self):
        return self._current_version

    @property
    def update_ready(self):
        return self._update_ready

    @property
    def update_version(self):
        return self._update_version

    @property
    def update_link(self):
        return self._update_link

    @current_version.setter
    def current_version(self, tuple_values):
        if tuple_values == None:
            self._current_version = None
            return
        elif type(tuple_values) is not tuple:
            try:
                tuple(tuple_values)
            except:
                raise ValueError(
                    "Not a tuple! current_version must be a tuple of integers")
        for i in tuple_values:
            if type(i) is not int:
                raise ValueError(
                    "Not an integer! current_version must be a tuple of integers")
        self._current_version = tuple(tuple_values)

    def set_check_interval(self, enable=False, months=0, days=14, hours=0, minutes=0):
        # enabled = False, default initially will not check against frequency
        # if enabled, default is then 2 weeks

        if type(enable) is not bool:
            raise ValueError("Enable must be a boolean value")
        if type(months) is not int:
            raise ValueError("Months must be an integer value")
        if type(days) is not int:
            raise ValueError("Days must be an integer value")
        if type(hours) is not int:
            raise ValueError("Hours must be an integer value")
        if type(minutes) is not int:
            raise ValueError("Minutes must be an integer value")

        if enable == False:
            self._check_interval_enable = False
        else:
            self._check_interval_enable = True

        self._check_interval_months = months
        self._check_interval_days = days
        self._check_interval_hours = hours
        self._check_interval_minutes = minutes

    @property
    def check_interval(self):
        return (self._check_interval_enable,
                self._check_interval_months,
                self._check_interval_days,
                self._check_interval_hours,
                self._check_interval_minutes)

    @property
    def error(self):
        return self._error

    @property
    def error_msg(self):
        return self._error_msg

    @property
    def version_min_update(self):
        return self._version_min_update

    @version_min_update.setter
    def version_min_update(self, value):
        if value == None:
            self._version_min_update = None
            return
        if type(value) != type((1, 2, 3)):
            raise ValueError("Version minimum must be a tuple")
        else:
            # potentially check entries are integers
            self._version_min_update = value

    @property
    def version_max_update(self):
        return self._version_max_update

    @version_max_update.setter
    def version_max_update(self, value):
        if value == None:
            self._version_max_update = None
            return
        if type(value) != type((1, 2, 3)):
            raise ValueError("Version maximum must be a tuple")
        else:
            # potentially check entries are integers
            self._version_max_update = value

    @property
    def backup_current(self):
        return self._backup_current

    @backup_current.setter
    def backup_current(self, value):
        if value == None:
            self._backup_current = False
            return
        else:
            self._backup_current = value

    @property
    def backup_ignore_patterns(self):
        return self._backup_ignore_patterns

    @backup_ignore_patterns.setter
    def backup_ignore_patterns(self, value):
        if value == None:
            self._backup_ignore_patterns = None
            return
        elif type(value) != type(['list']):
            raise ValueError("Backup pattern must be in list format")
        else:
            self._backup_ignore_patterns = value

    # -------------------------------------------------------------------------
    # Parameter validation related functions
    # -------------------------------------------------------------------------

    def check_is_url(self, url):
        if not ("http://" in url or "https://" in url):
            return False
        if "." not in url:
            return False
        return True

    def get_tag_names(self):
        tag_names = []
        self.get_tags(self)
        for tag in self._tags:
            tag_names.append(tag["name"])
        return tag_names

    # declare how the class gets printed

    def __repr__(self):
        return "<Module updater from {a}>".format(a=__file__)

    def __str__(self):
        return "Updater, with user: {a}, repository: {b}, url: {c}".format(
            a=self._user,
            b=self._repo, c=self.form_repo_url())

    # -------------------------------------------------------------------------
    # API-related functions
    # -------------------------------------------------------------------------

    def form_repo_url(self):
        return self._engine.form_repo_url(self)

    def form_tags_url(self):
        return self._engine.form_tags_url(self)

    def form_branch_url(self, branch):
        return self._engine.form_branch_url(branch, self)

    def get_tags(self):
        request = self.form_tags_url()
        if self._verbose: print("Getting tags from server")

        # get all tags, internet call
        all_tags = self._engine.parse_tags(self.get_api(request), self)
        self._prefiltered_tag_count = len(all_tags)

        # pre-process to skip tags
        if self.skip_tag != None:
            self._tags = [tg for tg in all_tags if self.skip_tag(self, tg) == False]
        else:
            self._tags = all_tags

        # get additional branches too, if needed, and place in front
        # does NO checking here whether branch is valid
        if self._include_branches == True:
            temp_branches = self._include_branch_list.copy()
            temp_branches.reverse()
            for branch in temp_branches:
                request = self.form_branch_url(branch)
                include = {
                    "name": branch.title(),
                    "zipball_url": request
                }
                self._tags = [include] + self._tags  # append to front

        if self._tags == None:
            # some error occurred
            self._tag_latest = None
            self._tags = []
            return
        elif self._prefiltered_tag_count == 0 and self._include_branches == False:
            self._tag_latest = None
            self._error = "No releases found"
            self._error_msg = "No releases or tags found on this repository"
            if self._verbose: print("No releases or tags found on this repository")
        elif self._prefiltered_tag_count == 0 and self._include_branches == True:
            self._tag_latest = self._tags[0]
            if self._verbose:
                branch = self._include_branch_list[0]
                print("{} branch found, no releases".format(branch), self._tags[0])
        elif len(self._tags) == 0 and self._prefiltered_tag_count > 0:
            self._tag_latest = None
            self._error = "No releases available"
            self._error_msg = "No versions found within compatible version range"
            if self._verbose: print("No versions found within compatible version range")
        else:
            if self._include_branches == False:
                self._tag_latest = self._tags[0]
                if self._verbose: print("Most recent tag found:", self._tags[0]['name'])
            else:
                # don't return branch if in list
                n = len(self._include_branch_list)
                self._tag_latest = self._tags[n]  # guaranteed at least len()=n+1
                if self._verbose: print("Most recent tag found:", self._tags[n]['name'])

    # all API calls to base url
    def get_raw(self, url):
        # print("Raw request:", url)
        request = urllib.request.Request(url)

        # setup private request headers if appropriate
        if self._engine.token != None:
            if self._engine.name == "gitlab":
                request.add_header('PRIVATE-TOKEN', self._engine.token)
            else:
                if self._verbose: print("Tokens not setup for engine yet")

        # run the request
        try:
            result = urllib.request.urlopen(request)
        except urllib.error.HTTPError as e:
            self._error = "HTTP error"
            self._error_msg = str(e.code)
            self._update_ready = None
        except urllib.error.URLError as e:
            self._error = "URL error, check internet connection"
            self._error_msg = str(e.reason)
            self._update_ready = None
            return None
        else:
            result_string = result.read()
            result.close()
            return result_string.decode()

    # result of all api calls, decoded into json format
    def get_api(self, url):
        # return the json version
        get = None
        get = self.get_raw(url)
        if get != None:
            try:
                return json.JSONDecoder().decode(get)
            except Exception as e:
                self._error = "API response has invalid JSON format"
                self._error_msg = str(e.reason)
                self._update_ready = None
                return None
        else:
            return None

    # create a working directory and download the new files
    def stage_repository(self, url):

        local = os.path.join(self._updater_path, "update_staging")
        error = None

        # make/clear the staging folder
        # ensure the folder is always "clean"
        if self._verbose: print("Preparing staging folder for download:\n", local)
        if os.path.isdir(local) == True:
            try:
                shutil.rmtree(local)
                os.makedirs(local)
            except:
                error = "failed to remove existing staging directory"
        else:
            try:
                os.makedirs(local)
            except:
                error = "failed to create staging directory"

        if error != None:
            if self._verbose: print("Error: Aborting update, " + error)
            self._error = "Update aborted, staging path error"
            self._error_msg = "Error: {}".format(error)
            return False

        if self._backup_current == True:
            self.create_backup()
        if self._verbose: print("Now retrieving the new source zip")

        self._source_zip = os.path.join(local, "source.zip")

        if self._verbose: print("Starting download update zip")
        try:
            request = urllib.request.Request(url)

            # setup private token if appropriate
            if self._engine.token != None:
                if self._engine.name == "gitlab":
                    request.add_header('PRIVATE-TOKEN', self._engine.token)
                else:
                    if self._verbose: print("Tokens not setup for selected engine yet")
            self.urlretrieve(urllib.request.urlopen(request), self._source_zip)
            # add additional checks on file size being non-zero
            if self._verbose: print("Successfully downloaded update zip")
            return True
        except Exception as e:
            self._error = "Error retrieving download, bad link?"
            self._error_msg = "Error: {}".format(e)
            if self._verbose:
                print("Error retrieving download, bad link?")
                print("Error: {}".format(e))
            return False

    def create_backup(self):
        if self._verbose: print("Backing up current addon folder")
        local = os.path.join(self._updater_path, "backup")
        tempdest = os.path.join(self._addon_root,
                                os.pardir,
                                self._addon + "_updater_backup_temp")

        if os.path.isdir(local) == True:
            shutil.rmtree(local)
        if self._verbose: print("Backup destination path: ", local)

        # make the copy
        if self._backup_ignore_patterns != None:
            shutil.copytree(
                self._addon_root, tempdest,
                ignore=shutil.ignore_patterns(*self._backup_ignore_patterns))
        else:
            shutil.copytree(self._addon_root, tempdest)
        shutil.move(tempdest, local)

        # save the date for future ref
        now = datetime.now()
        self._json["backup_date"] = "{m}-{d}-{yr}".format(
            m=now.strftime("%B"), d=now.day, yr=now.year)
        self.save_updater_json()

    def restore_backup(self):
        if self._verbose: print("Restoring backup")

        if self._verbose: print("Backing up current addon folder")
        backuploc = os.path.join(self._updater_path, "backup")
        tempdest = os.path.join(self._addon_root,
                                os.pardir,
                                self._addon + "_updater_backup_temp")
        tempdest = os.path.abspath(tempdest)

        # make the copy
        shutil.move(backuploc, tempdest)
        shutil.rmtree(self._addon_root)
        os.rename(tempdest, self._addon_root)

        self._json["backup_date"] = ""
        self._json["just_restored"] = True
        self._json["just_updated"] = True
        self.save_updater_json()

        self.reload_addon()

    def unpack_staged_zip(self, clean=False):

        if os.path.isfile(self._source_zip) == False:
            if self._verbose: print("Error, update zip not found")
            return -1

        # clear the existing source folder in case previous files remain
        try:
            shutil.rmtree(os.path.join(self._updater_path, "source"))
            os.makedirs(os.path.join(self._updater_path, "source"))
            if self._verbose: print("Source folder cleared and recreated")
        except:
            pass

        if self._verbose: print("Begin extracting source")
        if zipfile.is_zipfile(self._source_zip):
            with zipfile.ZipFile(self._source_zip) as zf:
                # extractall is no longer a security hazard, below is safe
                zf.extractall(os.path.join(self._updater_path, "source"))
        else:
            if self._verbose:
                print("Not a zip file, future add support for just .py files")
            raise ValueError("Resulting file is not a zip")
        if self._verbose: print("Extracted source")

        # either directly in root of zip, or one folder level deep
        unpath = os.path.join(self._updater_path, "source")
        if os.path.isfile(os.path.join(unpath, "__init__.py")) == False:
            dirlist = os.listdir(unpath)
            if len(dirlist) > 0:
                unpath = os.path.join(unpath, dirlist[0])

            # smarter check for additional sub folders for a single folder
            # containing __init__.py
            if os.path.isfile(os.path.join(unpath, "__init__.py")) == False:
                if self._verbose:
                    print("not a valid addon found")
                    print("Paths:")
                    print(dirlist)

                raise ValueError("__init__ file not found in new source")

        # now commence merging in the two locations:
        # note this MAY not be accurate, as updater files could be placed elsewhere
        origpath = os.path.dirname(__file__)

        # merge code with running addon directory, using blender default behavior
        # plus any modifiers indicated by user (e.g. force remove/keep)
        self.deepMergeDirectory(origpath, unpath, clean)

        # Now save the json state
        #  Change to True, to trigger the handler on other side
        #  if allowing reloading within same blender instance
        self._json["just_updated"] = True
        self.save_updater_json()
        self.reload_addon()
        self._update_ready = False

    # merge folder 'merger' into folder 'base' without deleting existing
    def deepMergeDirectory(self, base, merger, clean=False):
        if not os.path.exists(base):
            if self._verbose: print("Base path does not exist")
            return -1
        elif not os.path.exists(merger):
            if self._verbose: print("Merger path does not exist")
            return -1

        # paths to be aware of and not overwrite/remove/etc
        staging_path = os.path.join(self._updater_path, "update_staging")
        backup_path = os.path.join(self._updater_path, "backup")
        json_path = os.path.join(self._updater_path, "updater_status.json")

        # If clean install is enabled, clear existing files ahead of time
        # note: will not delete the update.json, update folder, staging, or staging
        # but will delete all other folders/files in addon directory
        error = None
        if clean == True:
            try:
                # implement clearing of all folders/files, except the
                # updater folder and updater json
                # Careful, this deletes entire subdirectories recursively...
                # make sure that base is not a high level shared folder, but
                # is dedicated just to the addon itself
                if self._verbose: print("clean=True, clearing addon folder to fresh install state")

                # remove root files and folders (except update folder)
                files = [f for f in os.listdir(base) if os.path.isfile(os.path.join(base, f))]
                folders = [f for f in os.listdir(base) if os.path.isdir(os.path.join(base, f))]

                for f in files:
                    os.remove(os.path.join(base, f))
                    print("Clean removing file {}".format(os.path.join(base, f)))
                for f in folders:
                    if os.path.join(base, f) == self._updater_path: continue
                    shutil.rmtree(os.path.join(base, f))
                    print("Clean removing folder and contents {}".format(os.path.join(base, f)))

            except error:
                error = "failed to create clean existing addon folder"
                print(error, str(e))

        # Walk through the base addon folder for rules on pre-removing
        # but avoid removing/altering backup and updater file
        for path, dirs, files in os.walk(base):
            # prune ie skip updater folder
            dirs[:] = [d for d in dirs if os.path.join(path, d) not in [self._updater_path]]
            for file in files:
                for ptrn in self.remove_pre_update_patterns:
                    if fnmatch.filter([file], ptrn):
                        try:
                            fl = os.path.join(path, file)
                            os.remove(fl)
                            if self._verbose: print("Pre-removed file " + file)
                        except OSError:
                            print("Failed to pre-remove " + file)

        # Walk through the temp addon sub folder for replacements
        # this implements the overwrite rules, which apply after
        # the above pre-removal rules. This also performs the
        # actual file copying/replacements
        for path, dirs, files in os.walk(merger):
            # verify this structure works to prune updater sub folder overwriting
            dirs[:] = [d for d in dirs if os.path.join(path, d) not in [self._updater_path]]
            relPath = os.path.relpath(path, merger)
            destPath = os.path.join(base, relPath)
            if not os.path.exists(destPath):
                os.makedirs(destPath)
            for file in files:
                # bring in additional logic around copying/replacing
                # Blender default: overwrite .py's, don't overwrite the rest
                destFile = os.path.join(destPath, file)
                srcFile = os.path.join(path, file)

                # decide whether to replace if file already exists, and copy new over
                if os.path.isfile(destFile):
                    # otherwise, check each file to see if matches an overwrite pattern
                    replaced = False
                    for ptrn in self._overwrite_patterns:
                        if fnmatch.filter([destFile], ptrn):
                            replaced = True
                            break
                    if replaced:
                        os.remove(destFile)
                        os.rename(srcFile, destFile)
                        if self._verbose: print("Overwrote file " + os.path.basename(destFile))
                    else:
                        if self._verbose: print(
                            "Pattern not matched to " + os.path.basename(destFile) + ", not overwritten")
                else:
                    # file did not previously exist, simply move it over
                    os.rename(srcFile, destFile)
                    if self._verbose: print("New file " + os.path.basename(destFile))

        # now remove the temp staging folder and downloaded zip
        try:
            shutil.rmtree(staging_path)
        except:
            error = "Error: Failed to remove existing staging directory, consider manually removing " + staging_path
            if self._verbose: print(error)

    def reload_addon(self):
        # if post_update false, skip this function
        # else, unload/reload addon & trigger popup
        if self._auto_reload_post_update == False:
            print("Restart blender to reload addon and complete update")
            return

        if self._verbose: print("Reloading addon...")
        addon_utils.modules(refresh=True)
        bpy.utils.refresh_script_paths()

        # not allowed in restricted context, such as register module
        # toggle to refresh
        bpy.ops.wm.addon_disable(module=self._addon_package)
        bpy.ops.wm.addon_refresh()
        bpy.ops.wm.addon_enable(module=self._addon_package)

    # -------------------------------------------------------------------------
    # Other non-api functions and setups
    # -------------------------------------------------------------------------

    def clear_state(self):
        self._update_ready = None
        self._update_link = None
        self._update_version = None
        self._source_zip = None
        self._error = None
        self._error_msg = None

    # custom urlretrieve implementation
    def urlretrieve(self, urlfile, filepath):
        chunk = 1024 * 8
        f = open(filepath, "wb")
        while 1:
            data = urlfile.read(chunk)
            if not data:
                # print("done.")
                break
            f.write(data)
        # print("Read %s bytes"%len(data))
        f.close()

    def version_tuple_from_text(self, text):
        if text == None: return ()

        # should go through string and remove all non-integers,
        # and for any given break split into a different section
        segments = []
        tmp = ''
        for l in str(text):
            if l.isdigit() == False:
                if len(tmp) > 0:
                    segments.append(int(tmp))
                    tmp = ''
            else:
                tmp += l
        if len(tmp) > 0:
            segments.append(int(tmp))

        if len(segments) == 0:
            if self._verbose: print("No version strings found text: ", text)
            if self._include_branches == False:
                return ()
            else:
                return (text)
        return tuple(segments)

    # called for running check in a background thread
    def check_for_update_async(self, callback=None):

        if self._json != None and "update_ready" in self._json:
            if self._json["update_ready"] == True:
                self._update_ready = True
                self._update_link = self._json["version_text"]["link"]
                self._update_version = str(self._json["version_text"]["version"])
                # cached update
                callback(True)
                return

        # do the check
        if self._check_interval_enable == False:
            return
        elif self._async_checking == True:
            if self._verbose: print("Skipping async check, already started")
            return  # already running the bg thread
        elif self._update_ready == None:
            self.start_async_check_update(False, callback)

    def check_for_update_now(self, callback=None):

        self._error = None
        self._error_msg = None

        if self._verbose:
            print("Check update pressed, first getting current status")
        if self._async_checking == True:
            if self._verbose: print("Skipping async check, already started")
            return  # already running the bg thread
        elif self._update_ready == None:
            self.start_async_check_update(True, callback)
        else:
            self._update_ready = None
            self.start_async_check_update(True, callback)

    # this function is not async, will always return in sequential fashion
    # but should have a parent which calls it in another thread
    def check_for_update(self, now=False):
        if self._verbose: print("Checking for update function")

        # clear the errors if any
        self._error = None
        self._error_msg = None

        # avoid running again in, just return past result if found
        # but if force now check, then still do it
        if self._update_ready != None and now == False:
            return (self._update_ready, self._update_version, self._update_link)

        if self._current_version == None:
            raise ValueError("current_version not yet defined")
        if self._repo == None:
            raise ValueError("repo not yet defined")
        if self._user == None:
            raise ValueError("username not yet defined")

        self.set_updater_json()  # self._json

        if now == False and self.past_interval_timestamp() == False:
            if self._verbose:
                print("Aborting check for updated, check interval not reached")
            return (False, None, None)

        # check if using tags or releases
        # note that if called the first time, this will pull tags from online
        if self._fake_install == True:
            if self._verbose:
                print("fake_install = True, setting fake version as ready")
            self._update_ready = True
            self._update_version = "(999,999,999)"
            self._update_link = "http://127.0.0.1"

            return (self._update_ready, self._update_version, self._update_link)

        # primary internet call
        self.get_tags()  # sets self._tags and self._tag_latest

        self._json["last_check"] = str(datetime.now())
        self.save_updater_json()

        # can be () or ('master') in addition to branches, and version tag
        new_version = self.version_tuple_from_text(self.tag_latest)

        if len(self._tags) == 0:
            self._update_ready = False
            self._update_version = None
            self._update_link = None
            return (False, None, None)
        elif self._include_branches == False:
            link = self._tags[0]["zipball_url"]  # potentially other sources
        else:
            n = len(self._include_branch_list)
            if len(self._tags) == n:
                # effectively means no tags found on repo
                # so provide the first one as default
                link = self._tags[0]["zipball_url"]  # potentially other sources
            else:
                link = self._tags[n]["zipball_url"]  # potentially other sources

        if new_version == ():
            self._update_ready = False
            self._update_version = None
            self._update_link = None
            return (False, None, None)
        elif str(new_version).lower() in self._include_branch_list:
            # handle situation where master/whichever branch is included
            # however, this code effectively is not triggered now
            # as new_version will only be tag names, not branch names
            if self._include_branch_autocheck == False:
                # don't offer update as ready,
                # but set the link for the default
                # branch for installing
                self._update_ready = True
                self._update_version = new_version
                self._update_link = link
                self.save_updater_json()
                return (True, new_version, link)
            else:
                raise ValueError("include_branch_autocheck: NOT YET DEVELOPED")
            # bypass releases and look at timestamp of last update
            # from a branch compared to now, see if commit values
            # match or not.

        else:
            # situation where branches not included

            if new_version > self._current_version:
                self._update_ready = True
                self._update_version = new_version
                self._update_link = link
                self.save_updater_json()
                return (True, new_version, link)

        # elif new_version != self._current_version:
        # 	self._update_ready = False
        # 	self._update_version = new_version
        # 	self._update_link = link
        # 	self.save_updater_json()
        # 	return (True, new_version, link)

        # if no update, set ready to False from None
        self._update_ready = False
        self._update_version = None
        self._update_link = None
        return (False, None, None)

    def set_tag(self, name):
        tg = None
        for tag in self._tags:
            if name == tag["name"]:
                tg = tag
                break
        if tg == None:
            raise ValueError("Version tag not found: " + revert_tag)
        new_version = self.version_tuple_from_text(self.tag_latest)
        self._update_version = new_version
        self._update_link = tg["zipball_url"]

    def run_update(self, force=False, revert_tag=None, clean=False, callback=None):
        # revert_tag: could e.g. get from drop down list
        # different versions of the addon to revert back to
        # clean: not used, but in future could use to totally refresh addon
        self._json["update_ready"] = False
        self._json["ignore"] = False  # clear ignore flag
        self._json["version_text"] = {}

        if revert_tag != None:
            self.set_tag(revert_tag)
            self._update_ready = True

        # clear the errors if any
        self._error = None
        self._error_msg = None

        if self._verbose: print("Running update")

        if self._fake_install == True:
            # change to True, to trigger the reload/"update installed" handler
            if self._verbose:
                print("fake_install=True")
                print("Just reloading and running any handler triggers")
            self._json["just_updated"] = True
            self.save_updater_json()
            if self._backup_current == True:
                self.create_backup()
            self.reload_addon()
            self._update_ready = False
            res = True  # fake "success" zip download flag

        elif force == False:
            if self._update_ready != True:
                if self._verbose: print("Update stopped, new version not ready")
                return "Update stopped, new version not ready"
            elif self._update_link == None:
                # this shouldn't happen if update is ready
                if self._verbose: print("Update stopped, update link unavailable")
                return "Update stopped, update link unavailable"

            if self._verbose and revert_tag == None:
                print("Staging update")
            elif self._verbose:
                print("Staging install")

            res = self.stage_repository(self._update_link)
            if res != True:
                print("Error in staging repository: " + str(res))
                if callback != None: callback(self._error_msg)
                return self._error_msg
            self.unpack_staged_zip(clean)

        else:
            if self._update_link == None:
                if self._verbose: print("Update stopped, could not get link")
                return "Update stopped, could not get link"
            if self._verbose: print("Forcing update")

            res = self.stage_repository(self._update_link)
            if res != True:
                print("Error in staging repository: " + str(res))
                if callback != None: callback(self._error_msg)
                return self._error_msg
            self.unpack_staged_zip(clean)
        # would need to compare against other versions held in tags

        # run the front-end's callback if provided
        if callback != None: callback()

        # return something meaningful, 0 means it worked
        return 0

    def past_interval_timestamp(self):
        if self._check_interval_enable == False:
            return True  # ie this exact feature is disabled

        if "last_check" not in self._json or self._json["last_check"] == "":
            return True
        else:
            now = datetime.now()
            last_check = datetime.strptime(self._json["last_check"],
                                           "%Y-%m-%d %H:%M:%S.%f")
            next_check = last_check
            offset = timedelta(
                days=self._check_interval_days + 30 * self._check_interval_months,
                hours=self._check_interval_hours,
                minutes=self._check_interval_minutes
            )

            delta = (now - offset) - last_check
            if delta.total_seconds() > 0:
                if self._verbose:
                    print("{} Updater: Time to check for updates!".format(self._addon))
                return True
            else:
                if self._verbose:
                    print("{} Updater: Determined it's not yet time to check for updates".format(self._addon))
                return False

    def set_updater_json(self):
        if self._updater_path == None:
            raise ValueError("updater_path is not defined")
        elif os.path.isdir(self._updater_path) == False:
            os.makedirs(self._updater_path)

        jpath = os.path.join(self._updater_path, "updater_status.json")
        if os.path.isfile(jpath):
            with open(jpath) as data_file:
                self._json = json.load(data_file)
                if self._verbose: print("{} Updater: Read in json settings from file".format(self._addon))
        else:
            # set data structure
            self._json = {
                "last_check": "",
                "backup_date": "",
                "update_ready": False,
                "ignore": False,
                "just_restored": False,
                "just_updated": False,
                "version_text": {}
            }
            self.save_updater_json()

    def save_updater_json(self):
        # first save the state
        if self._update_ready == True:
            if type(self._update_version) == type((0, 0, 0)):
                self._json["update_ready"] = True
                self._json["version_text"]["link"] = self._update_link
                self._json["version_text"]["version"] = self._update_version
            else:
                self._json["update_ready"] = False
                self._json["version_text"] = {}
        else:
            self._json["update_ready"] = False
            self._json["version_text"] = {}

        jpath = os.path.join(self._updater_path, "updater_status.json")
        outf = open(jpath, 'w')
        data_out = json.dumps(self._json, indent=4)
        outf.write(data_out)
        outf.close()
        if self._verbose:
            print(self._addon + ": Wrote out updater json settings to file, with the contents:")
            print(self._json)

    def json_reset_postupdate(self):
        self._json["just_updated"] = False
        self._json["update_ready"] = False
        self._json["version_text"] = {}
        self.save_updater_json()

    def json_reset_restore(self):
        self._json["just_restored"] = False
        self._json["update_ready"] = False
        self._json["version_text"] = {}
        self.save_updater_json()
        self._update_ready = None  # reset so you could check update again

    def ignore_update(self):
        self._json["ignore"] = True
        self.save_updater_json()

    # -------------------------------------------------------------------------
    # ASYNC stuff
    # -------------------------------------------------------------------------

    def start_async_check_update(self, now=False, callback=None):
        if self._async_checking == True:
            return
        if self._verbose: print("{} updater: Starting background checking thread".format(self._addon))
        check_thread = threading.Thread(target=self.async_check_update,
                                        args=(now, callback,))
        check_thread.daemon = True
        self._check_thread = check_thread
        check_thread.start()

        return True

    def async_check_update(self, now, callback=None):
        self._async_checking = True
        if self._verbose: print("{} BG thread: Checking for update now in background".format(self._addon))
        # time.sleep(3)  # to test background, in case internet too fast to tell
        # try:
        self.check_for_update(now=now)
        # except Exception as exception:
        # 	print("Checking for update error:")
        # 	print(exception)
        # 	self._update_ready = False
        # 	self._update_version = None
        # 	self._update_link = None
        # 	self._error = "Error occurred"
        # 	self._error_msg = "Encountered an error while checking for updates"

        if self._verbose:
            print("{} BG thread: Finished checking for update, doing callback".format(self._addon))
        if callback != None: callback(self._update_ready)
        self._async_checking = False
        self._check_thread = None

    def stop_async_check_update(self):
        if self._check_thread != None:
            try:
                if self._verbose: print("Thread will end in normal course.")
            # however, "There is no direct kill method on a thread object."
            # better to let it run its course
            # self._check_thread.stop()
            except:
                pass
        self._async_checking = False
        self._error = None
        self._error_msg = None

# -----------------------------------------------------------------------------
# Updater Engines
# -----------------------------------------------------------------------------

class BitbucketEngine(object):

    def __init__(self):
        self.api_url = 'https://api.bitbucket.org'
        self.token = None
        self.name = "bitbucket"

    def form_repo_url(self, updater):
        return self.api_url + "/2.0/repositories/" + updater.user + "/" + updater.repo

    def form_tags_url(self, updater):
        return self.form_repo_url(updater) + "/refs/tags?sort=-name"

    def form_branch_url(self, branch, updater):
        return self.get_zip_url(branch, updater)

    def get_zip_url(self, name, updater):
        return "https://bitbucket.org/{user}/{repo}/get/{name}.zip".format(
            user=updater.user,
            repo=updater.repo,
            name=name)

    def parse_tags(self, response, updater):
        if response == None:
            return []
        return [{"name": tag["name"], "zipball_url": self.get_zip_url(tag["name"], updater)} for tag in
                response["values"]]

class GithubEngine(object):

    def __init__(self):
        self.api_url = 'https://api.github.com'
        self.token = None
        self.name = "github"

    def form_repo_url(self, updater):
        return "{}{}{}{}{}".format(self.api_url, "/repos/", updater.user,
                                   "/", updater.repo)

    def form_tags_url(self, updater):
        return "{}{}".format(self.form_repo_url(updater), "/tags")

    def form_branch_list_url(self, updater):
        return "{}{}".format(self.form_repo_url(updater), "/branches")

    def form_branch_url(self, branch, updater):
        return "{}{}{}".format(self.form_repo_url(updater),
                               "/zipball/", branch)

    def parse_tags(self, response, updater):
        return response

class GitlabEngine(object):

    def __init__(self):
        self.api_url = 'https://gitlab.com'
        self.token = None
        self.name = "gitlab"

    def form_repo_url(self, updater):
        return "{}{}{}".format(self.api_url, "/api/v3/projects/", updater.repo)

    def form_tags_url(self, updater):
        return "{}{}".format(self.form_repo_url(updater), "/repository/tags")

    def form_branch_list_url(self, updater):
        # does not validate branch name.
        return "{}{}".format(
            self.form_repo_url(updater),
            "/repository/branches")

    def form_branch_url(self, branch, updater):
        # Could clash with tag names and if it does, it will
        # download TAG zip instead of branch zip to get
        # direct path, would need.
        return "{}{}{}".format(
            self.form_repo_url(updater),
            "/repository/archive.zip?sha=",
            branch)

    def get_zip_url(self, sha, updater):
        return "{base}/repository/archive.zip?sha:{sha}".format(
            base=self.form_repo_url(updater),
            sha=sha)

    # def get_commit_zip(self, id, updater):
    # 	return self.form_repo_url(updater)+"/repository/archive.zip?sha:"+id

    def parse_tags(self, response, updater):
        if response == None:
            return []
        return [{"name": tag["name"], "zipball_url": self.get_zip_url(tag["commit"]["id"], updater)} for tag in
                response]

# -----------------------------------------------------------------------------
# The module-shared class instance,
# should be what's imported to other files
# -----------------------------------------------------------------------------

Updater = Singleton_updater()


#! /usr/bin/env python3

from collections import Counter

count = 0

hexamer_5 = Counter()
hexamer_3 = Counter()

import sys

filename = sys.argv[1]

for line in open(filename):
    line = line.rstrip()

    if count == 0:
        name = line
        count += 1

    elif count == 1:
        seq = line
        count += 1

    elif count == 2:
        count += 1

    elif count == 3:
        qual = line
        count = 0

def fiveprime_hexamer(filename):

    hexamer_5 = seq[0:6]
    hexamers_5[hexamer_5] += 1

    for hexamer, count in hexamers_5.most_common(1):
        print("The most common 5'end hexamer sequence is", hexamer)

def threeprime_hexamer(filename):

    hexamer_3 = seq[-6:]
    hexamers_3[hexamer_3] += 1

    for hexamer, count in hexamers_3.most_common(1):
         print("The most common 3'end hexamer sequence is", hexamer)

fiveprime_hexamer(filename)
threeprime_hexamer(filename)


from __future__ import print_function, division

def autowrapt_decimal(_module):
    import sys
    import cdecimal

    sys.modules['decimal'] = cdecimal
    print('module decimal patched to {}'.format(sys.modules['decimal'].__name__))
    cdecimal.tmp_context = cdecimal.Context

    def patched(*args, **kwargs):
        if 'rounding' in kwargs and kwargs['rounding'] is None:
            kwargs['rounding'] = 6
        return cdecimal.tmp_context(*args, **kwargs)
    cdecimal.Context = patched

    print('cdecimal.Context monkey-patch for boto(if rounding=None, then 6 passed to Context instead) applied\n')

def autowrapt_psycopg2(module):
    import ujson
    module.json = ujson
    print('psycopg2._json.json monkey-patched to ujson instead of json\n')

def autowrapt_dynamodb(module):
    from decimal import Decimal
    from boto.dynamodb.types import Binary

    def deepcopy(data):
        """Expected dicts nesting level 1. Lists, sets, and nested dicts
         contains only simple types"""
        res = {}
        for k, v in data.items():
            if type(v) in [Decimal, Binary, str, int, unicode, long, bool]:
                res[k] = v
            elif type(v) in [set, dict, list]:
                res[k] = type(v)(v)
            elif v is None:
                res[k] = v
            else:
                raise ValueError('Unknown type: {}'.format(type(v)))
        return res

    module.deepcopy = deepcopy
    print('boto.dynamodb2.items.deepcopy monkey-patched\n')


# -*- coding: utf-8 -*-
#%% NumPyの読み込み
import numpy as np
#   SciPyのstatsモジュールの読み込み
import scipy.stats as st
#   Pandasの読み込み
import pandas as pd
#   PyMCの読み込み
import pymc as pm
#   MatplotlibのPyplotモジュールの読み込み
import matplotlib.pyplot as plt
#   日本語フォントの設定
from matplotlib.font_manager import FontProperties
import sys
if sys.platform.startswith('win'):
    FontPath = 'C:\\Windows\\Fonts\\meiryo.ttc'
elif sys.platform.startswith('darwin' ):
    FontPath = '/System/Library/Fonts/ヒラギノ角ゴシック W4.ttc'
elif sys.platform.startswith('linux'):
    FontPath = '/usr/share/fonts/truetype/takao-gothic/TakaoPGothic.ttf'
else:
    sys.exit('このPythonコードが対応していないOSを使用しています．')
jpfont = FontProperties(fname=FontPath)
#   PandasからMatplotlibへのコンバーター
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
#%% 使用電力量データの読み込み
"""
    電灯電力需要実績月報・用途別使用電力量・販売電力合計・10社計
    電気事業連合会ウェブサイト・電力統計情報より入手
    http://www.fepc.or.jp/library/data/tokei/index.html
"""
data = pd.read_csv('electricity.csv', index_col=0)
y0 = np.log(data.values.reshape((data.shape[0]//3, 3)).sum(axis=1))
y = 100 * (y0 - y0[0])
n = y.size
series_date = pd.date_range(start='1/1/1989', periods=n, freq='Q')
#%% 確率的トレンド+季節変動
trend_coef = np.array([2.0, -1.0])
seasonal_coef = np.array([-1.0, -1.0, -1.0])
timeseries_decomp = pm.Model()
with timeseries_decomp:
    sigma = pm.HalfCauchy('sigma', beta=1.0)
    tau = pm.HalfCauchy('tau', beta=1.0)
    omega = pm.HalfCauchy('omega', beta=1.0)
    trend = pm.AR('trend', trend_coef, sigma=tau, shape=n)
    seasonal = pm.AR('seasonal', seasonal_coef, sigma=omega, shape=n)
    observation = pm.Normal('y', mu=trend+seasonal, sigma=sigma, observed=y)
#%% 事後分布からのサンプリング
param_names = ['sigma', 'tau', 'omega']
n_draws = 5000
n_chains = 4
n_tune = 2000
with timeseries_decomp:
    trace = pm.sample(draws=n_draws, chains=n_chains, tune=n_tune,
                      target_accept=0.95, random_seed=123)
    print(pm.summary(trace, var_names=param_names))
#%% 事後分布のグラフの作成
series_name = ['原系列', '平滑値', 'トレンド', '季節変動', 'ノイズ']
k = len(param_names)
fig1, ax1 = plt.subplots(k, 2, num=1, figsize=(8, 1.5*k), facecolor='w')
for index in range(k):
    mc_trace = trace.posterior[param_names[index]].values.flatten()
    x_min = mc_trace.min() - 0.2 * np.abs(mc_trace.min())
    x_max =  mc_trace.max() + 0.2 * np.abs(mc_trace.max())
    x = np.linspace(x_min, x_max, 250)
    posterior = st.gaussian_kde(mc_trace).evaluate(x)
    ax1[index, 0].plot(mc_trace, 'k-', linewidth=0.1)
    ax1[index, 0].set_xlim(1, n_draws*n_chains)
    ax1[index, 0].set_ylabel('$\\{label:s}$'.format(label=param_names[index]),
                             fontproperties=jpfont)
    ax1[index, 1].plot(x, posterior, 'k-')
    ax1[index, 1].set_xlim(x_min, x_max)
    ax1[index, 1].set_ylim(0, 1.1*posterior.max())
    ax1[index, 1].set_ylabel('確率密度', fontproperties=jpfont)
ax1[k-1, 0].set_xlabel('乱数系列', fontproperties=jpfont)
ax1[k-1, 1].set_xlabel('周辺事後分布', fontproperties=jpfont)
plt.tight_layout()
plt.savefig('pybayes_fig_decomp_posterior.png', dpi=300)
plt.show()
#%% 時系列の分解
trend = trace.posterior['trend'].values.reshape(n_draws*n_chains, n).mean(axis=0)
seasonal = trace.posterior['seasonal'].values.reshape(n_draws*n_chains, n).mean(axis=0)
noise = y - trend - seasonal
series = np.vstack((y, trend + seasonal, trend, seasonal, noise)).T
results = pd.DataFrame(series, index=series_date, columns=series_name)
fig2, ax2 = plt.subplots(4, 1, sharex='col',
                         num=2, figsize=(8, 6), facecolor='w')
for index in range(4):
    ts_name = series_name[index+1]
    ax2[index].plot(results[ts_name], 'k-', label=ts_name)
    ax2[index].set_ylabel(ts_name, fontproperties=jpfont)
ax2[0].plot(results[series_name[0]], 'k:', label=series_name[0])
ax2[0].set_xlim(series_date[0], series_date[-1])
ax2[0].legend(loc='lower right', frameon=False, prop=jpfont)
plt.tight_layout()
plt.savefig('pybayes_fig_decomp_timeseries.png', dpi=300)
plt.show()


from datetime import datetime
import os

import database_func

class Date:
    """
    Adds date based on what file it is.
    Params::
    filename - The Full path to file oncluding file name
    """

    def __init__(self, folder, filename):
        self.folder = folder
        self.filename = filename
        self.path = "F:\Work_Python_Scripts\Windows_info\\"
        self.date = str(datetime.now().strftime("%Y-%m-%d %H-%M"))

    def format_date(self):
        temp_list = []
        new = self.path + self.folder + "\\" + self.date + ".txt"
        old = self.path +  self.folder + "\\" + self.filename + ".txt"
        self.rename(old, new)

    def rename(self, old, new):
        os.rename(old, new)

def get_computername():
    """
    Returns a Computer name from text file which is then iterated over in
    powershell()
    """
    with open (r'f:\Work_Python_Scripts\Windows_info\powershell\NoBOM.txt', 'r') as f:

        x = f.read().splitlines()

        return x

def add_computer():
    """
    Reads in Text file created by powershell() assigns it to a variable then
    passes the Info to Computer Class which creates an object.
    Lastly the function passes the returned object to db.update_data
    """

    file1 = 'f:\Work_Python_Scripts\Windows_info\powershell\SysConfig.txt'
    file = r'f:\Work_Python_Scripts\Windows_info\powershell\ou.txt'

    try:
        with open(file1, 'r') as f1:

            text = f1.read().splitlines()

            if text:
                name = text[0]
                username = text[1]
                windows = text[2]
                cpu = text[3]
                currentamount = text[4]
                totalslots = text[5]
                lastlogon = text[6]
                ipaddress = text[7]
                splitted = text[8].split(",")
                location = splitted[2][3:]

                computer = database_func.Computer(name=name, username=username,
                                                  windows=windows, cpu=cpu,
                                                  currentamount=currentamount,
                                                  totalslots=totalslots,
                                                  lastlogon=lastlogon,
                                                  ipaddress=ipaddress,
                                                  location=location)
                database_func.update_data(computer)

            else:
                print("")

    except FileNotFoundError:
        pass

def log_main(added, updated, not_connect, deleted):

    log = r"F:\Work_Python_Scripts\Windows_info\logs\main.txt"

    with open (log, 'w') as f:
        f.write("Computers that were added\n")
        f.write("-" * 95)
        f.write("\n")
        f.writelines(f"{line}\n" for line in added)
        f.write("Computers that were updated\n")
        f.write("-" * 95)
        f.write("\n")
        f.writelines(f"{line}\n" for line in updated)
        f.write("Computers that could not be connect to\n")
        f.write("-" * 95)
        f.write("\n")
        f.writelines(f"{line}\n" for line in not_connect)
        f.write("Deleted Computers\n")
        f.write("-" * 95)
        f.write("\n")
        f.writelines(f"{line}\n" for line in deleted)
    p = Date("logs", "main")
    p.format_date()

def change_directory(cwd):
    powershell=r"F:\Work_Python_Scripts\Windows_Info\powershell\\"
    python=r"F:\Work_Python_Scripts\Windows_info\python\\"
    logs=r"F:\Work_Python_Scripts\Windows_info\logs\\"

    if  cwd == python:
        os.chdir(python)

    elif cwd == powershell:
        os.chdir(powershell)

    else:
        os.chdir(logs)


import tensorflow as tf
from sklearn.feature_extraction.text import TfidfVectorizer
import os
import pickle
import numpy as np
from data_preparation import TOP_K

NGRAM_RANGE = (1, 2)
TOKEN_MODE = 'word'
CATEGORY_LIST = ['Autism', 'Ethnicity', 'Gender Equality', 'LGBTQ+', 'Religion']
THRESHHOLD_OFFSET_FACTOR=1.5

path = os.path.join(os.path.dirname(__file__), "TAG_mlp_model.h5")
my_model = tf.keras.models.load_model(path)

kwargs = {
        'ngram_range': NGRAM_RANGE,  # Use 1-grams + 2-grams.
        'dtype': 'int32',
        'strip_accents': 'unicode',
        'decode_error': 'replace',
        'analyzer': TOKEN_MODE,  # Split text into word tokens.
}

vectorizer = TfidfVectorizer(**kwargs, max_features = TOP_K)
feature_list = pickle.load(open("feature.pkl", "rb"))
vectorizer.fit(feature_list)

def is_covid_problem(problem):
    problem = problem.lower()
    corona_words = ['corona', 'pandemic', 'quarantine', 'epidemic', 'virus', 'covid']
    if any(corona_word in problem for corona_word in corona_words):
        return True

def predict_category(problem):
    list_x = [problem]
    x = vectorizer.transform(list_x)
    x = x.toarray()
    result = my_model.predict(x)
    result_list = list(result[0])
    number_of_tags = len(CATEGORY_LIST)
    threshhold = (1/number_of_tags) * THRESHHOLD_OFFSET_FACTOR
    tags = []
    print(result_list)
    for tag in result_list:
        if tag > threshhold:
            tags.append(CATEGORY_LIST[result_list.index(tag)])
    if is_covid_problem(problem):
        tags.append("Covid-19")
    return tags

my_model.summary()
while True:
    input_x = input("What bothers you?: ")
    print(predict_category(input_x))

AI_Dev/AI versions/KI sub v1/predict_category.py


"""
"""
import unittest
import os
import logging
import logging.config
import socket
import time
import tempfile
import shutil
from httplib import HTTPException
from httplib import BadStatusLine, IncompleteRead

from nose.plugins.attrib import attr

from WMCore.Services.Service import Service
from WMCore.Services.Requests import Requests
from WMCore.Algorithms import Permissions
from WMQuality.TestInitCouchApp import TestInitCouchApp as TestInit
import cherrypy

class CrappyServer(object):
    def truncated(self):
        cherrypy.response.headers['Content-Length'] = 500
        return "Hello World!"
    truncated.exposed = True

class SlowServer(object):
    def slow(self):
        time.sleep(300)
        return "Hello World!"
    slow.exposed = True

class CrappyRequest(Requests):
    def makeRequest(self, uri=None, data={}, verb='GET', incoming_headers={},
                     encoder=True, decoder=True, contentType=None):
        # METAL \m/
        raise BadStatusLine(666)

class RegularServer(object):
    def regular(self):
        return "This is silly."
    regular.exposed = True

class BackupServer(object):
    def regular(self):
        return "This is nuts."
    regular.exposed = True

class ServiceTest(unittest.TestCase):
    def setUp(self):
        """
        Setup for unit tests
        """
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()
        testname = self.id().split('.')[-1]

        logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    datefmt='%m-%d %H:%M',
                    filename='service_unittests.log',
                    filemode='w')

        logger_name = 'Service%s' % testname.replace('test', '', 1)

        self.logger = logging.getLogger(logger_name)

        #self.cache_path = tempfile.mkdtemp()
        test_dict = {'logger': self.logger,
                     'endpoint': 'https://github.com/dmwm'}

        self.myService = Service(test_dict)

        test_dict['endpoint'] = 'http://cmssw-test.cvs.cern.ch/cgi-bin/cmssw.cgi'
        self.myService2 = Service(test_dict)
        self.testUrl = 'http://cern.ch'

        self.port = 8888
        cherrypy.config.update({'server.socket_port': self.port})

    def tearDown(self):
        self.testInit.delWorkDir()
        # There was old code here to see if the test passed and send a message to
        # self.logger.info It broke in 2.7, so if needed find a supported way to do it
        return

    def testClear(self):
        """
        Populate the cache, and then check that it's deleted
        """
        f = self.myService.refreshCache('testClear', '/WMCore/blob/master/setup.py#L11')
        assert os.path.exists(f.name)
        f.close()

        self.myService.clearCache('testClear')
        assert not os.path.exists(f.name)

    def testClearAndRepopulate(self):
        """
        Populate the cache, and then check that it's deleted
        """
        f = self.myService.refreshCache('testClear', '/WMCore/blob/master/setup.py#L11')
        assert os.path.exists(f.name)
        f.close()

        self.myService.clearCache('testClear')
        assert not os.path.exists(f.name)

        f = self.myService.refreshCache('testClear', '/WMCore/blob/master/setup.py#L11')
        assert os.path.exists(f.name)
        f.close()

    def testCachePath(self):
        cache_path = tempfile.mkdtemp()
        dict = {'logger': self.logger,
                'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
                'cachepath' : cache_path,
                'req_cache_path': '%s/requests' % cache_path
                }
        service = Service(dict)
        # We append hostname to the cachepath, so that we can talk to two
        # services on different hosts
        self.assertEqual(service['cachepath'],
                         '%s/cmssw.cvs.cern.ch' % dict['cachepath'] )
        shutil.rmtree(cache_path, ignore_errors = True)

    @attr("integration")
    def testCacheLifetime(self):
        """Cache deleted if created by Service - else left alone"""
        dict = {'logger': self.logger,
                'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
                'cacheduration': 100}
        os.environ.pop('TMPDIR', None) # Mac sets this by default
        service = Service(dict)
        cache_path = service['cachepath']
        self.assertTrue(os.path.isdir(cache_path))
        del service
        self.assertFalse(os.path.exists(cache_path))

        cache_path = tempfile.mkdtemp()
        dict['cachepath'] = cache_path
        service = Service(dict)
        del service
        self.assertTrue(os.path.isdir(cache_path))
        Permissions.owner_readwriteexec(cache_path)

    def testCachePermissions(self):
        """Raise error if pre-defined cache permission loose"""
        cache_path = tempfile.mkdtemp()
        sub_cache_path = os.path.join(cache_path, 'cmssw.cvs.cern.ch')
        os.makedirs(sub_cache_path, 0o777)
        dict = {'logger': self.logger,
                'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
                'cacheduration': 100,
                'cachepath' : cache_path}
        self.assertRaises(AssertionError, Service, dict)

    def testCacheDuration(self):
        dict = {'logger': self.logger,
                'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
                'cacheduration': 100,
                #'cachepath' : self.cache_path,
                #'req_cache_path': '%s/requests' % self.cache_path
                }
        service = Service(dict)
        self.assertEqual( service['cacheduration'] ,  dict['cacheduration'] )

    def testNoCacheDuration(self):
        dict = {'logger': self.logger,
                'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
                'cacheduration': None,
                #'cachepath' : self.cache_path,
                #'req_cache_path': '%s/requests' % self.cache_path
                }
        service = Service(dict)
        self.assertEqual( service['cacheduration'] ,  dict['cacheduration'] )

    def testSocketTimeout(self):
        dict = {'logger': self.logger,
                'endpoint': 'https://github.com/dmwm',
                'cacheduration': None,
                'timeout': 10,
                }
        service = Service(dict)
        deftimeout = socket.getdefaulttimeout()
        service.getData('%s/socketresettest' % self.testDir, '/WMCore/blob/master/setup.py#L11')
        assert deftimeout == socket.getdefaulttimeout()

    @attr("integration")
    def testStaleCache(self):

        dict = {'logger': self.logger,
                'endpoint':'http://cmssw.cvs.cern.ch',
                'cacheduration': 0.0002,
                'maxcachereuse': 0.001,
                'timeout': 10,
                'usestalecache': True,
                #'cachepath' : self.cache_path,
                #'req_cache_path': '%s/requests' % self.cache_path
                }
        service = Service(dict)
        cache = 'stalecachetest'

        # Start test from a clear cache
        service.clearCache(cache)

        cachefile = service.cacheFileName(cache)

        # first check that the exception raises when the file doesn't exist
        self.logger.info('first call to refreshCache - should fail')

        self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')

        cacheddata = 'this data is mouldy'
        f = open(cachefile, 'w')
        f.write(cacheddata)
        f.close()

        self.logger.info('second call to refreshCache - should pass')
        data = service.refreshCache(cache, '/lies').read()
        self.assertEqual(cacheddata, data)

        # sleep a while so the file expires in the cache
        # FIXME: RACY
        time.sleep(2)
        self.logger.info('third call to refreshCache - should return stale cache')
        data = service.refreshCache(cache, '/lies').read()
        self.assertEqual(cacheddata, data)

        # sleep a while longer so the cache is dead
        # FIXME: RACY
        time.sleep(5)
        self.logger.info('fourth call to refreshCache - cache should be dead')
        self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')

        # touch the file and expire it
        f = open(cachefile, 'w')
        f.write('foo')
        f.close()
        time.sleep(2)

        self.logger.info('fifth call to refreshCache - do not use stale cache')
        # now our service cache is less permissive, the following should fail
        service['usestalecache'] = False
        self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')

        service.cacheFileName(cache)

    def testCacheFileName(self):
        """Hash url + data to get cache file name"""
        hashes = {}
        inputdata = [{}, {'fred' : 'fred'},
                     {'fred' : 'fred', 'carl' : [1, 2]},
                     {'fred' : 'fred', 'carl' : ["1", "2"]},
                     {'fred' : 'fred', 'carl' : ["1", "2"], 'jim' : {}}
                     ]
        for data in inputdata:
            thishash = self.myService.cacheFileName('bob', inputdata = data)
            thishash2 = self.myService2.cacheFileName('bob', inputdata = data)
            self.assertNotEqual(thishash, thishash2)
            self.assertTrue(thishash not in hashes, '%s is not unique' % thishash)
            self.assertTrue(thishash2 not in hashes,
                         '%s is not unique' % thishash2)
            hashes[thishash], hashes[thishash2] = None, None

    def testNoCache(self):
        """Cache disabled"""
        dict = {'logger': self.logger,
                'endpoint': 'https://github.com/dmwm',
                'cachepath': None,
                }
        service = Service(dict)

        self.assertEqual(service['cachepath'], dict['cachepath'])
        self.assertEqual(service['requests']['cachepath'], dict['cachepath'])
        self.assertEqual(service['requests']['req_cache_path'], dict['cachepath'])

        out = service.refreshCache('shouldntbeused', '/').read()
        self.assertTrue('html' in out)

    @attr("integration")
    def testTruncatedResponse(self):
        """
        _TruncatedResponse_

        """
        cherrypy.tree.mount(CrappyServer())
        cherrypy.engine.start()
        FORMAT = '%(message)s'
        logging.basicConfig(format=FORMAT)
        dummyLogger = logging.getLogger('john')
        test_dict = {'logger': self.logger,'endpoint':'http://127.0.0.1:%i/truncated' % self.port,
                     'usestalecache': True}
        myService = Service(test_dict)
        self.assertRaises(IncompleteRead, myService.getData, 'foo', '')
        cherrypy.engine.exit()
        cherrypy.engine.stop()

    @attr("integration")
    def testSlowResponse(self):
        """
        _SlowResponse_

        """
        cherrypy.tree.mount(SlowServer())
        cherrypy.engine.start()
        FORMAT = '%(message)s'
        logging.basicConfig(format=FORMAT)
        dummyLogger = logging.getLogger('john')
        test_dict = {'logger': self.logger,'endpoint':'http://127.0.0.1:%i/slow' % self.port,
                     'usestalecache': True}
        myService = Service(test_dict)
        startTime = int(time.time())
        self.assertRaises(socket.timeout, myService.getData, 'foo', '')
        self.assertTrue(int(time.time()) - startTime < 130,
                        "Error: Timeout took too long")
        cherrypy.engine.exit()
        cherrypy.engine.stop()

    def testBadStatusLine(self):
        """
        _BadStatusLine_

        """
        FORMAT = '%(message)s'
        logging.basicConfig(format=FORMAT)
        dummyLogger = logging.getLogger('john')
        test_dict = {'logger': self.logger,'endpoint':'http://127.0.0.1:%i/badstatus' % self.port,
                     'usestalecache': True}
        myService = Service(test_dict)
        # Have to fudge the status line in the Request object as cherrypy won't
        # Allow bad statuses to be raised
        myService['requests'] = CrappyRequest('http://bad.com', {})
        self.assertRaises(BadStatusLine, myService.getData, 'foo', '')

    @attr("integration")
    def testZ_InterruptedConnection(self):
        """
        _InterruptedConnection_

        What happens if we shut down the server while
        the connection is still active?

        Confirm that the cache works as expected
        """

        cherrypy.tree.mount(RegularServer(), "/reg1")
        cherrypy.engine.start()
        FORMAT = '%(message)s'
        logging.basicConfig(format=FORMAT)
        dummyLogger = logging.getLogger('john')
        test_dict = {'logger': self.logger,'endpoint':'http://127.0.0.1:%i/reg1/regular' % self.port,
                     'usestalecache': True, "cacheduration": 0.005}
        myService = Service(test_dict)
        self.assertRaises(HTTPException, myService.getData, 'foo', 'THISISABADURL')

        data = myService.refreshCache('foo', '')
        dataString = data.read()
        self.assertEqual(dataString, "This is silly.")
        data.close()

        # Now stop the server and confirm that it is down
        cherrypy.server.stop()
        self.assertRaises(socket.error, myService.forceRefresh, 'foo', '')

        # Make sure we can still read from the cache
        data = myService.refreshCache('foo', '')
        dataString = data.read()
        self.assertEqual(dataString, "This is silly.")
        data.close()

        # Mount a backup server
        del cherrypy.tree.apps['/reg1']
        cherrypy.tree.mount(BackupServer(), "/reg1")

        # Expire cache
        time.sleep(30)
        self.assertRaises(socket.error, myService.forceRefresh, 'foo', '')

        # get expired cache results while the server is down
        data = myService.refreshCache('foo', '')
        dataString = data.read()
        self.assertEqual(dataString, "This is silly.")
        data.close()

        # Restart server
        cherrypy.server.start()

        # Confirm new server is in place
        data = myService.refreshCache('foo', '')
        dataString = data.read()
        self.assertEqual(dataString, "This is nuts.")
        data.close()

        cherrypy.engine.exit()
        cherrypy.engine.stop()

        return

if __name__ == '__main__':
    unittest.main()


import unittest

class DefocalImage(object):

    def __init__(self, intraImg=None, extraImg=None):
        """

        Initialize the DefocalImage class.

        Keyword Arguments:
            intraImg {[ndarray]} -- Intra-defocal image. (default: {None})
            extraImg {[ndarray]} -- Extra-defocal image. (default: {None})
        """

        # Defocal images
        self.intraImg = intraImg
        self.extraImg = extraImg

    def setImg(self, intraImg=None, extraImg=None):
        """

        Set the image.

        Keyword Arguments:
            intraImg {[ndarray]} -- Intra-defocal image. (default: {None})
            extraImg {[ndarray]} -- Extra-defocal image. (default: {None})
        """

        if (intraImg is not None):
            self.intraImg = intraImg

        if (extraImg is not None):
            self.extraImg = extraImg

class DonutImage(DefocalImage):

    def __init__(self, starId, pixelX, pixelY, fieldX, fieldY, intraImg=None, extraImg=None):
        """

        Initialize the DonutImage class.

        Arguments:
            starId {[int]} -- star ID.
            pixelX {[float]} -- Pixel x.
            pixelY {[float]} -- Pixel y.
            fieldX {[float]} -- Field x in degree.
            fieldY {[float]} -- Field y in degree.

        Keyword Arguments:
            intraImg {[ndarray]} -- Intra-defocal image. (default: {None})
            extraImg {[ndarray]} -- Extra-defocal image. (default: {None})
        """

        super(DonutImage, self).__init__(intraImg=intraImg, extraImg=extraImg)
        self.starId = int(starId)
        self.pixelX = pixelX
        self.pixelY = pixelY
        self.fieldX = fieldX
        self.fieldY = fieldY

        # Wavefront eror in annular Zk in nm (z4-z22)
        self.zer4UpNm = None

    def setWfErr(self, zer4UpNm):
        """

        Set the wavefront error in annular Zk in nm (z4-z22).

        Arguments:
            zer4UpNm {[ndarray]} -- z4 to z22 in nm.
        """

        self.zer4UpNm = zer4UpNm

class DefocalImageTest(unittest.TestCase):

    """
    Test the function of DefocalImage.
    """

    def setUp(self):

        self.defocalImg = DefocalImage()

    def testFunction(self):

        intraImg = 1
        extraImg = 2
        self.defocalImg.setImg(intraImg=intraImg, extraImg=extraImg)
        self.assertEqual(self.defocalImg.intraImg, intraImg)
        self.assertEqual(self.defocalImg.extraImg, extraImg)

class DonutImageTest(unittest.TestCase):

    """
    Test the function of DonutImage.
    """

    def setUp(self):

        starId = 0
        pixelX = 1
        pixelY = 1
        fieldX = 2
        fieldY = 2
        self.donutImg = DonutImage(starId, pixelX, pixelY, fieldX, fieldY)

    def testFunction(self):

        self.assertEqual(self.donutImg.pixelX, self.donutImg.pixelY)

        zer4UpNm = 10
        self.donutImg.setWfErr(zer4UpNm)
        self.assertEqual(self.donutImg.zer4UpNm, zer4UpNm)

if __name__ == "__main__":

    # Do the unit test
    unittest.main()


from __future__ import print_function

import os
import platform
import shutil
import sys
from glob import glob
from subprocess import CalledProcessError
from subprocess import check_call
from subprocess import check_output
from time import sleep

from charms.layer.execd import execd_preinstall

def lsb_release():
    """Return /etc/lsb-release in a dict

    Based on host env, there are two methods:
    1. For Ubuntu, read /etc/lsb-release, contents in xenial 16.04 for example:
        DISTRIB_ID=Ubuntu
        DISTRIB_RELEASE=16.04
        DISTRIB_CODENAME=xenial
        DISTRIB_DESCRIPTION="Ubuntu 16.04.2 LTS"

    2. For CenTos AND RHEL, read /etc/redhat-release, one string:
        CentOS Linux release 7.3.1611 (Core)

    Returns:
        dict: Dictionary presenting the host OS. Keys are:
            1. `DISTRIB_ID`: `Ubuntu` or `CentOS`
            2. `DISTRIB_RELEASE`: eg. `16.04`, `7.3.1611`
            3. `DISTRIB_CODENAME`: eg. `xenial`, `CentOS7.3.1611`
            4. `DISTRIB_DESCRIPTION`: eg. `Ubuntu 16.04.2 LTS`
    """
    d = {}
    me = platform.linux_distribution()[0]
    if 'ubuntu' in me.lower():
        # DISTRIB_ID=Ubuntu
        # DISTRIB_RELEASE=16.04
        # DISTRIB_CODENAME=xenial
        # DISTRIB_DESCRIPTION="Ubuntu 16.04.2 LTS"
        with open('/etc/lsb-release', 'r') as lsb:
            for l in lsb:
                k, v = l.split('=')
                d[k.strip()] = v.strip()
    elif 'cent' in me.lower():
        if os.path.exists('/etc/redhat-release'):
            # http://www.binarytides.com/command-check-centos-version/
            # TODO: need verify this method reading release info of CentOS/RHEL
            # file content:
            #     CentOS Linux release 7.3.1611 (Core)
            with open('/etc/redhat-release', 'r') as lsb:
                for l in lsb:
                    if 'centos' in l.lower():
                        tmp = l.split(' ')  # split by white space
                        d['DISTRIB_ID'] = tmp[0]  # CentOS
                        d['DISTRIB_RELEASE'] = tmp[-2]  # 7.3.1611
                        d['DISTRIB_CODENAME'] = tmp[0] + tmp[-2]  # CentOS7.3.1611
                        d['DISTRIB_DESCRIPTIOIN'] = l  # original string
                        break
        # This is a fallback, if /etc/rethat-release doesn't exist
        else:
            d['DISTRIB_ID'] = 'CentOS'
            d['DISTRIB_RELEASE'] = ''  # unknown?
            d['DISTRIB_CODENAME'] = 'CentOS'
            d['DISTRIB_DESCRIPTIOIN'] = 'CentOS'

    return d

def bootstrap_charm_deps():
    """
    Set up the base charm dependencies so that the reactive system can run.
    """
    # execd must happen first, before any attempt to install packages or
    # access the network, because sites use this hook to do bespoke
    # configuration and install secrets so the rest of this bootstrap
    # and the charm itself can actually succeed. This call does nothing
    # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d.
    execd_preinstall()

    # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts
    charm_dir = os.environ['JUJU_CHARM_DIR']
    os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin')
    venv = os.path.abspath('../.venv')
    vbin = os.path.join(venv, 'bin')
    vpip = os.path.join(vbin, 'pip')  # system default pip
    vpy = os.path.join(vbin, 'python')

    # ".bootstrapped" is a flag file. If it exists, meaning some other charms
    # have already done these steps so there is no need to go through
    # these steps again.
    if os.path.exists('wheelhouse/.bootstrapped'):
        activate_venv()
        return

    # determine host env
    dist = lsb_release()['DISTRIB_ID'].lower()

    # bootstrap wheelhouse
    if os.path.exists('wheelhouse'):
        with open('/root/.pydistutils.cfg', 'w') as fp:
            # make sure that easy_install also only uses the wheelhouse
            # (see https://github.com/pypa/pip/issues/410)
            fp.writelines([
                "[easy_install]\n",
                "allow_hosts = ''\n",
                "find_links = file://{}/wheelhouse/\n".format(charm_dir),
            ])

        # Pre-install packages based on host env.
        if 'ubuntu' in dist:
            # Python27, required by CentOS7, pylxca
            apt_install([
                'python',  # default python pkg
                'python-pip',
                'python-setuptools',
                'python-yaml',
                'python-dev',
            ])

        elif 'cent' in dist:
            apt_install([
                'epel-release',
                'python-setuptools',
                'python-pip',
                'python-yaml',
                'python-devel',
            ])

        # include packages defined in layer.yaml
        from charms import layer
        cfg = layer.options('basic')
        apt_install(cfg.get('packages', []))

        # If using python virtualenv on Ubuntu host
        if 'ubuntu' in dist and cfg.get('use_venv'):
            if not os.path.exists(venv):
                series = lsb_release()['DISTRIB_CODENAME']
                if series in ('precise', 'trusty'):
                    apt_install(['python-virtualenv'])
                else:
                    apt_install(['virtualenv'])
                cmd = ['virtualenv', '-ppython3', '--never-download', venv]
                if cfg.get('include_system_packages'):
                    cmd.append('--system-site-packages')
                check_call(cmd)
            os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']])
            pip = vpip

        # If installing python virtualenv on CentOS host
        elif 'cent' in dist and cfg.get('use_venv'):
            # TODO: how to install virtualenv in CentOS?
            pass

        # If NOT using virtualenv
        elif not cfg.get('use_venv'):
            if 'ubuntu' in dist:
                pip = 'pip'  # Ubuntu using pip

                # save a copy of system pip to prevent `pip3 install -U pip`
                # from changing it
                if os.path.exists('/usr/bin/pip'):
                    shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save')
            elif 'cent' in dist:
                pip = 'pip'  # CentOS using default pip

        # need newer pip, to fix spurious Double Requirement error:
        # https://github.com/pypa/pip/issues/56
        check_call([pip, 'install', '-U', 'pip'])
        check_call([pip, 'install', '-U', 'simplejson'])
        check_call([pip, 'install', '-U', 'requests-toolbelt'])
        check_call([pip, 'install', '-U', '--user', 'setuptools', 'pip'])
        check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', 'pip'])

        # install the rest of the wheelhouse deps
        output = check_output([pip, 'install', '-U', '--no-index', '-f',
                               'wheelhouse'] + glob('wheelhouse/*'))

        if not cfg.get('use_venv'):
            # restore system pip to prevent `pip3 install -U pip`
            # from changing it
            if os.path.exists('/usr/bin/pip.save'):
                shutil.copy2('/usr/bin/pip.save', '/usr/bin/pip')
                os.remove('/usr/bin/pip.save')
        os.remove('/root/.pydistutils.cfg')

        # flag us as having already bootstrapped so we don't do it again
        open('wheelhouse/.bootstrapped', 'w').close()

        # Ensure that the newly bootstrapped libs are available.
        # Note: this only seems to be an issue with namespace packages.
        # Non-namespace-package libs (e.g., charmhelpers) are available
        # without having to reload the interpreter. :/
        sys.path.append('/usr/local/lib/python2.7/dist-packages')
        reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0])

def activate_venv():
    """
    Activate the venv if enabled in ``layer.yaml``.

    This is handled automatically for normal hooks, but actions might
    need to invoke this manually, using something like:

        # Load modules from $JUJU_CHARM_DIR/lib
        import sys
        sys.path.append('lib')

        from charms.layer.basic import activate_venv
        activate_venv()

    This will ensure that modules installed in the charm's
    virtual environment are available to the action.
    """
    venv = os.path.abspath('../.venv')
    vbin = os.path.join(venv, 'bin')
    vpy = os.path.join(vbin, 'python')
    from charms import layer
    cfg = layer.options('basic')
    if cfg.get('use_venv') and '.venv' not in sys.executable:
        # activate the venv
        os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']])
        reload_interpreter(vpy)

def reload_interpreter(python):
    """
    Reload the python interpreter to ensure that all deps are available.

    Newly installed modules in namespace packages sometimes seemt to
    not be picked up by Python 3.
    """
    os.execve(python, [python] + list(sys.argv), os.environ)

def apt_install(packages):
    """
    Install apt packages.

    This ensures a consistent set of options that are often missed but
    should really be set.
    """
    if isinstance(packages, (str, bytes)):
        packages = [packages]

    env = os.environ.copy()

    if 'DEBIAN_FRONTEND' not in env:
        env['DEBIAN_FRONTEND'] = 'noninteractive'

    # determine host env
    dist = lsb_release()['DISTRIB_ID'].lower()

    if 'ubuntu' in dist:
        pkg_cmd = 'apt-get'
        say_yes = '--assume-yes'
        options = ['--option=Dpkg::Options::=--force-confold', ]
        install_cmd = 'install'
    elif 'cent' in dist:
        pkg_cmd = 'yum'
        say_yes = '--assumeyes'
        options = []
        install_cmd = 'install'

    cmd = [pkg_cmd] + options + [say_yes] + [install_cmd]

    # Try cmd 3 times
    for attempt in range(3):
        try:
            check_call(cmd + packages, env=env)
        except CalledProcessError:
            if attempt == 2:  # third attempt
                raise
            sleep(5)
        else:
            break

def init_config_states():
    import yaml
    from charmhelpers.core import hookenv
    from charms.reactive import set_state
    from charms.reactive import toggle_state
    config = hookenv.config()
    config_defaults = {}
    config_defs = {}
    config_yaml = os.path.join(hookenv.charm_dir(), 'config.yaml')
    if os.path.exists(config_yaml):
        with open(config_yaml) as fp:
            config_defs = yaml.safe_load(fp).get('options', {})
            config_defaults = {key: value.get('default')
                               for key, value in config_defs.items()}
    for opt in config_defs.keys():
        if config.changed(opt):
            set_state('config.changed')
            set_state('config.changed.{}'.format(opt))
        toggle_state('config.set.{}'.format(opt), config.get(opt))
        toggle_state('config.default.{}'.format(opt),
                     config.get(opt) == config_defaults[opt])
    hookenv.atexit(clear_config_states)

def clear_config_states():
    from charmhelpers.core import hookenv, unitdata
    from charms.reactive import remove_state
    config = hookenv.config()
    remove_state('config.changed')
    for opt in config.keys():
        remove_state('config.changed.{}'.format(opt))
        remove_state('config.set.{}'.format(opt))
        remove_state('config.default.{}'.format(opt))
    unitdata.kv().flush()


# -*- coding: utf-8 -*-

from collective.transmogrifier.interfaces import ISection
from collective.transmogrifier.interfaces import ISectionBlueprint
from collective.transmogrifier.utils import defaultMatcher
from zope.interface import classProvides
from zope.interface import implements

class WorkflowConverter(object):
    classProvides(ISectionBlueprint)
    implements(ISection)

    def __init__(self, transmogrifier, name, options, previous):
        self.transmogrifier = transmogrifier
        self.name = name
        self.options = options
        self.previous = previous
        self.context = transmogrifier.context
        self.pathkey = defaultMatcher(options, 'path-key', name, 'path')

    def __iter__(self):
        for item in self.previous:
            pathkey = self.pathkey(*item.keys())[0]
            if not pathkey:
                yield item
                continue

            path = item[pathkey]
            obj = self.context.unrestrictedTraverse(path, None)
            if obj is None:
                yield item
                continue

            yield item


'''
Created on Sep 28, 2014

@author: jesse
'''

from pygame.threads import Thread
import logging
import time

logging.basicConfig(level = logging.WARNING)

class Parameters(object):
    '''
    Takes care reading and writing parameters.
    '''

    toc = None
    parameters = {}
    _cf = None

    def __init__(self, cf):
        '''
        Creates a new set of sensors
        '''
        self._cf = cf

    def setup(self):
        # Print the param TOC
        self.toc = self._cf.param.toc.toc
        for group in sorted(self.toc.keys()):
            for param in sorted(self.toc[group].keys()):
                self._cf.param.add_update_callback(group = group, name = param,
                                           cb = self.updateValue)

    def start(self):
        t = Thread(target = self.updateLoop)
        t.daemon = True
        t.start()

    def updateValue(self, name, value):
        parts = name.split(".")
        if parts[0] not in self.parameters:
            self.parameters[parts[0]] = {}
        group = self.parameters[parts[0]]
        group[parts[1]] = value

    def requestUpdate(self):
        for group in sorted(self.toc.keys()):
            for param in sorted(self.toc[group].keys()):
                self._cf.param.request_param_update("{0}.{1}".format(group, param))

    def setValue(self, paramname, value):
        self._cf.param.set_value(paramname, "{:.2f}".format(value))
        self._cf.param.request_param_update(paramname)

    def updateLoop(self):
        while True:
            self.requestUpdate()
            time.sleep(5)


import subprocess
from gwpy.segment import SegmentList

def fw_segments(fw_name='fw0',scp=False):
    ''' Return the segment list when the frame writer could write data.

    Information about when the fw0 write data is given by a text file
    named "fw0-latest.txt". This text file is generated by a script
    written by T. Yamamoto. Then we can make a segment list by
    translating this file.

    MEMO:"fw0-latest.txt" is saved on /users/DGS/Frame.

    Parameters
    ----------
    fw_name : `str`, optional
        Name of the frame writer. KAGRA have a fw0 and fw1. Default is fw0.
    scp : `str`, optional
        If download from control machine, please choose True. In default, False.

    Returns
    -------
    ok : `gwpy.segments.SegmentList`
        SegmentList whihch only contain segments when fw has write data on main strage.
    '''
    if scp:
        cmd = 'scp controls@k1ctr7:/users/DGS/Frame/{0}-latest.txt ./ '.format(fw_name)
        download = subprocess.call(cmd, shell=True)

    cmd = "less ./{0}-latest.txt".format(fw_name)
    cmd += " | awk '{if ($4>86400) print $1 , $2}' > tmp_{0}.txt".format(fw_name)
    make_tmp_txt = subprocess.call(cmd,shell=True)
    ok = SegmentList.read('tmp_{0}.txt'.format(fw_name))
    remove_tmp_txt = subprocess.call("rm tmp_{0}.txt".format(fw_name), shell=True)
    return ok