# -*- coding: utf-8 -*-
import scrapy
from time import sleep

class DoubanSpider(scrapy.Spider):
    name = 'DouBan'
    allowed_domains = ['douban.com']
    start_urls = ['https://book.douban.com/top250?start=0']

    def parse(self, response):
        sleep(3)
        book_stars = response.xpath('//span[@class="rating_nums"]').xpath('string(.)').extract()
        book_names = []
        for name in response.xpath('//div[@class="pl2"]/a').xpath('string(.)').extract():
            book_names.append(name.replace('\n','').replace(' ',''))
        next_url = response.xpath('//span[@class="next"]/a/@href').extract_first()
        for n,s in zip(book_names,book_stars):
            yield {
                'book_name':n,
                'book_star':s
            }
        yield scrapy.Request(next_url,callback=self.parse)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import click
from pypiup.requirement import Requirement

class Requirements(object):

    def __init__(self, requirements_file):
        self.file = self.read_file(requirements_file)
        self.requirements = []

    def should_ignore_requirement(self, requirement):
        if requirement.startswith("#") or requirement == "":
            return True
        return False

    def read_file(self, requirements_filename):
        click.secho("\nAttempting to read %s ...\n" % requirements_filename, fg='magenta')

        try:
            requirements_file = open(requirements_filename)
            requirements_file_lines = requirements_file.read().splitlines()
            with click.progressbar(
                    iterable=requirements_file_lines,
                    label='Getting requirements details',
                    bar_template='%(label)s %(bar)s | %(info)s',
                    fill_char=click.style(u'█', fg='cyan'),
                    empty_char=' ') as bar:
                requirements = []
                for line in bar:
                    if not self.should_ignore_requirement(line):
                        requirement = Requirement(line)
                        requirements.append(requirement)
                self.requirements = requirements
                self.show_details()
                self.show_stats()
                requirements_file.close()
        except IOError:
            return click.secho("Could not find %s. No such file or directory.\n" % (requirements_filename), fg='red')

    def show_details(self):
        click.echo()
        for requirement in self.requirements:
            requirement.output_details()

    def show_stats(self):
        stat_total = len(self.requirements)
        stat_uptodate = 0
        stat_needs_update = 0
        stat_invalid_semver = 0
        stat_invalid = 0

        for requirement in self.requirements:
            if requirement.valid:
                if requirement.status == "UPTODATE":
                    stat_uptodate += 1
                elif requirement.status == "NEEDS_UPDATE":
                    stat_needs_update += 1
                elif requirement.status == "INVALID_SEMVER":
                    stat_invalid_semver += 1
            else:
                stat_invalid += 1

        click.echo(click.style("\nStatistics", bold=True, underline=True, fg='yellow'))
        click.echo("  Total Requirements: %s" % stat_total)
        click.echo("  Up to date: %s" % stat_uptodate)
        click.echo("  Update available: %s" % stat_needs_update)
        click.echo("  Invalid Semver: %s" % stat_invalid_semver)
        click.echo("  Non PyPI Requirements: %s" % stat_invalid)


import numpy as np
from openslide import open_slide
from openslide.deepzoom import DeepZoomGenerator
import os
import sys

# to remove leading and trailing whitespace
image_path = sys.argv[1]
# split the line into words
words = image_path.split("/")
svs_file = words[len(words)-1]#"23-168-001-004.svs"
slide = open_slide(f"{image_path}")
current_svs_file_nm = svs_file.split(".")[0]
print(current_svs_file_nm)
tiles = DeepZoomGenerator(slide, tile_size=1024, overlap=0, limit_bounds=False)
level = tiles.level_count - 1
cols, rows = tiles.level_tiles[level]
#os.system(f'mkdir processing/tiles/{current_svs_file_nm}_{level}')
# we are looping over the words array and printing the word
# with the count of 1 to the STDOUT
for col in range(cols):
    for row in range(rows):
    # write the results to STDOUT (standard output);
    # what we output here will be the input for the
    # Reduce step, i.e. the input for reducer.py
        img = tiles.get_tile(level, (0, 0))
        img_d = list(img.getdata())
        #dt=np.dtype
        np_img = np.asarray(img)
        data = np_img.tobytes()
        print(f"{level}_{str(col)}_{str(row)}_{current_svs_file_nm}\t{data}")