patch 9.0.0322: crash when no errors and 'quickfixtextfunc' is set

Problem:    Crash when no errors and 'quickfixtextfunc' is set.
Solution:   Do not handle errors if there aren't any.

qf_fill_buffer(qf_list_T *qfl, buf_T *buf, qfline_T *old_last, int qf_winid)
{
    linenr_T	lnum;
    qfline_T	*qfp;
    int		old_KeyTyped = KeyTyped;
    list_T	*qftf_list = NULL;
    listitem_T	*qftf_li = NULL;

    if (old_last == NULL)
    {
	if (buf != curbuf)
	{
	    internal_error("qf_fill_buffer()");
	    return;
	}

	// delete all existing lines
	while ((curbuf->b_ml.ml_flags & ML_EMPTY) == 0)
	    (void)ml_delete((linenr_T)1);
    }

    // Check if there is anything to display
    if (qfl != NULL && qfl->qf_start != NULL)
    {
	char_u		dirname[MAXPATHL];
	int		invalid_val = FALSE;
	int		prev_bufnr = -1;

	*dirname = NUL;

	// Add one line for each error
	if (old_last == NULL)
	{
	    qfp = qfl->qf_start;
	    lnum = 0;
	}
	else
	{
	    if (old_last->qf_next != NULL)
		qfp = old_last->qf_next;
	    else
		qfp = old_last;
	    lnum = buf->b_ml.ml_line_count;
	}

	qftf_list = call_qftf_func(qfl, qf_winid, (long)(lnum + 1),
							(long)qfl->qf_count);
	if (qftf_list != NULL)
	    qftf_li = qftf_list->lv_first;

	while (lnum < qfl->qf_count)
	{
	    char_u	*qftf_str = NULL;

	    // Use the text supplied by the user defined function (if any).
	    // If the returned value is not string, then ignore the rest
	    // of the returned values and use the default.
	    if (qftf_li != NULL && !invalid_val)
	    {
		qftf_str = tv_get_string_chk(&qftf_li->li_tv);
		if (qftf_str == NULL)
		    invalid_val = TRUE;
	    }

	    if (qf_buf_add_line(buf, lnum, qfp, dirname,
			prev_bufnr != qfp->qf_fnum, qftf_str) == FAIL)
		break;

	    prev_bufnr = qfp->qf_fnum;
	    ++lnum;
	    qfp = qfp->qf_next;
	    if (qfp == NULL)
		break;

	    if (qftf_li != NULL)
		qftf_li = qftf_li->li_next;
	}

	if (old_last == NULL)
	    // Delete the empty line which is now at the end
	    (void)ml_delete(lnum + 1);
    }

    // correct cursor position
    check_lnums(TRUE);

    if (old_last == NULL)
    {
	// Set the 'filetype' to "qf" each time after filling the buffer.
	// This resembles reading a file into a buffer, it's more logical when
	// using autocommands.
	++curbuf_lock;
	set_option_value_give_err((char_u *)"ft",
						0L, (char_u *)"qf", OPT_LOCAL);
	curbuf->b_p_ma = FALSE;

	keep_filetype = TRUE;		// don't detect 'filetype'
	apply_autocmds(EVENT_BUFREADPOST, (char_u *)"quickfix", NULL,
							       FALSE, curbuf);
	apply_autocmds(EVENT_BUFWINENTER, (char_u *)"quickfix", NULL,
							       FALSE, curbuf);
	keep_filetype = FALSE;
	--curbuf_lock;

	// make sure it will be redrawn
	redraw_curbuf_later(UPD_NOT_VALID);
    }

    // Restore KeyTyped, setting 'filetype' may reset it.
    KeyTyped = old_KeyTyped;
}

Use After Free in GitHub repository vim/vim prior to 9.0.0322.


Sanitize inputs to bookmark-cmd (#591)

Newsbeuter didn't properly shell-escape the arguments passed to
bookmarking command, which allows a remote attacker to perform remote
code execution by crafting an RSS item whose title and/or URL contain
something interpretable by the shell (most notably subshell
invocations.)

This has been reported by Jeriko One <jeriko.one@gmx.us>, complete with
PoC and a patch.

This vulnerability was assigned CVE-2017-12904.

void omg_a_child_died(int /* sig */) {
	pid_t pid;
	int stat;
	while ((pid = waitpid(-1,&stat,WNOHANG)) > 0) { }
	::signal(SIGCHLD, omg_a_child_died); /* in case of unreliable signals */
}

Improper Neutralization of Special Elements used in an OS Command in bookmarking function of Newsbeuter versions 0.7 through 2.9 allows remote attackers to perform user-assisted code execution by crafting an RSS item that includes shell code in its title and/or URL.


std::string quote_empty(const std::string& input) {
	if (input.empty()) {
		return "''";
	} else {
		return input;
	}
}


void controller::reload_range(unsigned int start, unsigned int end, unsigned int size, bool unattended) {

	std::vector<unsigned int> v;
	for (unsigned int i=start; i<=end; ++i)
		v.push_back(i);

	auto extract = [](std::string& s, const std::string& url) {
		size_t p = url.find("//");
		p = (p == std::string::npos) ? 0 : p+2;
		std::string suff(url.substr(p));
		p = suff.find('/');
		s = suff.substr(0, p);
	};

	std::sort(v.begin(), v.end(), [&](unsigned int a, unsigned int b) {
		std::string domain1, domain2;
		extract(domain1, feeds[a]->rssurl());
		extract(domain2, feeds[b]->rssurl());
		std::reverse(domain1.begin(), domain1.end());
		std::reverse(domain2.begin(), domain2.end());
		return domain1 < domain2;
	});

	curl_handle easyhandle;

	for (auto i : v) {
		LOG(level::DEBUG, "controller::reload_range: reloading feed #%u", i);
		this->reload(i, size, unattended, &easyhandle);
	}
}


void controller::export_read_information(const std::string& readinfofile) {
	std::vector<std::string> guids = rsscache->get_read_item_guids();

	std::fstream f;
	f.open(readinfofile.c_str(), std::fstream::out);
	if (f.is_open()) {
		for (auto guid : guids) {
			f << guid << std::endl;
		}
	}
}


void controller::setup_dirs(bool silent) {
	const char * env_home;
	if (!(env_home = ::getenv("HOME"))) {
		struct passwd * spw = ::getpwuid(::getuid());
		if (spw) {
			env_home = spw->pw_dir;
		} else {
			std::cerr << _("Fatal error: couldn't determine home directory!") << std::endl;
			std::cerr << strprintf::fmt(_("Please set the HOME environment variable or add a valid user for UID %u!"), ::getuid()) << std::endl;
			::exit(EXIT_FAILURE);
		}
	}

	config_dir = env_home;
	config_dir.append(NEWSBEUTER_PATH_SEP);
	config_dir.append(NEWSBEUTER_CONFIG_SUBDIR);

	if (setup_dirs_xdg(env_home, silent))
		return;

	mkdir(config_dir.c_str(),0700); // create configuration directory if it doesn't exist

	url_file = config_dir + std::string(NEWSBEUTER_PATH_SEP) + url_file;
	cache_file = config_dir + std::string(NEWSBEUTER_PATH_SEP) + cache_file;
	lock_file = cache_file + LOCK_SUFFIX;
	config_file = config_dir + std::string(NEWSBEUTER_PATH_SEP) + config_file;
	queue_file = config_dir + std::string(NEWSBEUTER_PATH_SEP) + queue_file;

	searchfile = strprintf::fmt("%s%shistory.search", config_dir, NEWSBEUTER_PATH_SEP);
	cmdlinefile = strprintf::fmt("%s%shistory.cmdline", config_dir, NEWSBEUTER_PATH_SEP);
}


void controller::run(int argc, char * argv[]) {
	int c;

	::signal(SIGINT, ctrl_c_action);
	::signal(SIGPIPE, ignore_signal);
	::signal(SIGHUP, ctrl_c_action);
	::signal(SIGCHLD, omg_a_child_died);

	bool do_import = false, do_export = false, cachefile_given_on_cmdline = false, do_vacuum = false;
	std::string importfile;
	bool do_read_import = false, do_read_export = false;
	std::string readinfofile;
	unsigned int show_version = 0;

	bool silent = false;
	bool execute_cmds = false;

	static const char getopt_str[] = "i:erhqu:c:C:d:l:vVoxXI:E:";
	static const struct option longopts[] = {
		{"cache-file"      , required_argument, 0, 'c'},
		{"config-file"     , required_argument, 0, 'C'},
		{"execute"         , required_argument, 0, 'x'},
		{"export-to-file"  , required_argument, 0, 'E'},
		{"export-to-opml"  , no_argument      , 0, 'e'},
		{"help"            , no_argument      , 0, 'h'},
		{"import-from-file", required_argument, 0, 'I'},
		{"import-from-opml", required_argument, 0, 'i'},
		{"log-file"        , required_argument, 0, 'd'},
		{"log-level"       , required_argument, 0, 'l'},
		{"quiet"           , no_argument      , 0, 'q'},
		{"refresh-on-start", no_argument      , 0, 'r'},
		{"url-file"        , required_argument, 0, 'u'},
		{"vacuum"          , no_argument      , 0, 'X'},
		{"version"         , no_argument      , 0, 'v'},
		{0                 , 0                , 0,  0 }
	};

	/* First of all, let's check for options that imply silencing of the
	 * output: import, export, command execution and, well, quiet mode */
	while ((c = ::getopt_long(argc, argv, getopt_str, longopts, nullptr)) != -1) {
		if (strchr("iexq", c) != nullptr) {
			silent = true;
			break;
		}
	}

	setup_dirs(silent);

	/* Now that silencing's set up, let's rewind to the beginning of argv and
	 * process the options */
	optind = 1;

	while ((c = ::getopt_long(argc, argv, getopt_str, longopts, nullptr)) != -1) {
		switch (c) {
		case ':': /* fall-through */
		case '?': /* missing option */
			usage(argv[0]);
			break;
		case 'i':
			if (do_export)
				usage(argv[0]);
			do_import = true;
			importfile = optarg;
			break;
		case 'r':
			refresh_on_start = true;
			break;
		case 'e':
			if (do_import)
				usage(argv[0]);
			do_export = true;
			break;
		case 'h':
			usage(argv[0]);
			break;
		case 'u':
			url_file = optarg;
			break;
		case 'c':
			cache_file = optarg;
			lock_file = std::string(cache_file) + LOCK_SUFFIX;
			cachefile_given_on_cmdline = true;
			break;
		case 'C':
			config_file = optarg;
			break;
		case 'X':
			do_vacuum = true;
			break;
		case 'v':
		case 'V':
			show_version++;
			break;
		case 'x':
			execute_cmds = true;
			break;
		case 'q':
			break;
		case 'd':
			logger::getInstance().set_logfile(optarg);
			break;
		case 'l': {
			level l = static_cast<level>(atoi(optarg));
			if (l > level::NONE && l <= level::DEBUG) {
				logger::getInstance().set_loglevel(l);
			} else {
				std::cerr << strprintf::fmt(_("%s: %d: invalid loglevel value"), argv[0], l) << std::endl;
				::std::exit(EXIT_FAILURE);
			}
		}
		break;
		case 'I':
			if (do_read_export)
				usage(argv[0]);
			do_read_import = true;
			readinfofile = optarg;
			break;
		case 'E':
			if (do_read_import)
				usage(argv[0]);
			do_read_export = true;
			readinfofile = optarg;
			break;
		default:
			std::cout << strprintf::fmt(_("%s: unknown option - %c"), argv[0], static_cast<char>(c)) << std::endl;
			usage(argv[0]);
			break;
		}
	};

	if (show_version) {
		version_information(argv[0], show_version);
	}

	if (do_import) {
		LOG(level::INFO,"Importing OPML file from %s",importfile);
		urlcfg = new file_urlreader(url_file);
		urlcfg->reload();
		import_opml(importfile);
		return;
	}

	LOG(level::INFO, "nl_langinfo(CODESET): %s", nl_langinfo(CODESET));

	if (!do_export) {

		if (!silent)
			std::cout << strprintf::fmt(_("Starting %s %s..."), PROGRAM_NAME, PROGRAM_VERSION) << std::endl;

		pid_t pid;
		if (!utils::try_fs_lock(lock_file, pid)) {
			if (pid > 0) {
				LOG(level::ERROR,"an instance is already running: pid = %u",pid);
			} else {
				LOG(level::ERROR,"something went wrong with the lock: %s", strerror(errno));
			}
			if (!execute_cmds) {
				std::cout << strprintf::fmt(_("Error: an instance of %s is already running (PID: %u)"), PROGRAM_NAME, pid) << std::endl;
			}
			return;
		}
	}

	if (!silent)
		std::cout << _("Loading configuration...");
	std::cout.flush();

	cfg.register_commands(cfgparser);
	colorman.register_commands(cfgparser);

	keymap keys(KM_NEWSBEUTER);
	cfgparser.register_handler("bind-key",&keys);
	cfgparser.register_handler("unbind-key",&keys);
	cfgparser.register_handler("macro", &keys);

	cfgparser.register_handler("ignore-article",&ign);
	cfgparser.register_handler("always-download",&ign);
	cfgparser.register_handler("reset-unread-on-update",&ign);

	cfgparser.register_handler("define-filter",&filters);
	cfgparser.register_handler("highlight", &rxman);
	cfgparser.register_handler("highlight-article", &rxman);

	try {
		cfgparser.parse("/etc/" PROGRAM_NAME "/config");
		cfgparser.parse(config_file);
	} catch (const configexception& ex) {
		LOG(level::ERROR,"an exception occurred while parsing the configuration file: %s",ex.what());
		std::cout << ex.what() << std::endl;
		utils::remove_fs_lock(lock_file);
		return;
	}

	update_config();

	if (!silent)
		std::cout << _("done.") << std::endl;

	// create cache object
	std::string cachefilepath = cfg.get_configvalue("cache-file");
	if (cachefilepath.length() > 0 && !cachefile_given_on_cmdline) {
		cache_file = cachefilepath.c_str();

		// ok, we got another cache file path via the configuration
		// that means we need to remove the old lock file, assemble
		// the new lock file's name, and then try to lock it.
		utils::remove_fs_lock(lock_file);
		lock_file = std::string(cache_file) + LOCK_SUFFIX;

		pid_t pid;
		if (!utils::try_fs_lock(lock_file, pid)) {
			if (pid > 0) {
				LOG(level::ERROR,"an instance is already running: pid = %u",pid);
			} else {
				LOG(level::ERROR,"something went wrong with the lock: %s", strerror(errno));
			}
			std::cout << strprintf::fmt(_("Error: an instance of %s is already running (PID: %u)"), PROGRAM_NAME, pid) << std::endl;
			return;
		}
	}

	if (!silent) {
		std::cout << _("Opening cache...");
		std::cout.flush();
	}
	try {
		rsscache = new cache(cache_file,&cfg);
	} catch (const dbexception& e) {
		std::cerr << strprintf::fmt(_("Error: opening the cache file `%s' failed: %s"), cache_file, e.what()) << std::endl;
		utils::remove_fs_lock(lock_file);
		::exit(EXIT_FAILURE);
	}

	if (!silent) {
		std::cout << _("done.") << std::endl;
	}

	std::string type = cfg.get_configvalue("urls-source");
	if (type == "local") {
		urlcfg = new file_urlreader(url_file);
	} else if (type == "opml") {
		urlcfg = new opml_urlreader(&cfg);
	} else if (type == "oldreader") {
		api = new oldreader_api(&cfg);
		urlcfg = new oldreader_urlreader(&cfg, url_file, api);
	} else if (type == "ttrss") {
		api = new ttrss_api(&cfg);
		urlcfg = new ttrss_urlreader(url_file, api);
	} else if (type == "newsblur") {
		api = new newsblur_api(&cfg);
		urlcfg = new newsblur_urlreader(url_file, api);
	} else if (type == "feedhq") {
		api = new feedhq_api(&cfg);
		urlcfg = new feedhq_urlreader(&cfg, url_file, api);
	} else if (type == "ocnews") {
		api = new ocnews_api(&cfg);
		urlcfg = new ocnews_urlreader(url_file, api);
	} else {
		LOG(level::ERROR,"unknown urls-source `%s'", urlcfg->get_source());
	}

	if (!do_export && !silent) {
		std::cout << strprintf::fmt(_("Loading URLs from %s..."), urlcfg->get_source());
		std::cout.flush();
	}
	if (api) {
		if (!api->authenticate()) {
			std::cout << "Authentication failed." << std::endl;
			utils::remove_fs_lock(lock_file);
			return;
		}
	}
	urlcfg->reload();
	if (!do_export && !silent) {
		std::cout << _("done.") << std::endl;
	}

	if (urlcfg->get_urls().size() == 0) {
		LOG(level::ERROR,"no URLs configured.");
		std::string msg;
		if (type == "local") {
			msg = strprintf::fmt(_("Error: no URLs configured. Please fill the file %s with RSS feed URLs or import an OPML file."), url_file);
		} else if (type == "opml") {
			msg = strprintf::fmt(_("It looks like the OPML feed you subscribed contains no feeds. Please fill it with feeds, and try again."));
		} else if (type == "oldreader") {
			msg = strprintf::fmt(_("It looks like you haven't configured any feeds in your The Old Reader account. Please do so, and try again."));
		} else if (type == "ttrss") {
			msg = strprintf::fmt(_("It looks like you haven't configured any feeds in your Tiny Tiny RSS account. Please do so, and try again."));
		} else if (type == "newsblur") {
			msg = strprintf::fmt(_("It looks like you haven't configured any feeds in your NewsBlur account. Please do so, and try again."));
		} else {
			assert(0); // shouldn't happen
		}
		std::cout << msg << std::endl << std::endl;
		usage(argv[0]);
	}

	if (!do_export && !do_vacuum && !silent)
		std::cout << _("Loading articles from cache...");
	if (do_vacuum)
		std::cout << _("Opening cache...");
	std::cout.flush();

	if (do_vacuum) {
		std::cout << _("done.") << std::endl;
		std::cout << _("Cleaning up cache thoroughly...");
		std::cout.flush();
		rsscache->do_vacuum();
		std::cout << _("done.") << std::endl;
		utils::remove_fs_lock(lock_file);
		return;
	}

	unsigned int i=0;
	for (auto url : urlcfg->get_urls()) {
		try {
			bool ignore_disp = (cfg.get_configvalue("ignore-mode") == "display");
			std::shared_ptr<rss_feed> feed = rsscache->internalize_rssfeed(url, ignore_disp ? &ign : nullptr);
			feed->set_tags(urlcfg->get_tags(url));
			feed->set_order(i);
			std::lock_guard<std::mutex> feedslock(feeds_mutex);
			feeds.push_back(feed);
		} catch(const dbexception& e) {
			std::cout << _("Error while loading feeds from database: ") << e.what() << std::endl;
			utils::remove_fs_lock(lock_file);
			return;
		} catch(const std::string& str) {
			std::cout << strprintf::fmt(_("Error while loading feed '%s': %s"), url, str) << std::endl;
			utils::remove_fs_lock(lock_file);
			return;
		}
		i++;
	}

	std::vector<std::string> tags = urlcfg->get_alltags();

	if (!do_export && !silent)
		std::cout << _("done.") << std::endl;

	// if configured, we fill all query feeds with some data; no need to sort it, it will be refilled when actually opening it.
	if (cfg.get_configvalue_as_bool("prepopulate-query-feeds")) {
		std::cout << _("Prepopulating query feeds...");
		std::cout.flush();
		std::lock_guard<std::mutex> feedslock(feeds_mutex);
		for (auto feed : feeds) {
			if (feed->rssurl().substr(0,6) == "query:") {
				feed->update_items(get_all_feeds_unlocked());
			}
		}
		std::cout << _("done.") << std::endl;
	}

	sort_feeds();

	if (do_export) {
		export_opml();
		utils::remove_fs_lock(lock_file);
		return;
	}

	if (do_read_import) {
		LOG(level::INFO,"Importing read information file from %s",readinfofile);
		std::cout << _("Importing list of read articles...");
		std::cout.flush();
		import_read_information(readinfofile);
		std::cout << _("done.") << std::endl;
		return;
	}

	if (do_read_export) {
		LOG(level::INFO,"Exporting read information file to %s",readinfofile);
		std::cout << _("Exporting list of read articles...");
		std::cout.flush();
		export_read_information(readinfofile);
		std::cout << _("done.") << std::endl;
		return;
	}

	// hand over the important objects to the view
	v->set_config_container(&cfg);
	v->set_keymap(&keys);
	v->set_tags(tags);

	if (execute_cmds) {
		execute_commands(argv, optind);
		utils::remove_fs_lock(lock_file);
		return;
	}

	// if the user wants to refresh on startup via configuration file, then do so,
	// but only if -r hasn't been supplied.
	if (!refresh_on_start && cfg.get_configvalue_as_bool("refresh-on-startup")) {
		refresh_on_start = true;
	}

	formaction::load_histories(searchfile, cmdlinefile);

	// run the view
	v->run();

	unsigned int history_limit = cfg.get_configvalue_as_int("history-limit");
	LOG(level::DEBUG, "controller::run: history-limit = %u", history_limit);
	formaction::save_histories(searchfile, cmdlinefile, history_limit);

	if (!silent) {
		std::cout << _("Cleaning up cache...");
		std::cout.flush();
	}
	try {
		std::lock_guard<std::mutex> feedslock(feeds_mutex);
		rsscache->cleanup_cache(feeds);
		if (!silent) {
			std::cout << _("done.") << std::endl;
		}
	} catch (const dbexception& e) {
		LOG(level::USERERROR, "Cleaning up cache failed: %s", e.what());
		if (!silent) {
			std::cout << _("failed: ") << e.what() << std::endl;
		}
	}

	utils::remove_fs_lock(lock_file);
}


void controller::compute_unread_numbers(unsigned int& unread_feeds, unsigned int& unread_articles) {
	unread_feeds = 0;
	unread_articles = 0;
	for (auto feed : feeds) {
		unsigned int items = feed->unread_item_count();
		if (items > 0) {
			++unread_feeds;
			unread_articles += items;
		}
	}
}


unsigned int controller::get_feed_count_per_tag(const std::string& tag) {
	unsigned int count = 0;
	std::lock_guard<std::mutex> feedslock(feeds_mutex);

	for (auto feed : feeds) {
		if (feed->matches_tag(tag)) {
			count++;
		}
	}

	return count;
}


bool controller::is_valid_podcast_type(const std::string& /* mimetype */) {
	return true;
}


void controller::reload_all(bool unattended) {
	unsigned int unread_feeds, unread_articles;
	compute_unread_numbers(unread_feeds, unread_articles);
	unsigned int num_threads = cfg.get_configvalue_as_int("reload-threads");
	time_t t1, t2, dt;

	unsigned int size;

	{
		std::lock_guard<std::mutex> feedlock(feeds_mutex);
		for (auto feed : feeds) {
			feed->reset_status();
		}
		size = feeds.size();
	}

	if (num_threads < 1)
		num_threads = 1;

	if (num_threads > size) {
		num_threads = size;
	}

	t1 = time(nullptr);

	LOG(level::DEBUG,"controller::reload_all: starting with reload all...");
	if (num_threads <= 1) {
		this->reload_range(0, size-1, size, unattended);
	} else {
		std::vector<std::pair<unsigned int, unsigned int>> partitions = utils::partition_indexes(0, size-1, num_threads);
		std::vector<std::thread> threads;
		LOG(level::DEBUG, "controller::reload_all: starting reload threads...");
		for (unsigned int i=0; i<num_threads-1; i++) {
			threads.push_back(std::thread(reloadrangethread(this, partitions[i].first, partitions[i].second, size, unattended)));
		}
		LOG(level::DEBUG, "controller::reload_all: starting my own reload...");
		this->reload_range(partitions[num_threads-1].first, partitions[num_threads-1].second, size, unattended);
		LOG(level::DEBUG, "controller::reload_all: joining other threads...");
		for (size_t i=0; i<threads.size(); i++) {
			threads[i].join();
		}
	}

	// refresh query feeds (update and sort)
	LOG(level::DEBUG, "controller::reload_all: refresh query feeds");
	for (auto feed : feeds) {
		v->prepare_query_feed(feed);
	}
	v->force_redraw();

	sort_feeds();
	update_feedlist();

	t2 = time(nullptr);
	dt = t2 - t1;
	LOG(level::INFO, "controller::reload_all: reload took %d seconds", dt);

	unsigned int unread_feeds2, unread_articles2;
	compute_unread_numbers(unread_feeds2, unread_articles2);
	bool notify_always = cfg.get_configvalue_as_bool("notify-always");
	if (notify_always || unread_feeds2 > unread_feeds || unread_articles2 > unread_articles) {
		int article_count = unread_articles2 - unread_articles;
		int feed_count = unread_feeds2 - unread_feeds;

		LOG(level::DEBUG, "unread article count: %d", article_count);
		LOG(level::DEBUG, "unread feed count: %d", feed_count);

		fmtstr_formatter fmt;
		fmt.register_fmt('f', std::to_string(unread_feeds2));
		fmt.register_fmt('n', std::to_string(unread_articles2));
		fmt.register_fmt('d', std::to_string(article_count >= 0 ? article_count : 0));
		fmt.register_fmt('D', std::to_string(feed_count >= 0 ? feed_count : 0));
		this->notify(fmt.do_format(cfg.get_configvalue("notify-format")));
	}
}


void controller::start_reload_all_thread(std::vector<int> * indexes) {
	LOG(level::INFO,"starting reload all thread");
	std::thread t(downloadthread(this, indexes));
	t.detach();
}


void controller::mark_all_read(const std::string& feedurl) {
	try {
		rsscache->mark_all_read(feedurl);
	} catch (const dbexception& e) {
		v->show_error(strprintf::fmt(_("Error: couldn't mark all feeds read: %s"), e.what()));
		return;
	}

	std::lock_guard<std::mutex> feedslock(feeds_mutex);
	for (const auto& feed : feeds) {
		std::lock_guard<std::mutex> lock(feed->item_mutex);

		if (feedurl.length() > 0 && feed->rssurl() != feedurl)
			continue;

		if (feed->total_item_count() > 0) {
			if (api) {
				api->mark_all_read(feed->rssurl());
			}
			for (auto item : feed->items()) {
				item->set_unread_nowrite(false);
			}
		}

		// no point in going on - there is only one feed with a given URL
		if (feedurl.length() > 0) break;
	}
}


void controller::usage(char * argv0) {
	auto msg =
	    strprintf::fmt(_("%s %s\nusage: %s [-i <file>|-e] [-u <urlfile>] "
	    "[-c <cachefile>] [-x <command> ...] [-h]\n"),
	    PROGRAM_NAME,
	    PROGRAM_VERSION,
	    argv0);
	std::cout << msg;

	struct arg {
		const char name;
		const std::string longname;
		const std::string params;
		const std::string desc;
	};

	static const std::vector<arg> args = {
		{ 'e', "export-to-opml"  , ""                , _s("export OPML feed to stdout") }                                                 ,
		{ 'r', "refresh-on-start", ""                , _s("refresh feeds on start") }                                                     ,
		{ 'i', "import-from-opml", _s("<file>")      , _s("import OPML file") }                                                           ,
		{ 'u', "url-file"        , _s("<urlfile>")   , _s("read RSS feed URLs from <urlfile>") }                                          ,
		{ 'c', "cache-file"      , _s("<cachefile>") , _s("use <cachefile> as cache file") }                                              ,
		{ 'C', "config-file"     , _s("<configfile>"), _s("read configuration from <configfile>") }                                       ,
		{ 'X', "vacuum"          , ""                , _s("compact the cache") }                                                  ,
		{ 'x', "execute"         , _s("<command>..."), _s("execute list of commands") }                                                   ,
		{ 'q', "quiet"           , ""                , _s("quiet startup") }                                                              ,
		{ 'v', "version"         , ""                , _s("get version information") }                                                    ,
		{ 'l', "log-level"       , _s("<loglevel>")  , _s("write a log with a certain loglevel (valid values: 1 to 6)") }                 ,
		{ 'd', "log-file"        , _s("<logfile>")   , _s("use <logfile> as output log file") }                                           ,
		{ 'E', "export-to-file"  , _s("<file>")      , _s("export list of read articles to <file>") }                                     ,
		{ 'I', "import-from-file", _s("<file>")      , _s("import list of read articles from <file>") }                                   ,
		{ 'h', "help"            , ""                , _s("this help") }
	};

	for (const auto & a : args) {
		std::string longcolumn("-");
		longcolumn += a.name;
		longcolumn += ", --" + a.longname;
		longcolumn += a.params.size() > 0 ? "=" + a.params : "";
		std::cout << "\t" << longcolumn;
		for (unsigned int j = 0; j < utils::gentabs(longcolumn); j++) {
			std::cout << "\t";
		}
		std::cout << a.desc << std::endl;
	}

	::exit(EXIT_FAILURE);
}


void controller::update_visible_feeds() {
	std::lock_guard<std::mutex> feedslock(feeds_mutex);
	v->update_visible_feeds(feeds);
}


bool controller::setup_dirs_xdg(const char *env_home, bool silent) {
	const char *env_xdg_config;
	const char *env_xdg_data;
	std::string xdg_config_dir;
	std::string xdg_data_dir;

	env_xdg_config = ::getenv("XDG_CONFIG_HOME");
	if (env_xdg_config) {
		xdg_config_dir = env_xdg_config;
	} else {
		xdg_config_dir = env_home;
		xdg_config_dir.append(NEWSBEUTER_PATH_SEP);
		xdg_config_dir.append(".config");
	}

	env_xdg_data = ::getenv("XDG_DATA_HOME");
	if (env_xdg_data) {
		xdg_data_dir = env_xdg_data;
	} else {
		xdg_data_dir = env_home;
		xdg_data_dir.append(NEWSBEUTER_PATH_SEP);
		xdg_data_dir.append(".local");
		xdg_data_dir.append(NEWSBEUTER_PATH_SEP);
		xdg_data_dir.append("share");
	}

	xdg_config_dir.append(NEWSBEUTER_PATH_SEP);
	xdg_config_dir.append(NEWSBEUTER_SUBDIR_XDG);

	xdg_data_dir.append(NEWSBEUTER_PATH_SEP);
	xdg_data_dir.append(NEWSBEUTER_SUBDIR_XDG);

	bool config_dir_exists = 0 == access(xdg_config_dir.c_str(), R_OK | X_OK);

	if (!config_dir_exists) {
		if (!silent) {
			std::cerr
				<< strprintf::fmt(
					   _("XDG: configuration directory '%s' not accessible, "
						 "using '%s' instead."),
					   xdg_config_dir,
					   config_dir)
				<< std::endl;
		}

		return false;
	}

	/* Invariant: config dir exists.
	 *
	 * At this point, we're confident we'll be using XDG. We don't check if
	 * data dir exists, because if it doesn't we'll create it. */

	config_dir = xdg_config_dir;

	// create data directory if it doesn't exist
	utils::mkdir_parents(xdg_data_dir, 0700);

	/* in config */
	url_file = config_dir + std::string(NEWSBEUTER_PATH_SEP) + url_file;
	config_file = config_dir + std::string(NEWSBEUTER_PATH_SEP) + config_file;

	/* in data */
	cache_file = xdg_data_dir + std::string(NEWSBEUTER_PATH_SEP) + cache_file;
	lock_file = cache_file + LOCK_SUFFIX;
	queue_file = xdg_data_dir + std::string(NEWSBEUTER_PATH_SEP) + queue_file;
	searchfile = strprintf::fmt("%s%shistory.search", xdg_data_dir, NEWSBEUTER_PATH_SEP);
	cmdlinefile = strprintf::fmt("%s%shistory.cmdline", xdg_data_dir, NEWSBEUTER_PATH_SEP);

	return true;
}


void controller::notify(const std::string& msg) {
	if (cfg.get_configvalue_as_bool("notify-screen")) {
		LOG(level::DEBUG, "controller:notify: notifying screen");
		std::cout << "\033^" << msg << "\033\\";
		std::cout.flush();
	}
	if (cfg.get_configvalue_as_bool("notify-xterm")) {
		LOG(level::DEBUG, "controller:notify: notifying xterm");
		std::cout << "\033]2;" << msg << "\033\\";
		std::cout.flush();
	}
	if (cfg.get_configvalue_as_bool("notify-beep")) {
		LOG(level::DEBUG, "controller:notify: notifying beep");
		::beep();
	}
	if (cfg.get_configvalue("notify-program").length() > 0) {
		std::string prog = cfg.get_configvalue("notify-program");
		LOG(level::DEBUG, "controller:notify: notifying external program `%s'", prog);
		utils::run_command(prog, msg);
	}
}


std::vector<std::shared_ptr<rss_item>> controller::search_for_items(const std::string& query, std::shared_ptr<rss_feed> feed) {
	std::vector<std::shared_ptr<rss_item>> items;
	LOG(level::DEBUG, "controller::search_for_items: setting feed pointers");
	if (feed != nullptr && feed->rssurl().substr(0,6) == "query:") {
		for (auto item : feed->items()) {
			if (!item->deleted()
				&& (item->title().find(query) != std::string::npos
				    || item->description().find(query) != std::string::npos))
			{
				std::shared_ptr<rss_item> newitem(new rss_item(nullptr));
				newitem->set_guid(item->guid());
				newitem->set_title(item->title());
				newitem->set_author(item->author());
				newitem->set_link(item->link());

				newitem->set_pubDate(item->pubDate_timestamp());

				newitem->set_size(item->size());
				newitem->set_unread(item->unread());
				newitem->set_feedurl(item->feedurl());

				newitem->set_enclosure_url(item->enclosure_url());
				newitem->set_enclosure_type(item->enclosure_type());
				newitem->set_enqueued(item->enqueued());
				newitem->set_flags(item->flags());
				newitem->set_base(item->get_base());

				newitem->set_feedptr(item->get_feedptr());
				newitem->set_cache(get_cache());

				items.push_back(newitem);
			}
		}
	} else {
		items = rsscache->search_for_items(query, (feed != nullptr ? feed->rssurl() : ""));
		for (auto item : items) {
			item->set_feedptr(get_feed_by_url(item->feedurl()));
		}
	}
	return items;
}


std::shared_ptr<rss_feed> controller::get_feed_by_url(const std::string& feedurl) {
	for (auto feed : feeds) {
		if (feedurl == feed->rssurl())
			return feed;
	}
	LOG(level::ERROR, "controller:get_feed_by_url failed for %s", feedurl);
	return std::shared_ptr<rss_feed>();
}


void controller::import_read_information(const std::string& readinfofile) {
	std::vector<std::string> guids;

	std::ifstream f(readinfofile.c_str());
	std::string line;
	getline(f,line);
	if (!f.is_open()) {
		return;
	}
	while (f.is_open() && !f.eof()) {
		guids.push_back(line);
		getline(f, line);
	}
	rsscache->mark_items_read_by_guid(guids);
}


void controller::enqueue_url(const std::string& url, std::shared_ptr<rss_feed> feed) {
	bool url_found = false;
	std::fstream f;
	f.open(queue_file.c_str(), std::fstream::in);
	if (f.is_open()) {
		do {
			std::string line;
			getline(f, line);
			if (!f.eof() && line.length() > 0) {
				std::vector<std::string> fields = utils::tokenize_quoted(line);
				if (!fields.empty() && fields[0] == url) {
					url_found = true;
					break;
				}
			}
		} while (!f.eof());
		f.close();
	}
	if (!url_found) {
		f.open(queue_file.c_str(), std::fstream::app | std::fstream::out);
		std::string filename = generate_enqueue_filename(url, feed);
		f << url << " " << stfl::quote(filename) << std::endl;
		f.close();
	}
}


void controller::execute_commands(char ** argv, unsigned int i) {
	if (v->formaction_stack_size() > 0)
		v->pop_current_formaction();
	for (; argv[i]; ++i) {
		LOG(level::DEBUG, "controller::execute_commands: executing `%s'", argv[i]);
		std::string cmd(argv[i]);
		if (cmd == "reload") {
			reload_all(true);
		} else if (cmd == "print-unread") {
			std::cout << strprintf::fmt(_("%u unread articles"), rsscache->get_unread_count()) << std::endl;
		} else {
			std::cerr << strprintf::fmt(_("%s: %s: unknown command"), argv[0], argv[i]) << std::endl;
			::std::exit(EXIT_FAILURE);
		}
	}
}


std::string controller::generate_enqueue_filename(const std::string& url, std::shared_ptr<rss_feed> feed) {
	std::string dlformat = cfg.get_configvalue("download-path");
	if (dlformat[dlformat.length()-1] != NEWSBEUTER_PATH_SEP[0])
		dlformat.append(NEWSBEUTER_PATH_SEP);

	fmtstr_formatter fmt;
	fmt.register_fmt('n', feed->title());
	fmt.register_fmt('h', get_hostname_from_url(url));

	std::string dlpath = fmt.do_format(dlformat);

	char buf[2048];
	snprintf(buf, sizeof(buf), "%s", url.c_str());
	char * base = basename(buf);
	if (!base || strlen(base) == 0) {
		char lbuf[128];
		time_t t = time(nullptr);
		strftime(lbuf, sizeof(lbuf), "%Y-%b-%d-%H%M%S.unknown", localtime(&t));
		dlpath.append(lbuf);
	} else {
		dlpath.append(base);
	}
	return dlpath;
}


void controller::reload_indexes(const std::vector<int>& indexes, bool unattended) {
	scope_measure m1("controller::reload_indexes");
	unsigned int unread_feeds, unread_articles;
	compute_unread_numbers(unread_feeds, unread_articles);

	unsigned long size;
	{
		std::lock_guard<std::mutex> feedslock(feeds_mutex);
		size = feeds.size();
	}

	for (auto idx : indexes) {
		this->reload(idx, size, unattended);
	}

	unsigned int unread_feeds2, unread_articles2;
	compute_unread_numbers(unread_feeds2, unread_articles2);
	bool notify_always = cfg.get_configvalue_as_bool("notify-always");
	if (notify_always || unread_feeds2 != unread_feeds || unread_articles2 != unread_articles) {
		fmtstr_formatter fmt;
		fmt.register_fmt('f', std::to_string(unread_feeds2));
		fmt.register_fmt('n', std::to_string(unread_articles2));
		fmt.register_fmt('d', std::to_string(unread_articles2 - unread_articles));
		fmt.register_fmt('D', std::to_string(unread_feeds2 - unread_feeds));
		this->notify(fmt.do_format(cfg.get_configvalue("notify-format")));
	}
	if (!unattended)
		v->set_status("");
}


void controller::edit_urls_file() {
	const char * editor;

	editor = getenv("VISUAL");
	if (!editor)
		editor = getenv("EDITOR");
	if (!editor)
		editor = "vi";

	std::string cmdline = strprintf::fmt("%s \"%s\"", editor, utils::replace_all(url_file,"\"","\\\""));

	v->push_empty_formaction();
	stfl::reset();

	utils::run_interactively(cmdline, "controller::edit_urls_file");

	v->pop_current_formaction();

	reload_urls_file();
}


std::string controller::prepare_message(unsigned int pos, unsigned int max) {
	if (max > 0) {
		return strprintf::fmt("(%u/%u) ", pos, max);
	}
	return "";
}


void controller::rec_find_rss_outlines(xmlNode * node, std::string tag) {
	while (node) {
		std::string newtag = tag;

		if (strcmp((const char *)node->name, "outline")==0) {
			char * url = (char *)xmlGetProp(node, (const xmlChar *)"xmlUrl");
			if (!url) {
				url = (char *)xmlGetProp(node, (const xmlChar *)"url");
			}

			if (url) {
				LOG(level::DEBUG,"OPML import: found RSS outline with url = %s",url);

				std::string nurl = std::string(url);

				// Liferea uses a pipe to signal feeds read from the output of
				// a program in its OPMLs. Convert them to our syntax.
				if (*url == '|') {
					nurl = strprintf::fmt("exec:%s", url+1);
					LOG(level::DEBUG,"OPML import: liferea-style url %s converted to %s", url, nurl);
				}

				// Handle OPML filters.
				char * filtercmd = (char *)xmlGetProp(node, (const xmlChar *)"filtercmd");
				if (filtercmd) {
					LOG(level::DEBUG,"OPML import: adding filter command %s to url %s", filtercmd, nurl);
					nurl.insert(0, strprintf::fmt("filter:%s:", filtercmd));
					xmlFree(filtercmd);
				}

				xmlFree(url);
				// Filters and scripts may have arguments, so, quote them when needed.
				url = (char*) xmlStrdup((const xmlChar*)utils::quote_if_necessary(nurl).c_str());
				assert(url);

				bool found = false;

				LOG(level::DEBUG, "OPML import: size = %u", urlcfg->get_urls().size());
				if (urlcfg->get_urls().size() > 0) {
					for (auto u : urlcfg->get_urls()) {
						if (u == url) {
							found = true;
						}
					}
				}

				if (!found) {
					LOG(level::DEBUG,"OPML import: added url = %s",url);
					urlcfg->get_urls().push_back(std::string(url));
					if (tag.length() > 0) {
						LOG(level::DEBUG, "OPML import: appending tag %s to url %s", tag, url);
						urlcfg->get_tags(url).push_back(tag);
					}
				} else {
					LOG(level::DEBUG,"OPML import: url = %s is already in list",url);
				}
				xmlFree(url);
			} else {
				char * text = (char *)xmlGetProp(node, (const xmlChar *)"text");
				if (!text)
					text = (char *)xmlGetProp(node, (const xmlChar *)"title");
				if (text) {
					if (newtag.length() > 0) {
						newtag.append("/");
					}
					newtag.append(text);
					xmlFree(text);
				}
			}
		}
		rec_find_rss_outlines(node->children, newtag);

		node = node->next;
	}
}


std::vector<std::shared_ptr<rss_feed>> controller::get_all_feeds() {
	std::vector<std::shared_ptr<rss_feed>> tmpfeeds;
	{
		std::lock_guard<std::mutex> feedslock(feeds_mutex);
		tmpfeeds = feeds;
	}
	return tmpfeeds;
}


void controller::update_feedlist() {
	std::lock_guard<std::mutex> feedslock(feeds_mutex);
	v->set_feedlist(feeds);
}


std::vector<std::shared_ptr<rss_feed>> controller::get_all_feeds_unlocked() {
	return feeds;
}


void controller::write_item(std::shared_ptr<rss_item> item, const std::string& filename) {
	std::fstream f;
	f.open(filename.c_str(),std::fstream::out);
	if (!f.is_open())
		throw exception(errno);

	write_item(item, f);
}


void controller::export_opml() {
	xmlDocPtr root = xmlNewDoc((const xmlChar *)"1.0");
	xmlNodePtr opml_node = xmlNewDocNode(root, nullptr, (const xmlChar *)"opml", nullptr);
	xmlSetProp(opml_node, (const xmlChar *)"version", (const xmlChar *)"1.0");
	xmlDocSetRootElement(root, opml_node);

	xmlNodePtr head = xmlNewTextChild(opml_node, nullptr, (const xmlChar *)"head", nullptr);
	xmlNewTextChild(head, nullptr, (const xmlChar *)"title", (const xmlChar *)PROGRAM_NAME " - Exported Feeds");
	xmlNodePtr body = xmlNewTextChild(opml_node, nullptr, (const xmlChar *)"body", nullptr);

	for (auto feed : feeds) {
		if (!utils::is_special_url(feed->rssurl())) {
			std::string rssurl = feed->rssurl();
			std::string link = feed->link();
			std::string title = feed->title();

			xmlNodePtr outline = xmlNewTextChild(body, nullptr, (const xmlChar *)"outline", nullptr);
			xmlSetProp(outline, (const xmlChar *)"type", (const xmlChar *)"rss");
			xmlSetProp(outline, (const xmlChar *)"xmlUrl", (const xmlChar *)rssurl.c_str());
			xmlSetProp(outline, (const xmlChar *)"htmlUrl", (const xmlChar *)link.c_str());
			xmlSetProp(outline, (const xmlChar *)"title", (const xmlChar *)title.c_str());
		}
	}

	xmlSaveCtxtPtr savectx = xmlSaveToFd(1, nullptr, 1);
	xmlSaveDoc(savectx, root);
	xmlSaveClose(savectx);

	xmlFreeNode(opml_node);
}


void controller::version_information(const char * argv0, unsigned int level) {
	if (level<=1) {
		std::cout << PROGRAM_NAME << " " << PROGRAM_VERSION << " - " << PROGRAM_URL << std::endl;
		std::cout << "Copyright (C) 2006-2015 Andreas Krennmair" << std::endl;
		std::cout << "Copyright (C) 2015-2017 Alexander Batischev" << std::endl;
		std::cout << "Copyright (C) 2006-2017 Newsbeuter contributors" << std::endl;
		std::cout << std::endl;

		std::cout << _("newsbeuter is free software and licensed under the MIT/X Consortium License.") << std::endl;
		std::cout << strprintf::fmt(_("Type `%s -vv' for more information."), argv0) << std::endl << std::endl;

		struct utsname xuts;
		uname(&xuts);
		std::cout << PROGRAM_NAME << " " << PROGRAM_VERSION << std::endl;
		std::cout << "System: " << xuts.sysname << " " << xuts.release << " (" << xuts.machine << ")" << std::endl;
#if defined(__GNUC__) && defined(__VERSION__)
		std::cout << "Compiler: g++ " << __VERSION__ << std::endl;
#endif
		std::cout << "ncurses: " << curses_version() << " (compiled with " << NCURSES_VERSION << ")" << std::endl;
		std::cout << "libcurl: " << curl_version()  << " (compiled with " << LIBCURL_VERSION << ")" << std::endl;
		std::cout << "SQLite: " << sqlite3_libversion() << " (compiled with " << SQLITE_VERSION << ")" << std::endl;
		std::cout << "libxml2: compiled with " << LIBXML_DOTTED_VERSION << std::endl << std::endl;
	} else {
		std::cout << LICENSE_str << std::endl;
	}

	::exit(EXIT_SUCCESS);
}


void controller::mark_deleted(const std::string& guid, bool b) {
	rsscache->mark_item_deleted(guid, b);
}


void ctrl_c_action(int sig) {
	LOG(level::DEBUG,"caught signal %d",sig);
	if (SIGINT == sig) {
		ctrl_c_hit = 1;
	} else {
		stfl::reset();
		utils::remove_fs_lock(lock_file);
		::exit(EXIT_FAILURE);
	}
}


void controller::reload(unsigned int pos, unsigned int max, bool unattended, curl_handle *easyhandle) {
	LOG(level::DEBUG, "controller::reload: pos = %u max = %u", pos, max);
	if (pos < feeds.size()) {
		std::shared_ptr<rss_feed> oldfeed = feeds[pos];
		std::string errmsg;
		if (!unattended)
			v->set_status(strprintf::fmt(_("%sLoading %s..."), prepare_message(pos+1, max), utils::censor_url(oldfeed->rssurl())));

		bool ignore_dl = (cfg.get_configvalue("ignore-mode") == "download");

		rss_parser parser(oldfeed->rssurl(), rsscache, &cfg, ignore_dl ? &ign : nullptr, api);
		parser.set_easyhandle(easyhandle);
		LOG(level::DEBUG, "controller::reload: created parser");
		try {
			oldfeed->set_status(dl_status::DURING_DOWNLOAD);
			std::shared_ptr<rss_feed> newfeed = parser.parse();
			if (newfeed->total_item_count() > 0) {
				std::lock_guard<std::mutex> feedslock(feeds_mutex);
				save_feed(newfeed, pos);

				newfeed->clear_items();

				bool ignore_disp = (cfg.get_configvalue("ignore-mode") == "display");
				std::shared_ptr<rss_feed> feed = rsscache->internalize_rssfeed(oldfeed->rssurl(), ignore_disp ? &ign : nullptr);
				feed->set_tags(urlcfg->get_tags(oldfeed->rssurl()));
				feed->set_order(oldfeed->get_order());
				feeds[pos] = feed;
				enqueue_items(feed);

				oldfeed->clear_items();

				v->notify_itemlist_change(feeds[pos]);
				if (!unattended) {
					v->set_feedlist(feeds);
				}
			} else {
				LOG(level::DEBUG, "controller::reload: feed is empty");
			}
			oldfeed->set_status(dl_status::SUCCESS);
			v->set_status("");
		} catch (const dbexception& e) {
			errmsg = strprintf::fmt(_("Error while retrieving %s: %s"), utils::censor_url(oldfeed->rssurl()), e.what());
		} catch (const std::string& emsg) {
			errmsg = strprintf::fmt(_("Error while retrieving %s: %s"), utils::censor_url(oldfeed->rssurl()), emsg);
		} catch (rsspp::exception& e) {
			errmsg = strprintf::fmt(_("Error while retrieving %s: %s"), utils::censor_url(oldfeed->rssurl()), e.what());
		}
		if (errmsg != "") {
			oldfeed->set_status(dl_status::DL_ERROR);
			v->set_status(errmsg);
			LOG(level::USERERROR, "%s", errmsg);
		}
	} else {
		v->show_error(_("Error: invalid feed!"));
	}
}


std::string controller::write_temporary_item(std::shared_ptr<rss_item> item) {
	char filename[_POSIX_PATH_MAX];
	snprintf(filename, sizeof(filename), "/tmp/newsbeuter-article.XXXXXX");
	int fd = mkstemp(filename);
	if (fd != -1) {
		write_item(item, filename);
		close(fd);
		return std::string(filename);
	} else {
		return "";
	}
}


void controller::save_feed(std::shared_ptr<rss_feed> feed, unsigned int pos) {
	if (!feed->is_empty()) {
		LOG(level::DEBUG, "controller::save_feed: feed is nonempty, saving");
		rsscache->externalize_rssfeed(feed, ign.matches_resetunread(feed->rssurl()));
		LOG(level::DEBUG, "controller::save_feed: after externalize_rssfeed");

		bool ignore_disp = (cfg.get_configvalue("ignore-mode") == "display");
		feed = rsscache->internalize_rssfeed(feed->rssurl(), ignore_disp ? &ign : nullptr);
		LOG(level::DEBUG, "controller::save_feed: after internalize_rssfeed");
		feed->set_tags(urlcfg->get_tags(feed->rssurl()));
		{
			unsigned int order = feeds[pos]->get_order();
			std::lock_guard<std::mutex> itemlock(feeds[pos]->item_mutex);
			feeds[pos]->clear_items();
			feed->set_order(order);
		}
		feeds[pos] = feed;
		v->notify_itemlist_change(feeds[pos]);
	} else {
		LOG(level::DEBUG, "controller::save_feed: feed is empty, not saving");
	}
}


void controller::import_opml(const std::string& filename) {
	xmlDoc * doc = xmlReadFile(filename.c_str(), nullptr, 0);
	if (doc == nullptr) {
		std::cout
			<< strprintf::fmt(_("An error occurred while parsing %s."), filename)
			<< std::endl;
		return;
	}

	xmlNode * root = xmlDocGetRootElement(doc);

	for (xmlNode * node = root->children; node != nullptr; node = node->next) {
		if (strcmp((const char *)node->name, "body")==0) {
			LOG(level::DEBUG, "import_opml: found body");
			rec_find_rss_outlines(node->children, "");
			urlcfg->write_config();
		}
	}

	xmlFreeDoc(doc);
	std::cout << strprintf::fmt(_("Import of %s finished."), filename) << std::endl;
}


void controller::update_flags(std::shared_ptr<rss_item> item) {
	if (api) {
		api->update_article_flags(item->oldflags(), item->flags(), item->guid());
	}
	item->update_flags();
}


std::string controller::bookmark(
		const std::string& url,
		const std::string& title,
		const std::string& description,
		const std::string& feed_title)
{
	std::string bookmark_cmd = cfg.get_configvalue("bookmark-cmd");
	bool is_interactive = cfg.get_configvalue_as_bool("bookmark-interactive");
	if (bookmark_cmd.length() > 0) {
		std::string cmdline = strprintf::fmt("%s '%s' '%s' '%s' '%s'",
		                                       bookmark_cmd,
		                                       utils::replace_all(url,"'", "%27"),
		                                       utils::replace_all(title,"'", "%27"),
		                                       utils::replace_all(description,"'", "%27"),
		                                       utils::replace_all(feed_title,"'", "%27"));

		LOG(level::DEBUG, "controller::bookmark: cmd = %s", cmdline);

		if (is_interactive) {
			v->push_empty_formaction();
			stfl::reset();
			utils::run_interactively(cmdline, "controller::bookmark");
			v->pop_current_formaction();
			return "";
		} else {
			char * my_argv[4];
			my_argv[0] = const_cast<char *>("/bin/sh");
			my_argv[1] = const_cast<char *>("-c");
			my_argv[2] = const_cast<char *>(cmdline.c_str());
			my_argv[3] = nullptr;
			return utils::run_program(my_argv, "");
		}
	} else {
		return _("bookmarking support is not configured. Please set the configuration variable `bookmark-cmd' accordingly.");
	}
}


controller::~controller() {
	delete rsscache;
	delete urlcfg;
	delete api;

	std::lock_guard<std::mutex> feedslock(feeds_mutex);
	for (auto feed : feeds) {
		std::lock_guard<std::mutex> lock(feed->item_mutex);
		feed->clear_items();
	}
	feeds.clear();
}


void controller::load_configfile(const std::string& filename) {
	if (cfgparser.parse(filename, true)) {
		update_config();
	} else {
		v->show_error(strprintf::fmt(_("Error: couldn't open configuration file `%s'!"), filename));
	}
}


void controller::dump_config(const std::string& filename) {
	std::vector<std::string> configlines;
	cfg.dump_config(configlines);
	if (v) {
		v->get_keys()->dump_config(configlines);
	}
	ign.dump_config(configlines);
	filters.dump_config(configlines);
	colorman.dump_config(configlines);
	rxman.dump_config(configlines);
	std::fstream f;
	f.open(filename.c_str(), std::fstream::out);
	if (f.is_open()) {
		for (auto line : configlines) {
			f << line << std::endl;
		}
	}
}


void controller::write_item(std::shared_ptr<rss_item> item, std::ostream& ostr) {
	std::vector<std::pair<LineType, std::string>> lines;
	std::vector<linkpair> links; // not used

	std::string title(_("Title: "));
	title.append(item->title());
	lines.push_back(std::make_pair(LineType::wrappable, title));

	std::string author(_("Author: "));
	author.append(item->author());
	lines.push_back(std::make_pair(LineType::wrappable, author));

	std::string date(_("Date: "));
	date.append(item->pubDate());
	lines.push_back(std::make_pair(LineType::wrappable, date));

	std::string link(_("Link: "));
	link.append(item->link());
	lines.push_back(std::make_pair(LineType::softwrappable, link));

	if (item->enclosure_url() != "") {
		std::string dlurl(_("Podcast Download URL: "));
		dlurl.append(item->enclosure_url());
		lines.push_back(std::make_pair(LineType::softwrappable, dlurl));
	}

	lines.push_back(std::make_pair(LineType::wrappable, std::string("")));

	htmlrenderer rnd(true);
	rnd.render(item->description(), lines, links, item->feedurl());
	textformatter txtfmt;
	txtfmt.add_lines(lines);

	unsigned int width = cfg.get_configvalue_as_int("text-width");
	if (width == 0)
		width = 80;
	ostr << txtfmt.format_text_plain(width) << std::endl;
}


std::string controller::get_hostname_from_url(const std::string& url) {
	xmlURIPtr uri = xmlParseURI(url.c_str());
	std::string hostname;
	if (uri) {
		hostname = uri->server;
		xmlFreeURI(uri);
	}
	return hostname;
}


bool controller::trylock_reload_mutex() {
	if (reload_mutex.try_lock()) {
		LOG(level::DEBUG, "controller::trylock_reload_mutex succeeded");
		return true;
	}
	LOG(level::DEBUG, "controller::trylock_reload_mutex failed");
	return false;
}


void ignore_signal(int sig) {
	LOG(level::WARN, "caught signal %d but ignored it", sig);
}


void controller::mark_all_read(unsigned int pos) {
	if (pos < feeds.size()) {
		scope_measure m("controller::mark_all_read");
		std::lock_guard<std::mutex> feedslock(feeds_mutex);
		std::shared_ptr<rss_feed> feed = feeds[pos];
		if (feed->rssurl().substr(0,6) == "query:") {
			rsscache->mark_all_read(feed);
		} else {
			rsscache->mark_all_read(feed->rssurl());
			if (api) {
				api->mark_all_read(feed->rssurl());
			}
		}
		m.stopover("after rsscache->mark_all_read, before iteration over items");
		std::lock_guard<std::mutex> lock(feed->item_mutex);
		std::vector<std::shared_ptr<rss_item>>& items = feed->items();
		if (items.size() > 0) {
			bool notify = items[0]->feedurl() != feed->rssurl();
			LOG(level::DEBUG, "controller::mark_all_read: notify = %s", notify ? "yes" : "no");
			for (auto item : items) {
				item->set_unread_nowrite_notify(false, notify);
			}
		}
	}
}


std::shared_ptr<rss_feed> controller::get_feed(unsigned int pos) {
	std::lock_guard<std::mutex> feedslock(feeds_mutex);
	if (pos >= feeds.size()) {
		throw std::out_of_range(_("invalid feed index (bug)"));
	}
	std::shared_ptr<rss_feed> feed = feeds[pos];
	return feed;
}


void controller::set_view(view * vv) {
	v = vv;
}


void controller::enqueue_items(std::shared_ptr<rss_feed> feed) {
	if (!cfg.get_configvalue_as_bool("podcast-auto-enqueue"))
		return;
	std::lock_guard<std::mutex> lock(feed->item_mutex);
	for (auto item : feed->items()) {
		if (!item->enqueued() && item->enclosure_url().length() > 0) {
			LOG(level::DEBUG, "controller::enqueue_items: enclosure_url = `%s' enclosure_type = `%s'", item->enclosure_url(), item->enclosure_type());
			if (is_valid_podcast_type(item->enclosure_type()) && utils::is_http_url(item->enclosure_url())) {
				LOG(level::INFO, "controller::enqueue_items: enqueuing `%s'", item->enclosure_url());
				enqueue_url(item->enclosure_url(), feed);
				item->set_enqueued(true);
				rsscache->update_rssitem_unread_and_enqueued(item, feed->rssurl());
			}
		}
	}
}


void controller::update_config() {
	v->set_regexmanager(&rxman);
	v->update_bindings();

	if (colorman.colors_loaded()) {
		v->set_colors(colorman.get_fgcolors(), colorman.get_bgcolors(), colorman.get_attributes());
		v->apply_colors_to_all_formactions();
	}

	if (cfg.get_configvalue("error-log").length() > 0) {
		logger::getInstance().set_errorlogfile(cfg.get_configvalue("error-log"));
	}

}


unsigned int controller::get_pos_of_next_unread(unsigned int pos) {
	std::lock_guard<std::mutex> feedslock(feeds_mutex);
	for (pos++; pos < feeds.size(); pos++) {
		if (feeds[pos]->unread_item_count() > 0)
			break;
	}
	return pos;
}


void controller::reload_urls_file() {
	urlcfg->reload();
	std::vector<std::shared_ptr<rss_feed>> new_feeds;
	unsigned int i = 0;

	for (auto url : urlcfg->get_urls()) {
		bool found = false;
		for (auto feed : feeds) {
			if (url == feed->rssurl()) {
				found = true;
				feed->set_tags(urlcfg->get_tags(url));
				feed->set_order(i);
				new_feeds.push_back(feed);
				break;
			}
		}
		if (!found) {
			try {
				bool ignore_disp = (cfg.get_configvalue("ignore-mode") == "display");
				std::shared_ptr<rss_feed> new_feed = rsscache->internalize_rssfeed(url, ignore_disp ? &ign : nullptr);
				new_feed->set_tags(urlcfg->get_tags(url));
				new_feed->set_order(i);
				new_feeds.push_back(new_feed);
			} catch(const dbexception& e) {
				LOG(level::ERROR, "controller::reload_urls_file: caught exception: %s", e.what());
				throw;
			}
		}
		i++;
	}

	v->set_tags(urlcfg->get_alltags());

	{
		std::lock_guard<std::mutex> feedslock(feeds_mutex);
		feeds = new_feeds;
	}

	sort_feeds();

	update_feedlist();
}


controller::controller() : v(0), urlcfg(0), rsscache(0), url_file("urls"), cache_file("cache.db"), config_file("config"), queue_file("queue"), refresh_on_start(false), api(0) {
}


void controller::sort_feeds() {
	std::lock_guard<std::mutex> feedslock(feeds_mutex);
	std::vector<std::string> sortmethod_info = utils::tokenize(cfg.get_configvalue("feed-sort-order"), "-");
	std::string sortmethod = sortmethod_info[0];
	std::string direction = "desc";
	if (sortmethod_info.size() > 1)
		direction = sortmethod_info[1];
	if (sortmethod == "none") {
		std::stable_sort(feeds.begin(), feeds.end(), [](std::shared_ptr<rss_feed> a, std::shared_ptr<rss_feed> b) {
			return a->get_order() < b->get_order();
		});
	} else if (sortmethod == "firsttag") {
		std::stable_sort(feeds.begin(), feeds.end(), [](std::shared_ptr<rss_feed> a, std::shared_ptr<rss_feed> b) {
			if (a->get_firsttag().length() == 0 || b->get_firsttag().length() == 0) {
				return a->get_firsttag().length() > b->get_firsttag().length();
			}
			return strcasecmp(a->get_firsttag().c_str(), b->get_firsttag().c_str()) < 0;
		});
	} else if (sortmethod == "title") {
		std::stable_sort(feeds.begin(), feeds.end(), [](std::shared_ptr<rss_feed> a, std::shared_ptr<rss_feed> b) {
			return strcasecmp(a->title().c_str(), b->title().c_str()) < 0;
		});
	} else if (sortmethod == "articlecount") {
		std::stable_sort(feeds.begin(), feeds.end(), [](std::shared_ptr<rss_feed> a, std::shared_ptr<rss_feed> b) {
			return a->total_item_count() < b->total_item_count();
		});
	} else if (sortmethod == "unreadarticlecount") {
		std::stable_sort(feeds.begin(), feeds.end(), [](std::shared_ptr<rss_feed> a, std::shared_ptr<rss_feed> b) {
			return a->unread_item_count() < b->unread_item_count();
		});
	}
	if (direction == "asc") {
		std::reverse(feeds.begin(), feeds.end());
	}
}


void controller::mark_article_read(const std::string& guid, bool read) {
	if (api) {
		api->mark_article_read(guid, read);
	}
}


comics: make the files containing "--checkpoint-action=" unsupported

Fixes #257

comics_document_load (EvDocument *document,
		      const char *uri,
		      GError    **error)
{
	ComicsDocument *comics_document = COMICS_DOCUMENT (document);
	GSList *supported_extensions;
	gchar *std_out;
	gchar *mime_type;
	gchar **cb_files, *cb_file;
	gboolean success;
	int i, retval;
	GError *err = NULL;

	comics_document->archive = g_filename_from_uri (uri, NULL, error);
	if (!comics_document->archive)
		return FALSE;

	mime_type = ev_file_get_mime_type (uri, FALSE, &err);
	if (!mime_type) {
		if (err) {
			g_propagate_error (error, err);
		} else {
			g_set_error_literal (error,
					     EV_DOCUMENT_ERROR,
					     EV_DOCUMENT_ERROR_INVALID,
					     _("Unknown MIME Type"));
		}

		return FALSE;
	}

	if (!comics_check_decompress_command (mime_type, comics_document,
	error)) {
		g_free (mime_type);
		return FALSE;
	} else if (!comics_generate_command_lines (comics_document, error)) {
		   g_free (mime_type);
		return FALSE;
	}

	g_free (mime_type);

	/* Get list of files in archive */
	success = g_spawn_command_line_sync (comics_document->list_command,
					     &std_out, NULL, &retval, error);

	if (!success) {
		return FALSE;
	} else if (!WIFEXITED(retval) || WEXITSTATUS(retval) != EXIT_SUCCESS) {
		g_set_error_literal (error,
                                     EV_DOCUMENT_ERROR,
                                     EV_DOCUMENT_ERROR_INVALID,
                                     _("File corrupted"));
		return FALSE;
	}

	/* FIXME: is this safe against filenames containing \n in the archive ? */
	cb_files = g_strsplit (std_out, EV_EOL, 0);

	g_free (std_out);

	if (!cb_files) {
		g_set_error_literal (error,
				     EV_DOCUMENT_ERROR,
				     EV_DOCUMENT_ERROR_INVALID,
				     _("No files in archive"));
		return FALSE;
	}

        comics_document->page_names = g_ptr_array_sized_new (64);

	supported_extensions = get_supported_image_extensions ();
	for (i = 0; cb_files[i] != NULL; i++) {
		if (comics_document->offset != NO_OFFSET) {
			if (g_utf8_strlen (cb_files[i],-1) >
			    comics_document->offset) {
				cb_file =
					g_utf8_offset_to_pointer (cb_files[i],
						       comics_document->offset);
			} else {
				continue;
			}
		} else {
			cb_file = cb_files[i];
		}
		gchar *suffix = g_strrstr (cb_file, ".");
		if (!suffix)
			continue;
		suffix = g_ascii_strdown (suffix + 1, -1);
		if (g_slist_find_custom (supported_extensions, suffix,
					 (GCompareFunc) strcmp) != NULL) {
                        g_ptr_array_add (comics_document->page_names,
                                         g_strstrip (g_strdup (cb_file)));
		}
		g_free (suffix);
	}
	g_strfreev (cb_files);
	g_slist_foreach (supported_extensions, (GFunc) g_free, NULL);
	g_slist_free (supported_extensions);

	if (comics_document->page_names->len == 0) {
		g_set_error (error,
			     EV_DOCUMENT_ERROR,
			     EV_DOCUMENT_ERROR_INVALID,
			     _("No images found in archive %s"),
			     uri);
		return FALSE;
	}

        /* Now sort the pages */
        g_ptr_array_sort (comics_document->page_names, sort_page_names);

	return TRUE;
}


extract_argv (EvDocument *document, gint page)
{
	ComicsDocument *comics_document = COMICS_DOCUMENT (document);
	char **argv;
	char *command_line, *quoted_archive, *quoted_filename;
	GError *err = NULL;

	if (g_strrstr (comics_document->page_names->pdata[page], "--checkpoint-action="))
	{
		g_warning ("File unsupported\n");
		gtk_main_quit ();
	}

        if (page >= comics_document->page_names->len)
                return NULL;

	if (comics_document->regex_arg) {
		quoted_archive = g_shell_quote (comics_document->archive);
		quoted_filename =
			comics_regex_quote (comics_document->page_names->pdata[page]);
	} else {
		quoted_archive = g_shell_quote (comics_document->archive);
		quoted_filename = g_shell_quote (comics_document->page_names->pdata[page]);
	}

	command_line = g_strdup_printf ("%s %s %s",
					comics_document->extract_command,
					quoted_archive,
					quoted_filename);
	g_free (quoted_archive);
	g_free (quoted_filename);

	g_shell_parse_argv (command_line, NULL, &argv, &err);
	g_free (command_line);

	if (err) {
		g_warning (_("Error %s"), err->message);
		g_error_free (err);
		return NULL;
	}

	return argv;
}


comics_check_decompress_command	(gchar          *mime_type,
				 ComicsDocument *comics_document,
				 GError         **error)
{
	gboolean success;
	gchar *std_out, *std_err;
	gint retval;
	GError *err = NULL;

	/* FIXME, use proper cbr/cbz mime types once they're
	 * included in shared-mime-info */

	if (g_content_type_is_a (mime_type, "application/x-cbr") ||
	    g_content_type_is_a (mime_type, "application/x-rar")) {
	        /* The RARLAB provides a no-charge proprietary (freeware)
	        * decompress-only client for Linux called unrar. Another
		* option is a GPLv2-licensed command-line tool developed by
		* the Gna! project. Confusingly enough, the free software RAR
		* decoder is also named unrar. For this reason we need to add
		* some lines for disambiguation. Sorry for the added the
		* complexity but it's life :)
		* Finally, some distributions, like Debian, rename this free
		* option as unrar-free.
		* */
		comics_document->selected_command =
					g_find_program_in_path ("unrar");
		if (comics_document->selected_command) {
			/* We only use std_err to avoid printing useless error
			 * messages on the terminal */
			success =
				g_spawn_command_line_sync (
				              comics_document->selected_command,
							   &std_out, &std_err,
							   &retval, &err);
			if (!success) {
				g_propagate_error (error, err);
				g_error_free (err);
				return FALSE;
			/* I don't check retval status because RARLAB unrar
			 * doesn't have a way to return 0 without involving an
			 * operation with a file*/
			} else if (WIFEXITED (retval)) {
				if (g_strrstr (std_out,"freeware") != NULL)
					/* The RARLAB freeware client */
					comics_document->command_usage = RARLABS;
				else
					/* The Gna! free software client */
					comics_document->command_usage = GNAUNRAR;

				g_free (std_out);
				g_free (std_err);
				return TRUE;
			}
		}
		/* The Gna! free software client with Debian naming convention */
		comics_document->selected_command =
				g_find_program_in_path ("unrar-free");
		if (comics_document->selected_command) {
			comics_document->command_usage = GNAUNRAR;
			return TRUE;
		}
		comics_document->selected_command =
				g_find_program_in_path ("lsar");
		if (comics_document->selected_command) {
			comics_document->command_usage = UNARCHIVER;
			return TRUE;
		}
		comics_document->selected_command =
				g_find_program_in_path ("bsdtar");
		if (comics_document->selected_command) {
			comics_document->command_usage = TAR;
			return TRUE;
		}

	} else if (g_content_type_is_a (mime_type, "application/x-cbz") ||
		   g_content_type_is_a (mime_type, "application/zip")) {
		/* InfoZIP's unzip program */
		comics_document->selected_command =
				g_find_program_in_path ("unzip");
		comics_document->alternative_command =
				g_find_program_in_path ("zipnote");
		if (comics_document->selected_command &&
		    comics_document->alternative_command) {
			comics_document->command_usage = UNZIP;
			return TRUE;
		}
		comics_document->selected_command =
				g_find_program_in_path ("bsdtar");
		if (comics_document->selected_command) {
			comics_document->command_usage = TAR;
			return TRUE;
		}
		comics_document->selected_command =
				g_find_program_in_path ("lsar");
		if (comics_document->selected_command) {
			comics_document->command_usage = UNARCHIVER;
			return TRUE;
		}

	} else if (g_content_type_is_a (mime_type, "application/x-cb7") ||
		   g_content_type_is_a (mime_type, "application/x-7z-compressed")) {
		/* 7zr, 7za and 7z are the commands from the p7zip project able
		 * to decompress .7z files */
		comics_document->selected_command =
			g_find_program_in_path ("7zr");
		if (comics_document->selected_command) {
			comics_document->command_usage = P7ZIP;
			return TRUE;
		}
		comics_document->selected_command =
			g_find_program_in_path ("7za");
		if (comics_document->selected_command) {
			comics_document->command_usage = P7ZIP;
			return TRUE;
		}
		comics_document->selected_command =
			g_find_program_in_path ("7z");
		if (comics_document->selected_command) {
			comics_document->command_usage = P7ZIP;
			return TRUE;
		}
		comics_document->selected_command =
				g_find_program_in_path ("bsdtar");
		if (comics_document->selected_command) {
			comics_document->command_usage = TAR;
			return TRUE;
		}
		comics_document->selected_command =
				g_find_program_in_path ("lsar");
		if (comics_document->selected_command) {
			comics_document->command_usage = UNARCHIVER;
			return TRUE;
		}
	} else if (g_content_type_is_a (mime_type, "application/x-cbt") ||
		   g_content_type_is_a (mime_type, "application/x-tar")) {
		/* tar utility (Tape ARchive) */
		comics_document->selected_command =
				g_find_program_in_path ("tar");
		if (comics_document->selected_command) {
			comics_document->command_usage = TAR;
			return TRUE;
		}
		comics_document->selected_command =
				g_find_program_in_path ("bsdtar");
		if (comics_document->selected_command) {
			comics_document->command_usage = TAR;
			return TRUE;
		}
		comics_document->selected_command =
				g_find_program_in_path ("lsar");
		if (comics_document->selected_command) {
			comics_document->command_usage = UNARCHIVER;
			return TRUE;
		}
	} else {
		g_set_error (error,
			     EV_DOCUMENT_ERROR,
			     EV_DOCUMENT_ERROR_INVALID,
			     _("Not a comic book MIME type: %s"),
			     mime_type);
			     return FALSE;
	}
	g_set_error_literal (error,
			     EV_DOCUMENT_ERROR,
			     EV_DOCUMENT_ERROR_INVALID,
			     _("Can't find an appropriate command to "
			     "decompress this type of comic book"));
	return FALSE;
}


comics_document_thumbnails_get_dimensions (EvDocumentThumbnails *document,
					   EvRenderContext      *rc,
					   gint                 *width,
					   gint                 *height)
{
	gdouble page_width, page_height;

	comics_document_get_page_size (EV_DOCUMENT (document), rc->page,
				       &page_width, &page_height);

	if (rc->rotation == 90 || rc->rotation == 270) {
		*width = (gint) (page_height * rc->scale);
		*height = (gint) (page_width * rc->scale);
	} else {
		*width = (gint) (page_width * rc->scale);
		*height = (gint) (page_height * rc->scale);
	}
}


comics_document_render_pixbuf (EvDocument      *document,
			       EvRenderContext *rc)
{
	GdkPixbufLoader *loader;
	GdkPixbuf *rotated_pixbuf, *tmp_pixbuf;
	char **argv;
	guchar buf[4096];
	gboolean success;
	gint outpipe = -1;
	GPid child_pid;
	gssize bytes;
	gint width, height;
	gchar *filename;
	ComicsDocument *comics_document = COMICS_DOCUMENT (document);

	if (!comics_document->decompress_tmp) {
		argv = extract_argv (document, rc->page->index);
		success = g_spawn_async_with_pipes (NULL, argv, NULL,
						    G_SPAWN_SEARCH_PATH |
						    G_SPAWN_STDERR_TO_DEV_NULL,
						    NULL, NULL,
						    &child_pid,
						    NULL, &outpipe, NULL, NULL);
		g_strfreev (argv);
		g_return_val_if_fail (success == TRUE, NULL);

		loader = gdk_pixbuf_loader_new ();
		g_signal_connect (loader, "size-prepared",
				  G_CALLBACK (render_pixbuf_size_prepared_cb),
				  &rc->scale);

		while (outpipe >= 0) {
			bytes = read (outpipe, buf, 4096);

			if (bytes > 0) {
				gdk_pixbuf_loader_write (loader, buf, bytes,
				NULL);
			} else if (bytes <= 0) {
				close (outpipe);
				gdk_pixbuf_loader_close (loader, NULL);
				outpipe = -1;
			}
		}
		tmp_pixbuf = gdk_pixbuf_loader_get_pixbuf (loader);
		rotated_pixbuf =
			gdk_pixbuf_rotate_simple (tmp_pixbuf,
						  360 - rc->rotation);
		g_spawn_close_pid (child_pid);
		g_object_unref (loader);
	} else {
		filename =
			g_build_filename (comics_document->dir,
                                          (char *) comics_document->page_names->pdata[rc->page->index],
					  NULL);

		gdk_pixbuf_get_file_info (filename, &width, &height);

		tmp_pixbuf =
			gdk_pixbuf_new_from_file_at_size (
				    filename, width * (rc->scale) + 0.5,
				    height * (rc->scale) + 0.5, NULL);
		rotated_pixbuf =
			gdk_pixbuf_rotate_simple (tmp_pixbuf,
						  360 - rc->rotation);
		g_free (filename);
		g_object_unref (tmp_pixbuf);
	}
	return rotated_pixbuf;
}


render_pixbuf_size_prepared_cb (GdkPixbufLoader *loader,
				gint             width,
				gint             height,
				gpointer         data)
{
	double *scale = data;
	int w = (width  * (*scale) + 0.5);
	int h = (height * (*scale) + 0.5);

	gdk_pixbuf_loader_set_size (loader, w, h);
}


comics_document_thumbnails_get_thumbnail (EvDocumentThumbnails *document,
					  EvRenderContext      *rc,
					  gboolean              border)
{
	GdkPixbuf *thumbnail;

	thumbnail = comics_document_render_pixbuf (EV_DOCUMENT (document), rc);

	if (border) {
	      GdkPixbuf *tmp_pixbuf = thumbnail;

	      thumbnail = ev_document_misc_get_thumbnail_frame (-1, -1, tmp_pixbuf);
	      g_object_unref (tmp_pixbuf);
	}

	return thumbnail;
}


comics_generate_command_lines (ComicsDocument *comics_document,
			       GError         **error)
{
	gchar *quoted_file, *quoted_file_aux;
	gchar *quoted_command;
	ComicBookDecompressType type;

	type = comics_document->command_usage;
	comics_document->regex_arg = command_usage_def[type].regex_arg;
	quoted_command = g_shell_quote (comics_document->selected_command);
	if (comics_document->regex_arg) {
		quoted_file = comics_regex_quote (comics_document->archive);
		quoted_file_aux = g_shell_quote (comics_document->archive);
		comics_document->list_command =
			   g_strdup_printf (command_usage_def[type].list,
			                    comics_document->alternative_command,
			                    quoted_file_aux);
		g_free (quoted_file_aux);
	} else {
		quoted_file = g_shell_quote (comics_document->archive);
		comics_document->list_command =
				g_strdup_printf (command_usage_def[type].list,
				                 quoted_command, quoted_file);
	}
	comics_document->extract_command =
			    g_strdup_printf (command_usage_def[type].extract,
				             quoted_command);
	comics_document->offset = command_usage_def[type].offset;
	if (command_usage_def[type].decompress_tmp) {
		comics_document->dir = ev_mkdtemp ("atril-comics-XXXXXX", error);
                if (comics_document->dir == NULL)
                        return FALSE;

		/* unrar-free can't create directories, but ev_mkdtemp already created the dir */

		comics_document->decompress_tmp =
			g_strdup_printf (command_usage_def[type].decompress_tmp,
					 quoted_command, quoted_file,
					 comics_document->dir);
		g_free (quoted_file);
		g_free (quoted_command);

		if (!comics_decompress_temp_dir (comics_document->decompress_tmp,
		    comics_document->selected_command, error))
			return FALSE;
		else
			return TRUE;
	} else {
		g_free (quoted_file);
		g_free (quoted_command);
		return TRUE;
	}

}


comics_document_document_thumbnails_iface_init (EvDocumentThumbnailsInterface *iface)
{
	iface->get_thumbnail = comics_document_thumbnails_get_thumbnail;
	iface->get_dimensions = comics_document_thumbnails_get_dimensions;
}


sort_page_names (gconstpointer a,
                 gconstpointer b)
{
	const char *name_1, *name_2;
	gchar *key_1, *key_2;
	gboolean sort_last_1, sort_last_2;
	int compare;

	name_1 = * (const char **) a;
	name_2 = * (const char **) b;

	#define SORT_LAST_CHAR1 '.'
	#define SORT_LAST_CHAR2 '#'

	sort_last_1 = name_1[0] == SORT_LAST_CHAR1 || name_1[0] == SORT_LAST_CHAR2;
	sort_last_2 = name_2[0] == SORT_LAST_CHAR1 || name_2[0] == SORT_LAST_CHAR2;

	#undef SORT_LAST_CHAR1
	#undef SORT_LAST_CHAR2

	if (sort_last_1 && !sort_last_2)
	{
		compare = +1;
	}
	else if (!sort_last_1 && sort_last_2)
	{
		compare = -1;
	}
	else
	{
		key_1 = g_utf8_collate_key_for_filename (name_1, -1);
		key_2 = g_utf8_collate_key_for_filename (name_2, -1);

		compare = strcmp (key_1, key_2);

		g_free (key_1);
		g_free (key_2);
	}

	return compare;
}


sungem: switch to use qemu_receive_packet() for loopback

This patch switches to use qemu_receive_packet() which can detect
reentrancy and return early.

This is intended to address CVE-2021-3416.

Cc: Prasad J Pandit <ppandit@redhat.com>
Cc: qemu-stable@nongnu.org
Reviewed-by: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Reviewed-by: Alistair Francis <alistair.francis@wdc.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>

static void sungem_mmio_mif_write(void *opaque, hwaddr addr, uint64_t val,
                                  unsigned size)
{
    SunGEMState *s = opaque;

    if (!(addr <= 0x1c)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Write to unknown MIF register 0x%"HWADDR_PRIx"\n",
                      addr);
        return;
    }

    trace_sungem_mmio_mif_write(addr, val);

    /* Pre-write filter */
    switch (addr) {
    /* Read only registers */
    case MIF_STATUS:
    case MIF_SMACHINE:
        return; /* No actual write */
    case MIF_CFG:
        /* Maintain the RO MDI bits to advertize an MDIO PHY on MDI0 */
        val &= ~MIF_CFG_MDI1;
        val |= MIF_CFG_MDI0;
        break;
    }

    s->mifregs[addr >> 2] = val;

    /* Post write action */
    switch (addr) {
    case MIF_FRAME:
        s->mifregs[addr >> 2] = sungem_mii_op(s, val);
        break;
    }
}

A potential stack overflow via infinite loop issue was found in various NIC emulators of QEMU in versions up to and including 5.2.0. The issue occurs in loopback mode of a NIC wherein reentrant DMA checks get bypassed. A guest user/process may use this flaw to consume CPU cycles or crash the QEMU process on the host resulting in DoS scenario.


static void sungem_send_packet(SunGEMState *s, const uint8_t *buf,
                               int size)
{
    NetClientState *nc = qemu_get_queue(s->nic);

    if (s->macregs[MAC_XIFCFG >> 2] & MAC_XIFCFG_LBCK) {
        qemu_receive_packet(nc, buf, size);
    } else {
        qemu_send_packet(nc, buf, size);
    }
}


static void sungem_process_tx_desc(SunGEMState *s, struct gem_txd *desc)
{
    PCIDevice *d = PCI_DEVICE(s);
    uint32_t len;

    /* If it's a start of frame, discard anything we had in the
     * buffer and start again. This should be an error condition
     * if we had something ... for now we ignore it
     */
    if (desc->control_word & TXDCTRL_SOF) {
        if (s->tx_first_ctl) {
            trace_sungem_tx_unfinished();
        }
        s->tx_size = 0;
        s->tx_first_ctl = desc->control_word;
    }

    /* Grab data size */
    len = desc->control_word & TXDCTRL_BUFSZ;

    /* Clamp it to our max size */
    if ((s->tx_size + len) > MAX_PACKET_SIZE) {
        trace_sungem_tx_overflow();
        len = MAX_PACKET_SIZE - s->tx_size;
    }

    /* Read the data */
    pci_dma_read(d, desc->buffer, &s->tx_data[s->tx_size], len);
    s->tx_size += len;

    /* If end of frame, send packet */
    if (desc->control_word & TXDCTRL_EOF) {
        trace_sungem_tx_finished(s->tx_size);

        /* Handle csum */
        if (s->tx_first_ctl & TXDCTRL_CENAB) {
            sungem_do_tx_csum(s);
        }

        /* Send it */
        sungem_send_packet(s, s->tx_data, s->tx_size);

        /* No more pending packet */
        s->tx_size = 0;
        s->tx_first_ctl = 0;
    }
}


static uint32_t sungem_mii_op(SunGEMState *s, uint32_t val)
{
    uint8_t phy_addr, reg_addr, op;

    /* Ignore not start of frame */
    if ((val >> 30) != 1) {
        trace_sungem_mii_invalid_sof(val >> 30);
        return 0xffff;
    }
    phy_addr = (val & MIF_FRAME_PHYAD) >> 23;
    reg_addr = (val & MIF_FRAME_REGAD) >> 18;
    op = (val & MIF_FRAME_OP) >> 28;
    switch (op) {
    case 1:
        sungem_mii_write(s, phy_addr, reg_addr, val & MIF_FRAME_DATA);
        return val | MIF_FRAME_TALSB;
    case 2:
        return sungem_mii_read(s, phy_addr, reg_addr) | MIF_FRAME_TALSB;
    default:
        trace_sungem_mii_invalid_op(op);
    }
    return 0xffff | MIF_FRAME_TALSB;
}


static void sungem_eval_cascade_irq(SunGEMState *s)
{
    uint32_t stat, mask;

    mask = s->macregs[MAC_TXSTAT >> 2];
    stat = s->macregs[MAC_TXMASK >> 2];
    if (stat & ~mask) {
        sungem_update_status(s, GREG_STAT_TXMAC, true);
    } else {
        sungem_update_status(s, GREG_STAT_TXMAC, false);
    }

    mask = s->macregs[MAC_RXSTAT >> 2];
    stat = s->macregs[MAC_RXMASK >> 2];
    if (stat & ~mask) {
        sungem_update_status(s, GREG_STAT_RXMAC, true);
    } else {
        sungem_update_status(s, GREG_STAT_RXMAC, false);
    }

    mask = s->macregs[MAC_CSTAT >> 2];
    stat = s->macregs[MAC_MCMASK >> 2] & ~MAC_CSTAT_PTR;
    if (stat & ~mask) {
        sungem_update_status(s, GREG_STAT_MAC, true);
    } else {
        sungem_update_status(s, GREG_STAT_MAC, false);
    }
}


static void sungem_mmio_txdma_write(void *opaque, hwaddr addr, uint64_t val,
                                    unsigned size)
{
    SunGEMState *s = opaque;

    if (!(addr < 0x38) && !(addr >= 0x100 && addr <= 0x118)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Write to unknown TXDMA register 0x%"HWADDR_PRIx"\n",
                      addr);
        return;
    }

    trace_sungem_mmio_txdma_write(addr, val);

    /* Pre-write filter */
    switch (addr) {
    /* Read only registers */
    case TXDMA_TXDONE:
    case TXDMA_PCNT:
    case TXDMA_SMACHINE:
    case TXDMA_DPLOW:
    case TXDMA_DPHI:
    case TXDMA_FSZ:
    case TXDMA_FTAG:
        return; /* No actual write */
    }

    s->txdmaregs[addr >> 2] = val;

    /* Post write action */
    switch (addr) {
    case TXDMA_KICK:
        sungem_tx_kick(s);
        break;
    case TXDMA_CFG:
        sungem_update_masks(s);
        break;
    }
}


static void sungem_reset(DeviceState *dev)
{
    SunGEMState *s = SUNGEM(dev);

    sungem_reset_all(s, true);
}


static void sungem_update_masks(SunGEMState *s)
{
    uint32_t sz;

    sz = 1 << (((s->rxdmaregs[RXDMA_CFG >> 2] & RXDMA_CFG_RINGSZ) >> 1) + 5);
    s->rx_mask = sz - 1;

    sz = 1 << (((s->txdmaregs[TXDMA_CFG >> 2] & TXDMA_CFG_RINGSZ) >> 1) + 5);
    s->tx_mask = sz - 1;
}


static uint16_t sungem_mii_read(SunGEMState *s, uint8_t phy_addr,
                                uint8_t reg_addr)
{
    uint16_t val;

    val = __sungem_mii_read(s, phy_addr, reg_addr);

    trace_sungem_mii_read(phy_addr, reg_addr, val);

    return val;
}


static void sungem_do_tx_csum(SunGEMState *s)
{
    uint16_t start, off;
    uint32_t csum;

    start = (s->tx_first_ctl & TXDCTRL_CSTART) >> 15;
    off = (s->tx_first_ctl & TXDCTRL_COFF) >> 21;

    trace_sungem_tx_checksum(start, off);

    if (start > (s->tx_size - 2) || off > (s->tx_size - 2)) {
        trace_sungem_tx_checksum_oob();
        return;
    }

    csum = net_raw_checksum(s->tx_data + start, s->tx_size - start);
    stw_be_p(s->tx_data + off, csum);
}


static void sungem_mmio_pcs_write(void *opaque, hwaddr addr, uint64_t val,
                                  unsigned size)
{
    SunGEMState *s = opaque;

    if (!(addr <= 0x18) && !(addr >= 0x50 && addr <= 0x5c)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Write to unknown PCS register 0x%"HWADDR_PRIx"\n",
                      addr);
        return;
    }

    trace_sungem_mmio_pcs_write(addr, val);

    /* Pre-write filter */
    switch (addr) {
    /* Read only registers */
    case PCS_MIISTAT:
    case PCS_ISTAT:
    case PCS_SSTATE:
        return; /* No actual write */
    }

    s->pcsregs[addr >> 2] = val;
}


static void sungem_tx_kick(SunGEMState *s)
{
    PCIDevice *d = PCI_DEVICE(s);
    uint32_t comp, kick;
    uint32_t txdma_cfg, txmac_cfg, ints;
    uint64_t dbase;

    trace_sungem_tx_kick();

    /* Check that both TX MAC and TX DMA are enabled. We don't
     * handle DMA-less direct FIFO operations (we don't emulate
     * the FIFO at all).
     *
     * A write to TXDMA_KICK while DMA isn't enabled can happen
     * when the driver is resetting the pointer.
     */
    txdma_cfg = s->txdmaregs[TXDMA_CFG >> 2];
    txmac_cfg = s->macregs[MAC_TXCFG >> 2];
    if (!(txdma_cfg & TXDMA_CFG_ENABLE) ||
        !(txmac_cfg & MAC_TXCFG_ENAB)) {
        trace_sungem_tx_disabled();
        return;
    }

    /* XXX Test min frame size register ? */
    /* XXX Test max frame size register ? */

    dbase = s->txdmaregs[TXDMA_DBHI >> 2];
    dbase = (dbase << 32) | s->txdmaregs[TXDMA_DBLOW >> 2];

    comp = s->txdmaregs[TXDMA_TXDONE >> 2] & s->tx_mask;
    kick = s->txdmaregs[TXDMA_KICK >> 2] & s->tx_mask;

    trace_sungem_tx_process(comp, kick, s->tx_mask + 1);

    /* This is rather primitive for now, we just send everything we
     * can in one go, like e1000. Ideally we should do the sending
     * from some kind of background task
     */
    while (comp != kick) {
        struct gem_txd desc;

        /* Read the next descriptor */
        pci_dma_read(d, dbase + comp * sizeof(desc), &desc, sizeof(desc));

        /* Byteswap descriptor */
        desc.control_word = le64_to_cpu(desc.control_word);
        desc.buffer = le64_to_cpu(desc.buffer);
        trace_sungem_tx_desc(comp, desc.control_word, desc.buffer);

        /* Send it for processing */
        sungem_process_tx_desc(s, &desc);

        /* Interrupt */
        ints = GREG_STAT_TXDONE;
        if (desc.control_word & TXDCTRL_INTME) {
            ints |= GREG_STAT_TXINTME;
        }
        sungem_update_status(s, ints, true);

        /* Next ! */
        comp = (comp + 1) & s->tx_mask;
        s->txdmaregs[TXDMA_TXDONE >> 2] = comp;
    }

    /* We sent everything, set status/irq bit */
    sungem_update_status(s, GREG_STAT_TXALL, true);
}


static bool sungem_can_receive(NetClientState *nc)
{
    SunGEMState *s = qemu_get_nic_opaque(nc);
    uint32_t kick, done, rxdma_cfg, rxmac_cfg;
    bool full;

    rxmac_cfg = s->macregs[MAC_RXCFG >> 2];
    rxdma_cfg = s->rxdmaregs[RXDMA_CFG >> 2];

    /* If MAC disabled, can't receive */
    if ((rxmac_cfg & MAC_RXCFG_ENAB) == 0) {
        trace_sungem_rx_mac_disabled();
        return false;
    }
    if ((rxdma_cfg & RXDMA_CFG_ENABLE) == 0) {
        trace_sungem_rx_txdma_disabled();
        return false;
    }

    /* Check RX availability */
    kick = s->rxdmaregs[RXDMA_KICK >> 2];
    done = s->rxdmaregs[RXDMA_DONE >> 2];
    full = sungem_rx_full(s, kick, done);

    trace_sungem_rx_check(!full, kick, done);

    return !full;
}


static uint64_t sungem_mmio_mif_read(void *opaque, hwaddr addr, unsigned size)
{
    SunGEMState *s = opaque;
    uint32_t val;

    if (!(addr <= 0x1c)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Read from unknown MIF register 0x%"HWADDR_PRIx"\n",
                      addr);
        return 0;
    }

    val = s->mifregs[addr >> 2];

    trace_sungem_mmio_mif_read(addr, val);

    return val;
}


static bool sungem_rx_full(SunGEMState *s, uint32_t kick, uint32_t done)
{
    return kick == ((done + 1) & s->rx_mask);
}


static void sungem_update_status(SunGEMState *s, uint32_t bits, bool val)
{
    uint32_t stat;

    stat = s->gregs[GREG_STAT >> 2];
    if (val) {
        stat |= bits;
    } else {
        stat &= ~bits;
    }
    s->gregs[GREG_STAT >> 2] = stat;
    sungem_eval_irq(s);
}


static void sungem_eval_irq(SunGEMState *s)
{
    uint32_t stat, mask;

    mask = s->gregs[GREG_IMASK >> 2];
    stat = s->gregs[GREG_STAT >> 2] & ~GREG_STAT_TXNR;
    if (stat & ~mask) {
        pci_set_irq(PCI_DEVICE(s), 1);
    } else {
        pci_set_irq(PCI_DEVICE(s), 0);
    }
}


static void sungem_reset_rx(SunGEMState *s)
{
    trace_sungem_rx_reset();

    /* XXX Do RXCFG */
    /* XXX Check value */
    s->rxdmaregs[RXDMA_FSZ >> 2] = 0x140;
    s->rxdmaregs[RXDMA_DONE >> 2] = 0;
    s->rxdmaregs[RXDMA_KICK >> 2] = 0;
    s->rxdmaregs[RXDMA_CFG >> 2] = 0x1000010;
    s->rxdmaregs[RXDMA_PTHRESH >> 2] = 0xf8;
    s->rxdmaregs[RXDMA_BLANK >> 2] = 0;

    sungem_update_masks(s);
}


static int sungem_check_rx_mac(SunGEMState *s, const uint8_t *mac, uint32_t crc)
{
    uint32_t rxcfg = s->macregs[MAC_RXCFG >> 2];
    uint32_t mac0, mac1, mac2;

    /* Promisc enabled ? */
    if (rxcfg & MAC_RXCFG_PROM) {
        return rx_match_promisc;
    }

    /* Format MAC address into dwords */
    mac0 = (mac[4] << 8) | mac[5];
    mac1 = (mac[2] << 8) | mac[3];
    mac2 = (mac[0] << 8) | mac[1];

    trace_sungem_rx_mac_check(mac0, mac1, mac2);

    /* Is this a broadcast frame ? */
    if (mac0 == 0xffff && mac1 == 0xffff && mac2 == 0xffff) {
        return rx_match_bcast;
    }

    /* TODO: Implement address filter registers (or we don't care ?) */

    /* Is this a multicast frame ? */
    if (mac[0] & 1) {
        trace_sungem_rx_mac_multicast();

        /* Promisc group enabled ? */
        if (rxcfg & MAC_RXCFG_PGRP) {
            return rx_match_allmcast;
        }

        /* TODO: Check MAC control frames (or we don't care) ? */

        /* Check hash filter (somebody check that's correct ?) */
        if (rxcfg & MAC_RXCFG_HFE) {
            uint32_t hash, idx;

            crc >>= 24;
            idx = (crc >> 2) & 0x3c;
            hash = s->macregs[(MAC_HASH0 + idx) >> 2];
            if (hash & (1 << (15 - (crc & 0xf)))) {
                return rx_match_mcast;
            }
        }
        return rx_no_match;
    }

    /* Main MAC check */
    trace_sungem_rx_mac_compare(s->macregs[MAC_ADDR0 >> 2],
                                s->macregs[MAC_ADDR1 >> 2],
                                s->macregs[MAC_ADDR2 >> 2]);

    if (mac0 == s->macregs[MAC_ADDR0 >> 2] &&
        mac1 == s->macregs[MAC_ADDR1 >> 2] &&
        mac2 == s->macregs[MAC_ADDR2 >> 2]) {
        return rx_match_mac;
    }

    /* Alt MAC check */
    if (mac0 == s->macregs[MAC_ADDR3 >> 2] &&
        mac1 == s->macregs[MAC_ADDR4 >> 2] &&
        mac2 == s->macregs[MAC_ADDR5 >> 2]) {
        return rx_match_altmac;
    }

    return rx_no_match;
}


static ssize_t sungem_receive(NetClientState *nc, const uint8_t *buf,
                              size_t size)
{
    SunGEMState *s = qemu_get_nic_opaque(nc);
    PCIDevice *d = PCI_DEVICE(s);
    uint32_t mac_crc, done, kick, max_fsize;
    uint32_t fcs_size, ints, rxdma_cfg, rxmac_cfg, csum, coff;
    uint8_t smallbuf[60];
    struct gem_rxd desc;
    uint64_t dbase, baddr;
    unsigned int rx_cond;

    trace_sungem_rx_packet(size);

    rxmac_cfg = s->macregs[MAC_RXCFG >> 2];
    rxdma_cfg = s->rxdmaregs[RXDMA_CFG >> 2];
    max_fsize = s->macregs[MAC_MAXFSZ >> 2] & 0x7fff;

    /* If MAC or DMA disabled, can't receive */
    if (!(rxdma_cfg & RXDMA_CFG_ENABLE) ||
        !(rxmac_cfg & MAC_RXCFG_ENAB)) {
        trace_sungem_rx_disabled();
        return 0;
    }

    /* Size adjustment for FCS */
    if (rxmac_cfg & MAC_RXCFG_SFCS) {
        fcs_size = 0;
    } else {
        fcs_size = 4;
    }

    /* Discard frame smaller than a MAC or larger than max frame size
     * (when accounting for FCS)
     */
    if (size < 6 || (size + 4) > max_fsize) {
        trace_sungem_rx_bad_frame_size(size);
        /* XXX Increment error statistics ? */
        return size;
    }

    /* We don't drop too small frames since we get them in qemu, we pad
     * them instead. We should probably use the min frame size register
     * but I don't want to use a variable size staging buffer and I
     * know both MacOS and Linux use the default 64 anyway. We use 60
     * here to account for the non-existent FCS.
     */
    if (size < 60) {
        memcpy(smallbuf, buf, size);
        memset(&smallbuf[size], 0, 60 - size);
        buf = smallbuf;
        size = 60;
    }

    /* Get MAC crc */
    mac_crc = net_crc32_le(buf, ETH_ALEN);

    /* Packet isn't for me ? */
    rx_cond = sungem_check_rx_mac(s, buf, mac_crc);
    if (rx_cond == rx_no_match) {
        /* Just drop it */
        trace_sungem_rx_unmatched();
        return size;
    }

    /* Get ring pointers */
    kick = s->rxdmaregs[RXDMA_KICK >> 2] & s->rx_mask;
    done = s->rxdmaregs[RXDMA_DONE >> 2] & s->rx_mask;

    trace_sungem_rx_process(done, kick, s->rx_mask + 1);

    /* Ring full ? Can't receive */
    if (sungem_rx_full(s, kick, done)) {
        trace_sungem_rx_ringfull();
        return 0;
    }

    /* Note: The real GEM will fetch descriptors in blocks of 4,
     * for now we handle them one at a time, I think the driver will
     * cope
     */

    dbase = s->rxdmaregs[RXDMA_DBHI >> 2];
    dbase = (dbase << 32) | s->rxdmaregs[RXDMA_DBLOW >> 2];

    /* Read the next descriptor */
    pci_dma_read(d, dbase + done * sizeof(desc), &desc, sizeof(desc));

    trace_sungem_rx_desc(le64_to_cpu(desc.status_word),
                         le64_to_cpu(desc.buffer));

    /* Effective buffer address */
    baddr = le64_to_cpu(desc.buffer) & ~7ull;
    baddr |= (rxdma_cfg & RXDMA_CFG_FBOFF) >> 10;

    /* Write buffer out */
    pci_dma_write(d, baddr, buf, size);

    if (fcs_size) {
        /* Should we add an FCS ? Linux doesn't ask us to strip it,
         * however I believe nothing checks it... For now we just
         * do nothing. It's faster this way.
         */
    }

    /* Calculate the checksum */
    coff = (rxdma_cfg & RXDMA_CFG_CSUMOFF) >> 13;
    csum = net_raw_checksum((uint8_t *)buf + coff, size - coff);

    /* Build the updated descriptor */
    desc.status_word = (size + fcs_size) << 16;
    desc.status_word |= ((uint64_t)(mac_crc >> 16)) << 44;
    desc.status_word |= csum;
    if (rx_cond == rx_match_mcast) {
        desc.status_word |= RXDCTRL_HPASS;
    }
    if (rx_cond == rx_match_altmac) {
        desc.status_word |= RXDCTRL_ALTMAC;
    }
    desc.status_word = cpu_to_le64(desc.status_word);

    pci_dma_write(d, dbase + done * sizeof(desc), &desc, sizeof(desc));

    done = (done + 1) & s->rx_mask;
    s->rxdmaregs[RXDMA_DONE >> 2] = done;

    /* XXX Unconditionally set RX interrupt for now. The interrupt
     * mitigation timer might well end up adding more overhead than
     * helping here...
     */
    ints = GREG_STAT_RXDONE;
    if (sungem_rx_full(s, kick, done)) {
        ints |= GREG_STAT_RXNOBUF;
    }
    sungem_update_status(s, ints, true);

    return size;
}


static uint64_t sungem_mmio_pcs_read(void *opaque, hwaddr addr, unsigned size)
{
    SunGEMState *s = opaque;
    uint32_t val;

    if (!(addr <= 0x18) && !(addr >= 0x50 && addr <= 0x5c)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Read from unknown PCS register 0x%"HWADDR_PRIx"\n",
                      addr);
        return 0;
    }

    val = s->pcsregs[addr >> 2];

    trace_sungem_mmio_pcs_read(addr, val);

    return val;
}


static uint64_t sungem_mmio_mac_read(void *opaque, hwaddr addr, unsigned size)
{
    SunGEMState *s = opaque;
    uint32_t val;

    if (!(addr <= 0x134)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Read from unknown MAC register 0x%"HWADDR_PRIx"\n",
                      addr);
        return 0;
    }

    val = s->macregs[addr >> 2];

    trace_sungem_mmio_mac_read(addr, val);

    switch (addr) {
    case MAC_TXSTAT:
        /* Side effect, clear all */
        s->macregs[addr >> 2] = 0;
        sungem_update_status(s, GREG_STAT_TXMAC, false);
        break;
    case MAC_RXSTAT:
        /* Side effect, clear all */
        s->macregs[addr >> 2] = 0;
        sungem_update_status(s, GREG_STAT_RXMAC, false);
        break;
    case MAC_CSTAT:
        /* Side effect, interrupt bits */
        s->macregs[addr >> 2] &= MAC_CSTAT_PTR;
        sungem_update_status(s, GREG_STAT_MAC, false);
        break;
    }

    return val;
}


static void sungem_mii_write(SunGEMState *s, uint8_t phy_addr,
                             uint8_t reg_addr, uint16_t val)
{
    trace_sungem_mii_write(phy_addr, reg_addr, val);

    /* XXX TODO */
}


static void sungem_reset_tx(SunGEMState *s)
{
    trace_sungem_tx_reset();

    /* XXX Do TXCFG */
    /* XXX Check value */
    s->txdmaregs[TXDMA_FSZ >> 2] = 0x90;
    s->txdmaregs[TXDMA_TXDONE >> 2] = 0;
    s->txdmaregs[TXDMA_KICK >> 2] = 0;
    s->txdmaregs[TXDMA_CFG >> 2] = 0x118010;

    sungem_update_masks(s);

    s->tx_size = 0;
    s->tx_first_ctl = 0;
}


static void sungem_mmio_rxdma_write(void *opaque, hwaddr addr, uint64_t val,
                                    unsigned size)
{
    SunGEMState *s = opaque;

    if (!(addr <= 0x28) && !(addr >= 0x100 && addr <= 0x120)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Write to unknown RXDMA register 0x%"HWADDR_PRIx"\n",
                      addr);
        return;
    }

    trace_sungem_mmio_rxdma_write(addr, val);

    /* Pre-write filter */
    switch (addr) {
    /* Read only registers */
    case RXDMA_DONE:
    case RXDMA_PCNT:
    case RXDMA_SMACHINE:
    case RXDMA_DPLOW:
    case RXDMA_DPHI:
    case RXDMA_FSZ:
    case RXDMA_FTAG:
        return; /* No actual write */
    }

    s->rxdmaregs[addr >> 2] = val;

    /* Post write action */
    switch (addr) {
    case RXDMA_KICK:
        trace_sungem_rx_kick(val);
        break;
    case RXDMA_CFG:
        sungem_update_masks(s);
        if ((s->macregs[MAC_RXCFG >> 2] & MAC_RXCFG_ENAB) != 0 &&
            (s->rxdmaregs[RXDMA_CFG >> 2] & RXDMA_CFG_ENABLE) != 0) {
            qemu_flush_queued_packets(qemu_get_queue(s->nic));
        }
        break;
    }
}


static uint64_t sungem_mmio_rxdma_read(void *opaque, hwaddr addr, unsigned size)
{
    SunGEMState *s = opaque;
    uint32_t val;

    if (!(addr <= 0x28) && !(addr >= 0x100 && addr <= 0x120)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Read from unknown RXDMA register 0x%"HWADDR_PRIx"\n",
                      addr);
        return 0;
    }

    val = s->rxdmaregs[addr >> 2];

    trace_sungem_mmio_rxdma_read(addr, val);

    return val;
}


static void sungem_mmio_mac_write(void *opaque, hwaddr addr, uint64_t val,
                                  unsigned size)
{
    SunGEMState *s = opaque;

    if (!(addr <= 0x134)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Write to unknown MAC register 0x%"HWADDR_PRIx"\n",
                      addr);
        return;
    }

    trace_sungem_mmio_mac_write(addr, val);

    /* Pre-write filter */
    switch (addr) {
    /* Read only registers */
    case MAC_TXRST: /* Not technically read-only but will do for now */
    case MAC_RXRST: /* Not technically read-only but will do for now */
    case MAC_TXSTAT:
    case MAC_RXSTAT:
    case MAC_CSTAT:
    case MAC_PATMPS:
    case MAC_SMACHINE:
        return; /* No actual write */
    case MAC_MINFSZ:
        /* 10-bits implemented */
        val &= 0x3ff;
        break;
    }

    s->macregs[addr >> 2] = val;

    /* Post write action */
    switch (addr) {
    case MAC_TXMASK:
    case MAC_RXMASK:
    case MAC_MCMASK:
        sungem_eval_cascade_irq(s);
        break;
    case MAC_RXCFG:
        sungem_update_masks(s);
        if ((s->macregs[MAC_RXCFG >> 2] & MAC_RXCFG_ENAB) != 0 &&
            (s->rxdmaregs[RXDMA_CFG >> 2] & RXDMA_CFG_ENABLE) != 0) {
            qemu_flush_queued_packets(qemu_get_queue(s->nic));
        }
        break;
    }
}


static uint64_t sungem_mmio_txdma_read(void *opaque, hwaddr addr, unsigned size)
{
    SunGEMState *s = opaque;
    uint32_t val;

    if (!(addr < 0x38) && !(addr >= 0x100 && addr <= 0x118)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Read from unknown TXDMA register 0x%"HWADDR_PRIx"\n",
                      addr);
        return 0;
    }

    val = s->txdmaregs[addr >> 2];

    trace_sungem_mmio_txdma_read(addr, val);

    return val;
}


static void sungem_realize(PCIDevice *pci_dev, Error **errp)
{
    DeviceState *dev = DEVICE(pci_dev);
    SunGEMState *s = SUNGEM(pci_dev);
    uint8_t *pci_conf;

    pci_conf = pci_dev->config;

    pci_set_word(pci_conf + PCI_STATUS,
                 PCI_STATUS_FAST_BACK |
                 PCI_STATUS_DEVSEL_MEDIUM |
                 PCI_STATUS_66MHZ);

    pci_set_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID, 0x0);
    pci_set_word(pci_conf + PCI_SUBSYSTEM_ID, 0x0);

    pci_conf[PCI_INTERRUPT_PIN] = 1; /* interrupt pin A */
    pci_conf[PCI_MIN_GNT] = 0x40;
    pci_conf[PCI_MAX_LAT] = 0x40;

    sungem_reset_all(s, true);
    memory_region_init(&s->sungem, OBJECT(s), "sungem", SUNGEM_MMIO_SIZE);

    memory_region_init_io(&s->greg, OBJECT(s), &sungem_mmio_greg_ops, s,
                          "sungem.greg", SUNGEM_MMIO_GREG_SIZE);
    memory_region_add_subregion(&s->sungem, 0, &s->greg);

    memory_region_init_io(&s->txdma, OBJECT(s), &sungem_mmio_txdma_ops, s,
                          "sungem.txdma", SUNGEM_MMIO_TXDMA_SIZE);
    memory_region_add_subregion(&s->sungem, 0x2000, &s->txdma);

    memory_region_init_io(&s->rxdma, OBJECT(s), &sungem_mmio_rxdma_ops, s,
                          "sungem.rxdma", SUNGEM_MMIO_RXDMA_SIZE);
    memory_region_add_subregion(&s->sungem, 0x4000, &s->rxdma);

    memory_region_init_io(&s->mac, OBJECT(s), &sungem_mmio_mac_ops, s,
                          "sungem.mac", SUNGEM_MMIO_MAC_SIZE);
    memory_region_add_subregion(&s->sungem, 0x6000, &s->mac);

    memory_region_init_io(&s->mif, OBJECT(s), &sungem_mmio_mif_ops, s,
                          "sungem.mif", SUNGEM_MMIO_MIF_SIZE);
    memory_region_add_subregion(&s->sungem, 0x6200, &s->mif);

    memory_region_init_io(&s->pcs, OBJECT(s), &sungem_mmio_pcs_ops, s,
                          "sungem.pcs", SUNGEM_MMIO_PCS_SIZE);
    memory_region_add_subregion(&s->sungem, 0x9000, &s->pcs);

    pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->sungem);

    qemu_macaddr_default_if_unset(&s->conf.macaddr);
    s->nic = qemu_new_nic(&net_sungem_info, &s->conf,
                          object_get_typename(OBJECT(dev)),
                          dev->id, s);
    qemu_format_nic_info_str(qemu_get_queue(s->nic),
                             s->conf.macaddr.a);
}


static uint16_t __sungem_mii_read(SunGEMState *s, uint8_t phy_addr,
                                  uint8_t reg_addr)
{
    if (phy_addr != s->phy_addr) {
        return 0xffff;
    }
    /* Primitive emulation of a BCM5201 to please the driver,
     * ID is 0x00406210. TODO: Do a gigabit PHY like BCM5400
     */
    switch (reg_addr) {
    case MII_BMCR:
        return 0;
    case MII_PHYID1:
        return 0x0040;
    case MII_PHYID2:
        return 0x6210;
    case MII_BMSR:
        if (qemu_get_queue(s->nic)->link_down) {
            return MII_BMSR_100TX_FD  | MII_BMSR_AUTONEG;
        } else {
            return MII_BMSR_100TX_FD | MII_BMSR_AN_COMP |
                    MII_BMSR_AUTONEG | MII_BMSR_LINK_ST;
        }
    case MII_ANLPAR:
    case MII_ANAR:
        return MII_ANLPAR_TXFD;
    case 0x18: /* 5201 AUX status */
        return 3; /* 100FD */
    default:
        return 0;
    };
}


static void sungem_mmio_greg_write(void *opaque, hwaddr addr, uint64_t val,
                                   unsigned size)
{
    SunGEMState *s = opaque;

    if (!(addr < 0x20) && !(addr >= 0x1000 && addr <= 0x1010)) {
        qemu_log_mask(LOG_GUEST_ERROR,
                      "Write to unknown GREG register 0x%"HWADDR_PRIx"\n",
                      addr);
        return;
    }

    trace_sungem_mmio_greg_write(addr, val);

    /* Pre-write filter */
    switch (addr) {
    /* Read only registers */
    case GREG_SEBSTATE:
    case GREG_STAT:
    case GREG_STAT2:
    case GREG_PCIESTAT:
        return; /* No actual write */
    case GREG_IACK:
        val &= GREG_STAT_LATCH;
        s->gregs[GREG_STAT >> 2] &= ~val;
        sungem_eval_irq(s);
        return; /* No actual write */
    case GREG_PCIEMASK:
        val &= 0x7;
        break;
    }

    s->gregs[addr  >> 2] = val;

    /* Post write action */
    switch (addr) {
    case GREG_IMASK:
        /* Re-evaluate interrupt */
        sungem_eval_irq(s);
        break;
    case GREG_SWRST:
        switch (val & (GREG_SWRST_TXRST | GREG_SWRST_RXRST)) {
        case GREG_SWRST_RXRST:
            sungem_reset_rx(s);
            break;
        case GREG_SWRST_TXRST:
            sungem_reset_tx(s);
            break;
        case GREG_SWRST_RXRST | GREG_SWRST_TXRST:
            sungem_reset_all(s, false);
        }
        break;
    }
}