FreshRSS/app/Services/ImportService.php

<?php
declare(strict_types=1);

/**
 * Provide methods to import files.
 */
class FreshRSS_Import_Service {

	private readonly FreshRSS_CategoryDAO $catDAO;

	private readonly FreshRSS_FeedDAO $feedDAO;

	/** true if success, false otherwise */
	private bool $lastStatus;

	/**
	 * Initialize the service for the given user.
	 */
	public function __construct(?string $username = null) {
		$this->catDAO = FreshRSS_Factory::createCategoryDao($username);
		$this->feedDAO = FreshRSS_Factory::createFeedDao($username);
	}

	/** @return bool true if success, false otherwise */
	public function lastStatus(): bool {
		return $this->lastStatus;
	}

	/**
	 * This method parses and imports an OPML file.
	 *
	 * @param string $opml_file the OPML file content.
	 * @param FreshRSS_Category|null $forced_category force the feeds to be associated to this category.
	 * @param bool $dry_run true to not create categories and feeds in database.
	 * @param bool $trusted_source true when the OPML content is explicitly provided by the user (e.g. local file import);
	 *        false by default, and for content fetched from a remote source (e.g. dynamic OPML categories), in which case
	 *        security-relevant attributes (feed cURL parameters, nested dynamic categories) are ignored.
	 */
	public function importOpml(string $opml_file, ?FreshRSS_Category $forced_category = null, bool $dry_run = false, bool $trusted_source = false): void {
		if (function_exists('set_time_limit')) {
			@set_time_limit(300);
		}
		$this->lastStatus = true;
		$opml_array = [];
		try {
			$libopml = new \marienfressinaud\LibOpml\LibOpml(strict: false);
			/** @var array{body:array<array<mixed>>} $opml_array */
			$opml_array = $libopml->parseString($opml_file);
		} catch (\marienfressinaud\LibOpml\Exception $e) {
			self::log($e->getMessage());
			$this->lastStatus = false;
			return;
		}

		$this->catDAO->checkDefault();
		$default_category = $this->catDAO->getDefault();
		if ($default_category === null) {
			self::log('Cannot get the default category');
			$this->lastStatus = false;
			return;
		}

		// Get the categories by names so we can use this array to retrieve
		// existing categories later.
		$categories = $this->catDAO->listCategories(prePopulateFeeds: false);
		$categories_by_names = [];
		$highest_position = PHP_INT_MIN; // To order the numbers as best as possible in case of negative positions, we choose the lowest possible number first
		foreach ($categories as $category) {
			$categories_by_names[$category->name()] = $category;
			$position = $category->attributeInt('position') ?? PHP_INT_MIN;
			if ($position > $highest_position) {
				$highest_position = $position;
			}
		}
		if ($highest_position === PHP_INT_MIN) {
			// If it's still the same number, default to -1 instead
			$highest_position = -1; // will be incremented to 0
		}

		// Get current numbers of categories and feeds, and the limits to
		// verify the user can import its categories/feeds.
		$nb_categories = count($categories);
		$nb_feeds = count($this->feedDAO->listFeeds());
		$limits = FreshRSS_Context::systemConf()->limits;

		// Process the OPML outlines to get a list of categories and a list of
		// feeds elements indexed by their categories names.
		[$categories_elements, $categories_to_feeds] = $this->loadFromOutlines($opml_array['body'], '');

		foreach ($categories_to_feeds as $category_name => $feeds_elements) {
			$category_element = $categories_elements[$category_name] ?? null;

			$category = null;
			if ($forced_category !== null) {
				// If the category is forced, ignore the actual category name
				$category = $forced_category;
			} elseif (isset($categories_by_names[$category_name])) {
				// If the category already exists, get it from $categories_by_names
				$category = $categories_by_names[$category_name];
			} elseif (is_array($category_element)) {
				// Otherwise, create the category (if possible)
				$limit_reached = $nb_categories >= $limits['max_categories'];
				$can_create_category = FreshRSS_Context::$isCli || !$limit_reached;

				if ($can_create_category) {
					// Import category in the exact order as the outline's placement in the OPML, at the end of positioned categories
					$category = $this->createCategory($category_element, $dry_run, ++$highest_position, $trusted_source);
					if ($category !== null) {
						$categories_by_names[$category->name()] = $category;
						$nb_categories++;
					}
				} else {
					Minz_Log::warning(
						_t('feedback.sub.category.over_max', $limits['max_categories'])
					);
				}
			}

			if ($category === null) {
				// Category can be null if the feeds weren't in a category
				// outline, or if we weren't able to create the category.
				$category = $default_category;
			}

			// Then, create the feeds one by one and attach them to the
			// category we just got.
			foreach ($feeds_elements as $feed_element) {
				$limit_reached = $nb_feeds >= $limits['max_feeds'];
				$can_create_feed = FreshRSS_Context::$isCli || !$limit_reached;
				if (!$can_create_feed) {
					Minz_Log::warning(
						_t('feedback.sub.feed.over_max', $limits['max_feeds'])
					);
					$this->lastStatus = false;
					break;
				}

				if ($this->createFeed($feed_element, $category, $dry_run, $trusted_source) !== null) {
					// TODO what if the feed already exists in the database?
					$nb_feeds++;
				} else {
					$this->lastStatus = false;
				}
			}
		}
	}

	/**
	 * Create a feed from a feed element (i.e. OPML outline).
	 *
	 * @param array<string,string> $feed_elt An OPML element (must be a feed element).
	 * @param FreshRSS_Category $category The category to associate to the feed.
	 * @param bool $dry_run true to not create the feed in database.
	 * @param bool $trusted_source false to ignore security-relevant attributes (feed cURL parameters); see {@see FreshRSS_Import_Service::importOpml()}.
	 * @return FreshRSS_Feed|null The created feed, or null if it failed.
	 */
	private function createFeed(array $feed_elt, FreshRSS_Category $category, bool $dry_run, bool $trusted_source = false): ?FreshRSS_Feed {
		$url = Minz_Helper::htmlspecialchars_utf8($feed_elt['xmlUrl']);
		$name = $feed_elt['text'] ?? $feed_elt['title'] ?? '';
		$name = Minz_Helper::htmlspecialchars_utf8($name);
		$website = Minz_Helper::htmlspecialchars_utf8($feed_elt['htmlUrl'] ?? '');
		$description = Minz_Helper::htmlspecialchars_utf8($feed_elt['description'] ?? '');

		try {
			// Create a Feed object and add it in DB
			$feed = new FreshRSS_Feed($url);
			$feed->_category($category);
			$feed->_name($name);
			$feed->_website($website);
			$feed->_description($description);

			switch (strtolower($feed_elt['type'] ?? '')) {
				case strtolower(FreshRSS_Export_Service::TYPE_HTML_XPATH):
					$feed->_kind(FreshRSS_Feed::KIND_HTML_XPATH);
					break;
				case strtolower(FreshRSS_Export_Service::TYPE_XML_XPATH):
					$feed->_kind(FreshRSS_Feed::KIND_XML_XPATH);
					break;
				case strtolower(FreshRSS_Export_Service::TYPE_JSON_DOTNOTATION):
				case strtolower(FreshRSS_Export_Service::TYPE_JSON_DOTPATH):
					$feed->_kind(FreshRSS_Feed::KIND_JSON_DOTNOTATION);
					break;
				case strtolower(FreshRSS_Export_Service::TYPE_JSONFEED):
					$feed->_kind(FreshRSS_Feed::KIND_JSONFEED);
					break;
				case strtolower(FreshRSS_Export_Service::TYPE_HTML_XPATH_JSON_DOTNOTATION):
					$feed->_kind(FreshRSS_Feed::KIND_HTML_XPATH_JSON_DOTNOTATION);
					break;
				default:
					$feed->_kind(FreshRSS_Feed::KIND_RSS);
					break;
			}

			$feed->_priority(match (strtolower($feed_elt['frss:priority'] ?? '')) {
				FreshRSS_Export_Service::PRIORITY_IMPORTANT => FreshRSS_Feed::PRIORITY_IMPORTANT,
				FreshRSS_Export_Service::PRIORITY_MAIN_STREAM => FreshRSS_Feed::PRIORITY_MAIN_STREAM,
				FreshRSS_Export_Service::PRIORITY_CATEGORY => FreshRSS_Feed::PRIORITY_CATEGORY,
				FreshRSS_Export_Service::PRIORITY_FEED => FreshRSS_Feed::PRIORITY_FEED,
				FreshRSS_Export_Service::PRIORITY_HIDDEN => FreshRSS_Feed::PRIORITY_HIDDEN,
				default => FreshRSS_Feed::PRIORITY_MAIN_STREAM,
			});

			if (isset($feed_elt['frss:unicityCriteria']) && $feed_elt['frss:unicityCriteria'] !== 'id'
				&& preg_match('/^[a-z:_-]{2,64}$/', $feed_elt['frss:unicityCriteria'])) {
				$feed->_attribute('unicityCriteria', $feed_elt['frss:unicityCriteria']);
			}

			if (filter_var($feed_elt['frss:unicityCriteriaForced'] ?? '', FILTER_VALIDATE_BOOLEAN)) {
				$feed->_attribute('unicityCriteriaForced', true);
			}

			if (isset($feed_elt['frss:ttl'])) {
				// Signed refresh interval (TTL); a negative value indicates a muted feed
				$feed->_ttl((int)$feed_elt['frss:ttl']);
			}

			if (isset($feed_elt['frss:cssFullContent'])) {
				$feed->_pathEntries(Minz_Helper::htmlspecialchars_utf8($feed_elt['frss:cssFullContent']));
			}

			if (isset($feed_elt['frss:cssFullContentConditions'])) {
				$feed->_attribute(
					'path_entries_conditions',
					preg_split('/\R/u', $feed_elt['frss:cssFullContentConditions']) ?: []
				);
			}

			if (isset($feed_elt['frss:cssContentFilter']) || isset($feed_elt['frss:cssFullContentFilter'])) {
				$feed->_attribute('path_entries_filter', $feed_elt['frss:cssContentFilter'] ?? $feed_elt['frss:cssFullContentFilter']);
			}

			if (isset($feed_elt['frss:filtersActionRead'])) {
				$feed->_filtersAction(
					'read',
					preg_split('/\R/u', $feed_elt['frss:filtersActionRead']) ?: []
				);
			}

			$xPathSettings = [];
			if (isset($feed_elt['frss:xPathItem'])) {
				$xPathSettings['item'] = $feed_elt['frss:xPathItem'];
			}
			if (isset($feed_elt['frss:xPathItemTitle'])) {
				$xPathSettings['itemTitle'] = $feed_elt['frss:xPathItemTitle'];
			}
			if (isset($feed_elt['frss:xPathItemContent'])) {
				$xPathSettings['itemContent'] = $feed_elt['frss:xPathItemContent'];
			}
			if (isset($feed_elt['frss:xPathItemUri'])) {
				$xPathSettings['itemUri'] = $feed_elt['frss:xPathItemUri'];
			}
			if (isset($feed_elt['frss:xPathItemAuthor'])) {
				$xPathSettings['itemAuthor'] = $feed_elt['frss:xPathItemAuthor'];
			}
			if (isset($feed_elt['frss:xPathItemTimestamp'])) {
				$xPathSettings['itemTimestamp'] = $feed_elt['frss:xPathItemTimestamp'];
			}
			if (isset($feed_elt['frss:xPathItemTimeFormat'])) {
				$xPathSettings['itemTimeFormat'] = $feed_elt['frss:xPathItemTimeFormat'];
			}
			if (isset($feed_elt['frss:xPathItemThumbnail'])) {
				$xPathSettings['itemThumbnail'] = $feed_elt['frss:xPathItemThumbnail'];
			}
			if (isset($feed_elt['frss:xPathItemCategories'])) {
				$xPathSettings['itemCategories'] = $feed_elt['frss:xPathItemCategories'];
			}
			if (isset($feed_elt['frss:xPathItemUid'])) {
				$xPathSettings['itemUid'] = $feed_elt['frss:xPathItemUid'];
			}
			if (!empty($xPathSettings)) {
				$feed->_attribute('xpath', $xPathSettings);
			}

			$jsonSettings = [];
			if (isset($feed_elt['frss:jsonItem'])) {
				$jsonSettings['item'] = $feed_elt['frss:jsonItem'];
			}
			if (isset($feed_elt['frss:jsonItemTitle'])) {
				$jsonSettings['itemTitle'] = $feed_elt['frss:jsonItemTitle'];
			}
			if (isset($feed_elt['frss:jsonItemContent'])) {
				$jsonSettings['itemContent'] = $feed_elt['frss:jsonItemContent'];
			}
			if (isset($feed_elt['frss:jsonItemUri'])) {
				$jsonSettings['itemUri'] = $feed_elt['frss:jsonItemUri'];
			}
			if (isset($feed_elt['frss:jsonItemAuthor'])) {
				$jsonSettings['itemAuthor'] = $feed_elt['frss:jsonItemAuthor'];
			}
			if (isset($feed_elt['frss:jsonItemTimestamp'])) {
				$jsonSettings['itemTimestamp'] = $feed_elt['frss:jsonItemTimestamp'];
			}
			if (isset($feed_elt['frss:jsonItemTimeFormat'])) {
				$jsonSettings['itemTimeFormat'] = $feed_elt['frss:jsonItemTimeFormat'];
			}
			if (isset($feed_elt['frss:jsonItemThumbnail'])) {
				$jsonSettings['itemThumbnail'] = $feed_elt['frss:jsonItemThumbnail'];
			}
			if (isset($feed_elt['frss:jsonItemCategories'])) {
				$jsonSettings['itemCategories'] = $feed_elt['frss:jsonItemCategories'];
			}
			if (isset($feed_elt['frss:jsonItemUid'])) {
				$jsonSettings['itemUid'] = $feed_elt['frss:jsonItemUid'];
			}
			if (!empty($jsonSettings)) {
				$feed->_attribute('json_dotnotation', $jsonSettings);
			}
			$feed->_attribute('xPathToJson', $feed_elt['frss:xPathToJson'] ?? null);

			$curl_params = [];
			if (isset($feed_elt['frss:CURLOPT_COOKIE'])) {
				$curl_params[CURLOPT_COOKIE] = $feed_elt['frss:CURLOPT_COOKIE'];
			}
			if (isset($feed_elt['frss:CURLOPT_COOKIEFILE'])) {
				// Allow only an empty value just to enable the libcurl cookie engine
				$curl_params[CURLOPT_COOKIEFILE] = '';
			}
			if (isset($feed_elt['frss:CURLOPT_FOLLOWLOCATION'])) {
				$curl_params[CURLOPT_FOLLOWLOCATION] = (bool)$feed_elt['frss:CURLOPT_FOLLOWLOCATION'];
			}
			if (isset($feed_elt['frss:CURLOPT_HTTPHEADER'])) {
				$curl_params[CURLOPT_HTTPHEADER] = preg_split('/\R/u', $feed_elt['frss:CURLOPT_HTTPHEADER']) ?: [];
			}
			if (isset($feed_elt['frss:CURLOPT_MAXREDIRS'])) {
				$curl_params[CURLOPT_MAXREDIRS] = (int)$feed_elt['frss:CURLOPT_MAXREDIRS'];
			}
			if (isset($feed_elt['frss:CURLOPT_POST'])) {
				$curl_params[CURLOPT_POST] = (bool)$feed_elt['frss:CURLOPT_POST'];
			}
			if (isset($feed_elt['frss:CURLOPT_POSTFIELDS'])) {
				$curl_params[CURLOPT_POSTFIELDS] = $feed_elt['frss:CURLOPT_POSTFIELDS'];
			}
			if (isset($feed_elt['frss:CURLOPT_PROXY'])) {
				$curl_params[CURLOPT_PROXY] = $feed_elt['frss:CURLOPT_PROXY'];
			}
			if (isset($feed_elt['frss:CURLOPT_PROXYTYPE'])) {
				$curl_params[CURLOPT_PROXYTYPE] = (int)$feed_elt['frss:CURLOPT_PROXYTYPE'];
				if ($curl_params[CURLOPT_PROXYTYPE] === 3) {	// Legacy for NONE
					$curl_params[CURLOPT_PROXYTYPE] = -1;
				}
			}
			if (isset($feed_elt['frss:CURLOPT_USERAGENT'])) {
				$curl_params[CURLOPT_USERAGENT] = $feed_elt['frss:CURLOPT_USERAGENT'];
			}
			// Feed cURL parameters are only honored for OPML content explicitly provided by the user;
			// a remote (dynamic) OPML must not be able to configure the cURL behavior of the instance.
			if ($trusted_source && !empty($curl_params)) {
				$feed->_attribute('curl_params', FreshRSS_http_Util::sanitizeCurlParams($curl_params));
			}

			// Call the extension hook
			/** @var FreshRSS_Feed|null */
			$feed = Minz_ExtensionManager::callHook(Minz_HookType::FeedBeforeInsert, $feed);

			if ($dry_run) {
				if ($feed !== null) {
					$category->addFeed($feed);
				}
				return $feed;
			}

			if ($feed !== null) {
				// addFeedObject checks if feed is already in DB
				$id = $this->feedDAO->addFeedObject($feed);
				if ($id == false) {
					$this->lastStatus = false;
				} else {
					$feed->_id($id);
					$category->addFeed($feed);
					return $feed;
				}
			}
		} catch (FreshRSS_Feed_Exception $e) {
			self::log($e->getMessage());
			$this->lastStatus = false;
		}

		$clean_url = \SimplePie\Misc::url_remove_credentials($url);
		self::log("Cannot create {$clean_url} feed in category {$category->name()}");
		return null;
	}

	/**
	 * Create and return a category.
	 *
	 * @param array<string,string> $category_element An OPML element (must be a category element).
	 * @param bool $dry_run true to not create the category in database.
	 * @param bool $trusted_source false to ignore security-relevant attributes (dynamic OPML); see {@see FreshRSS_Import_Service::importOpml()}.
	 * @return FreshRSS_Category|null The created category, or null if it failed.
	 */
	private function createCategory(array $category_element, bool $dry_run, int $position, bool $trusted_source = false): ?FreshRSS_Category {
		$name = $category_element['text'] ?? $category_element['title'] ?? '';
		$name = Minz_Helper::htmlspecialchars_utf8($name);
		$category = new FreshRSS_Category($name);

		if ($trusted_source && isset($category_element['frss:opmlUrl'])) {
			$opml_url = FreshRSS_http_Util::checkUrl($category_element['frss:opmlUrl']);
			if ($opml_url != '') {
				$category->_kind(FreshRSS_Category::KIND_DYNAMIC_OPML);
				$category->_attribute('opml_url', $opml_url);
			}
		}

		$category->_attribute('position', $position);

		if ($dry_run) {
			return $category;
		}

		$id = $this->catDAO->addCategoryObject($category);
		if ($id !== false) {
			$category->_id($id);
			return $category;
		} else {
			self::log("Cannot create category {$category->name()}");
			$this->lastStatus = false;
			return null;
		}
	}

	/**
	 * Return the list of category and feed outlines by categories names.
	 *
	 * This method is applied to a list of outlines. It merges the different
	 * list of feeds from several outlines into one array.
	 *
	 * @param array<array<mixed>> $outlines The outlines from which to extract the outlines.
	 * @param string $parent_category_name The name of the parent category of the current outlines.
	 * @return array{0:array<string,array<string,string>>,1:array<string,list<array<string,string>>>}
	 */
	private function loadFromOutlines(array $outlines, string $parent_category_name): array {
		$categories_elements = [];
		$categories_to_feeds = [];

		foreach ($outlines as $outline) {
			if (!is_array($outline)) {
				continue;
			}
			// Get the categories and feeds from the child outline (it may
			// return several categories and feeds if the outline is a category).
			[$outline_categories, $outline_categories_to_feeds] = $this->loadFromOutline($outline, $parent_category_name);

			// Then, we merge the initial arrays with the arrays returned by
			// the outline.
			$categories_elements = array_merge($categories_elements, $outline_categories);

			foreach ($outline_categories_to_feeds as $category_name => $feeds) {
				if (!is_string($category_name) ||	// @phpstan-ignore function.alreadyNarrowedType (defensive)
					!is_array($feeds)) {	// @phpstan-ignore function.alreadyNarrowedType (defensive)
					continue;
				}
				if (!isset($categories_to_feeds[$category_name])) {
					$categories_to_feeds[$category_name] = [];
				}
				$categories_to_feeds[$category_name] = array_merge(
					$categories_to_feeds[$category_name],
					$feeds
				);
			}
		}

		return [$categories_elements, $categories_to_feeds];
	}

	/**
	 * Return the list of category and feed outlines by categories names.
	 *
	 * This method is applied to a specific outline. If the outline represents
	 * a category (i.e. @outlines key exists), it will reapply loadFromOutlines()
	 * to its children. If the outline represents a feed (i.e. xmlUrl key
	 * exists), it will add the outline to an array accessible by its category
	 * name.
	 *
	 * @param array<mixed> $outline The outline from which to extract the categories and feeds outlines.
	 * @param string $parent_category_name The name of the parent category of the current outline.
	 *
	 * @return array{0:array<string,array<string,string>>,1:array<string,list<array<string,string>>>}
	 */
	private function loadFromOutline(array $outline, string $parent_category_name): array {
		$categories_elements = [];
		$categories_to_feeds = [];

		if ($parent_category_name === '' && is_array($outline['category'] ?? null)) {
			// The outline has no parent category, but its OPML category
			// attribute is set, so we use it as the category name.
			// lib_opml parses this attribute as an array of strings, so we
			// rebuild a string here.
			$category_names = array_filter($outline['category'], 'is_string');
			$parent_category_name = implode(', ', $category_names);
			$categories_elements[$parent_category_name] = [
				'text' => $parent_category_name,
			];
		}

		if (is_array($outline['@outlines'] ?? null)) {
			// The outline has children, it’s probably a category
			if (!empty($outline['text']) && is_string($outline['text'])) {
				$category_name = $outline['text'];
			} elseif (!empty($outline['title']) && is_string($outline['title'])) {
				$category_name = $outline['title'];
			} else {
				$category_name = $parent_category_name;
			}

			$children = array_filter($outline['@outlines'], 'is_array');
			[$categories_elements, $categories_to_feeds] = $this->loadFromOutlines($children, $category_name);

			unset($outline['@outlines']);
			$categories_elements[$category_name] = array_filter($outline, static fn($value, $key) => is_string($key) && is_string($value), ARRAY_FILTER_USE_BOTH);
		}

		// The xmlUrl means it’s a feed URL: add the outline to the array if it exists.
		if (isset($outline['xmlUrl'])) {
			if (!isset($categories_to_feeds[$parent_category_name])) {
				$categories_to_feeds[$parent_category_name] = [];
			}
			$feed = array_filter($outline, static fn($value, $key) => is_string($key) && is_string($value), ARRAY_FILTER_USE_BOTH);
			$categories_to_feeds[$parent_category_name][] = $feed;
		}

		return [$categories_elements, $categories_to_feeds];
	}

	private static function log(string $message): void {
		if (FreshRSS_Context::$isCli) {
			fwrite(STDERR, "FreshRSS error during OPML import: {$message}\n");
		} else {
			Minz_Log::warning("Error during OPML import: {$message}");
		}
	}
}