Feed.php 33 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091
  1. <?php
  2. class FreshRSS_Feed extends Minz_Model {
  3. /**
  4. * Normal RSS or Atom feed
  5. * @var int
  6. */
  7. const KIND_RSS = 0;
  8. /**
  9. * Invalid RSS or Atom feed
  10. * @var int
  11. */
  12. const KIND_RSS_FORCED = 2;
  13. /**
  14. * Normal HTML with XPath scraping
  15. * @var int
  16. */
  17. const KIND_HTML_XPATH = 10;
  18. /**
  19. * Normal XML with XPath scraping
  20. * @var int
  21. */
  22. const KIND_XML_XPATH = 15;
  23. /**
  24. * Normal JSON with XPath scraping
  25. * @var int
  26. */
  27. const KIND_JSON_XPATH = 20;
  28. const PRIORITY_MAIN_STREAM = 10;
  29. const PRIORITY_NORMAL = 0;
  30. const PRIORITY_ARCHIVED = -10;
  31. const TTL_DEFAULT = 0;
  32. const ARCHIVING_RETENTION_COUNT_LIMIT = 10000;
  33. const ARCHIVING_RETENTION_PERIOD = 'P3M';
  34. /** @var int */
  35. private $id = 0;
  36. /** @var string */
  37. private $url = '';
  38. /** @var int */
  39. private $kind = 0;
  40. /** @var int */
  41. private $categoryId = 1;
  42. /** @var FreshRSS_Category|null */
  43. private $category;
  44. /** @var int */
  45. private $nbEntries = -1;
  46. /** @var int */
  47. private $nbNotRead = -1;
  48. /** @var int */
  49. private $nbPendingNotRead = 0;
  50. /** @var string */
  51. private $name = '';
  52. /** @var string */
  53. private $website = '';
  54. /** @var string */
  55. private $description = '';
  56. /** @var int */
  57. private $lastUpdate = 0;
  58. /** @var int */
  59. private $priority = self::PRIORITY_MAIN_STREAM;
  60. /** @var string */
  61. private $pathEntries = '';
  62. /** @var string */
  63. private $httpAuth = '';
  64. /** @var bool */
  65. private $error = false;
  66. /** @var int */
  67. private $ttl = self::TTL_DEFAULT;
  68. /** @var array<string,mixed> */
  69. private $attributes = [];
  70. /** @var bool */
  71. private $mute = false;
  72. /** @var string */
  73. private $hash = '';
  74. /** @var string */
  75. private $lockPath = '';
  76. /** @var string */
  77. private $hubUrl = '';
  78. /** @var string */
  79. private $selfUrl = '';
  80. /** @var array<FreshRSS_FilterAction> $filterActions */
  81. private $filterActions = null;
  82. public function __construct(string $url, bool $validate = true) {
  83. if ($validate) {
  84. $this->_url($url);
  85. } else {
  86. $this->url = $url;
  87. }
  88. }
  89. public static function example(): FreshRSS_Feed {
  90. $f = new FreshRSS_Feed('http://example.net/', false);
  91. $f->faviconPrepare();
  92. return $f;
  93. }
  94. public function id(): int {
  95. return $this->id;
  96. }
  97. public function hash(): string {
  98. if ($this->hash == '') {
  99. $salt = FreshRSS_Context::$system_conf->salt;
  100. $this->hash = hash('crc32b', $salt . $this->url);
  101. }
  102. return $this->hash;
  103. }
  104. public function url(bool $includeCredentials = true): string {
  105. return $includeCredentials ? $this->url : SimplePie_Misc::url_remove_credentials($this->url);
  106. }
  107. public function selfUrl(): string {
  108. return $this->selfUrl;
  109. }
  110. public function kind(): int {
  111. return $this->kind;
  112. }
  113. public function hubUrl(): string {
  114. return $this->hubUrl;
  115. }
  116. /**
  117. * @return FreshRSS_Category|null|false
  118. */
  119. public function category() {
  120. if ($this->category === null) {
  121. $catDAO = FreshRSS_Factory::createCategoryDao();
  122. $this->category = $catDAO->searchById($this->categoryId);
  123. }
  124. return $this->category;
  125. }
  126. public function categoryId(): int {
  127. return $this->categoryId;
  128. }
  129. /**
  130. * @return array<FreshRSS_Entry>|null
  131. * @deprecated
  132. */
  133. public function entries(): ?array {
  134. Minz_Log::warning(__method__ . ' is deprecated since FreshRSS 1.16.1!');
  135. $simplePie = $this->load(false, true);
  136. return $simplePie == null ? [] : iterator_to_array($this->loadEntries($simplePie));
  137. }
  138. public function name(bool $raw = false): string {
  139. return $raw || $this->name != '' ? $this->name : preg_replace('%^https?://(www[.])?%i', '', $this->url);
  140. }
  141. /** @return string HTML-encoded URL of the Web site of the feed */
  142. public function website(): string {
  143. return $this->website;
  144. }
  145. public function description(): string {
  146. return $this->description;
  147. }
  148. public function lastUpdate(): int {
  149. return $this->lastUpdate;
  150. }
  151. public function priority(): int {
  152. return $this->priority;
  153. }
  154. /** @return string HTML-encoded CSS selector */
  155. public function pathEntries(): string {
  156. return $this->pathEntries;
  157. }
  158. /**
  159. * @phpstan-return ($raw is true ? string : array{'username':string,'password':string})
  160. * @return array{'username':string,'password':string}|string
  161. */
  162. public function httpAuth(bool $raw = true) {
  163. if ($raw) {
  164. return $this->httpAuth;
  165. } else {
  166. $pos_colon = strpos($this->httpAuth, ':');
  167. $user = substr($this->httpAuth, 0, $pos_colon);
  168. $pass = substr($this->httpAuth, $pos_colon + 1);
  169. return array(
  170. 'username' => $user,
  171. 'password' => $pass
  172. );
  173. }
  174. }
  175. public function inError(): bool {
  176. return $this->error;
  177. }
  178. /**
  179. * @param bool $raw true for database version combined with mute information, false otherwise
  180. */
  181. public function ttl(bool $raw = false): int {
  182. if ($raw) {
  183. $ttl = $this->ttl;
  184. if ($this->mute && FreshRSS_Feed::TTL_DEFAULT === $ttl) {
  185. $ttl = FreshRSS_Context::$user_conf ? FreshRSS_Context::$user_conf->ttl_default : 3600;
  186. }
  187. return $ttl * ($this->mute ? -1 : 1);
  188. }
  189. return $this->ttl;
  190. }
  191. /**
  192. * @phpstan-return ($key is non-empty-string ? mixed : array<string,mixed>)
  193. * @return array<string,mixed>|mixed|null
  194. */
  195. public function attributes(string $key = '') {
  196. if ($key === '') {
  197. return $this->attributes;
  198. } else {
  199. return $this->attributes[$key] ?? null;
  200. }
  201. }
  202. public function mute(): bool {
  203. return $this->mute;
  204. }
  205. public function nbEntries(): int {
  206. if ($this->nbEntries < 0) {
  207. $feedDAO = FreshRSS_Factory::createFeedDao();
  208. $this->nbEntries = $feedDAO->countEntries($this->id());
  209. }
  210. return $this->nbEntries;
  211. }
  212. public function nbNotRead(bool $includePending = false): int {
  213. if ($this->nbNotRead < 0) {
  214. $feedDAO = FreshRSS_Factory::createFeedDao();
  215. $this->nbNotRead = $feedDAO->countNotRead($this->id());
  216. }
  217. return $this->nbNotRead + ($includePending ? $this->nbPendingNotRead : 0);
  218. }
  219. public function faviconPrepare(): void {
  220. require_once(LIB_PATH . '/favicons.php');
  221. $url = $this->website;
  222. if ($url == '') {
  223. $url = $this->url;
  224. }
  225. $txt = FAVICONS_DIR . $this->hash() . '.txt';
  226. if (@file_get_contents($txt) !== $url) {
  227. file_put_contents($txt, $url);
  228. }
  229. if (FreshRSS_Context::$isCli) {
  230. $ico = FAVICONS_DIR . $this->hash() . '.ico';
  231. $ico_mtime = @filemtime($ico);
  232. $txt_mtime = @filemtime($txt);
  233. if ($txt_mtime != false &&
  234. ($ico_mtime == false || $ico_mtime < $txt_mtime || ($ico_mtime < time() - (14 * 86400)))) {
  235. // no ico file or we should download a new one.
  236. $url = file_get_contents($txt);
  237. download_favicon($url, $ico) || touch($ico);
  238. }
  239. }
  240. }
  241. public static function faviconDelete(string $hash): void {
  242. $path = DATA_PATH . '/favicons/' . $hash;
  243. @unlink($path . '.ico');
  244. @unlink($path . '.txt');
  245. }
  246. public function favicon(): string {
  247. return Minz_Url::display('/f.php?' . $this->hash());
  248. }
  249. public function _id(int $value): void {
  250. $this->id = $value;
  251. }
  252. public function _url(string $value, bool $validate = true): void {
  253. $this->hash = '';
  254. $url = $value;
  255. if ($validate) {
  256. $url = checkUrl($url);
  257. }
  258. if ($url == '') {
  259. throw new FreshRSS_BadUrl_Exception($value);
  260. }
  261. $this->url = $url;
  262. }
  263. public function _kind(int $value): void {
  264. $this->kind = $value;
  265. }
  266. public function _category(?FreshRSS_Category $cat): void {
  267. $this->category = $cat;
  268. $this->categoryId = $this->category == null ? 0 : $this->category->id();
  269. }
  270. /** @param int|string $id */
  271. public function _categoryId($id): void {
  272. $this->category = null;
  273. $this->categoryId = intval($id);
  274. }
  275. public function _name(string $value): void {
  276. $this->name = $value == '' ? '' : trim($value);
  277. }
  278. public function _website(string $value, bool $validate = true): void {
  279. if ($validate) {
  280. $value = checkUrl($value);
  281. }
  282. if ($value == '') {
  283. $value = '';
  284. }
  285. $this->website = $value;
  286. }
  287. public function _description(string $value): void {
  288. $this->description = $value == '' ? '' : $value;
  289. }
  290. public function _lastUpdate(int $value): void {
  291. $this->lastUpdate = $value;
  292. }
  293. public function _priority(int $value): void {
  294. $this->priority = $value;
  295. }
  296. /** @param string $value HTML-encoded CSS selector */
  297. public function _pathEntries(string $value): void {
  298. $this->pathEntries = $value;
  299. }
  300. public function _httpAuth(string $value): void {
  301. $this->httpAuth = $value;
  302. }
  303. /** @param bool|int $value */
  304. public function _error($value): void {
  305. $this->error = (bool)$value;
  306. }
  307. public function _mute(bool $value): void {
  308. $this->mute = $value;
  309. }
  310. public function _ttl(int $value): void {
  311. $value = min($value, 100000000);
  312. $this->ttl = abs($value);
  313. $this->mute = $value < self::TTL_DEFAULT;
  314. }
  315. /** @param string|array<mixed>|bool|int|null $value Value, not HTML-encoded */
  316. public function _attributes(string $key, $value): void {
  317. if ($key == '') {
  318. if (is_string($value)) {
  319. $value = json_decode($value, true);
  320. }
  321. if (is_array($value)) {
  322. $this->attributes = $value;
  323. }
  324. } elseif ($value === null) {
  325. unset($this->attributes[$key]);
  326. } else {
  327. $this->attributes[$key] = $value;
  328. }
  329. }
  330. public function _nbNotRead(int $value): void {
  331. $this->nbNotRead = $value;
  332. }
  333. public function _nbEntries(int $value): void {
  334. $this->nbEntries = $value;
  335. }
  336. public function load(bool $loadDetails = false, bool $noCache = false): ?SimplePie {
  337. if ($this->url != '') {
  338. // @phpstan-ignore-next-line
  339. if (CACHE_PATH === false) {
  340. throw new Minz_FileNotExistException(
  341. 'CACHE_PATH',
  342. Minz_Exception::ERROR
  343. );
  344. } else {
  345. $url = htmlspecialchars_decode($this->url, ENT_QUOTES);
  346. if ($this->httpAuth != '') {
  347. $url = preg_replace('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $url);
  348. }
  349. $simplePie = customSimplePie($this->attributes());
  350. if (substr($url, -11) === '#force_feed') {
  351. $simplePie->force_feed(true);
  352. $url = substr($url, 0, -11);
  353. }
  354. $simplePie->set_feed_url($url);
  355. if (!$loadDetails) { //Only activates auto-discovery when adding a new feed
  356. $simplePie->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE);
  357. }
  358. if ($this->attributes('clear_cache')) {
  359. // Do not use `$simplePie->enable_cache(false);` as it would prevent caching in multiuser context
  360. $this->clearCache();
  361. }
  362. Minz_ExtensionManager::callHook('simplepie_before_init', $simplePie, $this);
  363. $mtime = $simplePie->init();
  364. if ((!$mtime) || $simplePie->error()) {
  365. $errorMessage = $simplePie->error();
  366. throw new FreshRSS_Feed_Exception(
  367. ($errorMessage == '' ? 'Unknown error for feed' : $errorMessage) . ' [' . $this->url . ']',
  368. $simplePie->status_code()
  369. );
  370. }
  371. $links = $simplePie->get_links('self');
  372. $this->selfUrl = empty($links[0]) ? '' : checkUrl($links[0]);
  373. if ($this->selfUrl == false) {
  374. $this->selfUrl = '';
  375. }
  376. $links = $simplePie->get_links('hub');
  377. $this->hubUrl = empty($links[0]) ? '' : checkUrl($links[0]);
  378. if ($this->hubUrl == false) {
  379. $this->hubUrl = '';
  380. }
  381. if ($loadDetails) {
  382. // si on a utilisé l’auto-discover, notre url va avoir changé
  383. $subscribe_url = $simplePie->subscribe_url(false);
  384. //HTML to HTML-PRE //ENT_COMPAT except '&'
  385. $title = strtr(html_only_entity_decode($simplePie->get_title()), array('<' => '&lt;', '>' => '&gt;', '"' => '&quot;'));
  386. $this->_name($title == '' ? $this->url : $title);
  387. $this->_website(html_only_entity_decode($simplePie->get_link()));
  388. $this->_description(html_only_entity_decode($simplePie->get_description()));
  389. } else {
  390. //The case of HTTP 301 Moved Permanently
  391. $subscribe_url = $simplePie->subscribe_url(true);
  392. }
  393. $clean_url = SimplePie_Misc::url_remove_credentials($subscribe_url);
  394. if ($subscribe_url !== null && $subscribe_url !== $url) {
  395. $this->_url($clean_url);
  396. }
  397. if (($mtime === true) || ($mtime > $this->lastUpdate) || $noCache) {
  398. //Minz_Log::debug('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url);
  399. return $simplePie;
  400. }
  401. //Minz_Log::debug('FreshRSS use cache for ' . $clean_url);
  402. }
  403. }
  404. return null;
  405. }
  406. /**
  407. * @return array<string>
  408. */
  409. public function loadGuids(SimplePie $simplePie): array {
  410. $hasUniqueGuids = true;
  411. $testGuids = [];
  412. $guids = [];
  413. $hasBadGuids = $this->attributes('hasBadGuids');
  414. $items = $simplePie->get_items();
  415. if (empty($items)) {
  416. return $guids;
  417. }
  418. for ($i = count($items) - 1; $i >= 0; $i--) {
  419. $item = $items[$i];
  420. if ($item == null) {
  421. continue;
  422. }
  423. $guid = safe_ascii($item->get_id(false, false));
  424. $hasUniqueGuids &= empty($testGuids['_' . $guid]);
  425. $testGuids['_' . $guid] = true;
  426. $guids[] = $guid;
  427. }
  428. if ($hasBadGuids != !$hasUniqueGuids) {
  429. $hasBadGuids = !$hasUniqueGuids;
  430. if ($hasBadGuids) {
  431. Minz_Log::warning('Feed has invalid GUIDs: ' . $this->url);
  432. } else {
  433. Minz_Log::warning('Feed has valid GUIDs again: ' . $this->url);
  434. }
  435. $feedDAO = FreshRSS_Factory::createFeedDao();
  436. $feedDAO->updateFeedAttribute($this, 'hasBadGuids', $hasBadGuids);
  437. }
  438. return $guids;
  439. }
  440. /** @return iterable<FreshRSS_Entry> */
  441. public function loadEntries(SimplePie $simplePie): iterable {
  442. $hasBadGuids = $this->attributes('hasBadGuids');
  443. $items = $simplePie->get_items();
  444. if (empty($items)) {
  445. return;
  446. }
  447. // We want chronological order and SimplePie uses reverse order.
  448. for ($i = count($items) - 1; $i >= 0; $i--) {
  449. $item = $items[$i];
  450. if ($item == null) {
  451. continue;
  452. }
  453. $title = html_only_entity_decode(strip_tags($item->get_title() ?? ''));
  454. $authors = $item->get_authors();
  455. $link = $item->get_permalink();
  456. $date = @strtotime($item->get_date() ?? '');
  457. //Tag processing (tag == category)
  458. $categories = $item->get_categories();
  459. $tags = array();
  460. if (is_array($categories)) {
  461. foreach ($categories as $category) {
  462. $text = html_only_entity_decode($category->get_label());
  463. //Some feeds use a single category with comma-separated tags
  464. $labels = explode(',', $text);
  465. if (is_array($labels)) {
  466. foreach ($labels as $label) {
  467. $tags[] = trim($label);
  468. }
  469. }
  470. }
  471. $tags = array_unique($tags);
  472. }
  473. $content = html_only_entity_decode($item->get_content());
  474. $attributeThumbnail = $item->get_thumbnail() ?? [];
  475. if (empty($attributeThumbnail['url'])) {
  476. $attributeThumbnail['url'] = '';
  477. }
  478. $attributeEnclosures = [];
  479. if (!empty($item->get_enclosures())) {
  480. foreach ($item->get_enclosures() as $enclosure) {
  481. $elink = $enclosure->get_link();
  482. if ($elink != '') {
  483. $etitle = $enclosure->get_title() ?? '';
  484. $credit = $enclosure->get_credit() ?? null;
  485. $description = $enclosure->get_description() ?? '';
  486. $mime = strtolower($enclosure->get_type() ?? '');
  487. $medium = strtolower($enclosure->get_medium() ?? '');
  488. $height = $enclosure->get_height();
  489. $width = $enclosure->get_width();
  490. $length = $enclosure->get_length();
  491. $attributeEnclosure = [
  492. 'url' => $elink,
  493. ];
  494. if ($etitle != '') $attributeEnclosure['title'] = $etitle;
  495. if ($credit != null) $attributeEnclosure['credit'] = $credit->get_name();
  496. if ($description != '') $attributeEnclosure['description'] = $description;
  497. if ($mime != '') $attributeEnclosure['type'] = $mime;
  498. if ($medium != '') $attributeEnclosure['medium'] = $medium;
  499. if ($length != '') $attributeEnclosure['length'] = intval($length);
  500. if ($height != '') $attributeEnclosure['height'] = intval($height);
  501. if ($width != '') $attributeEnclosure['width'] = intval($width);
  502. if (!empty($enclosure->get_thumbnails())) {
  503. foreach ($enclosure->get_thumbnails() as $thumbnail) {
  504. if ($thumbnail !== $attributeThumbnail['url']) {
  505. $attributeEnclosure['thumbnails'][] = $thumbnail;
  506. }
  507. }
  508. }
  509. $attributeEnclosures[] = $attributeEnclosure;
  510. }
  511. }
  512. }
  513. $guid = safe_ascii($item->get_id(false, false));
  514. unset($item);
  515. $authorNames = '';
  516. if (is_array($authors)) {
  517. foreach ($authors as $author) {
  518. $authorName = $author->name != '' ? $author->name : $author->email;
  519. if ($authorName != '') {
  520. $authorNames .= escapeToUnicodeAlternative(strip_tags($authorName), true) . '; ';
  521. }
  522. }
  523. }
  524. $authorNames = substr($authorNames, 0, -2);
  525. $entry = new FreshRSS_Entry(
  526. $this->id(),
  527. $hasBadGuids ? '' : $guid,
  528. $title == '' ? '' : $title,
  529. $authorNames,
  530. $content == '' ? '' : $content,
  531. $link == '' ? '' : $link,
  532. $date ? $date : time()
  533. );
  534. $entry->_tags($tags);
  535. $entry->_feed($this);
  536. if (!empty($attributeThumbnail['url'])) {
  537. $entry->_attributes('thumbnail', $attributeThumbnail);
  538. }
  539. $entry->_attributes('enclosures', $attributeEnclosures);
  540. $entry->hash(); //Must be computed before loading full content
  541. $entry->loadCompleteContent(); // Optionally load full content for truncated feeds
  542. yield $entry;
  543. }
  544. }
  545. public function loadHtmlXpath(): ?SimplePie {
  546. if ($this->url == '') {
  547. return null;
  548. }
  549. $feedSourceUrl = htmlspecialchars_decode($this->url, ENT_QUOTES);
  550. if ($this->httpAuth != '') {
  551. $feedSourceUrl = preg_replace('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $feedSourceUrl);
  552. }
  553. // Same naming conventions than https://rss-bridge.github.io/rss-bridge/Bridge_API/XPathAbstract.html
  554. // https://rss-bridge.github.io/rss-bridge/Bridge_API/BridgeAbstract.html#collectdata
  555. /** @var array<string,string> */
  556. $xPathSettings = $this->attributes('xpath');
  557. $xPathFeedTitle = $xPathSettings['feedTitle'] ?? '';
  558. $xPathItem = $xPathSettings['item'] ?? '';
  559. $xPathItemTitle = $xPathSettings['itemTitle'] ?? '';
  560. $xPathItemContent = $xPathSettings['itemContent'] ?? '';
  561. $xPathItemUri = $xPathSettings['itemUri'] ?? '';
  562. $xPathItemAuthor = $xPathSettings['itemAuthor'] ?? '';
  563. $xPathItemTimestamp = $xPathSettings['itemTimestamp'] ?? '';
  564. $xPathItemTimeFormat = $xPathSettings['itemTimeFormat'] ?? '';
  565. $xPathItemThumbnail = $xPathSettings['itemThumbnail'] ?? '';
  566. $xPathItemCategories = $xPathSettings['itemCategories'] ?? '';
  567. $xPathItemUid = $xPathSettings['itemUid'] ?? '';
  568. if ($xPathItem == '') {
  569. return null;
  570. }
  571. $cachePath = FreshRSS_Feed::cacheFilename($feedSourceUrl, $this->attributes(), $this->kind());
  572. $html = httpGet($feedSourceUrl, $cachePath,
  573. $this->kind() === FreshRSS_Feed::KIND_XML_XPATH ? 'xml' : 'html', $this->attributes());
  574. if (strlen($html) <= 0) {
  575. return null;
  576. }
  577. $view = new FreshRSS_View();
  578. $view->_path('index/rss.phtml');
  579. $view->internal_rendering = true;
  580. $view->rss_url = $feedSourceUrl;
  581. $view->entries = [];
  582. try {
  583. $doc = new DOMDocument();
  584. $doc->recover = true;
  585. $doc->strictErrorChecking = false;
  586. switch ($this->kind()) {
  587. case FreshRSS_Feed::KIND_HTML_XPATH:
  588. $doc->loadHTML($html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING);
  589. break;
  590. case FreshRSS_Feed::KIND_XML_XPATH:
  591. $doc->loadXML($html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING);
  592. break;
  593. default:
  594. return null;
  595. }
  596. $xpath = new DOMXPath($doc);
  597. $view->rss_title = $xPathFeedTitle == '' ? $this->name() :
  598. htmlspecialchars(@$xpath->evaluate('normalize-space(' . $xPathFeedTitle . ')'), ENT_COMPAT, 'UTF-8');
  599. $view->rss_base = htmlspecialchars(trim($xpath->evaluate('normalize-space(//base/@href)')), ENT_COMPAT, 'UTF-8');
  600. $nodes = $xpath->query($xPathItem);
  601. if (empty($nodes)) {
  602. return null;
  603. }
  604. foreach ($nodes as $node) {
  605. $item = [];
  606. $item['title'] = $xPathItemTitle == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemTitle . ')', $node);
  607. $item['content'] = '';
  608. if ($xPathItemContent != '') {
  609. $result = @$xpath->evaluate($xPathItemContent, $node);
  610. if ($result instanceof DOMNodeList) {
  611. // List of nodes, save as HTML
  612. $content = '';
  613. foreach ($result as $child) {
  614. $content .= $doc->saveHTML($child) . "\n";
  615. }
  616. $item['content'] = $content;
  617. } else {
  618. // Typed expression, save as-is
  619. $item['content'] = strval($result);
  620. }
  621. }
  622. $item['link'] = $xPathItemUri == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemUri . ')', $node);
  623. $item['author'] = $xPathItemAuthor == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemAuthor . ')', $node);
  624. $item['timestamp'] = $xPathItemTimestamp == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemTimestamp . ')', $node);
  625. if ($xPathItemTimeFormat != '') {
  626. $dateTime = DateTime::createFromFormat($xPathItemTimeFormat, $item['timestamp'] ?? '');
  627. if ($dateTime != false) {
  628. $item['timestamp'] = $dateTime->format(DateTime::ATOM);
  629. }
  630. }
  631. $item['thumbnail'] = $xPathItemThumbnail == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemThumbnail . ')', $node);
  632. if ($xPathItemCategories != '') {
  633. $itemCategories = @$xpath->query($xPathItemCategories, $node);
  634. if ($itemCategories) {
  635. foreach ($itemCategories as $itemCategory) {
  636. $item['categories'][] = $itemCategory->textContent;
  637. }
  638. }
  639. }
  640. if ($xPathItemUid != '') {
  641. $item['guid'] = @$xpath->evaluate('normalize-space(' . $xPathItemUid . ')', $node);
  642. }
  643. if (empty($item['guid'])) {
  644. $item['guid'] = 'urn:sha1:' . sha1($item['title'] . $item['content'] . $item['link']);
  645. }
  646. if ($item['title'] != '' || $item['content'] != '' || $item['link'] != '') {
  647. // HTML-encoding/escaping of the relevant fields (all except 'content')
  648. foreach (['author', 'categories', 'guid', 'link', 'thumbnail', 'timestamp', 'title'] as $key) {
  649. if (!empty($item[$key]) && is_string($item[$key])) {
  650. $item[$key] = Minz_Helper::htmlspecialchars_utf8($item[$key]);
  651. }
  652. }
  653. // CDATA protection
  654. $item['content'] = str_replace(']]>', ']]&gt;', $item['content']);
  655. $view->entries[] = FreshRSS_Entry::fromArray($item);
  656. }
  657. }
  658. } catch (Exception $ex) {
  659. Minz_Log::warning($ex->getMessage());
  660. return null;
  661. }
  662. $simplePie = customSimplePie();
  663. $simplePie->set_raw_data($view->renderToString());
  664. $simplePie->init();
  665. return $simplePie;
  666. }
  667. /**
  668. * To keep track of some new potentially unread articles since last commit+fetch from database
  669. */
  670. public function incPendingUnread(int $n = 1): void {
  671. $this->nbPendingNotRead += $n;
  672. }
  673. /**
  674. * Remember to call updateCachedValue($id_feed) or updateCachedValues() just after.
  675. * @return int|false the number of lines affected, or false if not applicable
  676. */
  677. public function keepMaxUnread() {
  678. $keepMaxUnread = $this->attributes('keep_max_n_unread');
  679. if ($keepMaxUnread === null) {
  680. $keepMaxUnread = FreshRSS_Context::$user_conf->mark_when['max_n_unread'];
  681. }
  682. if ($keepMaxUnread > 0 && $this->nbNotRead(false) + $this->nbPendingNotRead > $keepMaxUnread) {
  683. $feedDAO = FreshRSS_Factory::createFeedDao();
  684. return $feedDAO->keepMaxUnread($this->id(), max(0, $keepMaxUnread - $this->nbPendingNotRead));
  685. }
  686. return false;
  687. }
  688. /**
  689. * Applies the *mark as read upon gone* policy, if enabled.
  690. * Remember to call updateCachedValue($id_feed) or updateCachedValues() just after.
  691. * @return int|false the number of lines affected, or false if not applicable
  692. */
  693. public function markAsReadUponGone() {
  694. $readUponGone = $this->attributes('read_upon_gone');
  695. if ($readUponGone === null) {
  696. $readUponGone = FreshRSS_Context::$user_conf->mark_when['gone'];
  697. }
  698. if ($readUponGone) {
  699. $feedDAO = FreshRSS_Factory::createFeedDao();
  700. return $feedDAO->markAsReadUponGone($this->id());
  701. }
  702. return false;
  703. }
  704. /**
  705. * Remember to call updateCachedValue($id_feed) or updateCachedValues() just after
  706. * @return int|false
  707. */
  708. public function cleanOldEntries() {
  709. $archiving = $this->attributes('archiving');
  710. if ($archiving == null) {
  711. $catDAO = FreshRSS_Factory::createCategoryDao();
  712. $category = $catDAO->searchById($this->categoryId);
  713. $archiving = $category == null ? null : $category->attributes('archiving');
  714. if ($archiving == null) {
  715. $archiving = FreshRSS_Context::$user_conf->archiving;
  716. }
  717. }
  718. if (is_array($archiving)) {
  719. $entryDAO = FreshRSS_Factory::createEntryDao();
  720. $nb = $entryDAO->cleanOldEntries($this->id(), $archiving);
  721. if ($nb > 0) {
  722. Minz_Log::debug($nb . ' entries cleaned in feed [' . $this->url(false) . '] with: ' . json_encode($archiving));
  723. }
  724. return $nb;
  725. }
  726. return false;
  727. }
  728. /** @param array<string,mixed> $attributes */
  729. public static function cacheFilename(string $url, array $attributes, int $kind = FreshRSS_Feed::KIND_RSS): string {
  730. $simplePie = customSimplePie($attributes);
  731. $filename = $simplePie->get_cache_filename($url);
  732. if ($kind === FreshRSS_Feed::KIND_HTML_XPATH) {
  733. return CACHE_PATH . '/' . $filename . '.html';
  734. } elseif ($kind === FreshRSS_Feed::KIND_XML_XPATH) {
  735. return CACHE_PATH . '/' . $filename . '.xml';
  736. } else {
  737. return CACHE_PATH . '/' . $filename . '.spc';
  738. }
  739. }
  740. public function clearCache(): bool {
  741. return @unlink(FreshRSS_Feed::cacheFilename($this->url, $this->attributes(), $this->kind));
  742. }
  743. /** @return int|false */
  744. public function cacheModifiedTime() {
  745. return @filemtime(FreshRSS_Feed::cacheFilename($this->url, $this->attributes(), $this->kind));
  746. }
  747. public function lock(): bool {
  748. $this->lockPath = TMP_PATH . '/' . $this->hash() . '.freshrss.lock';
  749. if (file_exists($this->lockPath) && ((time() - @filemtime($this->lockPath)) > 3600)) {
  750. @unlink($this->lockPath);
  751. }
  752. if (($handle = @fopen($this->lockPath, 'x')) === false) {
  753. return false;
  754. }
  755. //register_shutdown_function('unlink', $this->lockPath);
  756. @fclose($handle);
  757. return true;
  758. }
  759. public function unlock(): bool {
  760. return @unlink($this->lockPath);
  761. }
  762. /**
  763. * @return array<FreshRSS_FilterAction>
  764. */
  765. public function filterActions(): array {
  766. if (empty($this->filterActions)) {
  767. $this->filterActions = array();
  768. $filters = $this->attributes('filters');
  769. if (is_array($filters)) {
  770. foreach ($filters as $filter) {
  771. $filterAction = FreshRSS_FilterAction::fromJSON($filter);
  772. if ($filterAction != null) {
  773. $this->filterActions[] = $filterAction;
  774. }
  775. }
  776. }
  777. }
  778. return $this->filterActions;
  779. }
  780. /**
  781. * @param array<FreshRSS_FilterAction>|null $filterActions
  782. */
  783. private function _filterActions(?array $filterActions): void {
  784. $this->filterActions = $filterActions;
  785. if (is_array($this->filterActions) && !empty($this->filterActions)) {
  786. $this->_attributes('filters', array_map(static function (?FreshRSS_FilterAction $af) {
  787. return $af == null ? null : $af->toJSON();
  788. }, $this->filterActions));
  789. } else {
  790. $this->_attributes('filters', null);
  791. }
  792. }
  793. /** @return array<FreshRSS_BooleanSearch> */
  794. public function filtersAction(string $action): array {
  795. $action = trim($action);
  796. if ($action == '') {
  797. return array();
  798. }
  799. $filters = array();
  800. $filterActions = $this->filterActions();
  801. for ($i = count($filterActions) - 1; $i >= 0; $i--) {
  802. $filterAction = $filterActions[$i];
  803. if ($filterAction != null && $filterAction->booleanSearch() != null &&
  804. $filterAction->actions() != null && in_array($action, $filterAction->actions(), true)) {
  805. $filters[] = $filterAction->booleanSearch();
  806. }
  807. }
  808. return $filters;
  809. }
  810. /**
  811. * @param array<string> $filters
  812. */
  813. public function _filtersAction(string $action, array $filters): void {
  814. $action = trim($action);
  815. if ($action == '' || !is_array($filters)) {
  816. return;
  817. }
  818. $filters = array_unique(array_map('trim', $filters));
  819. $filterActions = $this->filterActions();
  820. //Check existing filters
  821. for ($i = count($filterActions) - 1; $i >= 0; $i--) {
  822. $filterAction = $filterActions[$i];
  823. if ($filterAction == null || !is_array($filterAction->actions()) ||
  824. $filterAction->booleanSearch() == null || trim($filterAction->booleanSearch()->getRawInput()) == '') {
  825. array_splice($filterActions, $i, 1);
  826. continue;
  827. }
  828. $actions = $filterAction->actions();
  829. //Remove existing rules with same action
  830. for ($j = count($actions) - 1; $j >= 0; $j--) {
  831. if ($actions[$j] === $action) {
  832. array_splice($actions, $j, 1);
  833. }
  834. }
  835. //Update existing filter with new action
  836. for ($k = count($filters) - 1; $k >= 0; $k --) {
  837. $filter = $filters[$k];
  838. if ($filter === $filterAction->booleanSearch()->getRawInput()) {
  839. $actions[] = $action;
  840. array_splice($filters, $k, 1);
  841. }
  842. }
  843. //Save result
  844. if (empty($actions)) {
  845. array_splice($filterActions, $i, 1);
  846. } else {
  847. $filterAction->_actions($actions);
  848. }
  849. }
  850. //Add new filters
  851. for ($k = count($filters) - 1; $k >= 0; $k --) {
  852. $filter = $filters[$k];
  853. if ($filter != '') {
  854. $filterAction = FreshRSS_FilterAction::fromJSON(array(
  855. 'search' => $filter,
  856. 'actions' => array($action),
  857. ));
  858. if ($filterAction != null) {
  859. $filterActions[] = $filterAction;
  860. }
  861. }
  862. }
  863. if (empty($filterActions)) {
  864. $filterActions = null;
  865. }
  866. $this->_filterActions($filterActions);
  867. }
  868. //<WebSub>
  869. public function pubSubHubbubEnabled(): bool {
  870. $url = $this->selfUrl ? $this->selfUrl : $this->url;
  871. $hubFilename = PSHB_PATH . '/feeds/' . sha1($url) . '/!hub.json';
  872. if ($hubFile = @file_get_contents($hubFilename)) {
  873. $hubJson = json_decode($hubFile, true);
  874. if ($hubJson && empty($hubJson['error']) &&
  875. (empty($hubJson['lease_end']) || $hubJson['lease_end'] > time())) {
  876. return true;
  877. }
  878. }
  879. return false;
  880. }
  881. public function pubSubHubbubError(bool $error = true): bool {
  882. $url = $this->selfUrl ? $this->selfUrl : $this->url;
  883. $hubFilename = PSHB_PATH . '/feeds/' . sha1($url) . '/!hub.json';
  884. $hubFile = @file_get_contents($hubFilename);
  885. $hubJson = $hubFile ? json_decode($hubFile, true) : array();
  886. if (!isset($hubJson['error']) || $hubJson['error'] !== (bool)$error) {
  887. $hubJson['error'] = (bool)$error;
  888. file_put_contents($hubFilename, json_encode($hubJson));
  889. Minz_Log::warning('Set error to ' . ($error ? 1 : 0) . ' for ' . $url, PSHB_LOG);
  890. }
  891. return false;
  892. }
  893. /**
  894. * @return string|false
  895. */
  896. public function pubSubHubbubPrepare() {
  897. $key = '';
  898. if (Minz_Request::serverIsPublic(FreshRSS_Context::$system_conf->base_url) &&
  899. $this->hubUrl && $this->selfUrl && @is_dir(PSHB_PATH)) {
  900. $path = PSHB_PATH . '/feeds/' . sha1($this->selfUrl);
  901. $hubFilename = $path . '/!hub.json';
  902. if ($hubFile = @file_get_contents($hubFilename)) {
  903. $hubJson = json_decode($hubFile, true);
  904. if (!$hubJson || empty($hubJson['key']) || !ctype_xdigit($hubJson['key'])) {
  905. $text = 'Invalid JSON for WebSub: ' . $this->url;
  906. Minz_Log::warning($text);
  907. Minz_Log::warning($text, PSHB_LOG);
  908. return false;
  909. }
  910. if ((!empty($hubJson['lease_end'])) && ($hubJson['lease_end'] < (time() + (3600 * 23)))) { //TODO: Make a better policy
  911. $text = 'WebSub lease ends at '
  912. . date('c', empty($hubJson['lease_end']) ? time() : $hubJson['lease_end'])
  913. . ' and needs renewal: ' . $this->url;
  914. Minz_Log::warning($text);
  915. Minz_Log::warning($text, PSHB_LOG);
  916. $key = $hubJson['key']; //To renew our lease
  917. } elseif (((!empty($hubJson['error'])) || empty($hubJson['lease_end'])) &&
  918. (empty($hubJson['lease_start']) || $hubJson['lease_start'] < time() - (3600 * 23))) { //Do not renew too often
  919. $key = $hubJson['key']; //To renew our lease
  920. }
  921. } else {
  922. @mkdir($path, 0770, true);
  923. $key = sha1($path . FreshRSS_Context::$system_conf->salt);
  924. $hubJson = array(
  925. 'hub' => $this->hubUrl,
  926. 'key' => $key,
  927. );
  928. file_put_contents($hubFilename, json_encode($hubJson));
  929. @mkdir(PSHB_PATH . '/keys/', 0770, true);
  930. file_put_contents(PSHB_PATH . '/keys/' . $key . '.txt', $this->selfUrl);
  931. $text = 'WebSub prepared for ' . $this->url;
  932. Minz_Log::debug($text);
  933. Minz_Log::debug($text, PSHB_LOG);
  934. }
  935. $currentUser = Minz_User::name();
  936. if (FreshRSS_user_Controller::checkUsername($currentUser) && !file_exists($path . '/' . $currentUser . '.txt')) {
  937. touch($path . '/' . $currentUser . '.txt');
  938. }
  939. }
  940. return $key;
  941. }
  942. //Parameter true to subscribe, false to unsubscribe.
  943. public function pubSubHubbubSubscribe(bool $state): bool {
  944. if ($state) {
  945. $url = $this->selfUrl ? $this->selfUrl : $this->url;
  946. } else {
  947. $url = $this->url; //Always use current URL during unsubscribe
  948. }
  949. if ($url && (Minz_Request::serverIsPublic(FreshRSS_Context::$system_conf->base_url) || !$state)) {
  950. $hubFilename = PSHB_PATH . '/feeds/' . sha1($url) . '/!hub.json';
  951. $hubFile = @file_get_contents($hubFilename);
  952. if ($hubFile === false) {
  953. Minz_Log::warning('JSON not found for WebSub: ' . $this->url);
  954. return false;
  955. }
  956. $hubJson = json_decode($hubFile, true);
  957. if (!$hubJson || empty($hubJson['key']) || !ctype_xdigit($hubJson['key']) || empty($hubJson['hub'])) {
  958. Minz_Log::warning('Invalid JSON for WebSub: ' . $this->url);
  959. return false;
  960. }
  961. $callbackUrl = checkUrl(Minz_Request::getBaseUrl() . '/api/pshb.php?k=' . $hubJson['key']);
  962. if ($callbackUrl == '') {
  963. Minz_Log::warning('Invalid callback for WebSub: ' . $this->url);
  964. return false;
  965. }
  966. if (!$state) { //unsubscribe
  967. $hubJson['lease_end'] = time() - 60;
  968. file_put_contents($hubFilename, json_encode($hubJson));
  969. }
  970. $ch = curl_init();
  971. curl_setopt_array($ch, [
  972. CURLOPT_URL => $hubJson['hub'],
  973. CURLOPT_RETURNTRANSFER => true,
  974. CURLOPT_POSTFIELDS => http_build_query(array(
  975. 'hub.verify' => 'sync',
  976. 'hub.mode' => $state ? 'subscribe' : 'unsubscribe',
  977. 'hub.topic' => $url,
  978. 'hub.callback' => $callbackUrl,
  979. )),
  980. CURLOPT_USERAGENT => FRESHRSS_USERAGENT,
  981. CURLOPT_MAXREDIRS => 10,
  982. CURLOPT_FOLLOWLOCATION => true,
  983. CURLOPT_ENCODING => '', //Enable all encodings
  984. ]);
  985. $response = curl_exec($ch);
  986. $info = curl_getinfo($ch);
  987. Minz_Log::warning('WebSub ' . ($state ? 'subscribe' : 'unsubscribe') . ' to ' . $url .
  988. ' via hub ' . $hubJson['hub'] .
  989. ' with callback ' . $callbackUrl . ': ' . $info['http_code'] . ' ' . $response, PSHB_LOG);
  990. if (substr('' . $info['http_code'], 0, 1) == '2') {
  991. return true;
  992. } else {
  993. $hubJson['lease_start'] = time(); //Prevent trying again too soon
  994. $hubJson['error'] = true;
  995. file_put_contents($hubFilename, json_encode($hubJson));
  996. return false;
  997. }
  998. }
  999. return false;
  1000. }
  1001. //</WebSub>
  1002. }