Feed.php 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117
  1. <?php
  2. class FreshRSS_Feed extends Minz_Model {
  3. /**
  4. * Normal RSS or Atom feed
  5. * @var int
  6. */
  7. const KIND_RSS = 0;
  8. /**
  9. * Invalid RSS or Atom feed
  10. * @var int
  11. */
  12. const KIND_RSS_FORCED = 2;
  13. /**
  14. * Normal HTML with XPath scraping
  15. * @var int
  16. */
  17. const KIND_HTML_XPATH = 10;
  18. /**
  19. * Normal XML with XPath scraping
  20. * @var int
  21. */
  22. const KIND_XML_XPATH = 15;
  23. /**
  24. * Normal JSON with XPath scraping
  25. * @var int
  26. */
  27. const KIND_JSON_XPATH = 20;
  28. const PRIORITY_MAIN_STREAM = 10;
  29. const PRIORITY_NORMAL = 0;
  30. const PRIORITY_ARCHIVED = -10;
  31. const TTL_DEFAULT = 0;
  32. const ARCHIVING_RETENTION_COUNT_LIMIT = 10000;
  33. const ARCHIVING_RETENTION_PERIOD = 'P3M';
  34. /** @var int */
  35. private $id = 0;
  36. /** @var string */
  37. private $url = '';
  38. /** @var int */
  39. private $kind = 0;
  40. /** @var int */
  41. private $categoryId = 1;
  42. /** @var FreshRSS_Category|null */
  43. private $category;
  44. /** @var int */
  45. private $nbEntries = -1;
  46. /** @var int */
  47. private $nbNotRead = -1;
  48. /** @var int */
  49. private $nbPendingNotRead = 0;
  50. /** @var string */
  51. private $name = '';
  52. /** @var string */
  53. private $website = '';
  54. /** @var string */
  55. private $description = '';
  56. /** @var int */
  57. private $lastUpdate = 0;
  58. /** @var int */
  59. private $priority = self::PRIORITY_MAIN_STREAM;
  60. /** @var string */
  61. private $pathEntries = '';
  62. /** @var string */
  63. private $httpAuth = '';
  64. /** @var bool */
  65. private $error = false;
  66. /** @var int */
  67. private $ttl = self::TTL_DEFAULT;
  68. /** @var array<string,mixed> */
  69. private $attributes = [];
  70. /** @var bool */
  71. private $mute = false;
  72. /** @var string */
  73. private $hash = '';
  74. /** @var string */
  75. private $lockPath = '';
  76. /** @var string */
  77. private $hubUrl = '';
  78. /** @var string */
  79. private $selfUrl = '';
  80. /** @var array<FreshRSS_FilterAction> $filterActions */
  81. private $filterActions = null;
  82. public function __construct(string $url, bool $validate = true) {
  83. if ($validate) {
  84. $this->_url($url);
  85. } else {
  86. $this->url = $url;
  87. }
  88. }
  89. public static function example(): FreshRSS_Feed {
  90. $f = new FreshRSS_Feed('http://example.net/', false);
  91. $f->faviconPrepare();
  92. return $f;
  93. }
  94. public function id(): int {
  95. return $this->id;
  96. }
  97. public function hash(): string {
  98. if ($this->hash == '') {
  99. $salt = FreshRSS_Context::$system_conf->salt;
  100. $this->hash = hash('crc32b', $salt . $this->url);
  101. }
  102. return $this->hash;
  103. }
  104. public function url(bool $includeCredentials = true): string {
  105. return $includeCredentials ? $this->url : SimplePie_Misc::url_remove_credentials($this->url);
  106. }
  107. public function selfUrl(): string {
  108. return $this->selfUrl;
  109. }
  110. public function kind(): int {
  111. return $this->kind;
  112. }
  113. public function hubUrl(): string {
  114. return $this->hubUrl;
  115. }
  116. public function category(): ?FreshRSS_Category {
  117. if ($this->category === null) {
  118. $catDAO = FreshRSS_Factory::createCategoryDao();
  119. $this->category = $catDAO->searchById($this->categoryId);
  120. }
  121. return $this->category;
  122. }
  123. public function categoryId(): int {
  124. return $this->categoryId;
  125. }
  126. /**
  127. * @return array<FreshRSS_Entry>|null
  128. * @deprecated
  129. */
  130. public function entries(): ?array {
  131. Minz_Log::warning(__method__ . ' is deprecated since FreshRSS 1.16.1!');
  132. $simplePie = $this->load(false, true);
  133. return $simplePie == null ? [] : iterator_to_array($this->loadEntries($simplePie));
  134. }
  135. public function name(bool $raw = false): string {
  136. return $raw || $this->name != '' ? $this->name : preg_replace('%^https?://(www[.])?%i', '', $this->url);
  137. }
  138. /** @return string HTML-encoded URL of the Web site of the feed */
  139. public function website(): string {
  140. return $this->website;
  141. }
  142. public function description(): string {
  143. return $this->description;
  144. }
  145. public function lastUpdate(): int {
  146. return $this->lastUpdate;
  147. }
  148. public function priority(): int {
  149. return $this->priority;
  150. }
  151. /** @return string HTML-encoded CSS selector */
  152. public function pathEntries(): string {
  153. return $this->pathEntries;
  154. }
  155. /**
  156. * @phpstan-return ($raw is true ? string : array{'username':string,'password':string})
  157. * @return array{'username':string,'password':string}|string
  158. */
  159. public function httpAuth(bool $raw = true) {
  160. if ($raw) {
  161. return $this->httpAuth;
  162. } else {
  163. $pos_colon = strpos($this->httpAuth, ':');
  164. if ($pos_colon !== false) {
  165. $user = substr($this->httpAuth, 0, $pos_colon);
  166. $pass = substr($this->httpAuth, $pos_colon + 1);
  167. } else {
  168. $user = '';
  169. $pass = '';
  170. }
  171. return array(
  172. 'username' => $user,
  173. 'password' => $pass
  174. );
  175. }
  176. }
  177. public function inError(): bool {
  178. return $this->error;
  179. }
  180. /**
  181. * @param bool $raw true for database version combined with mute information, false otherwise
  182. */
  183. public function ttl(bool $raw = false): int {
  184. if ($raw) {
  185. $ttl = $this->ttl;
  186. if ($this->mute && FreshRSS_Feed::TTL_DEFAULT === $ttl) {
  187. $ttl = FreshRSS_Context::$user_conf ? FreshRSS_Context::$user_conf->ttl_default : 3600;
  188. }
  189. return $ttl * ($this->mute ? -1 : 1);
  190. }
  191. return $this->ttl;
  192. }
  193. /**
  194. * @phpstan-return ($key is non-empty-string ? mixed : array<string,mixed>)
  195. * @return array<string,mixed>|mixed|null
  196. */
  197. public function attributes(string $key = '') {
  198. if ($key === '') {
  199. return $this->attributes;
  200. } else {
  201. return $this->attributes[$key] ?? null;
  202. }
  203. }
  204. public function mute(): bool {
  205. return $this->mute;
  206. }
  207. public function nbEntries(): int {
  208. if ($this->nbEntries < 0) {
  209. $feedDAO = FreshRSS_Factory::createFeedDao();
  210. $this->nbEntries = $feedDAO->countEntries($this->id());
  211. }
  212. return $this->nbEntries;
  213. }
  214. public function nbNotRead(bool $includePending = false): int {
  215. if ($this->nbNotRead < 0) {
  216. $feedDAO = FreshRSS_Factory::createFeedDao();
  217. $this->nbNotRead = $feedDAO->countNotRead($this->id());
  218. }
  219. return $this->nbNotRead + ($includePending ? $this->nbPendingNotRead : 0);
  220. }
  221. public function faviconPrepare(): void {
  222. require_once(LIB_PATH . '/favicons.php');
  223. $url = $this->website;
  224. if ($url == '') {
  225. $url = $this->url;
  226. }
  227. $txt = FAVICONS_DIR . $this->hash() . '.txt';
  228. if (@file_get_contents($txt) !== $url) {
  229. file_put_contents($txt, $url);
  230. }
  231. if (FreshRSS_Context::$isCli) {
  232. $ico = FAVICONS_DIR . $this->hash() . '.ico';
  233. $ico_mtime = @filemtime($ico);
  234. $txt_mtime = @filemtime($txt);
  235. if ($txt_mtime != false &&
  236. ($ico_mtime == false || $ico_mtime < $txt_mtime || ($ico_mtime < time() - (14 * 86400)))) {
  237. // no ico file or we should download a new one.
  238. $url = file_get_contents($txt);
  239. if ($url == false || !download_favicon($url, $ico)) {
  240. touch($ico);
  241. }
  242. }
  243. }
  244. }
  245. public static function faviconDelete(string $hash): void {
  246. $path = DATA_PATH . '/favicons/' . $hash;
  247. @unlink($path . '.ico');
  248. @unlink($path . '.txt');
  249. }
  250. public function favicon(): string {
  251. return Minz_Url::display('/f.php?' . $this->hash());
  252. }
  253. public function _id(int $value): void {
  254. $this->id = $value;
  255. }
  256. public function _url(string $value, bool $validate = true): void {
  257. $this->hash = '';
  258. $url = $value;
  259. if ($validate) {
  260. $url = checkUrl($url);
  261. }
  262. if ($url == false) {
  263. throw new FreshRSS_BadUrl_Exception($value);
  264. }
  265. $this->url = $url;
  266. }
  267. public function _kind(int $value): void {
  268. $this->kind = $value;
  269. }
  270. public function _category(?FreshRSS_Category $cat): void {
  271. $this->category = $cat;
  272. $this->categoryId = $this->category == null ? 0 : $this->category->id();
  273. }
  274. /** @param int|string $id */
  275. public function _categoryId($id): void {
  276. $this->category = null;
  277. $this->categoryId = intval($id);
  278. }
  279. public function _name(string $value): void {
  280. $this->name = $value == '' ? '' : trim($value);
  281. }
  282. public function _website(string $value, bool $validate = true): void {
  283. if ($validate) {
  284. $value = checkUrl($value);
  285. }
  286. if ($value == false) {
  287. $value = '';
  288. }
  289. $this->website = $value;
  290. }
  291. public function _description(string $value): void {
  292. $this->description = $value == '' ? '' : $value;
  293. }
  294. public function _lastUpdate(int $value): void {
  295. $this->lastUpdate = $value;
  296. }
  297. public function _priority(int $value): void {
  298. $this->priority = $value;
  299. }
  300. /** @param string $value HTML-encoded CSS selector */
  301. public function _pathEntries(string $value): void {
  302. $this->pathEntries = $value;
  303. }
  304. public function _httpAuth(string $value): void {
  305. $this->httpAuth = $value;
  306. }
  307. /** @param bool|int $value */
  308. public function _error($value): void {
  309. $this->error = (bool)$value;
  310. }
  311. public function _mute(bool $value): void {
  312. $this->mute = $value;
  313. }
  314. public function _ttl(int $value): void {
  315. $value = min($value, 100000000);
  316. $this->ttl = abs($value);
  317. $this->mute = $value < self::TTL_DEFAULT;
  318. }
  319. /** @param string|array<mixed>|bool|int|null $value Value, not HTML-encoded */
  320. public function _attributes(string $key, $value): void {
  321. if ($key == '') {
  322. if (is_string($value)) {
  323. $value = json_decode($value, true);
  324. }
  325. if (is_array($value)) {
  326. $this->attributes = $value;
  327. }
  328. } elseif ($value === null) {
  329. unset($this->attributes[$key]);
  330. } else {
  331. $this->attributes[$key] = $value;
  332. }
  333. }
  334. public function _nbNotRead(int $value): void {
  335. $this->nbNotRead = $value;
  336. }
  337. public function _nbEntries(int $value): void {
  338. $this->nbEntries = $value;
  339. }
  340. public function load(bool $loadDetails = false, bool $noCache = false): ?SimplePie {
  341. if ($this->url != '') {
  342. // @phpstan-ignore-next-line
  343. if (CACHE_PATH == '') {
  344. throw new Minz_FileNotExistException(
  345. 'CACHE_PATH',
  346. Minz_Exception::ERROR
  347. );
  348. } else {
  349. $url = htmlspecialchars_decode($this->url, ENT_QUOTES);
  350. if ($this->httpAuth != '') {
  351. $url = preg_replace('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $url);
  352. }
  353. $simplePie = customSimplePie($this->attributes());
  354. if (substr($url, -11) === '#force_feed') {
  355. $simplePie->force_feed(true);
  356. $url = substr($url, 0, -11);
  357. }
  358. $simplePie->set_feed_url($url);
  359. if (!$loadDetails) { //Only activates auto-discovery when adding a new feed
  360. $simplePie->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE);
  361. }
  362. if ($this->attributes('clear_cache')) {
  363. // Do not use `$simplePie->enable_cache(false);` as it would prevent caching in multiuser context
  364. $this->clearCache();
  365. }
  366. Minz_ExtensionManager::callHook('simplepie_before_init', $simplePie, $this);
  367. $mtime = $simplePie->init();
  368. if ((!$mtime) || $simplePie->error()) {
  369. $errorMessage = $simplePie->error();
  370. if (empty($errorMessage)) {
  371. $errorMessage = '';
  372. } elseif (is_array($errorMessage)) {
  373. $errorMessage = json_encode($errorMessage, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_LINE_TERMINATORS) ?: '';
  374. }
  375. throw new FreshRSS_Feed_Exception(
  376. ($errorMessage == '' ? 'Unknown error for feed' : $errorMessage) .
  377. ' [' . $this->url . ']',
  378. $simplePie->status_code()
  379. );
  380. }
  381. $links = $simplePie->get_links('self');
  382. $this->selfUrl = empty($links[0]) ? '' : (checkUrl($links[0]) ?: '');
  383. $links = $simplePie->get_links('hub');
  384. $this->hubUrl = empty($links[0]) ? '' : (checkUrl($links[0]) ?: '');
  385. if ($loadDetails) {
  386. // si on a utilisé l’auto-discover, notre url va avoir changé
  387. $subscribe_url = $simplePie->subscribe_url(false);
  388. //HTML to HTML-PRE //ENT_COMPAT except '&'
  389. $title = strtr(html_only_entity_decode($simplePie->get_title()), array('<' => '&lt;', '>' => '&gt;', '"' => '&quot;'));
  390. $this->_name($title == '' ? $this->url : $title);
  391. $this->_website(html_only_entity_decode($simplePie->get_link()));
  392. $this->_description(html_only_entity_decode($simplePie->get_description()));
  393. } else {
  394. //The case of HTTP 301 Moved Permanently
  395. $subscribe_url = $simplePie->subscribe_url(true);
  396. }
  397. $clean_url = SimplePie_Misc::url_remove_credentials($subscribe_url);
  398. if ($subscribe_url !== null && $subscribe_url !== $url) {
  399. $this->_url($clean_url);
  400. }
  401. if (($mtime === true) || ($mtime > $this->lastUpdate) || $noCache) {
  402. //Minz_Log::debug('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url);
  403. return $simplePie;
  404. }
  405. //Minz_Log::debug('FreshRSS use cache for ' . $clean_url);
  406. }
  407. }
  408. return null;
  409. }
  410. /**
  411. * @return array<string>
  412. */
  413. public function loadGuids(SimplePie $simplePie): array {
  414. $hasUniqueGuids = true;
  415. $testGuids = [];
  416. $guids = [];
  417. $links = [];
  418. $hadBadGuids = $this->attributes('hasBadGuids');
  419. $items = $simplePie->get_items();
  420. if (empty($items)) {
  421. return $guids;
  422. }
  423. for ($i = count($items) - 1; $i >= 0; $i--) {
  424. $item = $items[$i];
  425. if ($item == null) {
  426. continue;
  427. }
  428. $guid = safe_ascii($item->get_id(false, false));
  429. $hasUniqueGuids &= empty($testGuids['_' . $guid]);
  430. $testGuids['_' . $guid] = true;
  431. $guids[] = $guid;
  432. $links[] = $item->get_permalink();
  433. }
  434. if ($hadBadGuids != !$hasUniqueGuids) {
  435. if ($hadBadGuids) {
  436. Minz_Log::warning('Feed has invalid GUIDs: ' . $this->url);
  437. } else {
  438. Minz_Log::warning('Feed has valid GUIDs again: ' . $this->url);
  439. }
  440. $feedDAO = FreshRSS_Factory::createFeedDao();
  441. $feedDAO->updateFeedAttribute($this, 'hasBadGuids', !$hasUniqueGuids);
  442. }
  443. return $hasUniqueGuids ? $guids : $links;
  444. }
  445. /** @return Traversable<FreshRSS_Entry> */
  446. public function loadEntries(SimplePie $simplePie): Traversable {
  447. $hasBadGuids = $this->attributes('hasBadGuids');
  448. $items = $simplePie->get_items();
  449. if (empty($items)) {
  450. return;
  451. }
  452. // We want chronological order and SimplePie uses reverse order.
  453. for ($i = count($items) - 1; $i >= 0; $i--) {
  454. $item = $items[$i];
  455. if ($item == null) {
  456. continue;
  457. }
  458. $title = html_only_entity_decode(strip_tags($item->get_title() ?? ''));
  459. $authors = $item->get_authors();
  460. $link = $item->get_permalink();
  461. $date = @strtotime((string)($item->get_date() ?? '')) ?: 0;
  462. //Tag processing (tag == category)
  463. $categories = $item->get_categories();
  464. $tags = array();
  465. if (is_array($categories)) {
  466. foreach ($categories as $category) {
  467. $text = html_only_entity_decode($category->get_label());
  468. //Some feeds use a single category with comma-separated tags
  469. $labels = explode(',', $text);
  470. if (!empty($labels)) {
  471. foreach ($labels as $label) {
  472. $tags[] = trim($label);
  473. }
  474. }
  475. }
  476. $tags = array_unique($tags);
  477. }
  478. $content = html_only_entity_decode($item->get_content());
  479. $attributeThumbnail = $item->get_thumbnail() ?? [];
  480. if (empty($attributeThumbnail['url'])) {
  481. $attributeThumbnail['url'] = '';
  482. }
  483. $attributeEnclosures = [];
  484. if (!empty($item->get_enclosures())) {
  485. foreach ($item->get_enclosures() as $enclosure) {
  486. $elink = $enclosure->get_link();
  487. if ($elink != '') {
  488. $etitle = $enclosure->get_title() ?? '';
  489. $credit = $enclosure->get_credit() ?? null;
  490. $description = $enclosure->get_description() ?? '';
  491. $mime = strtolower($enclosure->get_type() ?? '');
  492. $medium = strtolower($enclosure->get_medium() ?? '');
  493. $height = $enclosure->get_height();
  494. $width = $enclosure->get_width();
  495. $length = $enclosure->get_length();
  496. $attributeEnclosure = [
  497. 'url' => $elink,
  498. ];
  499. if ($etitle != '') $attributeEnclosure['title'] = $etitle;
  500. if ($credit != null) $attributeEnclosure['credit'] = $credit->get_name();
  501. if ($description != '') $attributeEnclosure['description'] = $description;
  502. if ($mime != '') $attributeEnclosure['type'] = $mime;
  503. if ($medium != '') $attributeEnclosure['medium'] = $medium;
  504. if ($length != '') $attributeEnclosure['length'] = intval($length);
  505. if ($height != '') $attributeEnclosure['height'] = intval($height);
  506. if ($width != '') $attributeEnclosure['width'] = intval($width);
  507. if (!empty($enclosure->get_thumbnails())) {
  508. foreach ($enclosure->get_thumbnails() as $thumbnail) {
  509. if ($thumbnail !== $attributeThumbnail['url']) {
  510. $attributeEnclosure['thumbnails'][] = $thumbnail;
  511. }
  512. }
  513. }
  514. $attributeEnclosures[] = $attributeEnclosure;
  515. }
  516. }
  517. }
  518. $guid = safe_ascii($item->get_id(false, false));
  519. unset($item);
  520. $authorNames = '';
  521. if (is_array($authors)) {
  522. foreach ($authors as $author) {
  523. $authorName = $author->name != '' ? $author->name : $author->email;
  524. if ($authorName != '') {
  525. $authorNames .= escapeToUnicodeAlternative(strip_tags($authorName), true) . '; ';
  526. }
  527. }
  528. }
  529. $authorNames = substr($authorNames, 0, -2);
  530. $entry = new FreshRSS_Entry(
  531. $this->id(),
  532. $hasBadGuids ? '' : $guid,
  533. $title == '' ? '' : $title,
  534. $authorNames,
  535. $content == '' ? '' : $content,
  536. $link == '' ? '' : $link,
  537. $date ? $date : time()
  538. );
  539. $entry->_tags($tags);
  540. $entry->_feed($this);
  541. if (!empty($attributeThumbnail['url'])) {
  542. $entry->_attributes('thumbnail', $attributeThumbnail);
  543. }
  544. $entry->_attributes('enclosures', $attributeEnclosures);
  545. $entry->hash(); //Must be computed before loading full content
  546. $entry->loadCompleteContent(); // Optionally load full content for truncated feeds
  547. yield $entry;
  548. }
  549. }
  550. public function loadHtmlXpath(): ?SimplePie {
  551. if ($this->url == '') {
  552. return null;
  553. }
  554. $feedSourceUrl = htmlspecialchars_decode($this->url, ENT_QUOTES);
  555. if ($this->httpAuth != '') {
  556. $feedSourceUrl = preg_replace('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $feedSourceUrl);
  557. }
  558. // Same naming conventions than https://rss-bridge.github.io/rss-bridge/Bridge_API/XPathAbstract.html
  559. // https://rss-bridge.github.io/rss-bridge/Bridge_API/BridgeAbstract.html#collectdata
  560. /** @var array<string,string> */
  561. $xPathSettings = $this->attributes('xpath');
  562. $xPathFeedTitle = $xPathSettings['feedTitle'] ?? '';
  563. $xPathItem = $xPathSettings['item'] ?? '';
  564. $xPathItemTitle = $xPathSettings['itemTitle'] ?? '';
  565. $xPathItemContent = $xPathSettings['itemContent'] ?? '';
  566. $xPathItemUri = $xPathSettings['itemUri'] ?? '';
  567. $xPathItemAuthor = $xPathSettings['itemAuthor'] ?? '';
  568. $xPathItemTimestamp = $xPathSettings['itemTimestamp'] ?? '';
  569. $xPathItemTimeFormat = $xPathSettings['itemTimeFormat'] ?? '';
  570. $xPathItemThumbnail = $xPathSettings['itemThumbnail'] ?? '';
  571. $xPathItemCategories = $xPathSettings['itemCategories'] ?? '';
  572. $xPathItemUid = $xPathSettings['itemUid'] ?? '';
  573. if ($xPathItem == '') {
  574. return null;
  575. }
  576. $cachePath = FreshRSS_Feed::cacheFilename($feedSourceUrl, $this->attributes(), $this->kind());
  577. $html = httpGet($feedSourceUrl, $cachePath,
  578. $this->kind() === FreshRSS_Feed::KIND_XML_XPATH ? 'xml' : 'html', $this->attributes());
  579. if (strlen($html) <= 0) {
  580. return null;
  581. }
  582. $view = new FreshRSS_View();
  583. $view->_path('index/rss.phtml');
  584. $view->internal_rendering = true;
  585. $view->rss_url = $feedSourceUrl;
  586. $view->entries = [];
  587. try {
  588. $doc = new DOMDocument();
  589. $doc->recover = true;
  590. $doc->strictErrorChecking = false;
  591. $ok = false;
  592. switch ($this->kind()) {
  593. case FreshRSS_Feed::KIND_HTML_XPATH:
  594. $ok = $doc->loadHTML($html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING) !== false;
  595. break;
  596. case FreshRSS_Feed::KIND_XML_XPATH:
  597. $ok = $doc->loadXML($html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING) !== false;
  598. break;
  599. }
  600. if (!$ok) {
  601. return null;
  602. }
  603. $xpath = new DOMXPath($doc);
  604. $view->rss_title = $xPathFeedTitle == '' ? $this->name() :
  605. htmlspecialchars(@$xpath->evaluate('normalize-space(' . $xPathFeedTitle . ')'), ENT_COMPAT, 'UTF-8');
  606. $view->rss_base = htmlspecialchars(trim($xpath->evaluate('normalize-space(//base/@href)')), ENT_COMPAT, 'UTF-8');
  607. $nodes = $xpath->query($xPathItem);
  608. if ($nodes === false || $nodes->length === 0) {
  609. return null;
  610. }
  611. foreach ($nodes as $node) {
  612. $item = [];
  613. $item['title'] = $xPathItemTitle == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemTitle . ')', $node);
  614. $item['content'] = '';
  615. if ($xPathItemContent != '') {
  616. $result = @$xpath->evaluate($xPathItemContent, $node);
  617. if ($result instanceof DOMNodeList) {
  618. // List of nodes, save as HTML
  619. $content = '';
  620. foreach ($result as $child) {
  621. $content .= $doc->saveHTML($child) . "\n";
  622. }
  623. $item['content'] = $content;
  624. } else {
  625. // Typed expression, save as-is
  626. $item['content'] = strval($result);
  627. }
  628. }
  629. $item['link'] = $xPathItemUri == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemUri . ')', $node);
  630. $item['author'] = $xPathItemAuthor == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemAuthor . ')', $node);
  631. $item['timestamp'] = $xPathItemTimestamp == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemTimestamp . ')', $node);
  632. if ($xPathItemTimeFormat != '') {
  633. $dateTime = DateTime::createFromFormat($xPathItemTimeFormat, $item['timestamp'] ?? '');
  634. if ($dateTime != false) {
  635. $item['timestamp'] = $dateTime->format(DateTime::ATOM);
  636. }
  637. }
  638. $item['thumbnail'] = $xPathItemThumbnail == '' ? '' : @$xpath->evaluate('normalize-space(' . $xPathItemThumbnail . ')', $node);
  639. if ($xPathItemCategories != '') {
  640. $itemCategories = @$xpath->query($xPathItemCategories, $node);
  641. if ($itemCategories !== false) {
  642. $item['tags'] = [];
  643. /** @var DOMNode $itemCategory */
  644. foreach ($itemCategories as $itemCategory) {
  645. $item['tags'][] = $itemCategory->textContent;
  646. }
  647. }
  648. }
  649. if ($xPathItemUid != '') {
  650. $item['guid'] = @$xpath->evaluate('normalize-space(' . $xPathItemUid . ')', $node);
  651. }
  652. if (empty($item['guid'])) {
  653. $item['guid'] = 'urn:sha1:' . sha1($item['title'] . $item['content'] . $item['link']);
  654. }
  655. if ($item['title'] != '' || $item['content'] != '' || $item['link'] != '') {
  656. // HTML-encoding/escaping of the relevant fields (all except 'content')
  657. foreach (['author', 'guid', 'link', 'thumbnail', 'timestamp', 'tags', 'title'] as $key) {
  658. if (!empty($item[$key]) && is_string($item[$key])) {
  659. $item[$key] = Minz_Helper::htmlspecialchars_utf8($item[$key]);
  660. }
  661. }
  662. // CDATA protection
  663. $item['content'] = str_replace(']]>', ']]&gt;', $item['content']);
  664. $view->entries[] = FreshRSS_Entry::fromArray($item);
  665. }
  666. }
  667. } catch (Exception $ex) {
  668. Minz_Log::warning($ex->getMessage());
  669. return null;
  670. }
  671. $simplePie = customSimplePie();
  672. $simplePie->set_raw_data($view->renderToString());
  673. $simplePie->init();
  674. return $simplePie;
  675. }
  676. /**
  677. * To keep track of some new potentially unread articles since last commit+fetch from database
  678. */
  679. public function incPendingUnread(int $n = 1): void {
  680. $this->nbPendingNotRead += $n;
  681. }
  682. /**
  683. * Remember to call updateCachedValue($id_feed) or updateCachedValues() just after.
  684. * @return int|false the number of lines affected, or false if not applicable
  685. */
  686. public function keepMaxUnread() {
  687. $keepMaxUnread = $this->attributes('keep_max_n_unread');
  688. if ($keepMaxUnread === null) {
  689. $keepMaxUnread = FreshRSS_Context::$user_conf->mark_when['max_n_unread'];
  690. }
  691. $keepMaxUnread = (int)$keepMaxUnread;
  692. if ($keepMaxUnread > 0 && $this->nbNotRead(false) + $this->nbPendingNotRead > $keepMaxUnread) {
  693. $feedDAO = FreshRSS_Factory::createFeedDao();
  694. return $feedDAO->keepMaxUnread($this->id(), max(0, $keepMaxUnread - $this->nbPendingNotRead));
  695. }
  696. return false;
  697. }
  698. /**
  699. * Applies the *mark as read upon gone* policy, if enabled.
  700. * Remember to call `updateCachedValue($id_feed)` or `updateCachedValues()` just after.
  701. * @return int|false the number of lines affected, or false if not applicable
  702. */
  703. public function markAsReadUponGone(bool $upstreamIsEmpty, int $maxTimestamp = 0) {
  704. $readUponGone = $this->attributes('read_upon_gone');
  705. if ($readUponGone === null) {
  706. $readUponGone = FreshRSS_Context::$user_conf->mark_when['gone'];
  707. }
  708. if (!$readUponGone) {
  709. return false;
  710. }
  711. if ($upstreamIsEmpty) {
  712. if ($maxTimestamp <= 0) {
  713. $maxTimestamp = time();
  714. }
  715. $entryDAO = FreshRSS_Factory::createEntryDao();
  716. $affected = $entryDAO->markReadFeed($this->id(), $maxTimestamp . '000000');
  717. } else {
  718. $feedDAO = FreshRSS_Factory::createFeedDao();
  719. $affected = $feedDAO->markAsReadUponGone($this->id());
  720. }
  721. if ($affected > 0) {
  722. Minz_Log::debug(__METHOD__ . " $affected items" . ($upstreamIsEmpty ? ' (all)' : '') . ' [' . $this->url(false) . ']');
  723. }
  724. return $affected;
  725. }
  726. /**
  727. * Remember to call `updateCachedValue($id_feed)` or `updateCachedValues()` just after
  728. * @return int|false
  729. */
  730. public function cleanOldEntries() {
  731. $archiving = $this->attributes('archiving');
  732. if ($archiving == null) {
  733. $catDAO = FreshRSS_Factory::createCategoryDao();
  734. $category = $catDAO->searchById($this->categoryId);
  735. $archiving = $category == null ? null : $category->attributes('archiving');
  736. if ($archiving == null) {
  737. $archiving = FreshRSS_Context::$user_conf->archiving;
  738. }
  739. }
  740. if (is_array($archiving)) {
  741. $entryDAO = FreshRSS_Factory::createEntryDao();
  742. $nb = $entryDAO->cleanOldEntries($this->id(), $archiving);
  743. if ($nb > 0) {
  744. Minz_Log::debug($nb . ' entries cleaned in feed [' . $this->url(false) . '] with: ' . json_encode($archiving));
  745. }
  746. return $nb;
  747. }
  748. return false;
  749. }
  750. /** @param array<string,mixed> $attributes */
  751. public static function cacheFilename(string $url, array $attributes, int $kind = FreshRSS_Feed::KIND_RSS): string {
  752. $simplePie = customSimplePie($attributes);
  753. $filename = $simplePie->get_cache_filename($url);
  754. if ($kind === FreshRSS_Feed::KIND_HTML_XPATH) {
  755. return CACHE_PATH . '/' . $filename . '.html';
  756. } elseif ($kind === FreshRSS_Feed::KIND_XML_XPATH) {
  757. return CACHE_PATH . '/' . $filename . '.xml';
  758. } else {
  759. return CACHE_PATH . '/' . $filename . '.spc';
  760. }
  761. }
  762. public function clearCache(): bool {
  763. return @unlink(FreshRSS_Feed::cacheFilename($this->url, $this->attributes(), $this->kind));
  764. }
  765. /** @return int|false */
  766. public function cacheModifiedTime() {
  767. return @filemtime(FreshRSS_Feed::cacheFilename($this->url, $this->attributes(), $this->kind));
  768. }
  769. public function lock(): bool {
  770. $this->lockPath = TMP_PATH . '/' . $this->hash() . '.freshrss.lock';
  771. if (file_exists($this->lockPath) && ((time() - (@filemtime($this->lockPath) ?: 0)) > 3600)) {
  772. @unlink($this->lockPath);
  773. }
  774. if (($handle = @fopen($this->lockPath, 'x')) === false) {
  775. return false;
  776. }
  777. //register_shutdown_function('unlink', $this->lockPath);
  778. @fclose($handle);
  779. return true;
  780. }
  781. public function unlock(): bool {
  782. return @unlink($this->lockPath);
  783. }
  784. /**
  785. * @return array<FreshRSS_FilterAction>
  786. */
  787. public function filterActions(): array {
  788. if (empty($this->filterActions)) {
  789. $this->filterActions = array();
  790. $filters = $this->attributes('filters');
  791. if (is_array($filters)) {
  792. foreach ($filters as $filter) {
  793. $filterAction = FreshRSS_FilterAction::fromJSON($filter);
  794. if ($filterAction != null) {
  795. $this->filterActions[] = $filterAction;
  796. }
  797. }
  798. }
  799. }
  800. return $this->filterActions;
  801. }
  802. /**
  803. * @param array<FreshRSS_FilterAction>|null $filterActions
  804. */
  805. private function _filterActions(?array $filterActions): void {
  806. $this->filterActions = $filterActions;
  807. if (is_array($this->filterActions) && !empty($this->filterActions)) {
  808. $this->_attributes('filters', array_map(static function (?FreshRSS_FilterAction $af) {
  809. return $af == null ? null : $af->toJSON();
  810. }, $this->filterActions));
  811. } else {
  812. $this->_attributes('filters', null);
  813. }
  814. }
  815. /** @return array<FreshRSS_BooleanSearch> */
  816. public function filtersAction(string $action): array {
  817. $action = trim($action);
  818. if ($action == '') {
  819. return array();
  820. }
  821. $filters = array();
  822. $filterActions = $this->filterActions();
  823. for ($i = count($filterActions) - 1; $i >= 0; $i--) {
  824. $filterAction = $filterActions[$i];
  825. if ($filterAction != null && $filterAction->booleanSearch() != null &&
  826. $filterAction->actions() != null && in_array($action, $filterAction->actions(), true)) {
  827. $filters[] = $filterAction->booleanSearch();
  828. }
  829. }
  830. return $filters;
  831. }
  832. /**
  833. * @param array<string> $filters
  834. */
  835. public function _filtersAction(string $action, array $filters): void {
  836. $action = trim($action);
  837. if ($action == '') {
  838. return;
  839. }
  840. $filters = array_unique(array_map('trim', $filters));
  841. $filterActions = $this->filterActions();
  842. //Check existing filters
  843. for ($i = count($filterActions) - 1; $i >= 0; $i--) {
  844. $filterAction = $filterActions[$i];
  845. if ($filterAction == null || !is_array($filterAction->actions()) ||
  846. $filterAction->booleanSearch() == null || trim($filterAction->booleanSearch()->getRawInput()) == '') {
  847. array_splice($filterActions, $i, 1);
  848. continue;
  849. }
  850. $actions = $filterAction->actions();
  851. //Remove existing rules with same action
  852. for ($j = count($actions) - 1; $j >= 0; $j--) {
  853. if ($actions[$j] === $action) {
  854. array_splice($actions, $j, 1);
  855. }
  856. }
  857. //Update existing filter with new action
  858. for ($k = count($filters) - 1; $k >= 0; $k --) {
  859. $filter = $filters[$k];
  860. if ($filter === $filterAction->booleanSearch()->getRawInput()) {
  861. $actions[] = $action;
  862. array_splice($filters, $k, 1);
  863. }
  864. }
  865. //Save result
  866. if (empty($actions)) {
  867. array_splice($filterActions, $i, 1);
  868. } else {
  869. $filterAction->_actions($actions);
  870. }
  871. }
  872. //Add new filters
  873. for ($k = count($filters) - 1; $k >= 0; $k --) {
  874. $filter = $filters[$k];
  875. if ($filter != '') {
  876. $filterAction = FreshRSS_FilterAction::fromJSON(array(
  877. 'search' => $filter,
  878. 'actions' => array($action),
  879. ));
  880. if ($filterAction != null) {
  881. $filterActions[] = $filterAction;
  882. }
  883. }
  884. }
  885. if (empty($filterActions)) {
  886. $filterActions = null;
  887. }
  888. $this->_filterActions($filterActions);
  889. }
  890. //<WebSub>
  891. public function pubSubHubbubEnabled(): bool {
  892. $url = $this->selfUrl ? $this->selfUrl : $this->url;
  893. $hubFilename = PSHB_PATH . '/feeds/' . sha1($url) . '/!hub.json';
  894. if ($hubFile = @file_get_contents($hubFilename)) {
  895. $hubJson = json_decode($hubFile, true);
  896. if ($hubJson && empty($hubJson['error']) &&
  897. (empty($hubJson['lease_end']) || $hubJson['lease_end'] > time())) {
  898. return true;
  899. }
  900. }
  901. return false;
  902. }
  903. public function pubSubHubbubError(bool $error = true): bool {
  904. $url = $this->selfUrl ? $this->selfUrl : $this->url;
  905. $hubFilename = PSHB_PATH . '/feeds/' . sha1($url) . '/!hub.json';
  906. $hubFile = @file_get_contents($hubFilename);
  907. $hubJson = $hubFile ? json_decode($hubFile, true) : array();
  908. if (!isset($hubJson['error']) || $hubJson['error'] !== $error) {
  909. $hubJson['error'] = $error;
  910. file_put_contents($hubFilename, json_encode($hubJson));
  911. Minz_Log::warning('Set error to ' . ($error ? 1 : 0) . ' for ' . $url, PSHB_LOG);
  912. }
  913. return false;
  914. }
  915. /**
  916. * @return string|false
  917. */
  918. public function pubSubHubbubPrepare() {
  919. $key = '';
  920. if (Minz_Request::serverIsPublic(FreshRSS_Context::$system_conf->base_url) &&
  921. $this->hubUrl && $this->selfUrl && @is_dir(PSHB_PATH)) {
  922. $path = PSHB_PATH . '/feeds/' . sha1($this->selfUrl);
  923. $hubFilename = $path . '/!hub.json';
  924. if ($hubFile = @file_get_contents($hubFilename)) {
  925. $hubJson = json_decode($hubFile, true);
  926. if (!$hubJson || empty($hubJson['key']) || !ctype_xdigit($hubJson['key'])) {
  927. $text = 'Invalid JSON for WebSub: ' . $this->url;
  928. Minz_Log::warning($text);
  929. Minz_Log::warning($text, PSHB_LOG);
  930. return false;
  931. }
  932. if ((!empty($hubJson['lease_end'])) && ($hubJson['lease_end'] < (time() + (3600 * 23)))) { //TODO: Make a better policy
  933. $text = 'WebSub lease ends at '
  934. . date('c', empty($hubJson['lease_end']) ? time() : $hubJson['lease_end'])
  935. . ' and needs renewal: ' . $this->url;
  936. Minz_Log::warning($text);
  937. Minz_Log::warning($text, PSHB_LOG);
  938. $key = $hubJson['key']; //To renew our lease
  939. } elseif (((!empty($hubJson['error'])) || empty($hubJson['lease_end'])) &&
  940. (empty($hubJson['lease_start']) || $hubJson['lease_start'] < time() - (3600 * 23))) { //Do not renew too often
  941. $key = $hubJson['key']; //To renew our lease
  942. }
  943. } else {
  944. @mkdir($path, 0770, true);
  945. $key = sha1($path . FreshRSS_Context::$system_conf->salt);
  946. $hubJson = array(
  947. 'hub' => $this->hubUrl,
  948. 'key' => $key,
  949. );
  950. file_put_contents($hubFilename, json_encode($hubJson));
  951. @mkdir(PSHB_PATH . '/keys/', 0770, true);
  952. file_put_contents(PSHB_PATH . '/keys/' . $key . '.txt', $this->selfUrl);
  953. $text = 'WebSub prepared for ' . $this->url;
  954. Minz_Log::debug($text);
  955. Minz_Log::debug($text, PSHB_LOG);
  956. }
  957. $currentUser = Minz_User::name();
  958. if (FreshRSS_user_Controller::checkUsername($currentUser) && !file_exists($path . '/' . $currentUser . '.txt')) {
  959. touch($path . '/' . $currentUser . '.txt');
  960. }
  961. }
  962. return $key;
  963. }
  964. //Parameter true to subscribe, false to unsubscribe.
  965. public function pubSubHubbubSubscribe(bool $state): bool {
  966. if ($state) {
  967. $url = $this->selfUrl ? $this->selfUrl : $this->url;
  968. } else {
  969. $url = $this->url; //Always use current URL during unsubscribe
  970. }
  971. if ($url && (Minz_Request::serverIsPublic(FreshRSS_Context::$system_conf->base_url) || !$state)) {
  972. $hubFilename = PSHB_PATH . '/feeds/' . sha1($url) . '/!hub.json';
  973. $hubFile = @file_get_contents($hubFilename);
  974. if ($hubFile === false) {
  975. Minz_Log::warning('JSON not found for WebSub: ' . $this->url);
  976. return false;
  977. }
  978. $hubJson = json_decode($hubFile, true);
  979. if (!$hubJson || empty($hubJson['key']) || !ctype_xdigit($hubJson['key']) || empty($hubJson['hub'])) {
  980. Minz_Log::warning('Invalid JSON for WebSub: ' . $this->url);
  981. return false;
  982. }
  983. $callbackUrl = checkUrl(Minz_Request::getBaseUrl() . '/api/pshb.php?k=' . $hubJson['key']);
  984. if ($callbackUrl == '') {
  985. Minz_Log::warning('Invalid callback for WebSub: ' . $this->url);
  986. return false;
  987. }
  988. if (!$state) { //unsubscribe
  989. $hubJson['lease_end'] = time() - 60;
  990. file_put_contents($hubFilename, json_encode($hubJson));
  991. }
  992. $ch = curl_init();
  993. curl_setopt_array($ch, [
  994. CURLOPT_URL => $hubJson['hub'],
  995. CURLOPT_RETURNTRANSFER => true,
  996. CURLOPT_POSTFIELDS => http_build_query(array(
  997. 'hub.verify' => 'sync',
  998. 'hub.mode' => $state ? 'subscribe' : 'unsubscribe',
  999. 'hub.topic' => $url,
  1000. 'hub.callback' => $callbackUrl,
  1001. )),
  1002. CURLOPT_USERAGENT => FRESHRSS_USERAGENT,
  1003. CURLOPT_MAXREDIRS => 10,
  1004. CURLOPT_FOLLOWLOCATION => true,
  1005. CURLOPT_ENCODING => '', //Enable all encodings
  1006. ]);
  1007. $response = curl_exec($ch);
  1008. $info = curl_getinfo($ch);
  1009. Minz_Log::warning('WebSub ' . ($state ? 'subscribe' : 'unsubscribe') . ' to ' . $url .
  1010. ' via hub ' . $hubJson['hub'] .
  1011. ' with callback ' . $callbackUrl . ': ' . $info['http_code'] . ' ' . $response, PSHB_LOG);
  1012. if (substr('' . $info['http_code'], 0, 1) == '2') {
  1013. return true;
  1014. } else {
  1015. $hubJson['lease_start'] = time(); //Prevent trying again too soon
  1016. $hubJson['error'] = true;
  1017. file_put_contents($hubFilename, json_encode($hubJson));
  1018. return false;
  1019. }
  1020. }
  1021. return false;
  1022. }
  1023. //</WebSub>
  1024. }