<?php

[0] Fix | Delete

/**

[1] Fix | Delete

* The crawler class.

[2] Fix | Delete

*

[3] Fix | Delete

* @package LiteSpeed

[4] Fix | Delete

* @since 1.1.0

[5] Fix | Delete

*/

[6] Fix | Delete

[7] Fix | Delete

namespace LiteSpeed;

[8] Fix | Delete

[9] Fix | Delete

defined( 'WPINC' ) || exit();

[10] Fix | Delete

[11] Fix | Delete

/**

[12] Fix | Delete

* Handles sitemap crawling, blacklisting, and async operations.

[13] Fix | Delete

*/

[14] Fix | Delete

class Crawler extends Root {

[15] Fix | Delete

[16] Fix | Delete

const LOG_TAG = '🕸️';

[17] Fix | Delete

[18] Fix | Delete

const TYPE_REFRESH_MAP = 'refresh_map';

[19] Fix | Delete

const TYPE_EMPTY = 'empty';

[20] Fix | Delete

const TYPE_BLACKLIST_EMPTY = 'blacklist_empty';

[21] Fix | Delete

const TYPE_BLACKLIST_DEL = 'blacklist_del';

[22] Fix | Delete

const TYPE_BLACKLIST_ADD = 'blacklist_add';

[23] Fix | Delete

const TYPE_START = 'start';

[24] Fix | Delete

const TYPE_RESET = 'reset';

[25] Fix | Delete

[26] Fix | Delete

const USER_AGENT = 'lscache_walker';

[27] Fix | Delete

const FAST_USER_AGENT = 'lscache_runner';

[28] Fix | Delete

const CHUNKS = 10000;

[29] Fix | Delete

[30] Fix | Delete

const STATUS_WAIT = 'W';

[31] Fix | Delete

const STATUS_HIT = 'H';

[32] Fix | Delete

const STATUS_MISS = 'M';

[33] Fix | Delete

const STATUS_BLACKLIST = 'B';

[34] Fix | Delete

const STATUS_NOCACHE = 'N';

[35] Fix | Delete

[36] Fix | Delete

/**

[37] Fix | Delete

* Sitemeta file slug.

[38] Fix | Delete

*

[39] Fix | Delete

* @var string

[40] Fix | Delete

*/

[41] Fix | Delete

private $_sitemeta = 'meta.data';

[42] Fix | Delete

[43] Fix | Delete

/**

[44] Fix | Delete

* Reset file full path.

[45] Fix | Delete

*

[46] Fix | Delete

* @var string

[47] Fix | Delete

*/

[48] Fix | Delete

private $_resetfile;

[49] Fix | Delete

[50] Fix | Delete

/**

[51] Fix | Delete

* Reason that ended current run.

[52] Fix | Delete

*

[53] Fix | Delete

* @var string

[54] Fix | Delete

*/

[55] Fix | Delete

private $_end_reason;

[56] Fix | Delete

[57] Fix | Delete

/**

[58] Fix | Delete

* Number of CPU cores.

[59] Fix | Delete

*

[60] Fix | Delete

* @var int

[61] Fix | Delete

*/

[62] Fix | Delete

private $_ncpu = 1;

[63] Fix | Delete

[64] Fix | Delete

/**

[65] Fix | Delete

* Server IP set in settings.

[66] Fix | Delete

*

[67] Fix | Delete

* @var string|null

[68] Fix | Delete

*/

[69] Fix | Delete

private $_server_ip;

[70] Fix | Delete

[71] Fix | Delete

/**

[72] Fix | Delete

* Crawler configuration.

[73] Fix | Delete

*

[74] Fix | Delete

* @var array

[75] Fix | Delete

*/

[76] Fix | Delete

private $_crawler_conf = [

[77] Fix | Delete

'cookies' => [],

[78] Fix | Delete

'headers' => [],

[79] Fix | Delete

'ua' => '',

[80] Fix | Delete

];

[81] Fix | Delete

[82] Fix | Delete

/**

[83] Fix | Delete

* Built crawler variants.

[84] Fix | Delete

*

[85] Fix | Delete

* @var array<int,array>

[86] Fix | Delete

*/

[87] Fix | Delete

private $_crawlers = [];

[88] Fix | Delete

[89] Fix | Delete

/**

[90] Fix | Delete

* Current allowed worker threads.

[91] Fix | Delete

*

[92] Fix | Delete

* @var int

[93] Fix | Delete

*/

[94] Fix | Delete

private $_cur_threads = -1;

[95] Fix | Delete

[96] Fix | Delete

/**

[97] Fix | Delete

* Max timestamp to run until.

[98] Fix | Delete

*

[99] Fix | Delete

* @var int

[100] Fix | Delete

*/

[101] Fix | Delete

private $_max_run_time;

[102] Fix | Delete

[103] Fix | Delete

/**

[104] Fix | Delete

* Last time threads were adjusted.

[105] Fix | Delete

*

[106] Fix | Delete

* @var int

[107] Fix | Delete

*/

[108] Fix | Delete

private $_cur_thread_time;

[109] Fix | Delete

[110] Fix | Delete

/**

[111] Fix | Delete

* Map-status list to batch-save.

[112] Fix | Delete

*

[113] Fix | Delete

* @var array

[114] Fix | Delete

*/

[115] Fix | Delete

private $_map_status_list = [

[116] Fix | Delete

'H' => [],

[117] Fix | Delete

'M' => [],

[118] Fix | Delete

'B' => [],

[119] Fix | Delete

'N' => [],

[120] Fix | Delete

];

[121] Fix | Delete

[122] Fix | Delete

/**

[123] Fix | Delete

* Summary cache.

[124] Fix | Delete

*

[125] Fix | Delete

* @var array

[126] Fix | Delete

*/

[127] Fix | Delete

protected $_summary;

[128] Fix | Delete

[129] Fix | Delete

/**

[130] Fix | Delete

* Initialize crawler, assign sitemap path.

[131] Fix | Delete

*

[132] Fix | Delete

* @since 1.1.0

[133] Fix | Delete

*/

[134] Fix | Delete

public function __construct() {

[135] Fix | Delete

if ( is_multisite() ) {

[136] Fix | Delete

$this->_sitemeta = 'meta' . get_current_blog_id() . '.data';

[137] Fix | Delete

}

[138] Fix | Delete

[139] Fix | Delete

$this->_resetfile = LITESPEED_STATIC_DIR . '/crawler/' . $this->_sitemeta . '.reset';

[140] Fix | Delete

[141] Fix | Delete

$this->_summary = self::get_summary();

[142] Fix | Delete

[143] Fix | Delete

$this->_ncpu = $this->_get_server_cpu();

[144] Fix | Delete

$this->_server_ip = $this->conf( Base::O_SERVER_IP );

[145] Fix | Delete

[146] Fix | Delete

self::debug( 'Init w/ CPU cores=' . $this->_ncpu );

[147] Fix | Delete

}

[148] Fix | Delete

[149] Fix | Delete

/**

[150] Fix | Delete

* Try get server CPUs.

[151] Fix | Delete

*

[152] Fix | Delete

* @since 5.2

[153] Fix | Delete

* @return int Number of cores detected.

[154] Fix | Delete

*/

[155] Fix | Delete

private function _get_server_cpu() {

[156] Fix | Delete

$cpuinfo_file = '/proc/cpuinfo';

[157] Fix | Delete

$setting_open_dir = ini_get( 'open_basedir' );

[158] Fix | Delete

if ( $setting_open_dir ) {

[159] Fix | Delete

return 1; // Server has limit.

[160] Fix | Delete

}

[161] Fix | Delete

[162] Fix | Delete

try {

[163] Fix | Delete

// phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged

[164] Fix | Delete

if (!@is_file($cpuinfo_file)) {

[165] Fix | Delete

return 1;

[166] Fix | Delete

}

[167] Fix | Delete

} catch ( \Exception $e ) {

[168] Fix | Delete

return 1;

[169] Fix | Delete

}

[170] Fix | Delete

[171] Fix | Delete

// Local system read; no WP alternative. Suppress sniff.

[172] Fix | Delete

// phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents

[173] Fix | Delete

$cpuinfo = file_get_contents( $cpuinfo_file );

[174] Fix | Delete

preg_match_all( '/^processor/m', $cpuinfo, $matches );

[175] Fix | Delete

$cnt = isset( $matches[0] ) ? count( $matches[0] ) : 0;

[176] Fix | Delete

return $cnt ? $cnt : 1;

[177] Fix | Delete

}

[178] Fix | Delete

[179] Fix | Delete

/**

[180] Fix | Delete

* Check whether the current crawler is active.

[181] Fix | Delete

*

[182] Fix | Delete

* @since 4.3

[183] Fix | Delete

* @param int $curr Crawler index.

[184] Fix | Delete

* @return bool Active state.

[185] Fix | Delete

*/

[186] Fix | Delete

public function is_active( $curr ) {

[187] Fix | Delete

$bypass_list = self::get_option( 'bypass_list', [] );

[188] Fix | Delete

return ! in_array( (int) $curr, $bypass_list, true );

[189] Fix | Delete

}

[190] Fix | Delete

[191] Fix | Delete

/**

[192] Fix | Delete

* Toggle the current crawler's active state and return the updated state.

[193] Fix | Delete

*

[194] Fix | Delete

* @since 4.3

[195] Fix | Delete

* @param int $curr Crawler index.

[196] Fix | Delete

* @return bool True if turned on, false if turned off.

[197] Fix | Delete

*/

[198] Fix | Delete

public function toggle_activeness( $curr ) {

[199] Fix | Delete

$bypass_list = self::get_option( 'bypass_list', [] );

[200] Fix | Delete

if ( in_array( (int) $curr, $bypass_list, true ) ) {

[201] Fix | Delete

// Remove it.

[202] Fix | Delete

$key = array_search( (int) $curr, $bypass_list, true );

[203] Fix | Delete

if ( false !== $key ) {

[204] Fix | Delete

unset( $bypass_list[ $key ] );

[205] Fix | Delete

$bypass_list = array_values( $bypass_list );

[206] Fix | Delete

self::update_option( 'bypass_list', $bypass_list );

[207] Fix | Delete

}

[208] Fix | Delete

return true;

[209] Fix | Delete

}

[210] Fix | Delete

[211] Fix | Delete

// Add it.

[212] Fix | Delete

$bypass_list[] = (int) $curr;

[213] Fix | Delete

self::update_option( 'bypass_list', $bypass_list );

[214] Fix | Delete

return false;

[215] Fix | Delete

}

[216] Fix | Delete

[217] Fix | Delete

/**

[218] Fix | Delete

* Clear bypassed list.

[219] Fix | Delete

*

[220] Fix | Delete

* @since 4.3

[221] Fix | Delete

* @access public

[222] Fix | Delete

* @return void

[223] Fix | Delete

*/

[224] Fix | Delete

public function clear_disabled_list() {

[225] Fix | Delete

self::update_option( 'bypass_list', [] );

[226] Fix | Delete

[227] Fix | Delete

$msg = __( 'Crawler disabled list is cleared! All crawlers are set to active! ', 'litespeed-cache' );

[228] Fix | Delete

Admin_Display::note( $msg );

[229] Fix | Delete

[230] Fix | Delete

self::debug( 'All crawlers are set to active...... ' );

[231] Fix | Delete

}

[232] Fix | Delete

[233] Fix | Delete

/**

[234] Fix | Delete

* Overwrite get_summary to init elements.

[235] Fix | Delete

*

[236] Fix | Delete

* @since 3.0

[237] Fix | Delete

* @access public

[238] Fix | Delete

*

[239] Fix | Delete

* @param string|false $field Field name to fetch or false to get all.

[240] Fix | Delete

* @return mixed Summary value/array or null if not found.

[241] Fix | Delete

*/

[242] Fix | Delete

public static function get_summary( $field = false ) {

[243] Fix | Delete

$_default = [

[244] Fix | Delete

'list_size' => 0,

[245] Fix | Delete

'last_update_time' => 0,

[246] Fix | Delete

'curr_crawler' => 0,

[247] Fix | Delete

'curr_crawler_beginning_time' => 0,

[248] Fix | Delete

'last_pos' => 0,

[249] Fix | Delete

'last_count' => 0,

[250] Fix | Delete

'last_crawled' => 0,

[251] Fix | Delete

'last_start_time' => 0,

[252] Fix | Delete

'last_status' => '',

[253] Fix | Delete

'is_running' => 0,

[254] Fix | Delete

'end_reason' => '',

[255] Fix | Delete

'meta_save_time' => 0,

[256] Fix | Delete

'pos_reset_check' => 0,

[257] Fix | Delete

'done' => 0,

[258] Fix | Delete

'this_full_beginning_time' => 0,

[259] Fix | Delete

'last_full_time_cost' => 0,

[260] Fix | Delete

'last_crawler_total_cost' => 0,

[261] Fix | Delete

'crawler_stats' => [], // this will store all crawlers hit/miss crawl status.

[262] Fix | Delete

];

[263] Fix | Delete

[264] Fix | Delete

wp_cache_delete( 'alloptions', 'options' ); // ensure the summary is current.

[265] Fix | Delete

$summary = parent::get_summary();

[266] Fix | Delete

$summary = array_merge( $_default, $summary );

[267] Fix | Delete

[268] Fix | Delete

if ( false === $field ) {

[269] Fix | Delete

return $summary;

[270] Fix | Delete

}

[271] Fix | Delete

[272] Fix | Delete

if ( array_key_exists( $field, $summary ) ) {

[273] Fix | Delete

return $summary[ $field ];

[274] Fix | Delete

}

[275] Fix | Delete

[276] Fix | Delete

return null;

[277] Fix | Delete

}

[278] Fix | Delete

[279] Fix | Delete

/**

[280] Fix | Delete

* Overwrite save_summary.

[281] Fix | Delete

*

[282] Fix | Delete

* @since 3.0

[283] Fix | Delete

* @access public

[284] Fix | Delete

*

[285] Fix | Delete

* @param array|false $data Data to save or false to save current.

[286] Fix | Delete

* @param bool $reload Whether to reload after saving.

[287] Fix | Delete

* @param bool $overwrite Whether to overwrite completely.

[288] Fix | Delete

* @return void

[289] Fix | Delete

*/

[290] Fix | Delete

public static function save_summary( $data = false, $reload = false, $overwrite = false ) {

[291] Fix | Delete

$instance = self::cls();

[292] Fix | Delete

$instance->_summary['meta_save_time'] = time();

[293] Fix | Delete

[294] Fix | Delete

if ( false === $data ) {

[295] Fix | Delete

$data = $instance->_summary;

[296] Fix | Delete

}

[297] Fix | Delete

[298] Fix | Delete

parent::save_summary( $data, $reload, $overwrite );

[299] Fix | Delete

[300] Fix | Delete

File::save( LITESPEED_STATIC_DIR . '/crawler/' . $instance->_sitemeta, wp_json_encode( $data ), true );

[301] Fix | Delete

}

[302] Fix | Delete

[303] Fix | Delete

/**

[304] Fix | Delete

* Cron start async crawling.

[305] Fix | Delete

*

[306] Fix | Delete

* @since 5.5

[307] Fix | Delete

* @return void

[308] Fix | Delete

*/

[309] Fix | Delete

public static function start_async_cron() {

[310] Fix | Delete

Task::async_call( 'crawler' );

[311] Fix | Delete

}

[312] Fix | Delete

[313] Fix | Delete

/**

[314] Fix | Delete

* Manually start async crawling.

[315] Fix | Delete

*

[316] Fix | Delete

* @since 5.5

[317] Fix | Delete

* @return void

[318] Fix | Delete

*/

[319] Fix | Delete

public static function start_async() {

[320] Fix | Delete

Task::async_call( 'crawler_force' );

[321] Fix | Delete

[322] Fix | Delete

$msg = __( 'Started async crawling', 'litespeed-cache' );

[323] Fix | Delete

Admin_Display::success( $msg );

[324] Fix | Delete

}

[325] Fix | Delete

[326] Fix | Delete

/**

[327] Fix | Delete

* Ajax crawl handler.

[328] Fix | Delete

*

[329] Fix | Delete

* @since 5.5

[330] Fix | Delete

* @param bool $manually_run Whether manually triggered.

[331] Fix | Delete

* @return void

[332] Fix | Delete

*/

[333] Fix | Delete

public static function async_handler( $manually_run = false ) {

[334] Fix | Delete

self::debug( '------------async-------------start_async_handler' );

[335] Fix | Delete

self::start( (bool) $manually_run );

[336] Fix | Delete

}

[337] Fix | Delete

[338] Fix | Delete

/**

[339] Fix | Delete

* Proceed crawling.

[340] Fix | Delete

*

[341] Fix | Delete

* @since 1.1.0

[342] Fix | Delete

* @access public

[343] Fix | Delete

*

[344] Fix | Delete

* @param bool $manually_run Whether manually triggered.

[345] Fix | Delete

* @return bool|void

[346] Fix | Delete

*/

[347] Fix | Delete

public static function start( $manually_run = false ) {

[348] Fix | Delete

if ( ! Router::can_crawl() ) {

[349] Fix | Delete

self::debug( '......crawler is NOT allowed by the server admin......' );

[350] Fix | Delete

return false;

[351] Fix | Delete

}

[352] Fix | Delete

[353] Fix | Delete

if ( $manually_run ) {

[354] Fix | Delete

self::debug( '......crawler manually ran......' );

[355] Fix | Delete

}

[356] Fix | Delete

[357] Fix | Delete

self::cls()->_crawl_data( (bool) $manually_run );

[358] Fix | Delete

}

[359] Fix | Delete

[360] Fix | Delete

/**

[361] Fix | Delete

* Crawling start.

[362] Fix | Delete

*

[363] Fix | Delete

* @since 1.1.0

[364] Fix | Delete

* @access private

[365] Fix | Delete

*

[366] Fix | Delete

* @param bool $manually_run Whether manually triggered.

[367] Fix | Delete

* @return void

[368] Fix | Delete

*/

[369] Fix | Delete

private function _crawl_data( $manually_run ) {

[370] Fix | Delete

if ( ! defined( 'LITESPEED_LANE_HASH' ) ) {

[371] Fix | Delete

define( 'LITESPEED_LANE_HASH', Str::rrand( 8 ) );

[372] Fix | Delete

}

[373] Fix | Delete

if ( $this->_check_valid_lane() ) {

[374] Fix | Delete

$this->_take_over_lane();

[375] Fix | Delete

} else {

[376] Fix | Delete

self::debug( '⚠️ lane in use' );

[377] Fix | Delete

return;

[378] Fix | Delete

}

[379] Fix | Delete

self::debug( '......crawler started......' );

[380] Fix | Delete

[381] Fix | Delete

// for the first time running.

[382] Fix | Delete

if ( ! $this->_summary || ! Data::cls()->tb_exist( 'crawler' ) || ! Data::cls()->tb_exist( 'crawler_blacklist' ) ) {

[383] Fix | Delete

$this->cls( 'Crawler_Map' )->gen();

[384] Fix | Delete

}

[385] Fix | Delete

[386] Fix | Delete

// if finished last time, regenerate sitemap.

[387] Fix | Delete

if ( 'touchedEnd' === $this->_summary['done'] ) {

[388] Fix | Delete

// check whole crawling interval.

[389] Fix | Delete

$last_finished_at = (int) $this->_summary['last_full_time_cost'] + (int) $this->_summary['this_full_beginning_time'];

[390] Fix | Delete

if ( ! $manually_run && ( time() - $last_finished_at ) < $this->conf( Base::O_CRAWLER_CRAWL_INTERVAL ) ) {

[391] Fix | Delete

self::debug( 'Cron abort: cache warmed already.' );

[392] Fix | Delete

$this->Release_lane();

[393] Fix | Delete

return;

[394] Fix | Delete

}

[395] Fix | Delete

self::debug( 'TouchedEnd. regenerate sitemap....' );

[396] Fix | Delete

$this->cls( 'Crawler_Map' )->gen();

[397] Fix | Delete

}

[398] Fix | Delete

[399] Fix | Delete

$crawlers = $this->list_crawlers();

[400] Fix | Delete

$crawlers_count = count( $crawlers );

[401] Fix | Delete

[402] Fix | Delete

// Skip the crawlers that in bypassed list.

[403] Fix | Delete

while ( ! $this->is_active( $this->_summary['curr_crawler'] ) && $this->_summary['curr_crawler'] < $crawlers_count ) {

[404] Fix | Delete

self::debug( 'Skipped the Crawler #' . $this->_summary['curr_crawler'] . ' ......' );

[405] Fix | Delete

$this->_summary['curr_crawler'] = (int) $this->_summary['curr_crawler'] + 1;

[406] Fix | Delete

}

[407] Fix | Delete

if ( $this->_summary['curr_crawler'] >= $crawlers_count ) {

[408] Fix | Delete

$this->_end_reason = 'end';

[409] Fix | Delete

$this->_terminate_running();

[410] Fix | Delete

$this->Release_lane();

[411] Fix | Delete

return;

[412] Fix | Delete

}

[413] Fix | Delete

[414] Fix | Delete

// In case crawlers are all done but not reload, reload it.

[415] Fix | Delete

if ( empty( $this->_summary['curr_crawler'] ) || empty( $this->_crawlers[ $this->_summary['curr_crawler'] ] ) ) {

[416] Fix | Delete

$this->_summary['curr_crawler'] = 0;

[417] Fix | Delete

$this->_summary['crawler_stats'][ $this->_summary['curr_crawler'] ] = [];

[418] Fix | Delete

}

[419] Fix | Delete

[420] Fix | Delete

$res = $this->load_conf();

[421] Fix | Delete

if ( ! $res ) {

[422] Fix | Delete

self::debug( 'Load conf failed' );

[423] Fix | Delete

$this->_terminate_running();

[424] Fix | Delete

$this->Release_lane();

[425] Fix | Delete

return;

[426] Fix | Delete

}

[427] Fix | Delete

[428] Fix | Delete

try {

[429] Fix | Delete

$this->_engine_start();

[430] Fix | Delete

$this->Release_lane();

[431] Fix | Delete

} catch ( \Exception $e ) {

[432] Fix | Delete

self::debug( '🛑 ' . $e->getMessage() );

[433] Fix | Delete

}

[434] Fix | Delete

}

[435] Fix | Delete

[436] Fix | Delete

/**

[437] Fix | Delete

* Load conf before running crawler.

[438] Fix | Delete

*

[439] Fix | Delete

* @since 3.0

[440] Fix | Delete

* @access private

[441] Fix | Delete

* @return bool True on success.

[442] Fix | Delete

*/

[443] Fix | Delete

private function load_conf() {

[444] Fix | Delete

$this->_crawler_conf['base'] = site_url();

[445] Fix | Delete

[446] Fix | Delete

$current_crawler = $this->_crawlers[ $this->_summary['curr_crawler'] ];

[447] Fix | Delete

[448] Fix | Delete

// Cookies.

[449] Fix | Delete

foreach ( $current_crawler as $k => $v ) {

[450] Fix | Delete

if ( 0 !== strpos( $k, 'cookie:' ) ) {

[451] Fix | Delete

continue;

[452] Fix | Delete

}

[453] Fix | Delete

[454] Fix | Delete

if ( '_null' === $v ) {

[455] Fix | Delete

continue;

[456] Fix | Delete

}

[457] Fix | Delete

[458] Fix | Delete

$this->_crawler_conf['cookies'][ substr( $k, 7 ) ] = $v;

[459] Fix | Delete

}

[460] Fix | Delete

[461] Fix | Delete

// WebP/AVIF simulation.

[462] Fix | Delete

if ( ! empty( $current_crawler['webp'] ) ) {

[463] Fix | Delete

$this->_crawler_conf['headers'][] = 'Accept: image/' . ( 2 === (int) $this->conf( Base::O_IMG_OPTM_WEBP ) ? 'avif' : 'webp' ) . ',*/*';

[464] Fix | Delete

}

[465] Fix | Delete

[466] Fix | Delete

// Mobile crawler.

[467] Fix | Delete

if ( ! empty( $current_crawler['mobile'] ) ) {

[468] Fix | Delete

$this->_crawler_conf['ua'] = 'Mobile iPhone';

[469] Fix | Delete

}

[470] Fix | Delete

[471] Fix | Delete

// Limit delay to use server setting.

[472] Fix | Delete

$this->_crawler_conf['run_delay'] = 500; // microseconds.

[473] Fix | Delete

if ( defined( 'LITESPEED_CRAWLER_USLEEP' ) && constant( 'LITESPEED_CRAWLER_USLEEP' ) > $this->_crawler_conf['run_delay'] ) {

[474] Fix | Delete

$this->_crawler_conf['run_delay'] = (int) constant( 'LITESPEED_CRAWLER_USLEEP' );

[475] Fix | Delete

}

[476] Fix | Delete

if ( isset( $_SERVER[ Base::ENV_CRAWLER_USLEEP ] ) ) {

[477] Fix | Delete

$env_usleep = absint( wp_unslash( $_SERVER[ Base::ENV_CRAWLER_USLEEP ] ) );

[478] Fix | Delete

if ( $env_usleep > (int) $this->_crawler_conf['run_delay'] ) {

[479] Fix | Delete

$this->_crawler_conf['run_delay'] = $env_usleep;

[480] Fix | Delete

}

[481] Fix | Delete

}

[482] Fix | Delete

[483] Fix | Delete

$this->_crawler_conf['run_duration'] = $this->get_crawler_duration();

[484] Fix | Delete

[485] Fix | Delete

$this->_crawler_conf['load_limit'] = (int) $this->conf( Base::O_CRAWLER_LOAD_LIMIT );

[486] Fix | Delete

if ( isset( $_SERVER[ Base::ENV_CRAWLER_LOAD_LIMIT_ENFORCE ] ) ) {

[487] Fix | Delete

$this->_crawler_conf['load_limit'] = absint( wp_unslash( $_SERVER[ Base::ENV_CRAWLER_LOAD_LIMIT_ENFORCE ] ) );

[488] Fix | Delete

} elseif ( isset( $_SERVER[ Base::ENV_CRAWLER_LOAD_LIMIT ] ) ) {

[489] Fix | Delete

$env_limit = absint( wp_unslash( $_SERVER[ Base::ENV_CRAWLER_LOAD_LIMIT ] ) );

[490] Fix | Delete

if ( $env_limit < (int) $this->_crawler_conf['load_limit'] ) {

[491] Fix | Delete

$this->_crawler_conf['load_limit'] = $env_limit;

[492] Fix | Delete

}

[493] Fix | Delete

}

[494] Fix | Delete

if ( 0 === (int) $this->_crawler_conf['load_limit'] ) {

[495] Fix | Delete

self::debug( '🛑 Terminated crawler due to load limit set to 0' );

[496] Fix | Delete

return false;

[497] Fix | Delete

}

[498] Fix | Delete

[499] Fix | Delete