From 9a6c50cc0d4ea841996316f3d4259c8838d790fe Mon Sep 17 00:00:00 2001 From: j-mendez Date: Thu, 22 Aug 2024 23:12:02 -0400 Subject: [PATCH] feat(config): expose wait for options --- Cargo.toml | 2 +- src/website.rs | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bebbf04..96fef54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2021" name = "spider_rs" -version = "0.0.45" +version = "0.0.46" repository = "https://github.com/spider-rs/spider-py" license = "MIT" description = "The fastest web crawler and indexer." diff --git a/src/website.rs b/src/website.rs index 1736cb8..72e68e3 100644 --- a/src/website.rs +++ b/src/website.rs @@ -3,7 +3,7 @@ use indexmap::IndexMap; use pyo3::prelude::*; use pyo3::types::PyDict; use spider::compact_str::CompactString; -use spider::configuration::WaitForIdleNetwork; +use spider::configuration::{WaitForDelay, WaitForIdleNetwork, WaitForSelector}; use spider::tokio::select; use spider::tokio::task::JoinHandle; use spider::utils::shutdown; @@ -718,6 +718,46 @@ impl Website { slf } + /// Wait for a delay. Should only be used for testing. This method does nothing if the `chrome` feature is not enabled. + pub fn with_wait_for_delay( + mut slf: PyRefMut<'_, Self>, + timeout: Option, + ) -> PyRefMut<'_, Self> { + slf + .inner + .configuration + .with_wait_for_delay(if timeout.is_some() { + let duration = Duration::from_millis(timeout.unwrap_or_default()); + Some(WaitForDelay::new(Some(duration))) + } else { + None + }); + + slf + } + + /// Wait for a CSS query selector. This method does nothing if the `chrome` feature is not enabled. + pub fn with_wait_for_selector( + mut slf: PyRefMut<'_, Self>, + selector: Option, + timeout: Option, + ) -> PyRefMut<'_, Self> { + slf + .inner + .configuration + .with_wait_for_selector(if timeout.is_some() { + let duration = Duration::from_millis(timeout.unwrap_or_default()); + Some(WaitForSelector::new( + Some(duration), + selector.unwrap_or_default().to_string(), + )) + } else { + None + }); + + slf + } + /// add external domains pub fn with_external_domains( mut slf: PyRefMut<'_, Self>,