Skip to content

Commit

Permalink
feat(config): expose wait for options
Browse files Browse the repository at this point in the history
  • Loading branch information
j-mendez committed Aug 23, 2024
1 parent 419531f commit 9a6c50c
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 2 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
edition = "2021"
name = "spider_rs"
version = "0.0.45"
version = "0.0.46"
repository = "https://github.com/spider-rs/spider-py"
license = "MIT"
description = "The fastest web crawler and indexer."
Expand Down
42 changes: 41 additions & 1 deletion src/website.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use indexmap::IndexMap;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use spider::compact_str::CompactString;
use spider::configuration::WaitForIdleNetwork;
use spider::configuration::{WaitForDelay, WaitForIdleNetwork, WaitForSelector};
use spider::tokio::select;
use spider::tokio::task::JoinHandle;
use spider::utils::shutdown;
Expand Down Expand Up @@ -718,6 +718,46 @@ impl Website {
slf
}

/// Wait for a delay. Should only be used for testing. This method does nothing if the `chrome` feature is not enabled.
pub fn with_wait_for_delay(
mut slf: PyRefMut<'_, Self>,
timeout: Option<u64>,
) -> PyRefMut<'_, Self> {
slf
.inner
.configuration
.with_wait_for_delay(if timeout.is_some() {
let duration = Duration::from_millis(timeout.unwrap_or_default());
Some(WaitForDelay::new(Some(duration)))
} else {
None
});

slf
}

/// Wait for a CSS query selector. This method does nothing if the `chrome` feature is not enabled.
pub fn with_wait_for_selector(
mut slf: PyRefMut<'_, Self>,
selector: Option<String>,
timeout: Option<u64>,
) -> PyRefMut<'_, Self> {
slf
.inner
.configuration
.with_wait_for_selector(if timeout.is_some() {
let duration = Duration::from_millis(timeout.unwrap_or_default());
Some(WaitForSelector::new(
Some(duration),
selector.unwrap_or_default().to_string(),
))
} else {
None
});

slf
}

/// add external domains
pub fn with_external_domains(
mut slf: PyRefMut<'_, Self>,
Expand Down

0 comments on commit 9a6c50c

Please sign in to comment.