From 0b10f631fc6a4b1a55c5f8c165d74cf9985550bf Mon Sep 17 00:00:00 2001 From: Eduard Tolosa Date: Fri, 11 Oct 2024 04:28:59 -0500 Subject: [PATCH] Feature and performance improvements. - Now, you can see the initial and final URL of the process. The initial URL is basically the host that you checked initially and the final URL is where the request finished. --- src/httplib.rs | 161 ++++++++++++++++++++----------------------------- src/lib.rs | 1 + src/structs.rs | 5 +- src/utils.rs | 2 +- 4 files changed, 70 insertions(+), 99 deletions(-) diff --git a/src/httplib.rs b/src/httplib.rs index c310e9d..f78ab81 100644 --- a/src/httplib.rs +++ b/src/httplib.rs @@ -3,94 +3,80 @@ use { structs::{HTTPFilters, HttpData, LibOptions}, utils, }, - async_recursion::async_recursion, futures::stream::StreamExt, rand::{distributions::Alphanumeric, thread_rng as rng, Rng}, reqwest::{ header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT}, redirect::Policy, - Client, Response, Url, + Client, Response, }, scraper::{Html, Selector}, std::collections::{HashMap, HashSet}, }; -#[allow(clippy::too_many_arguments)] -#[async_recursion] #[must_use] pub async fn return_http_data(options: &LibOptions) -> HashMap { - let threads = if options.hosts.len() < options.threads { - options.hosts.len() - } else { - options.threads - }; + let threads = options.hosts.len().min(options.threads); + + let filter_codes = options.filter_codes.as_deref().unwrap_or_default(); + let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default(); futures::stream::iter(options.hosts.clone().into_iter().map(|host| { - // Use a random user agent - let user_agent = utils::return_random_string(&options.user_agents); - - // Create futures - let https_send_fut = options - .client - .get(format!("https://{host}")) - .header(USER_AGENT, &user_agent); - - let http_send_fut = options - .client - .get(format!("http://{host}")) - .header(USER_AGENT, &user_agent); - - let mut http_data = HttpData::default(); - - let mut response = Option::::None; + let user_agent = utils::return_random_user_agent(&options.user_agents); async move { - if options.retries > 1 { - for _ in 0..options.retries { - if let Some(resp) = https_send_fut.try_clone() { - if let Ok(resp) = resp.send().await { - response = Some(resp); - break; - } - } else if let Some(resp) = http_send_fut.try_clone() { - if let Ok(resp) = resp.send().await { - response = Some(resp); - break; - } - } + let mut http_data = HttpData::default(); + let mut response = None; + + // Attempt both HTTPS and HTTP requests, retry if necessary + for _ in 0..options.retries { + let https_req = options + .client + .get(format!("https://{}", host)) + .header(USER_AGENT, &user_agent) + .send(); + + let http_req = options + .client + .get(format!("http://{}", host)) + .header(USER_AGENT, &user_agent) + .send(); + + response = https_req.await.or(http_req.await).ok(); + + if response.is_some() { + break; } - } else if let Ok(resp) = https_send_fut.send().await { - response = Some(resp); - } else if let Ok(resp) = http_send_fut.send().await { - response = Some(resp); } - match response { - Some(resp) => { - http_data.host_url = return_url(resp.url().clone()); - if options.assign_response_data { - http_data = - assign_response_data(http_data, resp, options.return_filters).await; - } else { - http_data.status_code = resp.status().as_u16(); - http_data.http_status = "ACTIVE".to_string(); - }; + if let Some(resp) = response { + http_data.checked_host = host.clone(); + http_data.final_url = resp.url().to_string(); + + if options.assign_response_data { + http_data = assign_response_data(http_data, resp, options.return_filters).await; + } else { + http_data.status_code = resp.status().as_u16(); + http_data.http_status = "ACTIVE".to_string(); } - None => { - http_data.http_status = "INACTIVE".to_string(); - } - }; + } else { + http_data.http_status = "INACTIVE".to_string(); + } if !options.quiet_flag - && (!http_data.host_url.is_empty() && options.conditional_response_code == 0) - || ((!http_data.host_url.is_empty() && options.conditional_response_code != 0) - && (http_data.status_code >= options.conditional_response_code - && http_data.status_code <= options.conditional_response_code + 99)) + && !http_data.checked_host.is_empty() + && (filter_codes.is_empty() + || filter_codes.contains(&http_data.status_code.to_string())) + && (exclude_codes.is_empty() + || !exclude_codes.contains(&http_data.status_code.to_string())) { if options.show_status_codes { - println!("{},{}", http_data.host_url, http_data.status_code); + println!( + "{},[{}],[{}]", + http_data.checked_host, http_data.final_url, http_data.status_code + ); } else { - println!("{}", http_data.host_url); + println!("{},[{}]", http_data.checked_host, http_data.final_url); } } (host, http_data) @@ -113,62 +99,45 @@ pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client { .timeout(std::time::Duration::from_secs(timeout)) .redirect(policy) .danger_accept_invalid_certs(true) - .trust_dns(true) .use_native_tls() .build() .expect("Failed to create HTTP client") } -#[must_use] -pub fn return_url(mut url: Url) -> String { - url.set_path(""); - url.set_query(None); - url.to_string() -} - #[allow(clippy::field_reassign_with_default)] pub async fn assign_response_data( mut http_data: HttpData, resp: Response, - return_filers: bool, + return_filters: bool, ) -> HttpData { let headers = resp.headers().clone(); let url = resp.url().clone(); http_data.http_status = "ACTIVE".to_string(); http_data.status_code = resp.status().as_u16(); - http_data.final_url = url.to_string(); - http_data.protocol = resp.url().scheme().to_string(); - http_data.content_type = if headers.contains_key(CONTENT_TYPE) { - headers[CONTENT_TYPE] - .to_str() - .unwrap_or_default() - .to_string() - } else { - String::new() - }; + http_data.protocol = url.scheme().to_string(); + http_data.content_type = headers + .get(CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string(); + + let full_body = (resp.text().await).unwrap_or_default(); + + http_data.content_length = headers + .get(CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()?.parse().ok()) + .unwrap_or_else(|| full_body.chars().count() as u64); http_data.headers = format!("{headers:?}"); - let full_body = resp.text().await.unwrap_or_default(); - - http_data.content_length = if headers.contains_key(CONTENT_LENGTH) { - headers[CONTENT_LENGTH] - .to_str() - .unwrap_or_default() - .parse() - .unwrap_or_default() - } else { - full_body.chars().count() as u64 - }; - return_title_and_body(&mut http_data, &full_body); - http_data.words_count = full_body.split(' ').count(); + http_data.words_count = full_body.split_whitespace().count(); http_data.lines = full_body.lines().count() + 1; - if return_filers { + if return_filters { let host = url.host_str().unwrap_or_default(); let client = return_http_client(5, 3); let user_agents = utils::user_agents(); diff --git a/src/lib.rs b/src/lib.rs index 739a5b1..d694d5b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod args; pub mod httplib; pub mod structs; pub mod utils; diff --git a/src/structs.rs b/src/structs.rs index e4aff0a..9228890 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -12,7 +12,7 @@ pub struct HTTPFilters { pub struct HttpData { pub http_status: String, pub status_code: u16, - pub host_url: String, + pub checked_host: String, pub final_url: String, pub protocol: String, pub title: String, @@ -35,7 +35,8 @@ pub struct LibOptions { pub retries: usize, pub threads: usize, pub return_filters: bool, - pub conditional_response_code: u16, + pub filter_codes: Option, + pub exclude_codes: Option, pub show_status_codes: bool, pub assign_response_data: bool, pub quiet_flag: bool, diff --git a/src/utils.rs b/src/utils.rs index 3734174..a34c17c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -34,7 +34,7 @@ pub fn user_agents() -> Vec { } #[must_use] -pub fn return_random_string(strings: &[String]) -> String { +pub fn return_random_user_agent(strings: &[String]) -> String { let empty_string = String::new(); strings .choose(&mut thread_rng())