Feature and performance improvements.

- Now, you can see the initial and final URL of the process. The initial URL is basically the host that you checked initially and the final URL is where the request finished.

src/httplib.rs

@@ -3,94 +3,80 @@ use {
         structs::{HTTPFilters, HttpData, LibOptions},
         utils,
     },
-    async_recursion::async_recursion,
     futures::stream::StreamExt,
     rand::{distributions::Alphanumeric, thread_rng as rng, Rng},
     reqwest::{
         header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT},
         redirect::Policy,
-        Client, Response, Url,
+        Client, Response,
     },
     scraper::{Html, Selector},
     std::collections::{HashMap, HashSet},
 };
 
-#[allow(clippy::too_many_arguments)]
-#[async_recursion]
 #[must_use]
 pub async fn return_http_data(options: &LibOptions) -> HashMap<String, HttpData> {
-    let threads = if options.hosts.len() < options.threads {
+    let threads = options.hosts.len().min(options.threads);
-        options.hosts.len()
+
-    } else {
+    let filter_codes = options.filter_codes.as_deref().unwrap_or_default();
-        options.threads
+    let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default();
-    };
 
     futures::stream::iter(options.hosts.clone().into_iter().map(|host| {
-        // Use a random user agent
+        let user_agent = utils::return_random_user_agent(&options.user_agents);
-        let user_agent = utils::return_random_string(&options.user_agents);
-
-        // Create futures
-        let https_send_fut = options
-            .client
-            .get(format!("https://{host}"))
-            .header(USER_AGENT, &user_agent);
-
-        let http_send_fut = options
-            .client
-            .get(format!("http://{host}"))
-            .header(USER_AGENT, &user_agent);
-
-        let mut http_data = HttpData::default();
-
-        let mut response = Option::<Response>::None;
 
         async move {
-            if options.retries > 1 {
+            let mut http_data = HttpData::default();
-                for _ in 0..options.retries {
+            let mut response = None;
-                    if let Some(resp) = https_send_fut.try_clone() {
+
-                        if let Ok(resp) = resp.send().await {
+            // Attempt both HTTPS and HTTP requests, retry if necessary
-                            response = Some(resp);
+            for _ in 0..options.retries {
-                            break;
+                let https_req = options
-                        }
+                    .client
-                    } else if let Some(resp) = http_send_fut.try_clone() {
+                    .get(format!("https://{}", host))
-                        if let Ok(resp) = resp.send().await {
+                    .header(USER_AGENT, &user_agent)
-                            response = Some(resp);
+                    .send();
-                            break;
+
-                        }
+                let http_req = options
-                    }
+                    .client
+                    .get(format!("http://{}", host))
+                    .header(USER_AGENT, &user_agent)
+                    .send();
+
+                response = https_req.await.or(http_req.await).ok();
+
+                if response.is_some() {
+                    break;
                 }
-            } else if let Ok(resp) = https_send_fut.send().await {
-                response = Some(resp);
-            } else if let Ok(resp) = http_send_fut.send().await {
-                response = Some(resp);
             }
 
-            match response {
+            if let Some(resp) = response {
-                Some(resp) => {
+                http_data.checked_host = host.clone();
-                    http_data.host_url = return_url(resp.url().clone());
+                http_data.final_url = resp.url().to_string();
-                    if options.assign_response_data {
+
-                        http_data =
+                if options.assign_response_data {
-                            assign_response_data(http_data, resp, options.return_filters).await;
+                    http_data = assign_response_data(http_data, resp, options.return_filters).await;
-                    } else {
+                } else {
-                        http_data.status_code = resp.status().as_u16();
+                    http_data.status_code = resp.status().as_u16();
-                        http_data.http_status = "ACTIVE".to_string();
+                    http_data.http_status = "ACTIVE".to_string();
-                    };
                 }
-                None => {
+            } else {
-                    http_data.http_status = "INACTIVE".to_string();
+                http_data.http_status = "INACTIVE".to_string();
-                }
+            }
-            };
 
             if !options.quiet_flag
-                && (!http_data.host_url.is_empty() && options.conditional_response_code == 0)
+                && !http_data.checked_host.is_empty()
-                || ((!http_data.host_url.is_empty() && options.conditional_response_code != 0)
+                && (filter_codes.is_empty()
-                    && (http_data.status_code >= options.conditional_response_code
+                    || filter_codes.contains(&http_data.status_code.to_string()))
-                        && http_data.status_code <= options.conditional_response_code + 99))
+                && (exclude_codes.is_empty()
+                    || !exclude_codes.contains(&http_data.status_code.to_string()))
             {
                 if options.show_status_codes {
-                    println!("{},{}", http_data.host_url, http_data.status_code);
+                    println!(
+                        "{},[{}],[{}]",
+                        http_data.checked_host, http_data.final_url, http_data.status_code
+                    );
                 } else {
-                    println!("{}", http_data.host_url);
+                    println!("{},[{}]", http_data.checked_host, http_data.final_url);
                 }
             }
             (host, http_data)
@@ -113,62 +99,45 @@ pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client {
         .timeout(std::time::Duration::from_secs(timeout))
         .redirect(policy)
         .danger_accept_invalid_certs(true)
-        .trust_dns(true)
         .use_native_tls()
         .build()
         .expect("Failed to create HTTP client")
 }
 
-#[must_use]
-pub fn return_url(mut url: Url) -> String {
-    url.set_path("");
-    url.set_query(None);
-    url.to_string()
-}
-
 #[allow(clippy::field_reassign_with_default)]
 pub async fn assign_response_data(
     mut http_data: HttpData,
     resp: Response,
-    return_filers: bool,
+    return_filters: bool,
 ) -> HttpData {
     let headers = resp.headers().clone();
     let url = resp.url().clone();
 
     http_data.http_status = "ACTIVE".to_string();
     http_data.status_code = resp.status().as_u16();
-
     http_data.final_url = url.to_string();
-    http_data.protocol = resp.url().scheme().to_string();
+    http_data.protocol = url.scheme().to_string();
-    http_data.content_type = if headers.contains_key(CONTENT_TYPE) {
+    http_data.content_type = headers
-        headers[CONTENT_TYPE]
+        .get(CONTENT_TYPE)
-            .to_str()
+        .and_then(|v| v.to_str().ok())
-            .unwrap_or_default()
+        .unwrap_or_default()
-            .to_string()
+        .to_string();
-    } else {
+
-        String::new()
+    let full_body = (resp.text().await).unwrap_or_default();
-    };
+
+    http_data.content_length = headers
+        .get(CONTENT_LENGTH)
+        .and_then(|v| v.to_str().ok()?.parse().ok())
+        .unwrap_or_else(|| full_body.chars().count() as u64);
 
     http_data.headers = format!("{headers:?}");
 
-    let full_body = resp.text().await.unwrap_or_default();
-
-    http_data.content_length = if headers.contains_key(CONTENT_LENGTH) {
-        headers[CONTENT_LENGTH]
-            .to_str()
-            .unwrap_or_default()
-            .parse()
-            .unwrap_or_default()
-    } else {
-        full_body.chars().count() as u64
-    };
-
     return_title_and_body(&mut http_data, &full_body);
 
-    http_data.words_count = full_body.split(' ').count();
+    http_data.words_count = full_body.split_whitespace().count();
     http_data.lines = full_body.lines().count() + 1;
 
-    if return_filers {
+    if return_filters {
         let host = url.host_str().unwrap_or_default();
         let client = return_http_client(5, 3);
         let user_agents = utils::user_agents();

src/lib.rs

@@ -1,3 +1,4 @@
+pub mod args;
 pub mod httplib;
 pub mod structs;
 pub mod utils;

src/structs.rs

@@ -12,7 +12,7 @@ pub struct HTTPFilters {
 pub struct HttpData {
     pub http_status: String,
     pub status_code: u16,
-    pub host_url: String,
+    pub checked_host: String,
     pub final_url: String,
     pub protocol: String,
     pub title: String,
@@ -35,7 +35,8 @@ pub struct LibOptions {
     pub retries: usize,
     pub threads: usize,
     pub return_filters: bool,
-    pub conditional_response_code: u16,
+    pub filter_codes: Option<String>,
+    pub exclude_codes: Option<String>,
     pub show_status_codes: bool,
     pub assign_response_data: bool,
     pub quiet_flag: bool,

src/utils.rs

@@ -34,7 +34,7 @@ pub fn user_agents() -> Vec<String> {
 }
 
 #[must_use]
-pub fn return_random_string(strings: &[String]) -> String {
+pub fn return_random_user_agent(strings: &[String]) -> String {
     let empty_string = String::new();
     strings
         .choose(&mut thread_rng())