diff --git a/Cargo.lock b/Cargo.lock index 7cd7767..dbebbfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "async-recursion" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.52" @@ -57,6 +68,12 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.1.0" @@ -90,6 +107,12 @@ dependencies = [ "vec_map", ] +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "core-foundation" version = "0.9.3" @@ -106,12 +129,73 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "cssparser" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa 0.4.8", + "matches", + "phf", + "proc-macro2", + "quote", + "smallvec", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "dtoa-short" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" + [[package]] name = "encoding_rs" version = "0.8.30" @@ -146,11 +230,13 @@ dependencies = [ name = "fhc" version = "0.6.1" dependencies = [ + "async-recursion", "clap", "futures", "openssl", - "rand", + "rand 0.8.5", "reqwest", + "scraper", "tokio", ] @@ -185,6 +271,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.21" @@ -274,6 +370,35 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.4" @@ -282,7 +407,7 @@ checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -339,6 +464,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "html5ever" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcf38a1a36118242d29b92e1b08ef84e67e4a5ed06e0a80be20e6a32bfed6b" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "http" version = "0.2.6" @@ -347,7 +486,7 @@ checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 1.0.1", ] [[package]] @@ -388,7 +527,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 1.0.1", "pin-project-lite", "socket2 0.4.4", "tokio", @@ -471,6 +610,12 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.1" @@ -531,6 +676,26 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -595,6 +760,18 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "ntapi" version = "0.3.7" @@ -694,6 +871,69 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_macros", + "phf_shared 0.8.0", + "proc-macro-hack", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared 0.8.0", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared 0.8.0", + "rand 0.7.3", +] + +[[package]] +name = "phf_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +dependencies = [ + "phf_generator", + "phf_shared 0.8.0", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.8" @@ -718,6 +958,18 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.36" @@ -744,14 +996,37 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.4" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", + "rand_pcg", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", - "rand_hc", + "rand_chacha 0.3.1", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -761,7 +1036,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -770,16 +1054,25 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom", + "getrandom 0.2.4", ] [[package]] name = "rand_hc" -version = "0.3.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -867,6 +1160,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustls" version = "0.20.3" @@ -910,6 +1212,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scraper" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "matches", + "selectors", + "smallvec", + "tendril", +] + [[package]] name = "sct" version = "0.7.0" @@ -943,6 +1261,32 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "matches", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", + "thin-slice", +] + +[[package]] +name = "semver" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" + [[package]] name = "serde" version = "1.0.136" @@ -955,7 +1299,7 @@ version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" dependencies = [ - "itoa", + "itoa 1.0.1", "ryu", "serde", ] @@ -967,11 +1311,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa", + "itoa 1.0.1", "ryu", "serde", ] +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -981,6 +1335,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.5" @@ -1020,6 +1380,38 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "string_cache" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33994d0838dc2d152d17a62adf608a869b5e846b65b389af7f3dbc1de45c5b26" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97" +dependencies = [ + "phf_generator", + "phf_shared 0.8.0", + "proc-macro2", + "quote", +] + [[package]] name = "strsim" version = "0.8.0" @@ -1051,6 +1443,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "tendril" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -1060,6 +1463,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thin-slice" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" + [[package]] name = "thiserror" version = "1.0.30" @@ -1202,7 +1611,7 @@ dependencies = [ "ipnet", "lazy_static", "log", - "rand", + "rand 0.8.5", "smallvec", "thiserror", "tinyvec", @@ -1222,7 +1631,7 @@ dependencies = [ "log", "lru-cache", "parking_lot", - "rand", + "rand 0.8.5", "resolv-conf", "smallvec", "thiserror", @@ -1287,6 +1696,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1309,6 +1724,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index ebd1bc5..5df5b55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,9 @@ futures = "0.3.21" clap = "2.33.4" reqwest = { version = "0.11.9", features = ["trust-dns", "rustls-tls", "native-tls"] } openssl = { version = "0.10.38", features = ["vendored"] } -rand = "0.8.4" +rand = "0.8.5" +scraper = "0.12.0" +async-recursion = "1.0.0" # https://github.com/bluejekyll/trust-dns/pull/1632 [patch.crates-io] diff --git a/src/httplib.rs b/src/httplib.rs index 03ae5a7..77bebcf 100644 --- a/src/httplib.rs +++ b/src/httplib.rs @@ -1,20 +1,31 @@ use std::collections::{HashMap, HashSet}; -use reqwest::Client; +use async_recursion::async_recursion; +use rand::{distributions::Alphanumeric, thread_rng as rng, Rng}; +use reqwest::{ + header::{CONTENT_LENGTH, CONTENT_TYPE}, + redirect::Policy, + Client, Response, +}; + +use crate::structs::HTTPFilters; use { crate::{structs::HttpData, utils}, futures::stream::StreamExt, reqwest::{self, header::USER_AGENT}, + scraper::{Html, Selector}, }; #[allow(clippy::too_many_arguments)] +#[async_recursion] pub async fn return_http_data( hosts: HashSet, client: Client, user_agents_list: Vec, retries: usize, mut threads: usize, + return_filers: bool, conditional_response_code: u16, show_status_codes: bool, quiet_flag: bool, @@ -45,33 +56,21 @@ pub async fn return_http_data( .send() .await { - http_data.host_url = https_url.clone(); - http_data.status_code = resp.status().as_u16(); - http_data.http_status = "ACTIVE".to_string(); - drop(resp) + http_data = assign_response_data(resp, return_filers).await; } else if let Ok(resp) = http_send_fut .try_clone() .expect("Failed to clone http future") .send() .await { - http_data.host_url = http_url.clone(); - http_data.status_code = resp.status().as_u16(); - http_data.http_status = "ACTIVE".to_string(); - drop(resp) + http_data = assign_response_data(resp, return_filers).await; } counter += 1 } } else if let Ok(resp) = https_send_fut.send().await { - http_data.host_url = https_url; - http_data.status_code = resp.status().as_u16(); - http_data.http_status = "ACTIVE".to_string(); - drop(resp) + http_data = assign_response_data(resp, return_filers).await; } else if let Ok(resp) = http_send_fut.send().await { - http_data.host_url = http_url; - http_data.status_code = resp.status().as_u16(); - http_data.http_status = "ACTIVE".to_string(); - drop(resp) + http_data = assign_response_data(resp, return_filers).await; } else { http_data.http_status = "INACTIVE".to_string(); } @@ -94,12 +93,155 @@ pub async fn return_http_data( .await } -pub fn return_http_client(timeout: u64) -> Client { +pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client { reqwest::Client::builder() .timeout(std::time::Duration::from_secs(timeout)) + .redirect(Policy::limited(max_redirects)) .danger_accept_invalid_certs(true) .trust_dns(true) .use_native_tls() .build() .expect("Failed to create HTTP client") } + +#[allow(clippy::field_reassign_with_default)] +pub async fn assign_response_data(resp: Response, return_filers: bool) -> HttpData { + let headers = resp.headers().clone(); + let mut http_data = HttpData::default(); + let mut url = resp.url().to_owned(); + + http_data.http_status = "ACTIVE".to_string(); + http_data.status_code = resp.status().as_u16(); + http_data.host_url = { + url.set_path(""); + url.set_query(None); + url.to_string() + }; + http_data.final_url = url.to_string(); + http_data.protocol = resp.url().scheme().to_string(); + http_data.content_type = if headers.contains_key(CONTENT_TYPE) { + headers[CONTENT_TYPE] + .to_str() + .unwrap_or_default() + .to_string() + } else { + "".to_string() + }; + + http_data.headers = format!("{:?}", headers); + + let full_body = resp.text().await.unwrap_or_default(); + + http_data.content_length = if headers.contains_key(CONTENT_LENGTH) { + headers[CONTENT_LENGTH] + .to_str() + .unwrap_or_default() + .parse() + .unwrap_or_default() + } else { + full_body.chars().count() as u64 + }; + + http_data.body = return_body(&full_body).await; + http_data.title = return_title(&full_body).await; + http_data.words_count = full_body.split(' ').count(); + http_data.lines = full_body.lines().count() + 1; + + if return_filers { + let host = url.host_str().unwrap_or_default(); + let client = return_http_client(5, 3); + let user_agents = utils::user_agents(); + http_data.bad_data = return_filters_data(host, client, user_agents).await; + } + + http_data +} + +pub async fn return_title(data: &str) -> String { + let document = Html::parse_document(data); + match Selector::parse("title") { + Ok(selector) => { + if let Some(title_element) = document.select(&selector).next() { + title_element.inner_html() + } else { + "NULL".to_string() + } + } + Err(err) => { + eprintln!("Failed to parse selector: {:?}", err); + String::new() + } + } +} + +pub async fn return_body(data: &str) -> String { + let document = Html::parse_document(data); + match Selector::parse("body") { + Ok(selector) => { + if let Some(body_element) = document.select(&selector).next() { + body_element.inner_html() + } else { + "NULL".to_string() + } + } + Err(err) => { + eprintln!("Failed to parse selector: {:?}", err); + String::new() + } + } +} + +pub async fn return_filters_data( + host: &str, + client: Client, + user_agents_list: Vec, +) -> HTTPFilters { + let mut urls_to_check = HashSet::new(); + let random_str = rng() + .sample_iter(Alphanumeric) + .take(16) + .map(char::from) + .collect::(); + let words = vec![ + "admin".to_string() + &random_str + "/", + ".htaccess".to_string() + &random_str, + random_str.to_string() + "/", + random_str.to_string(), + ]; + for word in words { + urls_to_check.insert(format!("{}/{}", &host, word)); + } + + let threads = urls_to_check.len(); + let mut http_filters = HTTPFilters::default(); + let data = return_http_data( + urls_to_check, + client, + user_agents_list, + 1, + threads, + false, + 0, + false, + true, + ) + .await; + + data.iter() + .map(|(_, http_data)| { + http_filters + .bad_http_lengths + .append(&mut vec![http_data.content_length.to_string()]); + http_filters.bad_words_numbers.append(&mut vec![http_data + .body + .split(' ') + .count() + .to_string()]); + http_filters + .bad_lines_numbers + .append(&mut vec![http_data.lines.to_string()]); + }) + .for_each(drop); + + http_filters +} diff --git a/src/main.rs b/src/main.rs index 8d62b7a..4397f3b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -109,7 +109,7 @@ async fn main() -> Result<(), Box> { let user_agents_list = utils::user_agents(); let show_status_codes = matches.is_present("show-codes"); - let client = httplib::return_http_client(timeout); + let client = httplib::return_http_client(timeout, 3); // Read stdin @@ -133,6 +133,7 @@ async fn main() -> Result<(), Box> { user_agents_list, retries, threads, + false, conditional_response_code, show_status_codes, false, diff --git a/src/structs.rs b/src/structs.rs index b629fb9..6b53636 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -1,16 +1,24 @@ -#[derive(Clone)] -pub struct HttpData { - pub http_status: String, - pub host_url: String, - pub status_code: u16, +#[allow(clippy::upper_case_acronyms)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Default)] +pub struct HTTPFilters { + pub bad_http_lengths: Vec, + pub bad_words_numbers: Vec, + pub bad_lines_numbers: Vec, } -impl Default for HttpData { - fn default() -> Self { - HttpData { - http_status: String::from(""), - host_url: String::from(""), - status_code: 0, - } - } +#[derive(Clone, Default)] +pub struct HttpData { + pub http_status: String, + pub status_code: u16, + pub host_url: String, + pub final_url: String, + pub protocol: String, + pub title: String, + pub content_type: String, + pub body: String, + pub headers: String, + pub content_length: u64, + pub words_count: usize, + pub lines: usize, + pub bad_data: HTTPFilters, }