mirror of
https://github.com/edu4rdshl/fhc.git
synced 2026-07-17 23:24:50 +00:00
Implementations for the Findomain Plus integration.
Signed-off-by: Edu4rdSHL <edu4rdshl@protonmail.com>
This commit is contained in:
Edu4rdSHL
parent
38efe67b5f
commit
2666cee126
5 changed files with 625 additions and 51 deletions
457
Cargo.lock
generated
457
Cargo.lock
generated
|
|
@ -11,6 +11,17 @@ dependencies = [
|
||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "async-recursion"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "async-trait"
|
name = "async-trait"
|
||||||
version = "0.1.52"
|
version = "0.1.52"
|
||||||
|
|
@ -57,6 +68,12 @@ version = "3.9.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
|
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteorder"
|
||||||
|
version = "1.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytes"
|
name = "bytes"
|
||||||
version = "1.1.0"
|
version = "1.1.0"
|
||||||
|
|
@ -90,6 +107,12 @@ dependencies = [
|
||||||
"vec_map",
|
"vec_map",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "convert_case"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "core-foundation"
|
name = "core-foundation"
|
||||||
version = "0.9.3"
|
version = "0.9.3"
|
||||||
|
|
@ -106,12 +129,73 @@ version = "0.8.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
|
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cssparser"
|
||||||
|
version = "0.27.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
|
||||||
|
dependencies = [
|
||||||
|
"cssparser-macros",
|
||||||
|
"dtoa-short",
|
||||||
|
"itoa 0.4.8",
|
||||||
|
"matches",
|
||||||
|
"phf",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"smallvec",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cssparser-macros"
|
||||||
|
version = "0.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
|
||||||
|
dependencies = [
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "data-encoding"
|
name = "data-encoding"
|
||||||
version = "2.3.2"
|
version = "2.3.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
|
checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "derive_more"
|
||||||
|
version = "0.99.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
|
||||||
|
dependencies = [
|
||||||
|
"convert_case",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"rustc_version",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dtoa"
|
||||||
|
version = "0.4.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dtoa-short"
|
||||||
|
version = "0.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6"
|
||||||
|
dependencies = [
|
||||||
|
"dtoa",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ego-tree"
|
||||||
|
version = "0.6.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "encoding_rs"
|
name = "encoding_rs"
|
||||||
version = "0.8.30"
|
version = "0.8.30"
|
||||||
|
|
@ -146,11 +230,13 @@ dependencies = [
|
||||||
name = "fhc"
|
name = "fhc"
|
||||||
version = "0.6.1"
|
version = "0.6.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"async-recursion",
|
||||||
"clap",
|
"clap",
|
||||||
"futures",
|
"futures",
|
||||||
"openssl",
|
"openssl",
|
||||||
"rand",
|
"rand 0.8.5",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
"scraper",
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -185,6 +271,16 @@ dependencies = [
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futf"
|
||||||
|
version = "0.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
|
||||||
|
dependencies = [
|
||||||
|
"mac",
|
||||||
|
"new_debug_unreachable",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures"
|
name = "futures"
|
||||||
version = "0.3.21"
|
version = "0.3.21"
|
||||||
|
|
@ -274,6 +370,35 @@ dependencies = [
|
||||||
"slab",
|
"slab",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fxhash"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "getopts"
|
||||||
|
version = "0.2.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-width",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "getrandom"
|
||||||
|
version = "0.1.16"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"libc",
|
||||||
|
"wasi 0.9.0+wasi-snapshot-preview1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "getrandom"
|
name = "getrandom"
|
||||||
version = "0.2.4"
|
version = "0.2.4"
|
||||||
|
|
@ -282,7 +407,7 @@ checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"libc",
|
"libc",
|
||||||
"wasi",
|
"wasi 0.10.2+wasi-snapshot-preview1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -339,6 +464,20 @@ dependencies = [
|
||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "html5ever"
|
||||||
|
version = "0.25.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "aafcf38a1a36118242d29b92e1b08ef84e67e4a5ed06e0a80be20e6a32bfed6b"
|
||||||
|
dependencies = [
|
||||||
|
"log",
|
||||||
|
"mac",
|
||||||
|
"markup5ever",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "http"
|
name = "http"
|
||||||
version = "0.2.6"
|
version = "0.2.6"
|
||||||
|
|
@ -347,7 +486,7 @@ checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"fnv",
|
"fnv",
|
||||||
"itoa",
|
"itoa 1.0.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -388,7 +527,7 @@ dependencies = [
|
||||||
"http-body",
|
"http-body",
|
||||||
"httparse",
|
"httparse",
|
||||||
"httpdate",
|
"httpdate",
|
||||||
"itoa",
|
"itoa 1.0.1",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"socket2 0.4.4",
|
"socket2 0.4.4",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|
@ -471,6 +610,12 @@ version = "2.3.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
|
checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "itoa"
|
||||||
|
version = "0.4.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "itoa"
|
name = "itoa"
|
||||||
version = "1.0.1"
|
version = "1.0.1"
|
||||||
|
|
@ -531,6 +676,26 @@ dependencies = [
|
||||||
"linked-hash-map",
|
"linked-hash-map",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "mac"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "markup5ever"
|
||||||
|
version = "0.10.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd"
|
||||||
|
dependencies = [
|
||||||
|
"log",
|
||||||
|
"phf",
|
||||||
|
"phf_codegen",
|
||||||
|
"string_cache",
|
||||||
|
"string_cache_codegen",
|
||||||
|
"tendril",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "match_cfg"
|
name = "match_cfg"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
|
|
@ -595,6 +760,18 @@ dependencies = [
|
||||||
"tempfile",
|
"tempfile",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "new_debug_unreachable"
|
||||||
|
version = "1.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nodrop"
|
||||||
|
version = "0.1.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ntapi"
|
name = "ntapi"
|
||||||
version = "0.3.7"
|
version = "0.3.7"
|
||||||
|
|
@ -694,6 +871,69 @@ version = "2.1.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "phf"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
|
||||||
|
dependencies = [
|
||||||
|
"phf_macros",
|
||||||
|
"phf_shared 0.8.0",
|
||||||
|
"proc-macro-hack",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "phf_codegen"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
|
||||||
|
dependencies = [
|
||||||
|
"phf_generator",
|
||||||
|
"phf_shared 0.8.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "phf_generator"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
|
||||||
|
dependencies = [
|
||||||
|
"phf_shared 0.8.0",
|
||||||
|
"rand 0.7.3",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "phf_macros"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"
|
||||||
|
dependencies = [
|
||||||
|
"phf_generator",
|
||||||
|
"phf_shared 0.8.0",
|
||||||
|
"proc-macro-hack",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "phf_shared"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
|
||||||
|
dependencies = [
|
||||||
|
"siphasher",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "phf_shared"
|
||||||
|
version = "0.10.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
|
||||||
|
dependencies = [
|
||||||
|
"siphasher",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pin-project-lite"
|
name = "pin-project-lite"
|
||||||
version = "0.2.8"
|
version = "0.2.8"
|
||||||
|
|
@ -718,6 +958,18 @@ version = "0.2.16"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
|
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "precomputed-hash"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-hack"
|
||||||
|
version = "0.5.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro2"
|
name = "proc-macro2"
|
||||||
version = "1.0.36"
|
version = "1.0.36"
|
||||||
|
|
@ -744,14 +996,37 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rand"
|
name = "rand"
|
||||||
version = "0.8.4"
|
version = "0.7.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
|
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom 0.1.16",
|
||||||
|
"libc",
|
||||||
|
"rand_chacha 0.2.2",
|
||||||
|
"rand_core 0.5.1",
|
||||||
|
"rand_hc",
|
||||||
|
"rand_pcg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand"
|
||||||
|
version = "0.8.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"rand_chacha",
|
"rand_chacha 0.3.1",
|
||||||
"rand_core",
|
"rand_core 0.6.3",
|
||||||
"rand_hc",
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_chacha"
|
||||||
|
version = "0.2.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
|
||||||
|
dependencies = [
|
||||||
|
"ppv-lite86",
|
||||||
|
"rand_core 0.5.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -761,7 +1036,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ppv-lite86",
|
"ppv-lite86",
|
||||||
"rand_core",
|
"rand_core 0.6.3",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_core"
|
||||||
|
version = "0.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom 0.1.16",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -770,16 +1054,25 @@ version = "0.6.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
|
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"getrandom",
|
"getrandom 0.2.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rand_hc"
|
name = "rand_hc"
|
||||||
version = "0.3.1"
|
version = "0.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
|
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rand_core",
|
"rand_core 0.5.1",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_pcg"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
|
||||||
|
dependencies = [
|
||||||
|
"rand_core 0.5.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -867,6 +1160,15 @@ dependencies = [
|
||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustc_version"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
|
||||||
|
dependencies = [
|
||||||
|
"semver",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls"
|
name = "rustls"
|
||||||
version = "0.20.3"
|
version = "0.20.3"
|
||||||
|
|
@ -910,6 +1212,22 @@ version = "1.1.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "scraper"
|
||||||
|
version = "0.12.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12"
|
||||||
|
dependencies = [
|
||||||
|
"cssparser",
|
||||||
|
"ego-tree",
|
||||||
|
"getopts",
|
||||||
|
"html5ever",
|
||||||
|
"matches",
|
||||||
|
"selectors",
|
||||||
|
"smallvec",
|
||||||
|
"tendril",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sct"
|
name = "sct"
|
||||||
version = "0.7.0"
|
version = "0.7.0"
|
||||||
|
|
@ -943,6 +1261,32 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "selectors"
|
||||||
|
version = "0.22.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"cssparser",
|
||||||
|
"derive_more",
|
||||||
|
"fxhash",
|
||||||
|
"log",
|
||||||
|
"matches",
|
||||||
|
"phf",
|
||||||
|
"phf_codegen",
|
||||||
|
"precomputed-hash",
|
||||||
|
"servo_arc",
|
||||||
|
"smallvec",
|
||||||
|
"thin-slice",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "semver"
|
||||||
|
version = "1.0.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde"
|
name = "serde"
|
||||||
version = "1.0.136"
|
version = "1.0.136"
|
||||||
|
|
@ -955,7 +1299,7 @@ version = "1.0.79"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
|
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"itoa",
|
"itoa 1.0.1",
|
||||||
"ryu",
|
"ryu",
|
||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
@ -967,11 +1311,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
|
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"form_urlencoded",
|
"form_urlencoded",
|
||||||
"itoa",
|
"itoa 1.0.1",
|
||||||
"ryu",
|
"ryu",
|
||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "servo_arc"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"
|
||||||
|
dependencies = [
|
||||||
|
"nodrop",
|
||||||
|
"stable_deref_trait",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "signal-hook-registry"
|
name = "signal-hook-registry"
|
||||||
version = "1.4.0"
|
version = "1.4.0"
|
||||||
|
|
@ -981,6 +1335,12 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "siphasher"
|
||||||
|
version = "0.3.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "slab"
|
name = "slab"
|
||||||
version = "0.4.5"
|
version = "0.4.5"
|
||||||
|
|
@ -1020,6 +1380,38 @@ version = "0.5.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
|
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "stable_deref_trait"
|
||||||
|
version = "1.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "string_cache"
|
||||||
|
version = "0.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "33994d0838dc2d152d17a62adf608a869b5e846b65b389af7f3dbc1de45c5b26"
|
||||||
|
dependencies = [
|
||||||
|
"lazy_static",
|
||||||
|
"new_debug_unreachable",
|
||||||
|
"parking_lot",
|
||||||
|
"phf_shared 0.10.0",
|
||||||
|
"precomputed-hash",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "string_cache_codegen"
|
||||||
|
version = "0.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97"
|
||||||
|
dependencies = [
|
||||||
|
"phf_generator",
|
||||||
|
"phf_shared 0.8.0",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "strsim"
|
name = "strsim"
|
||||||
version = "0.8.0"
|
version = "0.8.0"
|
||||||
|
|
@ -1051,6 +1443,17 @@ dependencies = [
|
||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tendril"
|
||||||
|
version = "0.4.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33"
|
||||||
|
dependencies = [
|
||||||
|
"futf",
|
||||||
|
"mac",
|
||||||
|
"utf-8",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "textwrap"
|
name = "textwrap"
|
||||||
version = "0.11.0"
|
version = "0.11.0"
|
||||||
|
|
@ -1060,6 +1463,12 @@ dependencies = [
|
||||||
"unicode-width",
|
"unicode-width",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thin-slice"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "thiserror"
|
name = "thiserror"
|
||||||
version = "1.0.30"
|
version = "1.0.30"
|
||||||
|
|
@ -1202,7 +1611,7 @@ dependencies = [
|
||||||
"ipnet",
|
"ipnet",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"log",
|
"log",
|
||||||
"rand",
|
"rand 0.8.5",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tinyvec",
|
"tinyvec",
|
||||||
|
|
@ -1222,7 +1631,7 @@ dependencies = [
|
||||||
"log",
|
"log",
|
||||||
"lru-cache",
|
"lru-cache",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"rand",
|
"rand 0.8.5",
|
||||||
"resolv-conf",
|
"resolv-conf",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
|
|
@ -1287,6 +1696,12 @@ dependencies = [
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "utf-8"
|
||||||
|
version = "0.7.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "vcpkg"
|
name = "vcpkg"
|
||||||
version = "0.2.15"
|
version = "0.2.15"
|
||||||
|
|
@ -1309,6 +1724,12 @@ dependencies = [
|
||||||
"try-lock",
|
"try-lock",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasi"
|
||||||
|
version = "0.9.0+wasi-snapshot-preview1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasi"
|
name = "wasi"
|
||||||
version = "0.10.2+wasi-snapshot-preview1"
|
version = "0.10.2+wasi-snapshot-preview1"
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,9 @@ futures = "0.3.21"
|
||||||
clap = "2.33.4"
|
clap = "2.33.4"
|
||||||
reqwest = { version = "0.11.9", features = ["trust-dns", "rustls-tls", "native-tls"] }
|
reqwest = { version = "0.11.9", features = ["trust-dns", "rustls-tls", "native-tls"] }
|
||||||
openssl = { version = "0.10.38", features = ["vendored"] }
|
openssl = { version = "0.10.38", features = ["vendored"] }
|
||||||
rand = "0.8.4"
|
rand = "0.8.5"
|
||||||
|
scraper = "0.12.0"
|
||||||
|
async-recursion = "1.0.0"
|
||||||
|
|
||||||
# https://github.com/bluejekyll/trust-dns/pull/1632
|
# https://github.com/bluejekyll/trust-dns/pull/1632
|
||||||
[patch.crates-io]
|
[patch.crates-io]
|
||||||
|
|
|
||||||
178
src/httplib.rs
178
src/httplib.rs
|
|
@ -1,20 +1,31 @@
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
use reqwest::Client;
|
use async_recursion::async_recursion;
|
||||||
|
use rand::{distributions::Alphanumeric, thread_rng as rng, Rng};
|
||||||
|
use reqwest::{
|
||||||
|
header::{CONTENT_LENGTH, CONTENT_TYPE},
|
||||||
|
redirect::Policy,
|
||||||
|
Client, Response,
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::structs::HTTPFilters;
|
||||||
|
|
||||||
use {
|
use {
|
||||||
crate::{structs::HttpData, utils},
|
crate::{structs::HttpData, utils},
|
||||||
futures::stream::StreamExt,
|
futures::stream::StreamExt,
|
||||||
reqwest::{self, header::USER_AGENT},
|
reqwest::{self, header::USER_AGENT},
|
||||||
|
scraper::{Html, Selector},
|
||||||
};
|
};
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
#[async_recursion]
|
||||||
pub async fn return_http_data(
|
pub async fn return_http_data(
|
||||||
hosts: HashSet<String>,
|
hosts: HashSet<String>,
|
||||||
client: Client,
|
client: Client,
|
||||||
user_agents_list: Vec<String>,
|
user_agents_list: Vec<String>,
|
||||||
retries: usize,
|
retries: usize,
|
||||||
mut threads: usize,
|
mut threads: usize,
|
||||||
|
return_filers: bool,
|
||||||
conditional_response_code: u16,
|
conditional_response_code: u16,
|
||||||
show_status_codes: bool,
|
show_status_codes: bool,
|
||||||
quiet_flag: bool,
|
quiet_flag: bool,
|
||||||
|
|
@ -45,33 +56,21 @@ pub async fn return_http_data(
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
http_data.host_url = https_url.clone();
|
http_data = assign_response_data(resp, return_filers).await;
|
||||||
http_data.status_code = resp.status().as_u16();
|
|
||||||
http_data.http_status = "ACTIVE".to_string();
|
|
||||||
drop(resp)
|
|
||||||
} else if let Ok(resp) = http_send_fut
|
} else if let Ok(resp) = http_send_fut
|
||||||
.try_clone()
|
.try_clone()
|
||||||
.expect("Failed to clone http future")
|
.expect("Failed to clone http future")
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
http_data.host_url = http_url.clone();
|
http_data = assign_response_data(resp, return_filers).await;
|
||||||
http_data.status_code = resp.status().as_u16();
|
|
||||||
http_data.http_status = "ACTIVE".to_string();
|
|
||||||
drop(resp)
|
|
||||||
}
|
}
|
||||||
counter += 1
|
counter += 1
|
||||||
}
|
}
|
||||||
} else if let Ok(resp) = https_send_fut.send().await {
|
} else if let Ok(resp) = https_send_fut.send().await {
|
||||||
http_data.host_url = https_url;
|
http_data = assign_response_data(resp, return_filers).await;
|
||||||
http_data.status_code = resp.status().as_u16();
|
|
||||||
http_data.http_status = "ACTIVE".to_string();
|
|
||||||
drop(resp)
|
|
||||||
} else if let Ok(resp) = http_send_fut.send().await {
|
} else if let Ok(resp) = http_send_fut.send().await {
|
||||||
http_data.host_url = http_url;
|
http_data = assign_response_data(resp, return_filers).await;
|
||||||
http_data.status_code = resp.status().as_u16();
|
|
||||||
http_data.http_status = "ACTIVE".to_string();
|
|
||||||
drop(resp)
|
|
||||||
} else {
|
} else {
|
||||||
http_data.http_status = "INACTIVE".to_string();
|
http_data.http_status = "INACTIVE".to_string();
|
||||||
}
|
}
|
||||||
|
|
@ -94,12 +93,155 @@ pub async fn return_http_data(
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn return_http_client(timeout: u64) -> Client {
|
pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client {
|
||||||
reqwest::Client::builder()
|
reqwest::Client::builder()
|
||||||
.timeout(std::time::Duration::from_secs(timeout))
|
.timeout(std::time::Duration::from_secs(timeout))
|
||||||
|
.redirect(Policy::limited(max_redirects))
|
||||||
.danger_accept_invalid_certs(true)
|
.danger_accept_invalid_certs(true)
|
||||||
.trust_dns(true)
|
.trust_dns(true)
|
||||||
.use_native_tls()
|
.use_native_tls()
|
||||||
.build()
|
.build()
|
||||||
.expect("Failed to create HTTP client")
|
.expect("Failed to create HTTP client")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::field_reassign_with_default)]
|
||||||
|
pub async fn assign_response_data(resp: Response, return_filers: bool) -> HttpData {
|
||||||
|
let headers = resp.headers().clone();
|
||||||
|
let mut http_data = HttpData::default();
|
||||||
|
let mut url = resp.url().to_owned();
|
||||||
|
|
||||||
|
http_data.http_status = "ACTIVE".to_string();
|
||||||
|
http_data.status_code = resp.status().as_u16();
|
||||||
|
http_data.host_url = {
|
||||||
|
url.set_path("");
|
||||||
|
url.set_query(None);
|
||||||
|
url.to_string()
|
||||||
|
};
|
||||||
|
http_data.final_url = url.to_string();
|
||||||
|
http_data.protocol = resp.url().scheme().to_string();
|
||||||
|
http_data.content_type = if headers.contains_key(CONTENT_TYPE) {
|
||||||
|
headers[CONTENT_TYPE]
|
||||||
|
.to_str()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string()
|
||||||
|
} else {
|
||||||
|
"".to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
http_data.headers = format!("{:?}", headers);
|
||||||
|
|
||||||
|
let full_body = resp.text().await.unwrap_or_default();
|
||||||
|
|
||||||
|
http_data.content_length = if headers.contains_key(CONTENT_LENGTH) {
|
||||||
|
headers[CONTENT_LENGTH]
|
||||||
|
.to_str()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.parse()
|
||||||
|
.unwrap_or_default()
|
||||||
|
} else {
|
||||||
|
full_body.chars().count() as u64
|
||||||
|
};
|
||||||
|
|
||||||
|
http_data.body = return_body(&full_body).await;
|
||||||
|
http_data.title = return_title(&full_body).await;
|
||||||
|
http_data.words_count = full_body.split(' ').count();
|
||||||
|
http_data.lines = full_body.lines().count() + 1;
|
||||||
|
|
||||||
|
if return_filers {
|
||||||
|
let host = url.host_str().unwrap_or_default();
|
||||||
|
let client = return_http_client(5, 3);
|
||||||
|
let user_agents = utils::user_agents();
|
||||||
|
http_data.bad_data = return_filters_data(host, client, user_agents).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
http_data
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn return_title(data: &str) -> String {
|
||||||
|
let document = Html::parse_document(data);
|
||||||
|
match Selector::parse("title") {
|
||||||
|
Ok(selector) => {
|
||||||
|
if let Some(title_element) = document.select(&selector).next() {
|
||||||
|
title_element.inner_html()
|
||||||
|
} else {
|
||||||
|
"NULL".to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
eprintln!("Failed to parse selector: {:?}", err);
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn return_body(data: &str) -> String {
|
||||||
|
let document = Html::parse_document(data);
|
||||||
|
match Selector::parse("body") {
|
||||||
|
Ok(selector) => {
|
||||||
|
if let Some(body_element) = document.select(&selector).next() {
|
||||||
|
body_element.inner_html()
|
||||||
|
} else {
|
||||||
|
"NULL".to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
eprintln!("Failed to parse selector: {:?}", err);
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn return_filters_data(
|
||||||
|
host: &str,
|
||||||
|
client: Client,
|
||||||
|
user_agents_list: Vec<String>,
|
||||||
|
) -> HTTPFilters {
|
||||||
|
let mut urls_to_check = HashSet::new();
|
||||||
|
let random_str = rng()
|
||||||
|
.sample_iter(Alphanumeric)
|
||||||
|
.take(16)
|
||||||
|
.map(char::from)
|
||||||
|
.collect::<String>();
|
||||||
|
let words = vec![
|
||||||
|
"admin".to_string() + &random_str + "/",
|
||||||
|
".htaccess".to_string() + &random_str,
|
||||||
|
random_str.to_string() + "/",
|
||||||
|
random_str.to_string(),
|
||||||
|
];
|
||||||
|
for word in words {
|
||||||
|
urls_to_check.insert(format!("{}/{}", &host, word));
|
||||||
|
}
|
||||||
|
|
||||||
|
let threads = urls_to_check.len();
|
||||||
|
let mut http_filters = HTTPFilters::default();
|
||||||
|
let data = return_http_data(
|
||||||
|
urls_to_check,
|
||||||
|
client,
|
||||||
|
user_agents_list,
|
||||||
|
1,
|
||||||
|
threads,
|
||||||
|
false,
|
||||||
|
0,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
data.iter()
|
||||||
|
.map(|(_, http_data)| {
|
||||||
|
http_filters
|
||||||
|
.bad_http_lengths
|
||||||
|
.append(&mut vec![http_data.content_length.to_string()]);
|
||||||
|
http_filters.bad_words_numbers.append(&mut vec![http_data
|
||||||
|
.body
|
||||||
|
.split(' ')
|
||||||
|
.count()
|
||||||
|
.to_string()]);
|
||||||
|
http_filters
|
||||||
|
.bad_lines_numbers
|
||||||
|
.append(&mut vec![http_data.lines.to_string()]);
|
||||||
|
})
|
||||||
|
.for_each(drop);
|
||||||
|
|
||||||
|
http_filters
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -109,7 +109,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let user_agents_list = utils::user_agents();
|
let user_agents_list = utils::user_agents();
|
||||||
let show_status_codes = matches.is_present("show-codes");
|
let show_status_codes = matches.is_present("show-codes");
|
||||||
|
|
||||||
let client = httplib::return_http_client(timeout);
|
let client = httplib::return_http_client(timeout, 3);
|
||||||
|
|
||||||
// Read stdin
|
// Read stdin
|
||||||
|
|
||||||
|
|
@ -133,6 +133,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
user_agents_list,
|
user_agents_list,
|
||||||
retries,
|
retries,
|
||||||
threads,
|
threads,
|
||||||
|
false,
|
||||||
conditional_response_code,
|
conditional_response_code,
|
||||||
show_status_codes,
|
show_status_codes,
|
||||||
false,
|
false,
|
||||||
|
|
|
||||||
|
|
@ -1,16 +1,24 @@
|
||||||
#[derive(Clone)]
|
#[allow(clippy::upper_case_acronyms)]
|
||||||
pub struct HttpData {
|
#[derive(Clone, Debug, Eq, PartialEq, Hash, Default)]
|
||||||
pub http_status: String,
|
pub struct HTTPFilters {
|
||||||
pub host_url: String,
|
pub bad_http_lengths: Vec<String>,
|
||||||
pub status_code: u16,
|
pub bad_words_numbers: Vec<String>,
|
||||||
|
pub bad_lines_numbers: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for HttpData {
|
#[derive(Clone, Default)]
|
||||||
fn default() -> Self {
|
pub struct HttpData {
|
||||||
HttpData {
|
pub http_status: String,
|
||||||
http_status: String::from(""),
|
pub status_code: u16,
|
||||||
host_url: String::from(""),
|
pub host_url: String,
|
||||||
status_code: 0,
|
pub final_url: String,
|
||||||
}
|
pub protocol: String,
|
||||||
}
|
pub title: String,
|
||||||
|
pub content_type: String,
|
||||||
|
pub body: String,
|
||||||
|
pub headers: String,
|
||||||
|
pub content_length: u64,
|
||||||
|
pub words_count: usize,
|
||||||
|
pub lines: usize,
|
||||||
|
pub bad_data: HTTPFilters,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue