From c4ab210c4d3f966b2eeb1181ff057e89ec77703e Mon Sep 17 00:00:00 2001 From: koalp Date: Sun, 4 Apr 2021 03:21:28 +0200 Subject: [PATCH] feat: add retrieval application and one newspaper A first example as well as some documentation have been added The first example builds an article location and download the article as an html String. The documentation explains how it has been designed and what is the goal of the application as well as it's intended architecture --- .gitattributes | 1 + .gitea/issue_template/bug_report.md | 22 + .gitea/issue_template/design_discussion.md | 19 + .gitea/issue_template/feature_request.md | 15 + .gitea/issue_template/question.md | 15 + .gitea/issue_template/refactor.md | 11 + .gitignore | 2 + .rustfmt.toml | 2 + Cargo.lock | 2153 +++++++++++++++++ Cargo.toml | 23 + README.md | 17 + crieur-retrieve/.gitignore | 1 + crieur-retrieve/Cargo.toml | 30 + crieur-retrieve/src/article_location.rs | 108 + crieur-retrieve/src/consts.rs | 83 + crieur-retrieve/src/errors.rs | 10 + crieur-retrieve/src/lib.rs | 17 + crieur-retrieve/src/newspaper.rs | 56 + crieur-retrieve/src/newspapers/mediapart.rs | 65 + crieur-retrieve/src/newspapers/mod.rs | 3 + crieur-retrieve/src/tools/download.rs | 57 + crieur-retrieve/src/tools/mod.rs | 5 + .../src/tools/self_contained_html.rs | 291 +++ crieur-retrieve/test_data/home.png | 3 + documentation/design/retrieve.md | 76 + documentation/design/scope.md | 48 + .../guides/add_a_newspaper_source.md | 1 + examples/cli_downloader.rs | 36 + justfile | 16 + src/main.rs | 19 + 30 files changed, 3205 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitea/issue_template/bug_report.md create mode 100644 .gitea/issue_template/design_discussion.md create mode 100644 .gitea/issue_template/feature_request.md create mode 100644 .gitea/issue_template/question.md create mode 100644 .gitea/issue_template/refactor.md create mode 100644 .gitignore create mode 100644 .rustfmt.toml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 crieur-retrieve/.gitignore create mode 100644 crieur-retrieve/Cargo.toml create mode 100644 crieur-retrieve/src/article_location.rs create mode 100644 crieur-retrieve/src/consts.rs create mode 100644 crieur-retrieve/src/errors.rs create mode 100644 crieur-retrieve/src/lib.rs create mode 100644 crieur-retrieve/src/newspaper.rs create mode 100644 crieur-retrieve/src/newspapers/mediapart.rs create mode 100644 crieur-retrieve/src/newspapers/mod.rs create mode 100644 crieur-retrieve/src/tools/download.rs create mode 100644 crieur-retrieve/src/tools/mod.rs create mode 100644 crieur-retrieve/src/tools/self_contained_html.rs create mode 100644 crieur-retrieve/test_data/home.png create mode 100644 documentation/design/retrieve.md create mode 100644 documentation/design/scope.md create mode 100644 documentation/guides/add_a_newspaper_source.md create mode 100644 examples/cli_downloader.rs create mode 100644 justfile create mode 100644 src/main.rs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..24a8e87 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.gitea/issue_template/bug_report.md b/.gitea/issue_template/bug_report.md new file mode 100644 index 0000000..2434293 --- /dev/null +++ b/.gitea/issue_template/bug_report.md @@ -0,0 +1,22 @@ +--- +name: "Bug report" +about: "This template is for reporting a bug" +title: "" +labels: +- "type::bug" +- "status::review_needed" + +--- + +**Description** +*write a concise bug description* + +**Steps to reproduce** +1. +2. + +**Expected behavior** +*describe what you expected to happen* + +**Configuration** +*paste the result of `stage --version` diff --git a/.gitea/issue_template/design_discussion.md b/.gitea/issue_template/design_discussion.md new file mode 100644 index 0000000..d5a006a --- /dev/null +++ b/.gitea/issue_template/design_discussion.md @@ -0,0 +1,19 @@ +--- +name: "Design discussion" +about: "For discussion about the design of features in the application, when there are several possibilities for implementation" +title: "" +labels: +- "type::discussion" +- "status::review_needed" + +--- + +*describe shortly the problem* + +## Requirements + +*list requirements that the feature have* + +## Propositions + +*explain the different implementation that you would propose for the feature* diff --git a/.gitea/issue_template/feature_request.md b/.gitea/issue_template/feature_request.md new file mode 100644 index 0000000..4ca4668 --- /dev/null +++ b/.gitea/issue_template/feature_request.md @@ -0,0 +1,15 @@ +--- +name: "Feature request" +about: "This template is for requesting a new feature" +title: "" +labels: +- "type::feature" +- "status::review_needed" + +--- + +*(if applicable) describe what problem or frustration you have currently* + +*describe what you would like to be able to do, or what solution you would like (you can propose several)* + +*(optional) additional context, comments or implementation propositions* diff --git a/.gitea/issue_template/question.md b/.gitea/issue_template/question.md new file mode 100644 index 0000000..92d384d --- /dev/null +++ b/.gitea/issue_template/question.md @@ -0,0 +1,15 @@ +--- +name: "Ask a question" +about: "If you have a question about the usage of the libraries or the tool" +title: "" +labels: +- "type::question" +- "status::review_needed" + +--- + +*ask your question* + +*describe what you have you read so far to try to answer this question ?* + +*(optional) would you think the is a in documentation ?* diff --git a/.gitea/issue_template/refactor.md b/.gitea/issue_template/refactor.md new file mode 100644 index 0000000..6eb7486 --- /dev/null +++ b/.gitea/issue_template/refactor.md @@ -0,0 +1,11 @@ +--- +name: "Refactor" +about: "For refactoring propositions" +title: "" +labels: +- "type::refactor" +- "status::review_needed" + +--- + +*explain why and what you want to refactor* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fedaa2b --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +.env diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000..98e7aee --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,2 @@ +format_strings = true +wrap_comments = true diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..4686eea --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2153 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" + +[[package]] +name = "async-trait" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bumpalo" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "const_fn" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "cookie" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627" +dependencies = [ + "time", + "version_check", +] + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "cow-utils" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" + +[[package]] +name = "crieur" +version = "0.1.0" +dependencies = [ + "anyhow", + "crieur-retrieve", + "dotenv", + "env_logger", + "log", + "tokio", +] + +[[package]] +name = "crieur-retrieve" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "bytes", + "cookie", + "derive_builder", + "env_logger", + "futures", + "html-minifier", + "hyper", + "hyper-rustls", + "indoc", + "itertools", + "log", + "lol_html", + "nipper", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "cssparser" +version = "0.25.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbe18ca4efb9ba3716c6da66cc3d7e673bf59fa576353011f48c4cfddbdd740e" +dependencies = [ + "autocfg 0.1.7", + "cssparser-macros 0.3.6", + "dtoa-short", + "itoa", + "matches", + "phf 0.7.24", + "proc-macro2", + "procedural-masquerade", + "quote", + "smallvec 0.6.14", + "syn", +] + +[[package]] +name = "cssparser" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +dependencies = [ + "cssparser-macros 0.6.0", + "dtoa-short", + "itoa", + "matches", + "phf 0.8.0", + "proc-macro2", + "quote", + "smallvec 1.6.1", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb1c84e87c717666564ec056105052331431803d606bd45529b28547b611eef" +dependencies = [ + "phf_codegen 0.7.24", + "proc-macro2", + "procedural-masquerade", + "quote", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct", +] + +[[package]] +name = "darling" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d6ddad5866bb2170686ed03f6839d31a76e5407d80b1c334a2c24618543ffa" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ced1fd13dc386d5a8315899de465708cf34ee2a6d9394654515214e67bb846" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7a1445d54b2f9792e3b31a3e715feabbace393f38dc4ffd49d94ee9bc487d5" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "derive_builder" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ef25735c9f0d0c547d2794701600c94abf030ecb740fad1673fa64461f3573" +dependencies = [ + "derive_builder_core", + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3150f1e84602847b99d3eeb702487fc364f7d6c94f634e944a68fdbaea09e457" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca1008bddefdc08d1e734aeb27b94f384390af261b4d1a8fb51fe19c577f05c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "derive_more" +version = "0.99.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b1b72f1263f214c0f823371768776c4f5841b942c9883aa8e5ec584fd0ba6" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "dtoa-short" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6" +dependencies = [ + "dtoa", +] + +[[package]] +name = "educe" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ed56329d95e524ef98177ad672881bdfe7f22f254eb6ae80deb6fdd2ab20c4" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "enum-ordinalize" +version = "3.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d52ff39419d3e16961ecfb9e32f5042bdaacf9a4cc553d2d688057117bae49b" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_logger" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "futf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c9c1ce3fa9336301af935ab852c437817d14cd33690446569392e65170aac3b" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" + +[[package]] +name = "futures-executor" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" + +[[package]] +name = "futures-macro" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" + +[[package]] +name = "futures-task" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" + +[[package]] +name = "futures-util" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "html-escape" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64794b2265e97e459334ed47a7b7369ce8e8ee3d3450c0c363a0b563fc92233" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "html-minifier" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae99b46762d01ce79973e90ff61541ee9d2b6179ff8f554c38602e48e8158088" +dependencies = [ + "cow-utils", + "educe", + "html-escape", + "minifier", +] + +[[package]] +name = "html5ever" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcf38a1a36118242d29b92e1b08ef84e67e4a5ed06e0a80be20e6a32bfed6b" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "http" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "webpki", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a75aeaaef0ce18b58056d306c27b07436fbb34b8816c53094b76dd81803136" +dependencies = [ + "unindent", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d855069fafbb9b344c0f962150cd2c1187975cb1c22c1522c240d8c4986714" + +[[package]] +name = "lock_api" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "lol_html" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b59f94556144354f6abfb3fe175e8f2da290329f254ab4019e5096875f6056e3" +dependencies = [ + "bitflags", + "cfg-if 0.1.10", + "cssparser 0.25.9", + "encoding_rs", + "hashbrown", + "lazy_static", + "lazycell", + "memchr", + "safemem", + "selectors 0.21.0", + "thiserror", +] + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "macro-utils" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e72f7deb758fea9ea7d290aebfa788763d0bffae12caa6406a25baaf8fa68a8" + +[[package]] +name = "markup5ever" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae38d669396ca9b707bfc3db254bc382ddb94f57cc5c235f34623a669a01dab" +dependencies = [ + "log", + "phf 0.8.0", + "phf_codegen 0.8.0", + "serde", + "serde_derive", + "serde_json", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "minifier" +version = "0.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdf618de5c9c98d4a7b2e0d1f1e44f82a19196cfd94040bb203621c25d28d98" +dependencies = [ + "macro-utils", +] + +[[package]] +name = "mio" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "nipper" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761382864693f4bb171abf9e8de181a320b00464a83a9a5071059057b1fe0116" +dependencies = [ + "cssparser 0.27.2", + "html5ever", + "markup5ever", + "selectors 0.22.0", + "tendril", +] + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-bigint" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d0a3d5e207573f948a9e5376662aa743a2ea13f7c50a554d7af443a73fbfeba" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec 1.6.1", + "winapi", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" +dependencies = [ + "phf_shared 0.7.24", +] + +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_macros", + "phf_shared 0.8.0", + "proc-macro-hack", +] + +[[package]] +name = "phf_codegen" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" +dependencies = [ + "phf_generator 0.7.24", + "phf_shared 0.7.24", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", +] + +[[package]] +name = "phf_generator" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared 0.7.24", + "rand 0.6.5", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared 0.8.0", + "rand 0.7.3", +] + +[[package]] +name = "phf_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher 0.2.3", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher 0.3.5", +] + +[[package]] +name = "pin-project" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "procedural-masquerade" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1383dff4092fe903ac180e391a8d4121cc48f08ccf850614b0290c6673b69d" + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg 0.1.2", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", + "rand_pcg 0.2.1", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b86b100bede4f651059740afc3b6cb83458d7401cb7c1ad96d8a11e91742c86" +dependencies = [ + "bitflags", + "cssparser 0.25.9", + "fxhash", + "log", + "matches", + "phf 0.7.24", + "phf_codegen 0.7.24", + "precomputed-hash", + "servo_arc", + "smallvec 0.6.14", + "thin-slice", +] + +[[package]] +name = "selectors" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" +dependencies = [ + "bitflags", + "cssparser 0.27.2", + "derive_more", + "fxhash", + "log", + "matches", + "phf 0.8.0", + "phf_codegen 0.8.0", + "precomputed-hash", + "servo_arc", + "smallvec 1.6.1", + "thin-slice", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" + +[[package]] +name = "serde_derive" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + +[[package]] +name = "siphasher" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27" + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "socket2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "string_cache" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb1139b5353f96e429e1a5e19fbaf663bddedaa06d1dbd49f82e352601209a" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "phf_shared 0.8.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", + "proc-macro2", + "quote", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ce15dd3ed8aa2f8eeac4716d6ef5ab58b6b9256db41d7e1a0224c2788e8fd87" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tendril" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thin-slice" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" + +[[package]] +name = "thiserror" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a8cbfbf47955132d0202d1662f49b2423ae35862aee471f3ba4b133358f372" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" +dependencies = [ + "autocfg 1.0.1", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5143d049e85af7fbc36f5454d990e62c2df705b3589f123b71f441b6b59f443f" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "unicode-bidi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "unindent" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "utf-8" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" + +[[package]] +name = "utf8-width" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasm-bindgen" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" + +[[package]] +name = "web-sys" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..be05e56 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,23 @@ +[workspace] + +members = [ + "crieur-retrieve", +] + + +[package] +name = "crieur" +version = "0.1.0" +authors = ["koalp "] +edition = "2018" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.40" +crieur-retrieve = {version = "0.1", path="crieur-retrieve"} +dotenv = "0.15.0" +env_logger = "0.8.3" +log = "0.4.14" +tokio = { version = "1.5.0", features = ["full"] } diff --git a/README.md b/README.md new file mode 100644 index 0000000..e4bfb50 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +Tools to retrieve articles from multiple newspaper you subscribed to. + +**This is a prototype, it isn't stable at all and you may not want to use it if you expect it to just work !** + +# How to use it + +First retrieve login cookies for websites and put it in a `.env` + +``` +cargo run --example=retrive_html_articles +``` + +# Documentation + +- 1. [Design](documentation/design/index.md) + - a. [Scope of the project](documentation/design/scope.md) + - b. [Retrieve](documentation/design/retrieve.md) diff --git a/crieur-retrieve/.gitignore b/crieur-retrieve/.gitignore new file mode 100644 index 0000000..03314f7 --- /dev/null +++ b/crieur-retrieve/.gitignore @@ -0,0 +1 @@ +Cargo.lock diff --git a/crieur-retrieve/Cargo.toml b/crieur-retrieve/Cargo.toml new file mode 100644 index 0000000..7b35a3c --- /dev/null +++ b/crieur-retrieve/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "crieur-retrieve" +description = "Retrive articles from newspapers websites" +authors = ["koalp "] +version = "0.1.0" +edition = "2018" +publish = false + +[dependencies] +anyhow = "1.0.40" +async-trait = "0.1.48" +thiserror = "1.0.24" +url = "2.2.1" +hyper = { version = "0.14.5", features = ["full"] } +hyper-rustls = "0.22.1" +cookie = "0.15.0" +lol_html = "0.3.0" +indoc = "1.0.3" +html-minifier = "3.0.9" +bytes = "1.0.1" +base64 = "0.13.0" +futures = "0.3.14" +derive_builder = "0.10.0" +nipper = "0.1.9" +log = "0.4.14" +env_logger = "0.8.3" +itertools = "0.10.0" + +[dev-dependencies] +tokio = "1.5.0" diff --git a/crieur-retrieve/src/article_location.rs b/crieur-retrieve/src/article_location.rs new file mode 100644 index 0000000..b98318b --- /dev/null +++ b/crieur-retrieve/src/article_location.rs @@ -0,0 +1,108 @@ +use std::convert::TryInto; +use std::ops::Deref; +use std::boxed::Box; + +use anyhow::{anyhow, Result}; +use url::{Host, Url}; +use log::info; + +use crate::newspaper::Newspaper; + +type Newspapers<'a> = Vec>; + +#[derive(Default)] +pub struct ArticleLocationBuilder<'a> { + url: Option, + newspapers: Option>, +} + +impl<'a> ArticleLocationBuilder<'a> { + pub fn new() -> Self { + Self::default() + } + + /// Adds an url corresponding to article location + /// + /// # Errors + /// + /// An error is returned if the could not be converted into an url + // TODO: move this to a defined error, remove anyhow ! + pub fn url<'e, U, E>(mut self, url: U) -> Result + where + U: TryInto + Send, + E: std::error::Error + Sync + Send + 'static, + { + let url = url.try_into()?; + self.url = Some(url); + Ok(self) + } + + /// Adds a newspaper to the list + pub fn newspaper(&mut self, newspaper: &'a T) -> &mut Self + where + T: 'a + Newspaper, + { + match &mut self.newspapers { + Some(newspapers) => newspapers.push(Box::new(newspaper)), + None => self.newspapers = Some(vec![Box::new(newspaper)]), + } + self + } + + /// Adds several newspapers to the list of accepted newspapers + //fn newspapers(&mut self, newspapers: Newspapers) -> Result<&mut Self> { + // let newspapers = match &self.newspapers { + // Some(current_newspapers) => newspapers + // .iter() + // .chain(current_newspapers.iter()) + // .map(|s| *(s.clone())) + // .collect::(), + // None => newspapers.into_iter().collect::>(), + // }; + // self.newspapers = Some(newspapers); + // Ok(self) + //} + + /// Builds the ArticleLocation by looking which newspaper + /// + /// # Errors + /// + /// The following errors can be returned + /// + /// - no newpspaper is given + /// - the url is not set + /// - the given url has no host + // TODO: move this to a defined error, remove anyhow ! + pub fn build(&self) -> Result> { + let url = Clone::clone(self.url.as_ref().ok_or(anyhow!( + "No url set. You can set it with the url() function" + ))?); + let host = url.host_str().ok_or(anyhow!("Given url has no host"))?; + let host = Host::parse(host)?; + let newspaper = self + .newspapers.as_ref() + .ok_or(anyhow!( + "A list of NewsPaper must be set. It can be set with newspapers() function" + ))? + .into_iter() + .find(|c| c.metadata().hosts.contains(&host)) + .ok_or(anyhow!("Newspaper couldn't be found"))?; + Ok(ArticleLocation { newspaper: newspaper.clone(), url }) + } +} + +pub struct ArticleLocation<'a> { + newspaper: Box<&'a dyn Newspaper>, + pub url: Url, +} + +impl<'a> ArticleLocation<'a> { + pub fn builder() -> ArticleLocationBuilder<'a> { + ArticleLocationBuilder::new() + } + + pub async fn retrieve_html(&self) -> Result { + info!("It will download from {}", self.url); + self.newspaper.retrieve_html(&self.url).await + } +} diff --git a/crieur-retrieve/src/consts.rs b/crieur-retrieve/src/consts.rs new file mode 100644 index 0000000..89c89fe --- /dev/null +++ b/crieur-retrieve/src/consts.rs @@ -0,0 +1,83 @@ +pub const EVENT_HANDLERS: &'static [&'static str] = &[ + // From https://www.w3.org/TR/html52/webappapis.html#event-handlers-on-elements-document-objects-and-window-objects + "onabort", + "onauxclick", + "oncancel", + "oncanplay", + "oncanplaythrough", + "onchange", + "onclick", + "onclose", + "oncuechange", + "ondblclick", + "ondrag", + "ondragend", + "ondragenter", + "ondragexit", + "ondragleave", + "ondragover", + "ondragstart", + "ondrop", + "ondurationchange", + "onemptied", + "onended", + "oninput", + "oninvalid", + "onkeydown", + "onkeypress", + "onkeyup", + "onloadeddata", + "onloadedmetadata", + "onloadend", + "onloadstart", + "onmousedown", + "onmouseenter", + "onmouseleave", + "onmousemove", + "onmouseout", + "onmouseover", + "onmouseup", + "onwheel", + "onpause", + "onplay", + "onplaying", + "onprogress", + "onratechange", + "onreset", + "onseeked", + "onseeking", + "onselect", + "onshow", + "onstalled", + "onsubmit", + "onsuspend", + "ontimeupdate", + "ontoggle", + "onvolumechange", + "onwaiting", + "onblur", + "onerror", + "onfocus", + "onload", + "onresize", + "onscroll", + "onafterprint", + "onbeforeprint", + "onbeforeunload", + "onhashchange", + "onlanguagechange", + "onmessage", + "onoffline", + "ononline", + "onpagehide", + "onpageshow", + "onrejectionhandled", + "onpopstate", + "onstorage", + "onunhandledrejection", + "onunload", + "oncut", + "oncopy", + "onpaste", + "onreadystatechange", +]; diff --git a/crieur-retrieve/src/errors.rs b/crieur-retrieve/src/errors.rs new file mode 100644 index 0000000..677a501 --- /dev/null +++ b/crieur-retrieve/src/errors.rs @@ -0,0 +1,10 @@ +use anyhow; +use thiserror; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Other(#[from] anyhow::Error), +} + +pub type Result = core::result::Result; diff --git a/crieur-retrieve/src/lib.rs b/crieur-retrieve/src/lib.rs new file mode 100644 index 0000000..7b9d058 --- /dev/null +++ b/crieur-retrieve/src/lib.rs @@ -0,0 +1,17 @@ +pub use url::Url; + +pub mod errors; + +mod tools; +pub use tools::{Download, Downloader}; + +pub mod newspaper; + +// TODO: move to another crate +mod newspapers; +pub use newspapers::Mediapart; + +mod article_location; +pub use article_location::ArticleLocation; + +mod consts; diff --git a/crieur-retrieve/src/newspaper.rs b/crieur-retrieve/src/newspaper.rs new file mode 100644 index 0000000..1da4007 --- /dev/null +++ b/crieur-retrieve/src/newspaper.rs @@ -0,0 +1,56 @@ +use anyhow::Result; +use async_trait::async_trait; +use derive_builder::Builder; +use url::Host; +pub use url::Url; + +enum Login { + Username(String, String), + Cookie(String), +} + +/// Contains metadata about a newspaper +// TODO: provide builder +#[derive(Debug, PartialEq, Default, Builder)] +#[builder(default)] +pub struct Metadata { + /// The hosts that can be corresponds to this newspaper + #[builder(setter(into))] + pub hosts: Vec, + /// The name of the newspaper, in lower case, without spaces + /// + /// As it should be unique and contain no spaces, it may be used for configuration purposes + #[builder(setter(into))] + pub lower_case_name: String, + /// The full name of the newspaper, in lower case, without spaces + #[builder(setter(into))] + pub name: String, +} + +impl Metadata { + pub fn builder() -> MetadataBuilder { + MetadataBuilder::default() + } +} + +#[async_trait] +pub trait Newspaper { + /// Returns a list of hosts that corresponds to the newspapers + fn metadata(&self) -> Metadata; + + /// Returns true if the Newspaper has complete access to the articles + /// + /// Usually, it will may tell you if you are logged in when newspaper have a paywall + async fn has_complete_access(&self) -> bool; + + /// Returns a newspaper structure + async fn new() -> Self + where + Self: Sized; + + /// Retrieve an article under the hmtl format + /// The article **must** be self-contained + async fn retrieve_html(&self, url: &Url) -> Result; + + // fn login(login: Login) +} diff --git a/crieur-retrieve/src/newspapers/mediapart.rs b/crieur-retrieve/src/newspapers/mediapart.rs new file mode 100644 index 0000000..8d817e9 --- /dev/null +++ b/crieur-retrieve/src/newspapers/mediapart.rs @@ -0,0 +1,65 @@ +use anyhow::Result; +use async_trait::async_trait; +use cookie::Cookie; +use url::Host; + +use crate::newspaper::{Metadata, Newspaper}; +use crate::tools; +use crate::Url; +use crate::{Download, Downloader}; + +#[derive(Debug, Clone, Default)] +pub struct Mediapart { + // TODO: remove this pub !! + pub login_cookie: Option<(String, String)>, +} + +fn str_to_host>(host: S) -> Host { + Host::Domain(host.into()) +} + +#[async_trait] +impl Newspaper for Mediapart { + fn metadata(&self) -> Metadata { + Metadata::builder() + .hosts(vec![ + str_to_host("mediapart.fr"), + str_to_host("www.mediapart.fr"), + ]) + .lower_case_name("mediapart") + .name("Médiapart") + .build() + .unwrap_or_default() + } + + async fn retrieve_html(&self, url: &Url) -> Result { + // TODO: add "?onglet=full" to the url if not + let cookies = if let Some((name, value)) = &self.login_cookie { + let cookie = Cookie::build(name, value).secure(true).finish(); + vec![cookie] + } else { + vec![] + }; + + // TODO: replace by builder + let downloader = Downloader { cookies }; + + let body = downloader.download(&url).await?; + let html = String::from_utf8(body.to_vec())?; + + // TODO: correction of usage of relative urls, and replace "" by the url + let single_page_html = tools::self_contained_html(&html, &downloader, &url).await; + Ok(single_page_html) + } + + async fn new() -> Self { + Self { + ..Default::default() + } + } + + async fn has_complete_access(&self) -> bool { + // TODO: check if we are logged using the cookie + true + } +} diff --git a/crieur-retrieve/src/newspapers/mod.rs b/crieur-retrieve/src/newspapers/mod.rs new file mode 100644 index 0000000..b7c9e05 --- /dev/null +++ b/crieur-retrieve/src/newspapers/mod.rs @@ -0,0 +1,3 @@ +mod mediapart; + +pub use mediapart::Mediapart; diff --git a/crieur-retrieve/src/tools/download.rs b/crieur-retrieve/src/tools/download.rs new file mode 100644 index 0000000..e00f77f --- /dev/null +++ b/crieur-retrieve/src/tools/download.rs @@ -0,0 +1,57 @@ +use std::error::Error as StdError; + +use anyhow::Result; +use async_trait::async_trait; +use bytes::Bytes; +use cookie::Cookie; +use hyper::{header, Body, Client, Method, Request}; +use thiserror::Error; +use url::Url; + +#[derive(Error, Debug)] +pub enum DownloadError { + #[error("Http error")] + HttpError(#[from] hyper::http::Error), + #[error("Hyper error")] + HyperError(#[from] hyper::Error), +} + +/// Downloads documents +#[async_trait] +pub trait Download { + type Error: StdError; + + /// Downloads a file from an url and returns the result as bytes + async fn download(&self, file_link: &Url) -> Result; +} + +/// Store several cookies +// TODO: add builder or new() funciton +#[derive(Debug, Clone)] +pub struct Downloader<'c> { + pub cookies: Vec>, +} + +#[async_trait] +impl<'c> Download for Downloader<'c> { + type Error = DownloadError; + + async fn download(&self, file_link: &Url) -> Result { + let https = hyper_rustls::HttpsConnector::with_native_roots(); + let client: Client<_, hyper::Body> = Client::builder().build(https); + + let mut req = Request::builder() + .method(Method::GET) + .uri(file_link.as_str()); + + for cookie in &self.cookies { + req = req.header(header::COOKIE, cookie.to_string()); + } + + let req = req.body(Body::empty())?; + + let resp = client.request(req).await?; + let body = hyper::body::to_bytes(resp).await?; + Ok(body) + } +} diff --git a/crieur-retrieve/src/tools/mod.rs b/crieur-retrieve/src/tools/mod.rs new file mode 100644 index 0000000..59381b1 --- /dev/null +++ b/crieur-retrieve/src/tools/mod.rs @@ -0,0 +1,5 @@ +mod download; +mod self_contained_html; + +pub use download::{Download, DownloadError, Downloader}; +pub use self_contained_html::self_contained_html; diff --git a/crieur-retrieve/src/tools/self_contained_html.rs b/crieur-retrieve/src/tools/self_contained_html.rs new file mode 100644 index 0000000..10ade69 --- /dev/null +++ b/crieur-retrieve/src/tools/self_contained_html.rs @@ -0,0 +1,291 @@ +use log::debug; +use std::fs::File; +use std::io::prelude::*; +use std::path::Path; + +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use base64; +use bytes::Bytes; +use futures::future::{JoinAll, OptionFuture}; +use html_minifier::HTMLMinifier; +use indoc::{formatdoc, indoc}; +use itertools::izip; +use nipper::Document; +use url::Url; + +use crate::consts::EVENT_HANDLERS; +use crate::errors; +use crate::Download; + +/// Makes an html page self-contained +/// +/// The `downloader` must implement `Download` and is used to download ressources that are +/// needed to make this page self-contained such as stylesheets or images. +/// +/// The function also removes all scripts on the page +pub async fn self_contained_html(html: S, downloader: &D, base_url: &Url) -> String +where + E: std::error::Error, + D: Download + Send, + S: AsRef, +{ + // TODO: split/refactor this function + // ¿ find a way to ease the |get urls| -> |async download| -> |replace| workflow ? + let (style_urls, html) = { + let document = Document::from(html.as_ref()); + + // ---- Remove scripts ---- + // + document.select("script").remove(); + + for event in EVENT_HANDLERS { + document + .select(format!("[{}]", event).as_str()) + .remove_attr(event); + } + + // ---- Replace stylesheets ---- + // + let stylesheets = document.select("link[href][rel=\"stylesheet\"]"); + let styles_url = stylesheets + .iter() + .map(|stylesheet| { + if let Some(src) = stylesheet.attr("href") { + //TODO: does it work with absolute urls ? + base_url.join(src.as_ref()).ok() + } else { + None + } + }) + .collect::>(); + (styles_url, String::from(document.html())) + }; + + let style_urls = style_urls.into_iter().map(|style_url| { + OptionFuture::from(style_url.map(|s| async move { downloader.download(&s).await.unwrap() })) + }); + let downloaded_styles = futures::future::join_all(style_urls).await; + + let html = { + let document = Document::from(&html); + let styles = document.select("link[href][rel=\"stylesheet\"]"); + + styles + .iter() + .zip(downloaded_styles.iter()) + .for_each(|(mut stylesheet, inner_css)| { + if let Some(inner_css) = inner_css { + let css = String::from_utf8(inner_css.to_vec()).unwrap(); + let css = format!("", css); + stylesheet.replace_with_html(css); + } else { + stylesheet.remove(); + } + }); + String::from(document.html()) + }; + + // ---- Replace imgs ---- + // + let image_urls = { + let document = Document::from(&html); + let imgs = document.select("img"); + + imgs.iter() + .map(|image| { + if let Some(src) = image.attr("src") { + base_url.join(src.as_ref()).ok() + } else { + None + } + }) + .collect::>() + }; + + let downloaded_images = image_urls.into_iter().map(|image_url| { + OptionFuture::from(image_url.map(|url| async move { + let data = downloader.download(&url).await.unwrap(); + (url, data) + })) + }); + let downloaded_images = futures::future::join_all(downloaded_images).await; + + let html = { + let document = Document::from(&html); + let imgs = document.select("img"); + + imgs.iter() + .zip(downloaded_images.iter()) + .for_each(|(mut img, data)| { + if let Some((url, data)) = data { + let data = base64::encode(data); + let extension = Path::new(url.path()).extension().unwrap().to_str().unwrap(); + img.set_attr("src", &format!("data:image/{};base64,{}", extension, data)); + } + }); + String::from(document.html()) + }; + + let mut minifier = HTMLMinifier::new(); + minifier.digest(html.as_str()).unwrap(); + + String::from_utf8(minifier.get_html().into()).unwrap() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init() { + let _ = env_logger::builder().is_test(true).try_init(); + } + + // TODO: the Dummy,Css and Png Downloaders don't really test the async scenario as + // they don't use futures : they don't call await. + // They should be testing the async scenario + struct DummyDownloader; + #[async_trait] + impl Download for DummyDownloader { + type Error = errors::Error; + async fn download(&self, _file_link: &Url) -> errors::Result { + Ok(Bytes::from("")) + } + } + + #[tokio::test] + async fn remove_scripts() -> Result<()> { + let html = ""; + let base_url = Url::parse("http://example.com")?; + let downloader = DummyDownloader {}; + assert_eq!( + self_contained_html(html, &downloader, &base_url).await, + "" + ); + Ok(()) + } + + #[tokio::test] + async fn remove_onevent_handlers() -> Result<()> { + init(); + let downloader = DummyDownloader {}; + let html = |onevent| { + formatdoc! {" + + + + + + + ", + onevent + } + }; + + let base_url = Url::parse("http://example.com")?; + for s in EVENT_HANDLERS { + assert_eq!( + self_contained_html(html(s), &downloader, &base_url).await, + "\n\n\n\n" + ); + } + Ok(()) + } + + struct CssDownloader; + #[async_trait] + impl Download for CssDownloader { + type Error = errors::Error; + async fn download(&self, _file_link: &Url) -> errors::Result { + Ok(indoc! {" + section#warning { + color: red; + }"} + .into()) + } + } + + #[tokio::test] + async fn download_css() -> Result<()> { + let downloader = CssDownloader {}; + + let html = indoc! {" + + + + + + + + "}; + + // FIXME: find why minify doesn't minify + let wanted_html = indoc! {" + + + + + + "}; + let mut minifier = HTMLMinifier::new(); + minifier.digest(wanted_html)?; + let minified = String::from_utf8(minifier.get_html().into())?; + + let base_url = Url::parse("http://example.com")?; + assert_eq!( + self_contained_html(html, &downloader, &base_url).await, + minified + ); + Ok(()) + } + + struct PngDownloader; + #[async_trait] + impl Download for PngDownloader { + type Error = errors::Error; + async fn download(&self, _file_link: &Url) -> errors::Result { + let image_path = Path::new("test_data/home.png"); + let mut image_file = File::open(&image_path).unwrap(); + let mut image_buf: Vec = vec![]; + image_file.read_to_end(&mut image_buf).unwrap(); + Ok(image_buf.into()) + } + } + + #[tokio::test] + async fn download_image_png() -> Result<()> { + let downloader = PngDownloader {}; + let html = indoc! {" + + + + \"an + + + "}; + + let wanted_html = indoc! {" + + + \"an + + "}; + let mut minifier = HTMLMinifier::new(); + minifier.digest(wanted_html)?; + let minified = String::from_utf8(minifier.get_html().into())?; + + let base_url = Url::parse("http://example.com")?; + assert_eq!( + self_contained_html(html, &downloader, &base_url).await, + minified + ); + Ok(()) + } +} diff --git a/crieur-retrieve/test_data/home.png b/crieur-retrieve/test_data/home.png new file mode 100644 index 0000000..f0d1dad --- /dev/null +++ b/crieur-retrieve/test_data/home.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5cf9656ed3af25127c9d98ce0b6227615f0b9cb6675b4b1f8bc1ed2e425dfcf +size 210 diff --git a/documentation/design/retrieve.md b/documentation/design/retrieve.md new file mode 100644 index 0000000..dd1d108 --- /dev/null +++ b/documentation/design/retrieve.md @@ -0,0 +1,76 @@ +--- +title: crieur-retrieve design +--- + +# Self-contained html + +Exporting the article as a self-contained may be the easier and more reliable , as it keep the +original ui of the newspaper, and do not require to export in a different format. + +Creating reusable methods to create a self-contained html page will make it easier to write +`Newspaper`s . Those methods would be part of a `crieur-retrieve-tool` library. + +The `self_contained_html` function have been created to do this. + +```rust +pub fn self_contained_html>() + html: S, + downloader: &dyn Fn(Url) -> Option, +) -> String +``` + +## Script removal + +Nothing should be executed by the exported html page. + +Scripts elements are contained in `