From a16dbbc790b9b2b681c57ae9999fdcc73902c472 Mon Sep 17 00:00:00 2001 From: koalp Date: Tue, 27 Apr 2021 04:32:27 +0200 Subject: [PATCH] feat: add basic chatbot A basic chabot application that downloads article from one newspaper have been added. It can download html pages and is called with !hmtl ArticleLocation have been refactored to own it's internal data. --- .gitea/issue_template/bug_report.md | 3 - .gitea/issue_template/feature_request.md | 5 +- Cargo.lock | 1079 ++++++++++++++++++- Cargo.toml | 2 + README.md | 2 +- crieur-chatbot/Cargo.toml | 19 + crieur-chatbot/src/chatbot.rs | 85 ++ crieur-chatbot/src/cli.rs | 32 + crieur-chatbot/src/handlers/html.rs | 94 ++ crieur-chatbot/src/handlers/mod.rs | 2 + crieur-chatbot/src/lib.rs | 10 + crieur-retrieve/src/article_location.rs | 53 +- crieur-retrieve/src/newspaper.rs | 4 +- crieur-retrieve/src/newspapers/mediapart.rs | 2 +- examples/cli_downloader.rs | 4 +- src/main.rs | 6 +- 16 files changed, 1361 insertions(+), 41 deletions(-) create mode 100644 crieur-chatbot/Cargo.toml create mode 100644 crieur-chatbot/src/chatbot.rs create mode 100644 crieur-chatbot/src/cli.rs create mode 100644 crieur-chatbot/src/handlers/html.rs create mode 100644 crieur-chatbot/src/handlers/mod.rs create mode 100644 crieur-chatbot/src/lib.rs diff --git a/.gitea/issue_template/bug_report.md b/.gitea/issue_template/bug_report.md index 2434293..c450e3d 100644 --- a/.gitea/issue_template/bug_report.md +++ b/.gitea/issue_template/bug_report.md @@ -17,6 +17,3 @@ labels: **Expected behavior** *describe what you expected to happen* - -**Configuration** -*paste the result of `stage --version` diff --git a/.gitea/issue_template/feature_request.md b/.gitea/issue_template/feature_request.md index 4ca4668..3746dda 100644 --- a/.gitea/issue_template/feature_request.md +++ b/.gitea/issue_template/feature_request.md @@ -3,13 +3,12 @@ name: "Feature request" about: "This template is for requesting a new feature" title: "" labels: -- "type::feature" +- "type::enhancement" - "status::review_needed" - --- *(if applicable) describe what problem or frustration you have currently* -*describe what you would like to be able to do, or what solution you would like (you can propose several)* +*describe what you would like to be able to do, or what solution you would like* *(optional) additional context, comments or implementation propositions* diff --git a/Cargo.lock b/Cargo.lock index 4686eea..db3830a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,71 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "aead" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" +dependencies = [ + "aes-soft", + "aesni", + "cipher", +] + +[[package]] +name = "aes-ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7729c3cde54d67063be556aeac75a81330d802f0259500ca40cb52967f975763" +dependencies = [ + "aes-soft", + "aesni", + "cipher", + "ctr", +] + +[[package]] +name = "aes-gcm" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aes-soft" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" +dependencies = [ + "cipher", + "opaque-debug", +] + +[[package]] +name = "aesni" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" +dependencies = [ + "cipher", + "opaque-debug", +] + [[package]] name = "ahash" version = "0.4.7" @@ -21,6 +87,12 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" +[[package]] +name = "assign" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" + [[package]] name = "async-trait" version = "0.1.48" @@ -32,6 +104,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3410529e8288c463bedb5930f82833bc0c90e5d2fe639a56582a4d09220b281" +dependencies = [ + "autocfg 1.0.1", +] + [[package]] name = "atty" version = "0.2.14" @@ -55,6 +136,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "backoff" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fe17f59a06fe8b87a6fc8bf53bb70b3aba76d7685f432487a68cd5552853625" +dependencies = [ + "futures-core", + "getrandom 0.2.2", + "instant", + "pin-project", + "rand 0.8.3", + "tokio", +] + [[package]] name = "base-x" version = "0.2.8" @@ -73,6 +168,15 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.6.1" @@ -109,6 +213,38 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chacha20" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed8738f14471a99f0e316c327e68fc82a3611cc2895fcb604b89eedaf8f39d95" +dependencies = [ + "cipher", + "zeroize", +] + +[[package]] +name = "chacha20poly1305" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1fc18e6d90c40164bf6c317476f2a98f04661e310e79830366b7e914c58a8e" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -118,6 +254,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "cmake" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb6210b637171dfba4cda12e579ac6dc73f5165ad56133e5d72ef3131f320855" +dependencies = [ + "cc", +] + [[package]] name = "const_fn" version = "0.4.6" @@ -162,11 +307,33 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "cpuid-bool" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crieur" version = "0.1.0" dependencies = [ "anyhow", + "crieur-chatbot", "crieur-retrieve", "dotenv", "env_logger", @@ -174,6 +341,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "crieur-chatbot" +version = "0.1.0" +dependencies = [ + "anyhow", + "crieur-retrieve", + "dotenv", + "log", + "matrix-sdk", + "mime", +] + [[package]] name = "crieur-retrieve" version = "0.1.0" @@ -199,6 +378,40 @@ dependencies = [ "url", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "crypto-mac" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "cssparser" version = "0.25.9" @@ -267,6 +480,15 @@ dependencies = [ "sct", ] +[[package]] +name = "ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" +dependencies = [ + "cipher", +] + [[package]] name = "darling" version = "0.12.3" @@ -302,6 +524,16 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + [[package]] name = "derive_builder" version = "0.10.0" @@ -346,6 +578,15 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "discard" version = "1.0.4" @@ -432,6 +673,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -442,6 +698,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -506,6 +772,15 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" +[[package]] +name = "futures-locks" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c4e684ddb2d8a4db5ca8a02b35156da129674ba4412b6f528698d58c594954" +dependencies = [ + "futures", +] + [[package]] name = "futures-macro" version = "0.3.14" @@ -530,6 +805,16 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +dependencies = [ + "gloo-timers", + "send_wrapper", +] + [[package]] name = "futures-util" version = "0.3.14" @@ -559,6 +844,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -567,7 +862,43 @@ checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "gloo-timers" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -607,6 +938,16 @@ dependencies = [ "libc", ] +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac", + "digest", +] + [[package]] name = "html-escape" version = "0.2.7" @@ -723,6 +1064,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -748,6 +1102,7 @@ checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg 1.0.1", "hashbrown", + "serde", ] [[package]] @@ -766,8 +1121,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "time", + "wasm-bindgen", + "web-sys", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itertools" version = "0.10.0" @@ -792,6 +1157,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "js_int" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcae89e078a96b781b38f36225bb3a174b8f6e905dfec550dd16a13539c82acc" +dependencies = [ + "serde", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -859,6 +1233,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e72f7deb758fea9ea7d290aebfa788763d0bffae12caa6406a25baaf8fa68a8" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "markup5ever" version = "0.10.0" @@ -882,6 +1262,95 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "matrix-sdk" +version = "0.2.0" +source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=ab180362c931606385dd53b73620d82ef2c3166d#ab180362c931606385dd53b73620d82ef2c3166d" +dependencies = [ + "backoff", + "bytes", + "dashmap", + "futures", + "futures-timer", + "http", + "matrix-sdk-base", + "matrix-sdk-common", + "mime", + "reqwest", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-futures", + "url", + "zeroize", +] + +[[package]] +name = "matrix-sdk-base" +version = "0.2.0" +source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=ab180362c931606385dd53b73620d82ef2c3166d#ab180362c931606385dd53b73620d82ef2c3166d" +dependencies = [ + "chacha20poly1305", + "dashmap", + "futures", + "hmac", + "matrix-sdk-common", + "matrix-sdk-crypto", + "pbkdf2", + "rand 0.8.3", + "serde", + "serde_json", + "sha2", + "sled", + "thiserror", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "matrix-sdk-common" +version = "0.2.0" +source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=ab180362c931606385dd53b73620d82ef2c3166d#ab180362c931606385dd53b73620d82ef2c3166d" +dependencies = [ + "async-trait", + "futures", + "futures-locks", + "instant", + "ruma", + "serde", + "tokio", + "uuid", + "wasm-bindgen-futures", +] + +[[package]] +name = "matrix-sdk-crypto" +version = "0.2.0" +source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=ab180362c931606385dd53b73620d82ef2c3166d#ab180362c931606385dd53b73620d82ef2c3166d" +dependencies = [ + "aes-ctr", + "aes-gcm", + "atomic", + "base64", + "byteorder", + "dashmap", + "futures", + "getrandom 0.2.2", + "hmac", + "matrix-sdk-common", + "olm-rs", + "pbkdf2", + "serde", + "serde_json", + "sha2", + "sled", + "thiserror", + "tracing", + "zeroize", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -894,6 +1363,21 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "memoffset" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "minifier" version = "0.0.39" @@ -925,6 +1409,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -999,18 +1501,73 @@ dependencies = [ "libc", ] +[[package]] +name = "olm-rs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8962a1fc909accf520991dcda872888554ecf16320097e02d3bd9981844a24ae" +dependencies = [ + "getrandom 0.2.2", + "olm-sys", + "serde", + "serde_json", + "zeroize", +] + +[[package]] +name = "olm-sys" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d29b3debd2916908e725d936ef5722d014a338a57c74e59dbf4c8c43855254d" +dependencies = [ + "cmake", +] + [[package]] name = "once_cell" version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + [[package]] name = "openssl-probe" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" +[[package]] +name = "openssl-sys" +version = "0.9.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.11.1" @@ -1036,6 +1593,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "paste" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" + +[[package]] +name = "pbkdf2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b8c0d71734018084da0c0354193a5edfb81b20d2d57a92c5b154aefc554a4a" +dependencies = [ + "crypto-mac", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1166,6 +1738,33 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "poly1305" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b7456bc1ad2d4cf82b3a016be4c2ac48daf11bf990c1603ebd447fe6f30fca8" +dependencies = [ + "cpuid-bool 0.2.0", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" +dependencies = [ + "cpuid-bool 0.2.0", + "opaque-debug", + "universal-hash", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1178,6 +1777,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "proc-macro-crate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fdbd1df62156fbc5945f4762632564d7d038153091c3fcf1067f6aef7cff92" +dependencies = [ + "thiserror", + "toml", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -1239,7 +1848,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -1247,6 +1856,18 @@ dependencies = [ "rand_pcg 0.2.1", ] +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + [[package]] name = "rand_chacha" version = "0.1.1" @@ -1267,6 +1888,16 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -1288,7 +1919,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", ] [[package]] @@ -1309,6 +1949,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + [[package]] name = "rand_isaac" version = "0.1.1" @@ -1406,6 +2055,49 @@ version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -1421,6 +2113,176 @@ dependencies = [ "winapi", ] +[[package]] +name = "ruma" +version = "0.0.3" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "assign", + "js_int", + "ruma-api", + "ruma-client-api", + "ruma-common", + "ruma-events", + "ruma-federation-api", + "ruma-identifiers", + "ruma-serde", +] + +[[package]] +name = "ruma-api" +version = "0.17.0-alpha.4" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "http", + "percent-encoding", + "ruma-api-macros", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "ruma-api-macros" +version = "0.17.0-alpha.4" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ruma-client-api" +version = "0.10.0-alpha.3" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "assign", + "http", + "js_int", + "maplit", + "percent-encoding", + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-common" +version = "0.5.0" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "indexmap", + "js_int", + "maplit", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", + "tracing", + "wildmatch", +] + +[[package]] +name = "ruma-events" +version = "0.22.0-alpha.3" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "js_int", + "ruma-common", + "ruma-events-macros", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-events-macros" +version = "0.22.0-alpha.3" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ruma-federation-api" +version = "0.1.0-alpha.2" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "js_int", + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-identifiers" +version = "0.19.0" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "paste", + "ruma-identifiers-macros", + "ruma-identifiers-validation", + "ruma-serde", + "ruma-serde-macros", + "serde", +] + +[[package]] +name = "ruma-identifiers-macros" +version = "0.19.0" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "proc-macro2", + "quote", + "ruma-identifiers-validation", + "syn", +] + +[[package]] +name = "ruma-identifiers-validation" +version = "0.3.0" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" + +[[package]] +name = "ruma-serde" +version = "0.3.1" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "form_urlencoded", + "itoa", + "js_int", + "ruma-serde-macros", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-serde-macros" +version = "0.3.1" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "rustc_version" version = "0.2.3" @@ -1570,11 +2432,20 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "serde" version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" @@ -1598,6 +2469,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "servo_arc" version = "0.1.1" @@ -1614,6 +2497,19 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +[[package]] +name = "sha2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpuid-bool 0.1.2", + "digest", + "opaque-debug", +] + [[package]] name = "signal-hook-registry" version = "1.3.0" @@ -1641,6 +2537,22 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "sled" +version = "0.34.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot", +] + [[package]] name = "smallvec" version = "0.6.14" @@ -1767,6 +2679,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" + [[package]] name = "syn" version = "1.0.68" @@ -1778,6 +2696,32 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "tendril" version = "0.4.2" @@ -1908,6 +2852,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.22.0" @@ -1933,6 +2887,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + [[package]] name = "tower-service" version = "0.3.1" @@ -1947,9 +2910,21 @@ checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.17" @@ -1959,12 +2934,28 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "try-lock" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "typenum" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" + [[package]] name = "unicode-bidi" version = "0.3.5" @@ -1995,6 +2986,16 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" +[[package]] +name = "universal-hash" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -2025,6 +3026,22 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.2", + "serde", +] + +[[package]] +name = "vcpkg" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" + [[package]] name = "version_check" version = "0.9.3" @@ -2047,6 +3064,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasm-bindgen" version = "0.2.73" @@ -2054,6 +3077,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" dependencies = [ "cfg-if 1.0.0", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -2072,6 +3097,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.73" @@ -2121,6 +3158,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "wildmatch" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6c48bd20df7e4ced539c12f570f937c6b4884928a87fee70a479d72f031d4e0" + [[package]] name = "winapi" version = "0.3.9" @@ -2151,3 +3194,33 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c1e130bebaeab2f23886bf9acbaca14b092408c452543c857f66399cd6dab1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] diff --git a/Cargo.toml b/Cargo.toml index be05e56..458df77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crieur-retrieve", + "crieur-chatbot", ] @@ -17,6 +18,7 @@ publish = false [dependencies] anyhow = "1.0.40" crieur-retrieve = {version = "0.1", path="crieur-retrieve"} +crieur-chatbot = {version = "0.1", path="crieur-chatbot"} dotenv = "0.15.0" env_logger = "0.8.3" log = "0.4.14" diff --git a/README.md b/README.md index e4bfb50..fc0677f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Tools to retrieve articles from multiple newspaper you subscribed to. First retrieve login cookies for websites and put it in a `.env` ``` -cargo run --example=retrive_html_articles +cargo run --example=cli_downloader ``` # Documentation diff --git a/crieur-chatbot/Cargo.toml b/crieur-chatbot/Cargo.toml new file mode 100644 index 0000000..e2f3631 --- /dev/null +++ b/crieur-chatbot/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "crieur-chatbot" +version = "0.1.0" +authors = ["koalp "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.40" +dotenv = "0.15.0" +crieur-retrieve = {version = "0.1.0", path = "../crieur-retrieve"} +mime = "0.3.16" +log = "0.4.14" + +[dependencies.matrix-sdk] +git = "https://github.com/matrix-org/matrix-rust-sdk" +rev = "ab180362c931606385dd53b73620d82ef2c3166d" +version = "0.2.0" diff --git a/crieur-chatbot/src/chatbot.rs b/crieur-chatbot/src/chatbot.rs new file mode 100644 index 0000000..cad6b1c --- /dev/null +++ b/crieur-chatbot/src/chatbot.rs @@ -0,0 +1,85 @@ +//! Chatbot +use std::convert::TryInto; + +use anyhow::Result; +use matrix_sdk::{ + self, async_trait, + events::{ + room::message::{MessageEventContent, MessageType, TextMessageEventContent}, + AnyMessageEventContent, SyncMessageEvent, + }, + room::Room, + Client, ClientConfig, EventHandler, SyncSettings, +}; + +use crate::Html; + +#[derive(Debug, Clone, Default)] +pub(crate) struct Builder { + user: String, + password: String, + homeserver: String, + //TODO: rooms + room: String, +} + +impl Builder { + fn new() -> Self { + Default::default() + } + + pub(crate) async fn connect(&self) -> Result { + let client = Client::new(self.homeserver.as_str())?; + client + .login(self.user.as_str(), self.password.as_str(), None, None) + .await?; + assert!(client.logged_in().await); + client + .join_room_by_id(&self.room.as_str().try_into()?) + .await?; + + Ok(Chatbot { client }) + } + + pub(crate) fn login( + &mut self, + user: &impl AsRef, + password: &impl AsRef, + ) -> &mut Self { + self.user = String::from(user.as_ref()); + self.password = String::from(password.as_ref()); + self + } + + pub(crate) fn homeserver(&mut self, homeserver: &impl AsRef) -> &mut Self { + self.homeserver = String::from(homeserver.as_ref()); + self + } + + pub(crate) fn room(&mut self, room: &impl AsRef) -> &mut Self { + self.room = String::from(room.as_ref()); + self + } +} + +#[derive(Debug, Clone)] +pub(crate) struct Chatbot { + client: Client, +} + +impl Chatbot { + pub(crate) fn builder() -> Builder { + Builder::new() + } + + pub(crate) async fn run(&self) -> Result<()> { + self.client.set_event_handler(Box::new(Html::new())).await; + + let mut settings = SyncSettings::default(); + if let Some(token) = self.client.sync_token().await { + settings = settings.token(token); + } + self.client.sync(settings).await; + Ok(()) + } +} diff --git a/crieur-chatbot/src/cli.rs b/crieur-chatbot/src/cli.rs new file mode 100644 index 0000000..f8ecf57 --- /dev/null +++ b/crieur-chatbot/src/cli.rs @@ -0,0 +1,32 @@ +use std::env; + +use anyhow::{bail, Result}; +use dotenv::dotenv; + +use crate::Chatbot; + +/// Runs the chatbot +pub async fn run() -> Result<()> { + dotenv().ok(); + + let (user, password, homeserver, room) = match ( + env::var("CRIEUR_MATRIX_USER"), + env::var("CRIEUR_MATRIX_PASSWORD"), + env::var("CRIEUR_MATRIX_HOMESERVER"), + env::var("CRIEUR_MATRIX_ROOM"), + ) { + (Ok(user), Ok(password), Ok(homeserver), Ok(room)) => (user, password, homeserver, room), + _ => bail!("Configuration incomplete, please set all required environment variables"), + }; + + let chatbot = Chatbot::builder() + .login(&user, &password) + .homeserver(&homeserver) + .room(&room) + .connect() + .await?; + + chatbot.run().await?; + + Ok(()) +} diff --git a/crieur-chatbot/src/handlers/html.rs b/crieur-chatbot/src/handlers/html.rs new file mode 100644 index 0000000..4c7ef7f --- /dev/null +++ b/crieur-chatbot/src/handlers/html.rs @@ -0,0 +1,94 @@ +use std::convert::TryInto; +use std::env; + +use anyhow::Result; +use log::info; +use matrix_sdk::{ + self, async_trait, + events::{ + room::message::{MessageEventContent, MessageType, TextMessageEventContent}, + AnyMessageEventContent, SyncMessageEvent, + }, + room::Room, + Client, ClientConfig, EventHandler, SyncSettings, +}; + +use crieur_retrieve::{newspaper::Newspaper, ArticleLocation, Mediapart, Url}; + +pub(crate) struct Html {} + +impl Html { + pub fn new() -> Self { + Self {} + } +} + +async fn send_article(url: U, room: matrix_sdk::room::Joined) +where + U: TryInto + Send, + E: std::error::Error + Sync + Send + 'static, +{ + //TODO: replace by async block when async block is stable + async fn article_html(url: U) -> Result + where + U: TryInto + Send, + E: std::error::Error + Sync + Send + 'static, + { + let article_str = ArticleLocation::builder() + .url(url)? + .build()? + .retrieve_html() + .await?; + Ok(article_str) + } + + let text_message = + |message| AnyMessageEventContent::RoomMessage(MessageEventContent::text_plain(message)); + + //TODO: replace occurences ok() by async and logging block when async block is stable + let article_html = match article_html(url).await { + Ok(url) => url, + Err(_) => { + room.send(text_message("Can't download the file"), None) + .await + .ok(); + return; + } + }; + + room.send_attachment( + "test.html", + &mime::TEXT_HTML_UTF_8, + &mut article_html.as_bytes(), + None, + ) + .await + .ok(); +} + +#[async_trait] +impl EventHandler for Html { + async fn on_room_message(&self, room: Room, event: &SyncMessageEvent) { + if let Room::Joined(room) = room { + let msg_body = if let SyncMessageEvent { + content: + MessageEventContent { + msgtype: MessageType::Text(TextMessageEventContent { body: msg_body, .. }), + .. + }, + .. + } = event + { + msg_body + } else { + return; + }; + info!("sending file"); + + match msg_body.split(' ').collect::>().as_slice() { + ["!html", url, ..] => send_article(*url, room).await, + _ => return, + } + } + } +} diff --git a/crieur-chatbot/src/handlers/mod.rs b/crieur-chatbot/src/handlers/mod.rs new file mode 100644 index 0000000..a75fb0f --- /dev/null +++ b/crieur-chatbot/src/handlers/mod.rs @@ -0,0 +1,2 @@ +mod html; +pub(crate) use html::Html; diff --git a/crieur-chatbot/src/lib.rs b/crieur-chatbot/src/lib.rs new file mode 100644 index 0000000..8646768 --- /dev/null +++ b/crieur-chatbot/src/lib.rs @@ -0,0 +1,10 @@ +//! Provides a matrix chatbot to download newspaper articles + +mod cli; +pub use cli::run; + +mod chatbot; +use chatbot::Chatbot; + +mod handlers; +use handlers::Html; diff --git a/crieur-retrieve/src/article_location.rs b/crieur-retrieve/src/article_location.rs index 657b2c3..734c947 100644 --- a/crieur-retrieve/src/article_location.rs +++ b/crieur-retrieve/src/article_location.rs @@ -1,21 +1,34 @@ use std::boxed::Box; use std::convert::TryInto; +use std::env; use anyhow::{anyhow, Result}; use log::info; use url::{Host, Url}; use crate::newspaper::Newspaper; +use crate::newspapers::Mediapart; -type Newspapers<'a> = Vec>; +type Newspapers = Vec>; -#[derive(Default)] -pub struct ArticleLocationBuilder<'a> { - url: Option, - newspapers: Option>, +fn default_newpapers() -> Newspapers { + let mut mediapart = Mediapart::new(); + + mediapart.login_cookie = Some(( + "MPRUUID".into(), + env::var("MEDIAPART_COOKIE").unwrap().into(), + )); + + vec![Box::new(mediapart)] } -impl<'a> ArticleLocationBuilder<'a> { +#[derive(Default)] +pub struct Builder { + url: Option, + newspapers: Option, +} + +impl Builder { pub fn new() -> Self { Self::default() } @@ -37,9 +50,9 @@ impl<'a> ArticleLocationBuilder<'a> { } /// Adds a newspaper to the list - pub fn newspaper(&mut self, newspaper: &'a T) -> &mut Self + pub fn newspaper(mut self, newspaper: T) -> Self where - T: 'a + Newspaper, + T: 'static + Newspaper, { match &mut self.newspapers { Some(newspapers) => newspapers.push(Box::new(newspaper)), @@ -72,7 +85,7 @@ impl<'a> ArticleLocationBuilder<'a> { /// - the url is not set /// - the given url has no host // TODO: move this to a defined error, remove anyhow ! - pub fn build(&self) -> Result> { + pub fn build(self) -> Result { let url = Clone::clone(self.url.as_ref().ok_or(anyhow!( "No url set. You can set it with the url() function" ))?); @@ -80,28 +93,22 @@ impl<'a> ArticleLocationBuilder<'a> { let host = Host::parse(host)?; let newspaper = self .newspapers - .as_ref() - .ok_or(anyhow!( - "A list of NewsPaper must be set. It can be set with newspapers() function" - ))? - .iter() + .unwrap_or(default_newpapers()) + .into_iter() .find(|c| c.metadata().hosts.contains(&host)) .ok_or(anyhow!("Newspaper couldn't be found"))?; - Ok(ArticleLocation { - newspaper: newspaper.clone(), - url, - }) + Ok(ArticleLocation { newspaper, url }) } } -pub struct ArticleLocation<'a> { - newspaper: Box<&'a dyn Newspaper>, +pub struct ArticleLocation { + newspaper: Box, pub url: Url, } -impl<'a> ArticleLocation<'a> { - pub fn builder() -> ArticleLocationBuilder<'a> { - ArticleLocationBuilder::new() +impl ArticleLocation { + pub fn builder() -> Builder { + Builder::new() } pub async fn retrieve_html(&self) -> Result { diff --git a/crieur-retrieve/src/newspaper.rs b/crieur-retrieve/src/newspaper.rs index cfadba0..532cc62 100644 --- a/crieur-retrieve/src/newspaper.rs +++ b/crieur-retrieve/src/newspaper.rs @@ -34,7 +34,7 @@ impl Metadata { } #[async_trait] -pub trait Newspaper { +pub trait Newspaper: Send + Sync { /// Returns a list of hosts that corresponds to the newspapers fn metadata(&self) -> Metadata; @@ -49,7 +49,7 @@ pub trait Newspaper { } /// Returns a newspaper structure - async fn new() -> Self + fn new() -> Self where Self: Sized; diff --git a/crieur-retrieve/src/newspapers/mediapart.rs b/crieur-retrieve/src/newspapers/mediapart.rs index 8070e7f..678a8ac 100644 --- a/crieur-retrieve/src/newspapers/mediapart.rs +++ b/crieur-retrieve/src/newspapers/mediapart.rs @@ -80,7 +80,7 @@ impl Newspaper for Mediapart { Ok(single_page_html) } - async fn new() -> Self { + fn new() -> Self { Self { ..Default::default() } diff --git a/examples/cli_downloader.rs b/examples/cli_downloader.rs index fffe05d..24341ba 100644 --- a/examples/cli_downloader.rs +++ b/examples/cli_downloader.rs @@ -17,7 +17,7 @@ async fn main() -> Result<()> { }; // TODO: remove this in favor of default newspapers - let mut mediapart = Mediapart::new().await + let mut mediapart = Mediapart::new() //.login(USERNAME, PASSWORD) // ; @@ -28,7 +28,7 @@ async fn main() -> Result<()> { // TODO: shorten this, maybe an helper function ? let article_location = ArticleLocation::builder() .url(url)? - .newspaper(&mediapart) + .newspaper(mediapart) .build()?; let article_str = article_location.retrieve_html().await?; diff --git a/src/main.rs b/src/main.rs index 03095a9..3b7e283 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use crieur_chatbot::run; use crieur_retrieve::{newspaper::Newspaper, Mediapart, Url}; use dotenv::dotenv; use std::env; @@ -7,13 +8,12 @@ use std::env; async fn main() -> Result<()> { dotenv().ok(); - let mut mediapart = Mediapart::new().await + let mut mediapart = Mediapart::new() //.login(USERNAME, PASSWORD) // ; mediapart.login_cookie = Some(("MPRUUID".into(), env::var("MEDIAPART_COOKIE")?)); - let url = Url::parse("https://www.mediapart.fr/journal/france/030421/l-hotel-dieu-patients-et-medecins-tentent-de-percer-les-mysteres-du-covid-long")?; - println!("{}", mediapart.retrieve_html(&url).await?); + run().await?; Ok(()) }