From a16dbbc790b9b2b681c57ae9999fdcc73902c472 Mon Sep 17 00:00:00 2001 From: koalp Date: Tue, 27 Apr 2021 04:32:27 +0200 Subject: [PATCH 1/3] feat: add basic chatbot A basic chabot application that downloads article from one newspaper have been added. It can download html pages and is called with !hmtl ArticleLocation have been refactored to own it's internal data. --- .gitea/issue_template/bug_report.md | 3 - .gitea/issue_template/feature_request.md | 5 +- Cargo.lock | 1079 ++++++++++++++++++- Cargo.toml | 2 + README.md | 2 +- crieur-chatbot/Cargo.toml | 19 + crieur-chatbot/src/chatbot.rs | 85 ++ crieur-chatbot/src/cli.rs | 32 + crieur-chatbot/src/handlers/html.rs | 94 ++ crieur-chatbot/src/handlers/mod.rs | 2 + crieur-chatbot/src/lib.rs | 10 + crieur-retrieve/src/article_location.rs | 53 +- crieur-retrieve/src/newspaper.rs | 4 +- crieur-retrieve/src/newspapers/mediapart.rs | 2 +- examples/cli_downloader.rs | 4 +- src/main.rs | 6 +- 16 files changed, 1361 insertions(+), 41 deletions(-) create mode 100644 crieur-chatbot/Cargo.toml create mode 100644 crieur-chatbot/src/chatbot.rs create mode 100644 crieur-chatbot/src/cli.rs create mode 100644 crieur-chatbot/src/handlers/html.rs create mode 100644 crieur-chatbot/src/handlers/mod.rs create mode 100644 crieur-chatbot/src/lib.rs diff --git a/.gitea/issue_template/bug_report.md b/.gitea/issue_template/bug_report.md index 2434293..c450e3d 100644 --- a/.gitea/issue_template/bug_report.md +++ b/.gitea/issue_template/bug_report.md @@ -17,6 +17,3 @@ labels: **Expected behavior** *describe what you expected to happen* - -**Configuration** -*paste the result of `stage --version` diff --git a/.gitea/issue_template/feature_request.md b/.gitea/issue_template/feature_request.md index 4ca4668..3746dda 100644 --- a/.gitea/issue_template/feature_request.md +++ b/.gitea/issue_template/feature_request.md @@ -3,13 +3,12 @@ name: "Feature request" about: "This template is for requesting a new feature" title: "" labels: -- "type::feature" +- "type::enhancement" - "status::review_needed" - --- *(if applicable) describe what problem or frustration you have currently* -*describe what you would like to be able to do, or what solution you would like (you can propose several)* +*describe what you would like to be able to do, or what solution you would like* *(optional) additional context, comments or implementation propositions* diff --git a/Cargo.lock b/Cargo.lock index 4686eea..db3830a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,71 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "aead" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" +dependencies = [ + "aes-soft", + "aesni", + "cipher", +] + +[[package]] +name = "aes-ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7729c3cde54d67063be556aeac75a81330d802f0259500ca40cb52967f975763" +dependencies = [ + "aes-soft", + "aesni", + "cipher", + "ctr", +] + +[[package]] +name = "aes-gcm" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aes-soft" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" +dependencies = [ + "cipher", + "opaque-debug", +] + +[[package]] +name = "aesni" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" +dependencies = [ + "cipher", + "opaque-debug", +] + [[package]] name = "ahash" version = "0.4.7" @@ -21,6 +87,12 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" +[[package]] +name = "assign" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" + [[package]] name = "async-trait" version = "0.1.48" @@ -32,6 +104,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3410529e8288c463bedb5930f82833bc0c90e5d2fe639a56582a4d09220b281" +dependencies = [ + "autocfg 1.0.1", +] + [[package]] name = "atty" version = "0.2.14" @@ -55,6 +136,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "backoff" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fe17f59a06fe8b87a6fc8bf53bb70b3aba76d7685f432487a68cd5552853625" +dependencies = [ + "futures-core", + "getrandom 0.2.2", + "instant", + "pin-project", + "rand 0.8.3", + "tokio", +] + [[package]] name = "base-x" version = "0.2.8" @@ -73,6 +168,15 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.6.1" @@ -109,6 +213,38 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chacha20" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed8738f14471a99f0e316c327e68fc82a3611cc2895fcb604b89eedaf8f39d95" +dependencies = [ + "cipher", + "zeroize", +] + +[[package]] +name = "chacha20poly1305" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1fc18e6d90c40164bf6c317476f2a98f04661e310e79830366b7e914c58a8e" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -118,6 +254,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "cmake" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb6210b637171dfba4cda12e579ac6dc73f5165ad56133e5d72ef3131f320855" +dependencies = [ + "cc", +] + [[package]] name = "const_fn" version = "0.4.6" @@ -162,11 +307,33 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "cpuid-bool" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crieur" version = "0.1.0" dependencies = [ "anyhow", + "crieur-chatbot", "crieur-retrieve", "dotenv", "env_logger", @@ -174,6 +341,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "crieur-chatbot" +version = "0.1.0" +dependencies = [ + "anyhow", + "crieur-retrieve", + "dotenv", + "log", + "matrix-sdk", + "mime", +] + [[package]] name = "crieur-retrieve" version = "0.1.0" @@ -199,6 +378,40 @@ dependencies = [ "url", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "crypto-mac" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "cssparser" version = "0.25.9" @@ -267,6 +480,15 @@ dependencies = [ "sct", ] +[[package]] +name = "ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" +dependencies = [ + "cipher", +] + [[package]] name = "darling" version = "0.12.3" @@ -302,6 +524,16 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + [[package]] name = "derive_builder" version = "0.10.0" @@ -346,6 +578,15 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "discard" version = "1.0.4" @@ -432,6 +673,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -442,6 +698,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -506,6 +772,15 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" +[[package]] +name = "futures-locks" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c4e684ddb2d8a4db5ca8a02b35156da129674ba4412b6f528698d58c594954" +dependencies = [ + "futures", +] + [[package]] name = "futures-macro" version = "0.3.14" @@ -530,6 +805,16 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +dependencies = [ + "gloo-timers", + "send_wrapper", +] + [[package]] name = "futures-util" version = "0.3.14" @@ -559,6 +844,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -567,7 +862,43 @@ checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "gloo-timers" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -607,6 +938,16 @@ dependencies = [ "libc", ] +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac", + "digest", +] + [[package]] name = "html-escape" version = "0.2.7" @@ -723,6 +1064,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -748,6 +1102,7 @@ checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg 1.0.1", "hashbrown", + "serde", ] [[package]] @@ -766,8 +1121,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "time", + "wasm-bindgen", + "web-sys", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itertools" version = "0.10.0" @@ -792,6 +1157,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "js_int" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcae89e078a96b781b38f36225bb3a174b8f6e905dfec550dd16a13539c82acc" +dependencies = [ + "serde", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -859,6 +1233,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e72f7deb758fea9ea7d290aebfa788763d0bffae12caa6406a25baaf8fa68a8" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "markup5ever" version = "0.10.0" @@ -882,6 +1262,95 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "matrix-sdk" +version = "0.2.0" +source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=ab180362c931606385dd53b73620d82ef2c3166d#ab180362c931606385dd53b73620d82ef2c3166d" +dependencies = [ + "backoff", + "bytes", + "dashmap", + "futures", + "futures-timer", + "http", + "matrix-sdk-base", + "matrix-sdk-common", + "mime", + "reqwest", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-futures", + "url", + "zeroize", +] + +[[package]] +name = "matrix-sdk-base" +version = "0.2.0" +source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=ab180362c931606385dd53b73620d82ef2c3166d#ab180362c931606385dd53b73620d82ef2c3166d" +dependencies = [ + "chacha20poly1305", + "dashmap", + "futures", + "hmac", + "matrix-sdk-common", + "matrix-sdk-crypto", + "pbkdf2", + "rand 0.8.3", + "serde", + "serde_json", + "sha2", + "sled", + "thiserror", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "matrix-sdk-common" +version = "0.2.0" +source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=ab180362c931606385dd53b73620d82ef2c3166d#ab180362c931606385dd53b73620d82ef2c3166d" +dependencies = [ + "async-trait", + "futures", + "futures-locks", + "instant", + "ruma", + "serde", + "tokio", + "uuid", + "wasm-bindgen-futures", +] + +[[package]] +name = "matrix-sdk-crypto" +version = "0.2.0" +source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=ab180362c931606385dd53b73620d82ef2c3166d#ab180362c931606385dd53b73620d82ef2c3166d" +dependencies = [ + "aes-ctr", + "aes-gcm", + "atomic", + "base64", + "byteorder", + "dashmap", + "futures", + "getrandom 0.2.2", + "hmac", + "matrix-sdk-common", + "olm-rs", + "pbkdf2", + "serde", + "serde_json", + "sha2", + "sled", + "thiserror", + "tracing", + "zeroize", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -894,6 +1363,21 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "memoffset" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "minifier" version = "0.0.39" @@ -925,6 +1409,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -999,18 +1501,73 @@ dependencies = [ "libc", ] +[[package]] +name = "olm-rs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8962a1fc909accf520991dcda872888554ecf16320097e02d3bd9981844a24ae" +dependencies = [ + "getrandom 0.2.2", + "olm-sys", + "serde", + "serde_json", + "zeroize", +] + +[[package]] +name = "olm-sys" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d29b3debd2916908e725d936ef5722d014a338a57c74e59dbf4c8c43855254d" +dependencies = [ + "cmake", +] + [[package]] name = "once_cell" version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + [[package]] name = "openssl-probe" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" +[[package]] +name = "openssl-sys" +version = "0.9.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.11.1" @@ -1036,6 +1593,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "paste" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" + +[[package]] +name = "pbkdf2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b8c0d71734018084da0c0354193a5edfb81b20d2d57a92c5b154aefc554a4a" +dependencies = [ + "crypto-mac", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1166,6 +1738,33 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "poly1305" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b7456bc1ad2d4cf82b3a016be4c2ac48daf11bf990c1603ebd447fe6f30fca8" +dependencies = [ + "cpuid-bool 0.2.0", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" +dependencies = [ + "cpuid-bool 0.2.0", + "opaque-debug", + "universal-hash", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1178,6 +1777,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "proc-macro-crate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fdbd1df62156fbc5945f4762632564d7d038153091c3fcf1067f6aef7cff92" +dependencies = [ + "thiserror", + "toml", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -1239,7 +1848,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -1247,6 +1856,18 @@ dependencies = [ "rand_pcg 0.2.1", ] +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + [[package]] name = "rand_chacha" version = "0.1.1" @@ -1267,6 +1888,16 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -1288,7 +1919,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", ] [[package]] @@ -1309,6 +1949,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + [[package]] name = "rand_isaac" version = "0.1.1" @@ -1406,6 +2055,49 @@ version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -1421,6 +2113,176 @@ dependencies = [ "winapi", ] +[[package]] +name = "ruma" +version = "0.0.3" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "assign", + "js_int", + "ruma-api", + "ruma-client-api", + "ruma-common", + "ruma-events", + "ruma-federation-api", + "ruma-identifiers", + "ruma-serde", +] + +[[package]] +name = "ruma-api" +version = "0.17.0-alpha.4" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "http", + "percent-encoding", + "ruma-api-macros", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "ruma-api-macros" +version = "0.17.0-alpha.4" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ruma-client-api" +version = "0.10.0-alpha.3" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "assign", + "http", + "js_int", + "maplit", + "percent-encoding", + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-common" +version = "0.5.0" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "indexmap", + "js_int", + "maplit", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", + "tracing", + "wildmatch", +] + +[[package]] +name = "ruma-events" +version = "0.22.0-alpha.3" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "js_int", + "ruma-common", + "ruma-events-macros", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-events-macros" +version = "0.22.0-alpha.3" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ruma-federation-api" +version = "0.1.0-alpha.2" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "js_int", + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "ruma-serde", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-identifiers" +version = "0.19.0" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "paste", + "ruma-identifiers-macros", + "ruma-identifiers-validation", + "ruma-serde", + "ruma-serde-macros", + "serde", +] + +[[package]] +name = "ruma-identifiers-macros" +version = "0.19.0" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "proc-macro2", + "quote", + "ruma-identifiers-validation", + "syn", +] + +[[package]] +name = "ruma-identifiers-validation" +version = "0.3.0" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" + +[[package]] +name = "ruma-serde" +version = "0.3.1" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "form_urlencoded", + "itoa", + "js_int", + "ruma-serde-macros", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-serde-macros" +version = "0.3.1" +source = "git+https://github.com/ruma/ruma?rev=c816630058ab625d93ebab294e9e6c02dd9d866c#c816630058ab625d93ebab294e9e6c02dd9d866c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "rustc_version" version = "0.2.3" @@ -1570,11 +2432,20 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "serde" version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" @@ -1598,6 +2469,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "servo_arc" version = "0.1.1" @@ -1614,6 +2497,19 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +[[package]] +name = "sha2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpuid-bool 0.1.2", + "digest", + "opaque-debug", +] + [[package]] name = "signal-hook-registry" version = "1.3.0" @@ -1641,6 +2537,22 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "sled" +version = "0.34.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot", +] + [[package]] name = "smallvec" version = "0.6.14" @@ -1767,6 +2679,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" + [[package]] name = "syn" version = "1.0.68" @@ -1778,6 +2696,32 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "tendril" version = "0.4.2" @@ -1908,6 +2852,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.22.0" @@ -1933,6 +2887,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + [[package]] name = "tower-service" version = "0.3.1" @@ -1947,9 +2910,21 @@ checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.17" @@ -1959,12 +2934,28 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "try-lock" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "typenum" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" + [[package]] name = "unicode-bidi" version = "0.3.5" @@ -1995,6 +2986,16 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" +[[package]] +name = "universal-hash" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -2025,6 +3026,22 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.2", + "serde", +] + +[[package]] +name = "vcpkg" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" + [[package]] name = "version_check" version = "0.9.3" @@ -2047,6 +3064,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasm-bindgen" version = "0.2.73" @@ -2054,6 +3077,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" dependencies = [ "cfg-if 1.0.0", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -2072,6 +3097,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.73" @@ -2121,6 +3158,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "wildmatch" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6c48bd20df7e4ced539c12f570f937c6b4884928a87fee70a479d72f031d4e0" + [[package]] name = "winapi" version = "0.3.9" @@ -2151,3 +3194,33 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c1e130bebaeab2f23886bf9acbaca14b092408c452543c857f66399cd6dab1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] diff --git a/Cargo.toml b/Cargo.toml index be05e56..458df77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crieur-retrieve", + "crieur-chatbot", ] @@ -17,6 +18,7 @@ publish = false [dependencies] anyhow = "1.0.40" crieur-retrieve = {version = "0.1", path="crieur-retrieve"} +crieur-chatbot = {version = "0.1", path="crieur-chatbot"} dotenv = "0.15.0" env_logger = "0.8.3" log = "0.4.14" diff --git a/README.md b/README.md index e4bfb50..fc0677f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Tools to retrieve articles from multiple newspaper you subscribed to. First retrieve login cookies for websites and put it in a `.env` ``` -cargo run --example=retrive_html_articles +cargo run --example=cli_downloader ``` # Documentation diff --git a/crieur-chatbot/Cargo.toml b/crieur-chatbot/Cargo.toml new file mode 100644 index 0000000..e2f3631 --- /dev/null +++ b/crieur-chatbot/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "crieur-chatbot" +version = "0.1.0" +authors = ["koalp "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.40" +dotenv = "0.15.0" +crieur-retrieve = {version = "0.1.0", path = "../crieur-retrieve"} +mime = "0.3.16" +log = "0.4.14" + +[dependencies.matrix-sdk] +git = "https://github.com/matrix-org/matrix-rust-sdk" +rev = "ab180362c931606385dd53b73620d82ef2c3166d" +version = "0.2.0" diff --git a/crieur-chatbot/src/chatbot.rs b/crieur-chatbot/src/chatbot.rs new file mode 100644 index 0000000..cad6b1c --- /dev/null +++ b/crieur-chatbot/src/chatbot.rs @@ -0,0 +1,85 @@ +//! Chatbot +use std::convert::TryInto; + +use anyhow::Result; +use matrix_sdk::{ + self, async_trait, + events::{ + room::message::{MessageEventContent, MessageType, TextMessageEventContent}, + AnyMessageEventContent, SyncMessageEvent, + }, + room::Room, + Client, ClientConfig, EventHandler, SyncSettings, +}; + +use crate::Html; + +#[derive(Debug, Clone, Default)] +pub(crate) struct Builder { + user: String, + password: String, + homeserver: String, + //TODO: rooms + room: String, +} + +impl Builder { + fn new() -> Self { + Default::default() + } + + pub(crate) async fn connect(&self) -> Result { + let client = Client::new(self.homeserver.as_str())?; + client + .login(self.user.as_str(), self.password.as_str(), None, None) + .await?; + assert!(client.logged_in().await); + client + .join_room_by_id(&self.room.as_str().try_into()?) + .await?; + + Ok(Chatbot { client }) + } + + pub(crate) fn login( + &mut self, + user: &impl AsRef, + password: &impl AsRef, + ) -> &mut Self { + self.user = String::from(user.as_ref()); + self.password = String::from(password.as_ref()); + self + } + + pub(crate) fn homeserver(&mut self, homeserver: &impl AsRef) -> &mut Self { + self.homeserver = String::from(homeserver.as_ref()); + self + } + + pub(crate) fn room(&mut self, room: &impl AsRef) -> &mut Self { + self.room = String::from(room.as_ref()); + self + } +} + +#[derive(Debug, Clone)] +pub(crate) struct Chatbot { + client: Client, +} + +impl Chatbot { + pub(crate) fn builder() -> Builder { + Builder::new() + } + + pub(crate) async fn run(&self) -> Result<()> { + self.client.set_event_handler(Box::new(Html::new())).await; + + let mut settings = SyncSettings::default(); + if let Some(token) = self.client.sync_token().await { + settings = settings.token(token); + } + self.client.sync(settings).await; + Ok(()) + } +} diff --git a/crieur-chatbot/src/cli.rs b/crieur-chatbot/src/cli.rs new file mode 100644 index 0000000..f8ecf57 --- /dev/null +++ b/crieur-chatbot/src/cli.rs @@ -0,0 +1,32 @@ +use std::env; + +use anyhow::{bail, Result}; +use dotenv::dotenv; + +use crate::Chatbot; + +/// Runs the chatbot +pub async fn run() -> Result<()> { + dotenv().ok(); + + let (user, password, homeserver, room) = match ( + env::var("CRIEUR_MATRIX_USER"), + env::var("CRIEUR_MATRIX_PASSWORD"), + env::var("CRIEUR_MATRIX_HOMESERVER"), + env::var("CRIEUR_MATRIX_ROOM"), + ) { + (Ok(user), Ok(password), Ok(homeserver), Ok(room)) => (user, password, homeserver, room), + _ => bail!("Configuration incomplete, please set all required environment variables"), + }; + + let chatbot = Chatbot::builder() + .login(&user, &password) + .homeserver(&homeserver) + .room(&room) + .connect() + .await?; + + chatbot.run().await?; + + Ok(()) +} diff --git a/crieur-chatbot/src/handlers/html.rs b/crieur-chatbot/src/handlers/html.rs new file mode 100644 index 0000000..4c7ef7f --- /dev/null +++ b/crieur-chatbot/src/handlers/html.rs @@ -0,0 +1,94 @@ +use std::convert::TryInto; +use std::env; + +use anyhow::Result; +use log::info; +use matrix_sdk::{ + self, async_trait, + events::{ + room::message::{MessageEventContent, MessageType, TextMessageEventContent}, + AnyMessageEventContent, SyncMessageEvent, + }, + room::Room, + Client, ClientConfig, EventHandler, SyncSettings, +}; + +use crieur_retrieve::{newspaper::Newspaper, ArticleLocation, Mediapart, Url}; + +pub(crate) struct Html {} + +impl Html { + pub fn new() -> Self { + Self {} + } +} + +async fn send_article(url: U, room: matrix_sdk::room::Joined) +where + U: TryInto + Send, + E: std::error::Error + Sync + Send + 'static, +{ + //TODO: replace by async block when async block is stable + async fn article_html(url: U) -> Result + where + U: TryInto + Send, + E: std::error::Error + Sync + Send + 'static, + { + let article_str = ArticleLocation::builder() + .url(url)? + .build()? + .retrieve_html() + .await?; + Ok(article_str) + } + + let text_message = + |message| AnyMessageEventContent::RoomMessage(MessageEventContent::text_plain(message)); + + //TODO: replace occurences ok() by async and logging block when async block is stable + let article_html = match article_html(url).await { + Ok(url) => url, + Err(_) => { + room.send(text_message("Can't download the file"), None) + .await + .ok(); + return; + } + }; + + room.send_attachment( + "test.html", + &mime::TEXT_HTML_UTF_8, + &mut article_html.as_bytes(), + None, + ) + .await + .ok(); +} + +#[async_trait] +impl EventHandler for Html { + async fn on_room_message(&self, room: Room, event: &SyncMessageEvent) { + if let Room::Joined(room) = room { + let msg_body = if let SyncMessageEvent { + content: + MessageEventContent { + msgtype: MessageType::Text(TextMessageEventContent { body: msg_body, .. }), + .. + }, + .. + } = event + { + msg_body + } else { + return; + }; + info!("sending file"); + + match msg_body.split(' ').collect::>().as_slice() { + ["!html", url, ..] => send_article(*url, room).await, + _ => return, + } + } + } +} diff --git a/crieur-chatbot/src/handlers/mod.rs b/crieur-chatbot/src/handlers/mod.rs new file mode 100644 index 0000000..a75fb0f --- /dev/null +++ b/crieur-chatbot/src/handlers/mod.rs @@ -0,0 +1,2 @@ +mod html; +pub(crate) use html::Html; diff --git a/crieur-chatbot/src/lib.rs b/crieur-chatbot/src/lib.rs new file mode 100644 index 0000000..8646768 --- /dev/null +++ b/crieur-chatbot/src/lib.rs @@ -0,0 +1,10 @@ +//! Provides a matrix chatbot to download newspaper articles + +mod cli; +pub use cli::run; + +mod chatbot; +use chatbot::Chatbot; + +mod handlers; +use handlers::Html; diff --git a/crieur-retrieve/src/article_location.rs b/crieur-retrieve/src/article_location.rs index 657b2c3..734c947 100644 --- a/crieur-retrieve/src/article_location.rs +++ b/crieur-retrieve/src/article_location.rs @@ -1,21 +1,34 @@ use std::boxed::Box; use std::convert::TryInto; +use std::env; use anyhow::{anyhow, Result}; use log::info; use url::{Host, Url}; use crate::newspaper::Newspaper; +use crate::newspapers::Mediapart; -type Newspapers<'a> = Vec>; +type Newspapers = Vec>; -#[derive(Default)] -pub struct ArticleLocationBuilder<'a> { - url: Option, - newspapers: Option>, +fn default_newpapers() -> Newspapers { + let mut mediapart = Mediapart::new(); + + mediapart.login_cookie = Some(( + "MPRUUID".into(), + env::var("MEDIAPART_COOKIE").unwrap().into(), + )); + + vec![Box::new(mediapart)] } -impl<'a> ArticleLocationBuilder<'a> { +#[derive(Default)] +pub struct Builder { + url: Option, + newspapers: Option, +} + +impl Builder { pub fn new() -> Self { Self::default() } @@ -37,9 +50,9 @@ impl<'a> ArticleLocationBuilder<'a> { } /// Adds a newspaper to the list - pub fn newspaper(&mut self, newspaper: &'a T) -> &mut Self + pub fn newspaper(mut self, newspaper: T) -> Self where - T: 'a + Newspaper, + T: 'static + Newspaper, { match &mut self.newspapers { Some(newspapers) => newspapers.push(Box::new(newspaper)), @@ -72,7 +85,7 @@ impl<'a> ArticleLocationBuilder<'a> { /// - the url is not set /// - the given url has no host // TODO: move this to a defined error, remove anyhow ! - pub fn build(&self) -> Result> { + pub fn build(self) -> Result { let url = Clone::clone(self.url.as_ref().ok_or(anyhow!( "No url set. You can set it with the url() function" ))?); @@ -80,28 +93,22 @@ impl<'a> ArticleLocationBuilder<'a> { let host = Host::parse(host)?; let newspaper = self .newspapers - .as_ref() - .ok_or(anyhow!( - "A list of NewsPaper must be set. It can be set with newspapers() function" - ))? - .iter() + .unwrap_or(default_newpapers()) + .into_iter() .find(|c| c.metadata().hosts.contains(&host)) .ok_or(anyhow!("Newspaper couldn't be found"))?; - Ok(ArticleLocation { - newspaper: newspaper.clone(), - url, - }) + Ok(ArticleLocation { newspaper, url }) } } -pub struct ArticleLocation<'a> { - newspaper: Box<&'a dyn Newspaper>, +pub struct ArticleLocation { + newspaper: Box, pub url: Url, } -impl<'a> ArticleLocation<'a> { - pub fn builder() -> ArticleLocationBuilder<'a> { - ArticleLocationBuilder::new() +impl ArticleLocation { + pub fn builder() -> Builder { + Builder::new() } pub async fn retrieve_html(&self) -> Result { diff --git a/crieur-retrieve/src/newspaper.rs b/crieur-retrieve/src/newspaper.rs index cfadba0..532cc62 100644 --- a/crieur-retrieve/src/newspaper.rs +++ b/crieur-retrieve/src/newspaper.rs @@ -34,7 +34,7 @@ impl Metadata { } #[async_trait] -pub trait Newspaper { +pub trait Newspaper: Send + Sync { /// Returns a list of hosts that corresponds to the newspapers fn metadata(&self) -> Metadata; @@ -49,7 +49,7 @@ pub trait Newspaper { } /// Returns a newspaper structure - async fn new() -> Self + fn new() -> Self where Self: Sized; diff --git a/crieur-retrieve/src/newspapers/mediapart.rs b/crieur-retrieve/src/newspapers/mediapart.rs index 8070e7f..678a8ac 100644 --- a/crieur-retrieve/src/newspapers/mediapart.rs +++ b/crieur-retrieve/src/newspapers/mediapart.rs @@ -80,7 +80,7 @@ impl Newspaper for Mediapart { Ok(single_page_html) } - async fn new() -> Self { + fn new() -> Self { Self { ..Default::default() } diff --git a/examples/cli_downloader.rs b/examples/cli_downloader.rs index fffe05d..24341ba 100644 --- a/examples/cli_downloader.rs +++ b/examples/cli_downloader.rs @@ -17,7 +17,7 @@ async fn main() -> Result<()> { }; // TODO: remove this in favor of default newspapers - let mut mediapart = Mediapart::new().await + let mut mediapart = Mediapart::new() //.login(USERNAME, PASSWORD) // ; @@ -28,7 +28,7 @@ async fn main() -> Result<()> { // TODO: shorten this, maybe an helper function ? let article_location = ArticleLocation::builder() .url(url)? - .newspaper(&mediapart) + .newspaper(mediapart) .build()?; let article_str = article_location.retrieve_html().await?; diff --git a/src/main.rs b/src/main.rs index 03095a9..3b7e283 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use crieur_chatbot::run; use crieur_retrieve::{newspaper::Newspaper, Mediapart, Url}; use dotenv::dotenv; use std::env; @@ -7,13 +8,12 @@ use std::env; async fn main() -> Result<()> { dotenv().ok(); - let mut mediapart = Mediapart::new().await + let mut mediapart = Mediapart::new() //.login(USERNAME, PASSWORD) // ; mediapart.login_cookie = Some(("MPRUUID".into(), env::var("MEDIAPART_COOKIE")?)); - let url = Url::parse("https://www.mediapart.fr/journal/france/030421/l-hotel-dieu-patients-et-medecins-tentent-de-percer-les-mysteres-du-covid-long")?; - println!("{}", mediapart.retrieve_html(&url).await?); + run().await?; Ok(()) } From 865b949b5f2c10c4ceccd455fd0a1d922a5788b8 Mon Sep 17 00:00:00 2001 From: koalp Date: Thu, 29 Apr 2021 02:06:14 +0200 Subject: [PATCH 2/3] feat: add builder for mediapart, document chatbot A builder for mediapart have been added. No generic builder have been created as there is no usecase yet. Some documentation have been added, roadmap and scope have been clarified and chatbot have been lightly documented. --- .drone.yml | 1 + README.md | 26 +++- crieur-chatbot/src/handlers/html.rs | 2 +- crieur-retrieve/src/article_location.rs | 18 ++- crieur-retrieve/src/lib.rs | 3 +- crieur-retrieve/src/newspaper.rs | 10 +- crieur-retrieve/src/newspapers/mediapart.rs | 54 ++++++-- crieur-retrieve/src/newspapers/mod.rs | 4 +- documentation/design/scope.md | 119 ++++++++++++------ documentation/guides/run_chatbot.md | 19 +++ .../reference/chatbot_configuration.md | 17 +++ .../reference/newspaper_configuration.md | 10 ++ examples/cli_downloader.rs | 16 ++- src/bin/crieur-chatbot.rs | 10 ++ src/main.rs | 19 --- 15 files changed, 228 insertions(+), 100 deletions(-) create mode 100644 documentation/guides/run_chatbot.md create mode 100644 documentation/reference/chatbot_configuration.md create mode 100644 documentation/reference/newspaper_configuration.md create mode 100644 src/bin/crieur-chatbot.rs delete mode 100644 src/main.rs diff --git a/.drone.yml b/.drone.yml index 0a95e84..e304ae4 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,6 +8,7 @@ steps: pull: true errignore: true commands: + - apt-get update && apt-get install -y cmake - rustup component add rustfmt - rustup component add clippy - cargo clippy diff --git a/README.md b/README.md index fc0677f..f5970e5 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,33 @@ Tools to retrieve articles from multiple newspaper you subscribed to. -**This is a prototype, it isn't stable at all and you may not want to use it if you expect it to just work !** +**This is a prototype, it isn't stable at all and you may not want to use it if +you expect it to just work !** # How to use it -First retrieve login cookies for websites and put it in a `.env` +First retrieve login cookies for websites and put it in a `.env` such as +explained in the [newspaper source configuration +documentation](./documentation/reference/newspaper_configuration.md) + +Then you can run run ``` cargo run --example=cli_downloader ``` +To know how to run the chatbot, please read the [chatbot +guide](./documentation/guides/run_chatbot.md) + # Documentation -- 1. [Design](documentation/design/index.md) - - a. [Scope of the project](documentation/design/scope.md) - - b. [Retrieve](documentation/design/retrieve.md) +- 1. Design + - a. [Scope of the project and roadmap](./documentation/design/scope.md) + - b. [Retrieve](./documentation/design/retrieve.md) +- 2. Guides + - a. [Add a newspaper a + source](./documentation/guides/add_a_newspaper_source.md) +- 3. Reference + - a. [Newspaper source + configuration](./documentation/reference/newspaper_configuration.md) + - b. [Chatbot + configuration](./documentation/reference/chatbot_configuration.md) diff --git a/crieur-chatbot/src/handlers/html.rs b/crieur-chatbot/src/handlers/html.rs index 4c7ef7f..5cad197 100644 --- a/crieur-chatbot/src/handlers/html.rs +++ b/crieur-chatbot/src/handlers/html.rs @@ -13,7 +13,7 @@ use matrix_sdk::{ Client, ClientConfig, EventHandler, SyncSettings, }; -use crieur_retrieve::{newspaper::Newspaper, ArticleLocation, Mediapart, Url}; +use crieur_retrieve::{ArticleLocation, Url}; pub(crate) struct Html {} diff --git a/crieur-retrieve/src/article_location.rs b/crieur-retrieve/src/article_location.rs index 734c947..1cd6178 100644 --- a/crieur-retrieve/src/article_location.rs +++ b/crieur-retrieve/src/article_location.rs @@ -7,19 +7,17 @@ use log::info; use url::{Host, Url}; use crate::newspaper::Newspaper; -use crate::newspapers::Mediapart; +use crate::newspapers::mediapart::{self, Mediapart}; type Newspapers = Vec>; -fn default_newpapers() -> Newspapers { - let mut mediapart = Mediapart::new(); +fn default_newpapers() -> Result { + let mpruiid = env::var("MEDIAPART_COOKIE")?.into(); + let mediapart = Mediapart::builder() + .login(mediapart::Login::MPRUUID(mpruiid)) + .build()?; - mediapart.login_cookie = Some(( - "MPRUUID".into(), - env::var("MEDIAPART_COOKIE").unwrap().into(), - )); - - vec![Box::new(mediapart)] + Ok(vec![Box::new(mediapart)]) } #[derive(Default)] @@ -93,7 +91,7 @@ impl Builder { let host = Host::parse(host)?; let newspaper = self .newspapers - .unwrap_or(default_newpapers()) + .unwrap_or(default_newpapers()?) .into_iter() .find(|c| c.metadata().hosts.contains(&host)) .ok_or(anyhow!("Newspaper couldn't be found"))?; diff --git a/crieur-retrieve/src/lib.rs b/crieur-retrieve/src/lib.rs index 7b9d058..31dc3a8 100644 --- a/crieur-retrieve/src/lib.rs +++ b/crieur-retrieve/src/lib.rs @@ -8,8 +8,7 @@ pub use tools::{Download, Downloader}; pub mod newspaper; // TODO: move to another crate -mod newspapers; -pub use newspapers::Mediapart; +pub mod newspapers; mod article_location; pub use article_location::ArticleLocation; diff --git a/crieur-retrieve/src/newspaper.rs b/crieur-retrieve/src/newspaper.rs index 532cc62..d78c81c 100644 --- a/crieur-retrieve/src/newspaper.rs +++ b/crieur-retrieve/src/newspaper.rs @@ -1,17 +1,10 @@ use anyhow::Result; use async_trait::async_trait; -use derive_builder::Builder; use url::Host; pub use url::Url; -enum Login { - Username(String, String), - Cookie(String), -} - /// Contains metadata about a newspaper -// TODO: provide builder -#[derive(Debug, PartialEq, Default, Builder)] +#[derive(Debug, PartialEq, Default, derive_builder::Builder)] #[builder(default)] pub struct Metadata { /// The hosts that can be corresponds to this newspaper @@ -28,6 +21,7 @@ pub struct Metadata { } impl Metadata { + /// Get metadata builder pub fn builder() -> MetadataBuilder { MetadataBuilder::default() } diff --git a/crieur-retrieve/src/newspapers/mediapart.rs b/crieur-retrieve/src/newspapers/mediapart.rs index 678a8ac..0933b6e 100644 --- a/crieur-retrieve/src/newspapers/mediapart.rs +++ b/crieur-retrieve/src/newspapers/mediapart.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use async_trait::async_trait; use cookie::Cookie; use url::Host; @@ -8,16 +8,46 @@ use crate::tools; use crate::Url; use crate::{Download, Downloader}; +pub enum Login { + Username(String, String), + MPRUUID(String), +} + #[derive(Debug, Clone, Default)] pub struct Mediapart { - // TODO: remove this pub !! - pub login_cookie: Option<(String, String)>, + login_cookie: (String, String), } fn str_to_host>(host: S) -> Host { Host::Domain(host.into()) } +#[derive(Debug, Clone, Default)] +pub struct Builder { + login_cookie: Option<(String, String)>, +} + +impl Builder { + pub fn login(&mut self, login: Login) -> &mut Self { + self.login_cookie = match login { + Login::Username(_username, _password) => { + unimplemented!("login using username and passwond not implemented") + } + Login::MPRUUID(cookie_value) => Some(("MPRUUID".into(), cookie_value)), + }; + self + } + + pub fn build(&self) -> Result { + match &self.login_cookie { + Some(login_cookie) => Ok(Mediapart { + login_cookie: login_cookie.clone(), + }), + None => Err(anyhow!("You have to log in to access this newspaper")), + } + } +} + #[async_trait] impl Newspaper for Mediapart { fn metadata(&self) -> Metadata { @@ -41,13 +71,10 @@ impl Newspaper for Mediapart { let mut url = url.clone(); url.set_query(Some(&query)); - // TODO: add "?onglet=full" to the url if not - let cookies = if let Some((name, value)) = &self.login_cookie { - let cookie = Cookie::build(name, value).secure(true).finish(); - vec![cookie] - } else { - vec![] - }; + let cookie = Cookie::build(&self.login_cookie.0, &self.login_cookie.1) + .secure(true) + .finish(); + let cookies = vec![cookie]; // TODO: replace by builder let downloader = Downloader { cookies }; @@ -74,7 +101,6 @@ impl Newspaper for Mediapart { "aside.cc-modal", ]; - // TODO: correction of usage of relative urls, and replace "" by the url let single_page_html = tools::self_contained_html(&html, &downloader, &url, &element_to_remove).await; Ok(single_page_html) @@ -91,3 +117,9 @@ impl Newspaper for Mediapart { true } } + +impl Mediapart { + pub fn builder() -> Builder { + Builder::default() + } +} diff --git a/crieur-retrieve/src/newspapers/mod.rs b/crieur-retrieve/src/newspapers/mod.rs index b7c9e05..7f44529 100644 --- a/crieur-retrieve/src/newspapers/mod.rs +++ b/crieur-retrieve/src/newspapers/mod.rs @@ -1,3 +1 @@ -mod mediapart; - -pub use mediapart::Mediapart; +pub mod mediapart; diff --git a/documentation/design/scope.md b/documentation/design/scope.md index bc8ca23..6417bf7 100644 --- a/documentation/design/scope.md +++ b/documentation/design/scope.md @@ -1,48 +1,97 @@ -This project mainly aims at providing an unified interface for several newspapers. Side -objectives are to provide web API and different clients like a webUI or chatbots. +--- +title: Scope of the project +--- -Several big components are planned for this project +This project mainly aims at providing an unified interface for several +newspapers. Side objectives are to provide web API and different clients like a +webUI or chatbots. -```dot -digraph G { - rankdir=TB - node [shape=rectangle, style=filled, color="#779988"] +Several big components are planned for this project (it is an initial draft and +may change later) : - subgraph cluster_frontend { - color = transparent - webui - chatbot - } +```plantuml +@startuml + +frame "backend" { + [Retrieval tools] as retrieval_tools + [Article representation] as article_repr + [Automatic retrieval] as auto_retrieve + [Atom/RSS adapters] as rss + [Cache DB] as cache + + [Newspaper\n(Mediapart, …)] as newspaper + () "Newspaper" as np_i + newspaper -up- np_i - webui -> api [color = red] - chatbot -> api [color = red] + [Article location] as article_location - subgraph cluster_backend { - label = "Backend\ncrieur binary" - labelloc = b - style=filled + [API] as api + () "API" as api_i + api -up- api_i - retrieve_tools [label="retrieve-tools"] - retrieve_adapters [label="retrieve-adapters"] - retrieve [label="retrieve-interface"] - auto_retrieve [label="automatic-retrieve"] - article_repr [label="article-representation\nRepresentation for articles"] - api - cache [label="Cache database"] - rss [label="Atom/RSS adapters"] + article_location ..> np_i - retrieve_tools -> retrieve_adapters - retrieve_adapters -> retrieve - retrieve_tools -> retrieve - rss -> auto_retrieve - article_repr -> retrieve_adapters + api -> article_location + api -> rss - retrieve -> api - auto_retrieve -> api - cache -> api + newspaper -> retrieval_tools: uses to implement - } + article_location --> article_repr :uses + + auto_retrieve --> rss: watches + auto_retrieve --> article_location + auto_retrieve --> cache: stores in } + +frame "Web ui" { + [Web UI] as webui + [HTML renderer] as html_rend + [Pdf exporter] as pdf_rend + [Articles] as articles + webui --> html_rend + webui --> pdf_rend + webui -> articles + articles ..> api_i +} + +[Chatbot] as chatbot + +chatbot ..> api_i + +actor User +User ..> webui +User ..> chatbot + +actor "Newspaper programmer" as newspaper_programmer +newspaper_programmer ..> newspaper: implements +@enduml ``` + +A task queue could be added later to space requests. + +# Implementation plan + +## Phase I +- [x] `Newspaper` interface : use to retrieve from newspaper websites +- [ ] minimal chatbot (uses libraries directly) +- [x] `ArticleLocation` : library for using several `Newspaper` and retrieving from +a given url. + +## Phase II +- [ ] Article Representation : having a (beta) unified representation for downloaded +articles + - [ ] adding this representation to Newpsaper + +## Phase III +- [ ] Cache +- [ ] Atom/rss adapters +- [ ] automatic retrieve + +## Phase IV +- [ ] API +- [ ] chatbot (uses api) + +## Phase V +- [ ] web ui diff --git a/documentation/guides/run_chatbot.md b/documentation/guides/run_chatbot.md new file mode 100644 index 0000000..b6f9322 --- /dev/null +++ b/documentation/guides/run_chatbot.md @@ -0,0 +1,19 @@ +--- +title: run the chatbot +--- + +1. You must first configure matrix login, every variable in [the_reference](../reference/chatbot_configuration.md) is mandatory. + +```env +CRIEUR_MATRIX_USER=user +CRIEUR_MATRIX_PASSWORD=password +CRIEUR_MATRIX_HOMESERVER=https://homeserv.er +CRIEUR_MATRIX_ROOM=roomid +``` + +You can put it in a `.env` file. + +2. run the chatbot +``` +cargo run --release --bin crieur-chatbot +``` diff --git a/documentation/reference/chatbot_configuration.md b/documentation/reference/chatbot_configuration.md new file mode 100644 index 0000000..3467dd9 --- /dev/null +++ b/documentation/reference/chatbot_configuration.md @@ -0,0 +1,17 @@ +--- +Title: Chatbot configuration reference +--- + +The chatbot is configured using environment variables + +CRIEUR_MATRIX_USER +: username of the matrix bot account + +CRIEUR_MATRIX_PASSWORD +: password of the matrix bot account + +CRIEUR_MATRIX_HOMESERVER +: homeserver of the matrix bot account + +CRIEUR_MATRIX_ROOM +: the room in which to listen to events diff --git a/documentation/reference/newspaper_configuration.md b/documentation/reference/newspaper_configuration.md new file mode 100644 index 0000000..8658087 --- /dev/null +++ b/documentation/reference/newspaper_configuration.md @@ -0,0 +1,10 @@ +--- +title: Newspapers configuration +--- + +The newspapers are configured using environment variables + +# Mediapart + +MEDIAPART_COOKIE +: sets the `MPRUUID` cookie, used to log in diff --git a/examples/cli_downloader.rs b/examples/cli_downloader.rs index 24341ba..4a4eefe 100644 --- a/examples/cli_downloader.rs +++ b/examples/cli_downloader.rs @@ -2,7 +2,11 @@ use std::convert::TryInto; use std::env; use anyhow::Result; -use crieur_retrieve::{newspaper::Newspaper, ArticleLocation, Mediapart, Url}; +use crieur_retrieve::{ + newspaper::Newspaper, + newspapers::mediapart::{self, Mediapart}, + ArticleLocation, Url, +}; use dotenv::dotenv; use log::info; @@ -17,12 +21,12 @@ async fn main() -> Result<()> { }; // TODO: remove this in favor of default newspapers - let mut mediapart = Mediapart::new() - //.login(USERNAME, PASSWORD) - // - ; - mediapart.login_cookie = Some(("MPRUUID".into(), env::var("MEDIAPART_COOKIE")?.into())); + let mpruiid = env::var("MEDIAPART_COOKIE")?.into(); + let mediapart = Mediapart::builder() + .login(mediapart::Login::MPRUUID(mpruiid)) + .build()?; + info!("Trying to download article from {}", url); // TODO: shorten this, maybe an helper function ? diff --git a/src/bin/crieur-chatbot.rs b/src/bin/crieur-chatbot.rs new file mode 100644 index 0000000..d0179be --- /dev/null +++ b/src/bin/crieur-chatbot.rs @@ -0,0 +1,10 @@ +use anyhow::Result; +use crieur_chatbot::run; +use dotenv::dotenv; + +#[tokio::main] +async fn main() -> Result<()> { + dotenv().ok(); + run().await?; + Ok(()) +} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 3b7e283..0000000 --- a/src/main.rs +++ /dev/null @@ -1,19 +0,0 @@ -use anyhow::Result; -use crieur_chatbot::run; -use crieur_retrieve::{newspaper::Newspaper, Mediapart, Url}; -use dotenv::dotenv; -use std::env; - -#[tokio::main] -async fn main() -> Result<()> { - dotenv().ok(); - - let mut mediapart = Mediapart::new() - //.login(USERNAME, PASSWORD) - // - ; - - mediapart.login_cookie = Some(("MPRUUID".into(), env::var("MEDIAPART_COOKIE")?)); - run().await?; - Ok(()) -} From 9dd501418ef7e61da7b0cce23154c898daafada7 Mon Sep 17 00:00:00 2001 From: koalp Date: Thu, 29 Apr 2021 02:14:30 +0200 Subject: [PATCH 3/3] chore(ci): add cmake to the test stage --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index e304ae4..d8799bb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -18,5 +18,6 @@ steps: pull: true errignore: true commands: + - apt-get update && apt-get install -y cmake - cargo test --all - cargo build