From 9de7c783ae6d3a92ea742d9aa20da7d8c01a3a8f Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Tue, 30 Mar 2021 16:11:36 +0000 Subject: [PATCH 1/9] job: add processing blacklist to global config --- Cargo.lock | 38 ++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/job.rs | 7 +++++++ src/settings.rs | 4 ++++ 4 files changed, 51 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 72a6705..c0de7d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,6 +56,15 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + [[package]] name = "anyhow" version = "1.0.40" @@ -503,10 +512,12 @@ dependencies = [ "hex", "hmac", "log 0.4.14", + "regex", "rocket", "rocket_contrib", "serde", "serde_json", + "serde_regex", "serde_yaml", "sha2", ] @@ -756,6 +767,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" + [[package]] name = "rocket" version = "0.4.7" @@ -874,6 +902,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + [[package]] name = "serde_yaml" version = "0.8.17" diff --git a/Cargo.toml b/Cargo.toml index 896e01a..cbbb9ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,9 +15,11 @@ anyhow = "1.0.40" hex = "0.4.3" hmac = "0.10.1" log = "0.4.14" +regex = "1" rocket = "0.4.7" rocket_contrib = { version = "0.4.7", features = [ "json" ] } serde = { version = "1.0.125", features = [ "derive" ] } serde_json = "1.0.64" +serde_regex = "1.1.0" serde_yaml = "0.8.17" sha2 = "0.9.3" diff --git a/src/job.rs b/src/job.rs index 2e91cf1..0477704 100644 --- a/src/job.rs +++ b/src/job.rs @@ -190,6 +190,13 @@ impl Job { } pub(crate) fn run(&mut self, homedir: &Path, config: &GlobalSettings) -> anyhow::Result<()> { + if config + .blacklist + .iter() + .any(|re| re.is_match(&self.repo.full_name)) + { + return Ok(()); + } let local_path = homedir.join(&self.repo.full_name); assert!(local_path.is_absolute()); self.local_path = Some(local_path); diff --git a/src/settings.rs b/src/settings.rs index bfe7744..976264f 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -10,4 +10,8 @@ pub(crate) struct GlobalSettings { /// List of remote stems to use for every repository #[serde(default)] pub additional_remotes: Vec, + /// List of regexes, if a repository's name matches any of the, it is not mirrored by `lohr` + /// even if it contains a `.lorh` file. + #[serde(with = "serde_regex")] + pub blacklist: Vec, } From d38e4556e14073a786ff282fcabd870f0fd24286 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Tue, 30 Mar 2021 18:09:11 +0000 Subject: [PATCH 2/9] nix: flake: add meta information in package --- flake.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flake.nix b/flake.nix index 330d2ff..2b631d1 100644 --- a/flake.nix +++ b/flake.nix @@ -37,6 +37,13 @@ defaultPackage = naersk-lib.buildPackage { src = ./.; pname = "lohr"; + + meta = with pkgs.lib; { + description = "A Git mirroring tool"; + homepage = "https://github.com/alarsyo/lohr"; + license = with licenses; [ mit asl20 ]; + platforms = platforms.unix; + }; }; defaultApp = flake-utils.lib.mkApp { From 6f63b4c95c26b85b040822cab5678772db42edc4 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Tue, 30 Mar 2021 18:46:14 +0000 Subject: [PATCH 3/9] main: add LOHR_CONFIG variable --- src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.rs b/src/main.rs index e71ec3d..ce09507 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,6 +52,9 @@ fn repo_updater(rx: Receiver, homedir: PathBuf, config: GlobalSettings) { fn parse_config(mut path: PathBuf) -> anyhow::Result { path.push("lohr-config"); path.set_extension("yaml"); + let path = env::var("LOHR_CONFIG") + .map(Into::into) + .unwrap_or_else(|_| path); let config = if let Ok(file) = File::open(path.as_path()) { serde_yaml::from_reader(file)? } else { From 8e7e0e9a84d12653f704ce7e203af27400c40634 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Tue, 30 Mar 2021 18:46:33 +0000 Subject: [PATCH 4/9] README: mention 'LOHR_CONFIG' --- README.org | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index 8231f47..ce868c4 100644 --- a/README.org +++ b/README.org @@ -73,8 +73,9 @@ variable. **** Extra remote configuration -=lohr= looks for a =lohr-config.yaml= file in its =LOHR_HOME= directory. This -file takes the following format: +=lohr= looks for a =lohr-config.yaml= file in its =LOHR_HOME= directory. The +=LOHR_CONFIG= variable takes precedence over looking into the state directory. +This file takes the following format: #+begin_src yaml default_remotes: @@ -83,12 +84,18 @@ default_remotes: additional_remotes: - "git@git.sr.ht:~user" + +blacklist: + - "private-.*" #+end_src - ~default_remotes~ is a list of remotes to use if no ~.lohr~ file is found in a repository. - ~additional_remotes~ is a list of remotes to add in any case, whether the original set of remotes is set via ~default_remotes~ or via a =.lohr= file. +- ~blacklist~ is a list of regular expressions to match against the full + repository names. Any that matches will not be mirrored, even if it contains a + `.lohr` file. Both settings take as input a list of "stems", i.e. incomplete remote addresses, to which the repo's name will be appended (so for example, if my From ff90b5fb2de9c389604117b809a2d9539ac3de1f Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 22:00:59 +0200 Subject: [PATCH 5/9] job: move blacklist processing to request --- src/job.rs | 7 ------- src/main.rs | 16 +++++++++++++++- src/settings.rs | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/job.rs b/src/job.rs index 0477704..2e91cf1 100644 --- a/src/job.rs +++ b/src/job.rs @@ -190,13 +190,6 @@ impl Job { } pub(crate) fn run(&mut self, homedir: &Path, config: &GlobalSettings) -> anyhow::Result<()> { - if config - .blacklist - .iter() - .any(|re| re.is_match(&self.repo.full_name)) - { - return Ok(()); - } let local_path = homedir.join(&self.repo.full_name); assert!(local_path.is_absolute()); self.local_path = Some(local_path); diff --git a/src/main.rs b/src/main.rs index ce09507..812615d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,7 +29,19 @@ struct JobSender(Mutex>); struct Secret(String); #[post("/", data = "")] -fn gitea_webhook(payload: SignedJson, sender: State) -> Status { +fn gitea_webhook( + payload: SignedJson, + sender: State, + config: State, +) -> Status { + if config + .blacklist + .iter() + .any(|re| re.is_match(&payload.repository.full_name)) + { + return Status::Ok; + } + { let sender = sender.0.lock().unwrap(); let repo = &payload.repository; @@ -74,6 +86,7 @@ fn main() -> anyhow::Result<()> { .expect("please provide a secret, otherwise anyone can send you a malicious webhook"); let config = parse_config(homedir.clone())?; + let config_state = config.clone(); thread::spawn(move || { repo_updater(receiver, homedir, config); @@ -83,6 +96,7 @@ fn main() -> anyhow::Result<()> { .mount("/", routes![gitea_webhook]) .manage(JobSender(Mutex::new(sender))) .manage(Secret(secret)) + .manage(config_state) .launch(); Ok(()) diff --git a/src/settings.rs b/src/settings.rs index 976264f..8dc71f9 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -2,7 +2,7 @@ use serde::Deserialize; pub(crate) type RepoUrl = String; // FIXME: probably needs a better type than this -#[derive(Default, Deserialize)] +#[derive(Clone, Default, Deserialize)] pub(crate) struct GlobalSettings { /// List of remote stems to use when no `.lohr` file is found #[serde(default)] From 422024d9193a4218383e7b7e34f7c9d18c40461a Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 22:51:51 +0200 Subject: [PATCH 6/9] main: support config override with CLI flag --- Cargo.lock | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 56 +++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 94 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0de7d0..316ab18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "anyhow" version = "1.0.40" @@ -152,6 +161,21 @@ dependencies = [ "generic-array", ] +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + [[package]] name = "cookie" version = "0.11.4" @@ -509,6 +533,7 @@ name = "lohr" version = "0.3.0" dependencies = [ "anyhow", + "clap", "hex", "hmac", "log 0.4.14", @@ -955,6 +980,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3015a7d0a5fd5105c91c3710d42f9ccf0abfb287d62206484dcc67f9569a6483" +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "subtle" version = "2.4.0" @@ -983,6 +1014,15 @@ dependencies = [ "unicode-xid 0.2.1", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "time" version = "0.1.43" @@ -1062,6 +1102,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.1.0" @@ -1095,6 +1141,12 @@ dependencies = [ "percent-encoding 1.0.1", ] +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index cbbb9ae..f68cf99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ repository = "https://github.com/alarsyo/lohr" [dependencies] anyhow = "1.0.40" +clap = "2.33.3" hex = "0.4.3" hmac = "0.10.1" log = "0.4.14" diff --git a/src/main.rs b/src/main.rs index 812615d..bedcdf7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,16 +2,17 @@ use std::env; use std::fs::File; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::{ mpsc::{channel, Receiver, Sender}, Mutex, }; use std::thread; -use rocket::{http::Status, post, routes, State}; - +use anyhow::Context; +use clap::{App, Arg}; use log::error; +use rocket::{http::Status, post, routes, State}; mod gitea; use gitea::GiteaWebHook; @@ -61,21 +62,46 @@ fn repo_updater(rx: Receiver, homedir: PathBuf, config: GlobalSettings) { } } -fn parse_config(mut path: PathBuf) -> anyhow::Result { - path.push("lohr-config"); - path.set_extension("yaml"); - let path = env::var("LOHR_CONFIG") - .map(Into::into) - .unwrap_or_else(|_| path); - let config = if let Ok(file) = File::open(path.as_path()) { - serde_yaml::from_reader(file)? - } else { - Default::default() +fn parse_config(home: &Path, flags: &clap::ArgMatches) -> anyhow::Result { + // prioritize CLI flag, then env var + let config_path = flags.value_of("config").map(PathBuf::from); + let config_path = config_path.or_else(|| env::var("LOHR_CONFIG").map(PathBuf::from).ok()); + + let file = match config_path { + Some(config_path) => File::open(&config_path).with_context(|| { + format!( + "could not open provided configuration file at {}", + config_path.display() + ) + })?, + None => { + // check if file exists in lohr home + let config_path = home.join("lohr-config.yaml"); + if !config_path.is_file() { + return Ok(Default::default()); + } + + File::open(config_path).context("failed to open configuration file in LOHR_HOME")? + } }; - Ok(config) + + serde_yaml::from_reader(file).context("could not parse configuration file") } fn main() -> anyhow::Result<()> { + let matches = App::new("lohr") + .version("0.3.0") + .about("Git mirroring daemon") + .arg( + Arg::with_name("config") + .short("c") + .long("config") + .value_name("FILE") + .help("Use a custom config file") + .takes_value(true), + ) + .get_matches(); + let (sender, receiver) = channel(); let homedir = env::var("LOHR_HOME").unwrap_or_else(|_| "./".to_string()); @@ -85,7 +111,7 @@ fn main() -> anyhow::Result<()> { let secret = env::var("LOHR_SECRET") .expect("please provide a secret, otherwise anyone can send you a malicious webhook"); - let config = parse_config(homedir.clone())?; + let config = parse_config(&homedir, &matches)?; let config_state = config.clone(); thread::spawn(move || { From aba153726ba4071136ffbe4a22fb2f19888fdb8a Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 22:53:33 +0200 Subject: [PATCH 7/9] main: log when skipping blacklisted repo --- src/main.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index bedcdf7..3856ee1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,7 +11,7 @@ use std::thread; use anyhow::Context; use clap::{App, Arg}; -use log::error; +use log::{error, info}; use rocket::{http::Status, post, routes, State}; mod gitea; @@ -40,6 +40,10 @@ fn gitea_webhook( .iter() .any(|re| re.is_match(&payload.repository.full_name)) { + info!( + "Ignoring webhook for repo {} which is blacklisted", + payload.repository.full_name + ); return Status::Ok; } From 54fafc6a463399646b3b073b8cbff08cfd8abd25 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 22:57:27 +0200 Subject: [PATCH 8/9] README: mention cli flag for configuration --- README.org | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index ce868c4..341d31b 100644 --- a/README.org +++ b/README.org @@ -73,8 +73,12 @@ variable. **** Extra remote configuration -=lohr= looks for a =lohr-config.yaml= file in its =LOHR_HOME= directory. The -=LOHR_CONFIG= variable takes precedence over looking into the state directory. +You can provide =lohr= with a YAML file containing additional configuration. You +can pass its path to the =--config= flag when launching =lohr=. If no +configuration is provided via a CLI flag, =lohr= will check the =LOHR_CONFIG= +environment variable. If the environment variable isn't set either, it will +check in =LOHR_HOME= is a =lohr-config.yaml= file exists, and try to load it. + This file takes the following format: #+begin_src yaml From 78194b69ad230a3bf6d679d17504a9cb40b16991 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 22:59:39 +0200 Subject: [PATCH 9/9] cargo: bump lockfile --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 316ab18..cc1a8dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -500,9 +500,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" +checksum = "56d855069fafbb9b344c0f962150cd2c1187975cb1c22c1522c240d8c4986714" [[package]] name = "linked-hash-map" @@ -913,7 +913,7 @@ checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.65", + "syn 1.0.67", ] [[package]] @@ -1005,9 +1005,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.65" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a1d708c221c5a612956ef9f75b37e454e88d1f7b899fbd3a18d4252012d663" +checksum = "6498a9efc342871f91cc2d0d694c674368b4ceb40f62b65a7a08c3792935e702" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9",