Compare commits

...

62 commits
v0.2.0 ... main

Author SHA1 Message Date
e0d2c988a0 flake: update all inputs 2025-04-01 13:42:49 +02:00
Bruno BELANYI
4d60965770 lohr: v0.4.6 2024-09-04 10:03:31 +00:00
Bruno BELANYI
a028104cb5 Merge pull request #7 from ambroisie/update-time
cargo: bump time to fix compilation error
2024-09-04 10:03:16 +00:00
Bruno BELANYI
aef4d65087 cargo: bump time to fix compilation error 2024-09-04 10:03:11 +00:00
24c0156b97 lohr: v0.4.5 2023-12-12 15:46:33 +01:00
298e895a0b cargo: bump lock again 2023-12-12 15:44:51 +01:00
b6445e6707 lohr: v0.4.4 2023-12-12 15:39:39 +01:00
926f48f979 switch to new rust-toolchain format 2023-12-12 15:37:36 +01:00
b04748a804 cargo: bump rocket to v0.5 2023-12-12 15:36:56 +01:00
77e5214561 lohr: v0.4.3 2023-12-12 15:24:22 +01:00
689c57a140 ci: make them runnable manually 2023-12-12 15:18:32 +01:00
66d965beee ci: update rust actions 2023-12-12 15:15:56 +01:00
708c350c9a ci: bump cachix workflow actions 2023-12-12 15:11:49 +01:00
075ca9a23c fix clippy lints 2023-12-12 15:06:59 +01:00
25eba69b16 cargo: bump dependencies 2023-12-12 15:01:58 +01:00
88609b24fe flake: add rust-analyzer to devshell 2023-12-12 14:57:37 +01:00
26eefe2ebe flake.lock: Update
Flake lock file updates:

• Updated input 'flake-compat':
    'github:edolstra/flake-compat/12c64ca55c1014cdc1b16ed5a804aa8576601ff2' (2021-08-02)
  → 'github:edolstra/flake-compat/0f9255e01c2351cc7d116c072cb317785dd33b33' (2023-10-04)
• Updated input 'flake-utils':
    'github:numtide/flake-utils/997f7efcb746a9c140ce1f13c72263189225f482' (2021-08-20)
  → 'github:numtide/flake-utils/4022d587cbbfd70fe950c1e2083a02621806a725' (2023-12-04)
• Added input 'flake-utils/systems':
    'github:nix-systems/default/da67096a3b9bf56a91d16901293e51ba5b49a27e' (2023-04-09)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/08ef0f28e3a41424b92ba1d203de64257a9fca6a' (2021-08-28)
  → 'github:NixOS/nixpkgs/e97b3e4186bcadf0ef1b6be22b8558eab1cdeb5d' (2023-12-11)
2023-12-12 14:51:44 +01:00
58503cc8b9 flake.lock: Update
Flake input changes:

* Updated 'flake-compat': 'github:edolstra/flake-compat/99f1c2157fba4bfe6211a321fd0ee43199025dbf' -> 'github:edolstra/flake-compat/12c64ca55c1014cdc1b16ed5a804aa8576601ff2'
* Updated 'flake-utils': 'github:numtide/flake-utils/5466c5bbece17adaab2d82fae80b46e807611bf3' -> 'github:numtide/flake-utils/997f7efcb746a9c140ce1f13c72263189225f482'
* Updated 'nixpkgs': 'github:NixOS/nixpkgs/7db379d016a9bad7b7da9b81103ab7454faa3344' -> 'github:NixOS/nixpkgs/08ef0f28e3a41424b92ba1d203de64257a9fca6a'
2021-09-04 18:18:31 +02:00
a48e59bedc github: fix cachix ci 2021-09-04 18:18:31 +02:00
c37a390f6c lohr: fix clippy lint 2021-09-04 18:16:38 +02:00
cd85ace566 github: trigger cachix build on flake change
This also covers version bumps because the version is in the flake
2021-09-04 18:14:14 +02:00
964975bada lohr: v0.4.2 2021-06-10 05:40:15 +02:00
b78c486a0e lohr: upgrade to Rocket v0.5.0-rc.1 2021-06-10 05:36:39 +02:00
7fad5a6a87 github: only trigger cachix build on tag 2021-06-04 17:48:39 +02:00
5f7d140b61 lohr: v0.4.1 2021-06-04 17:38:37 +02:00
1f6fe74d03 lohr: upgrade to latest Rocket master 2021-06-04 17:31:00 +02:00
fd50523f11 flake.lock: Update
Flake input changes:

* Updated 'nixpkgs': 'github:NixOS/nixpkgs/c0e881852006b132236cbf0301bd1939bb50867e' -> 'github:NixOS/nixpkgs/7db379d016a9bad7b7da9b81103ab7454faa3344'
2021-06-04 15:58:00 +02:00
d942dc1098 ci: setup cachix build 2021-04-08 04:24:48 +02:00
bc32f8a565 README: add note about missing versions on crates.io 2021-04-08 02:49:19 +02:00
6c4a496ede lohr: v0.4.0
This is the first lohr release that compiles on stable Rust, yay!
2021-04-08 02:49:19 +02:00
264b7baccd cargo: bump dependencies 2021-04-08 01:51:07 +02:00
776876233f gitea: use ssh_url instead of clone_url
Fixes #2
2021-04-08 01:46:52 +02:00
Bruno BELANYI
3fedfa3342 nix: flake: build package with vanilla 'nixpkgs' 2021-04-08 01:41:15 +02:00
378823a1fb ci: switch to stable toolchain 2021-04-08 01:41:15 +02:00
56f82c6467 README: remove mention of nightly compiler 2021-04-08 01:41:12 +02:00
3a93d9f994 lohr: build on rust stable
Switching to Rocket's master branch (soon to be 0.5.0) allows building
with a stable Rust compiler, yay!
2021-04-08 01:39:21 +02:00
da20e2c9ac cargo fmt
whoops
2021-03-31 21:04:30 +02:00
29c0a4abdf lohr: v0.3.3 2021-03-31 19:39:04 +02:00
1506ce1dac README: switch to Markdown
Unfortunately crates.io doesn't show Org READMEs, and GitHub's Org-mode
renderer isn't amazing either. So might as well switch to a well
supported format, even though I liked Org better.
2021-03-31 19:28:24 +02:00
fe52787f4d lohr: v0.3.2 2021-03-31 19:06:36 +02:00
e68fcf3e52 settings: blacklist isn't required 2021-03-31 18:57:50 +02:00
6a79949e28 cargo: try to reduce number of dependencies
serde_regex unconditionally uses all features of the regex crate, which
prevents optimizing the dependency graph further
2021-03-31 12:31:25 +02:00
933e164259 README: add section about installing lohr 2021-03-30 23:11:52 +02:00
a516d841e1 lohr: v0.3.1 2021-03-30 23:02:32 +02:00
78194b69ad cargo: bump lockfile 2021-03-30 22:59:39 +02:00
54fafc6a46 README: mention cli flag for configuration 2021-03-30 22:57:27 +02:00
aba153726b main: log when skipping blacklisted repo 2021-03-30 22:53:33 +02:00
422024d919 main: support config override with CLI flag 2021-03-30 22:51:51 +02:00
ff90b5fb2d job: move blacklist processing to request 2021-03-30 22:00:59 +02:00
Bruno BELANYI
8e7e0e9a84 README: mention 'LOHR_CONFIG' 2021-03-30 18:46:33 +00:00
Bruno BELANYI
6f63b4c95c main: add LOHR_CONFIG variable 2021-03-30 18:46:14 +00:00
Bruno BELANYI
d38e4556e1 nix: flake: add meta information in package 2021-03-30 18:40:32 +00:00
Bruno BELANYI
9de7c783ae job: add processing blacklist to global config 2021-03-30 16:28:00 +00:00
209e1e58c7 job: filter out empty lines from .lohr file 2021-03-30 15:27:15 +02:00
1be64262b2 main: remove solved TODO item 2021-03-30 14:45:43 +02:00
cc75f90a9a ci: fail on clippy warnings 2021-03-30 12:13:20 +02:00
fea9d61d2b signature: fix unused trait warning 2021-03-30 11:41:16 +02:00
ade177de54 README: mention shared secret in configuration 2021-03-30 11:38:58 +02:00
682e9c5bcc lohr: v0.3.0
This brings support for webhook signature verification, making lohr
usable in a production setting.
2021-03-30 11:33:42 +02:00
7e3c8b8f28 lohr: validate webhook signature
Previously lohr was unusable in a production setting, anyone could forge
a malicious webhook and either:

- mirror a private repo of yours to another remote they own
- wipe a repo of yours by forcing mirroring from an empty mirror

This is no longer the case!
2021-03-30 11:33:42 +02:00
7134b7700f lohr: v0.2.1 2021-03-30 01:12:50 +02:00
339c6e4f25 cargo: add repo and homepage metadata 2021-03-30 01:08:55 +02:00
15 changed files with 1612 additions and 840 deletions

20
.github/workflows/cachix.yaml vendored Normal file
View file

@ -0,0 +1,20 @@
name: "Cachix build"
on:
workflow_dispatch:
push:
paths:
- 'flake.nix'
- 'flake.lock'
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: cachix/install-nix-action@v24
- uses: cachix/cachix-action@v12
with:
name: alarsyo
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
- run: |
nix build --verbose

View file

@ -1,4 +1,4 @@
on: [push, pull_request]
on: [push, pull_request, workflow_dispatch]
name: CI
@ -8,62 +8,34 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v2
uses: actions/checkout@v4
- name: Install nightly toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: nightly
override: true
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
- name: Run cargo check
uses: actions-rs/cargo@v1
with:
command: check
- run: cargo check
test:
name: Test Suite
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v2
uses: actions/checkout@v4
- name: Install nightly toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: nightly
override: true
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
- name: Run cargo test
uses: actions-rs/cargo@v1
with:
command: test
- run: cargo test
lints:
name: Lints
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v2
uses: actions/checkout@v4
- name: Install nightly toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: nightly
override: true
components: rustfmt, clippy
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
- name: Run cargo fmt
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all -- --check
- name: Run cargo clippy
uses: actions-rs/clippy-check@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
args: --all-features
- run: cargo fmt --all -- --check
- run: cargo clippy --all-features -- -D warnings

1759
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -1,17 +1,34 @@
[package]
name = "lohr"
version = "0.2.0"
version = "0.4.6"
authors = ["Antoine Martin <antoine@alarsyo.net>"]
edition = "2018"
license = "Apache-2.0 OR MIT"
description = "A Git mirroring daemon"
homepage = "https://github.com/alarsyo/lohr"
repository = "https://github.com/alarsyo/lohr"
resolver = "2"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow = "1.0.40"
hex = "0.4.3"
hmac = "0.10.1"
log = "0.4.14"
rocket = "0.4.7"
rocket_contrib = { version = "0.4.7", features = [ "json" ] }
regex = "1"
serde = { version = "1.0.125", features = [ "derive" ] }
serde_json = "1.0.64"
serde_regex = "1.1.0"
serde_yaml = "0.8.17"
sha2 = "0.9.3"
[dependencies.rocket]
version = "0.5.0"
# don't need private-cookies
default-features = false
[dependencies.clap]
version = "2.33.3"
# no need for suggestions or color with only one argument
default-features = false

138
README.md Normal file
View file

@ -0,0 +1,138 @@
# lohr
`lohr` is a Git mirroring tool.
I created it to solve a simple problem I had: I host my own git server at
<https://git.alarsyo.net>, but want to mirror my public projects to GitHub /
GitLab, for backup and visibility purposes.
GitLab has a mirroring setting, but it doesn't allow for multiple mirrors, as
far as I know. I also wanted my instance to be the single source of truth.
## How it works
Gitea is setup to send webhooks to my `lohr` server on every push update. When
`lohr` receives a push, it clones the concerned repository, or updates it if
already cloned. Then it pushes the update to **all remotes listed** in the
[.lohr](.lohr) file at the repo root.
### Destructive
This is a very destructive process: anything removed from the single source of
truth is effectively removed from any mirror as well.
## Installing
`lohr` is [published on crates.io](https://crates.io/crates/lohr), so you can
install it with `cargo install`:
$ cargo install lohr
Note: currently this method won't get you the latest version of `lohr`, as it
depends on Rocket v0.5.0, which isn't released yet. Updated versions of `lohr`
will be published on crates.io as soon as Rocket v0.5.0 releases.
## Setup
### Quickstart
Setting up `lohr` should be quite simple:
1. Create a `Rocket.toml` file and [add your
configuration](https://rocket.rs/v0.4/guide/configuration/).
2. Export a secret variable:
$ export LOHR_SECRET=42 # please don't use this secret
3. Run `lohr`:
$ cargo run # or `cargo run --release` for production usage
4. Configure your favorite git server to send a webhook to `lohr`'s address on
every push event.
I used [Gitea's webhooks format](https://docs.gitea.io/en-us/webhooks/), but
I **think** they're similar to GitHub and GitLab's webhooks, so these should
work too! (If they don't, **please** file an issue!)
Don't forget to set the webhook secret to the one you chose above.
5. Add a `.lohr` file containing the remotes you want to mirror this repo to:
git@github.com:you/your_repo
and push it. That's it! `lohr` is mirroring your repo now.
### Configuration
#### Home directory
`lohr` needs a place to clone repos and store its data. By default, it's the
current directory, but you can set the `LOHR_HOME` environment variable to
customize it.
#### Shared secret
As shown in the quickstart guide, you **must** set the `LOHR_SECRET` environment
variable.
#### Extra remote configuration
You can provide `lohr` with a YAML file containing additional configuration. You
can pass its path to the `--config` flag when launching `lohr`. If no
configuration is provided via a CLI flag, `lohr` will check the `LOHR_CONFIG`
environment variable. If the environment variable isn't set either, it will
check in `LOHR_HOME` is a `lohr-config.yaml` file exists, and try to load it.
This file takes the following format:
``` yaml
default_remotes:
- "git@github:user"
- "git@gitlab:user"
additional_remotes:
- "git@git.sr.ht:~user"
blacklist:
- "private-.*"
```
- `default_remotes` is a list of remotes to use if no `.lohr` file is found in a
repository.
- `additional_remotes` is a list of remotes to add in any case, whether the
original set of remotes is set via `default_remotes` or via a `.lohr` file.
- `blacklist` is a list of regular expressions to match against the full
repository names. Any that matches will not be mirrored, even if it contains a
`.lohr` file.
Both settings take as input a list of "stems", i.e. incomplete remote addresses,
to which the repo's name will be appended (so for example, if my
`default_remotes` contains `git@github.com:alarsyo`, and a push event webhook is
received for repository `git@gitlab.com:some/long/path/repo_name`, then the
mirror destination will be `git@github.com:alarsyo/repo_name`.
## Contributing
I accept patches anywhere! Feel free to [open a GitHub Pull
Request](https://github.com/alarsyo/lohr/pulls), [a GitLab Merge
Request](https://gitlab.com/alarsyo/lohr/-/merge_requests), or [send me a patch
by email](https://lists.sr.ht/~alarsyo/lohr-dev)!
## Why lohr?
I was looking for a cool name, and thought about the Magic Mirror in Snow White.
Some **[furious wikipedia
searching](https://en.wikipedia.org/wiki/Magic_Mirror_(Snow_White))** later, I
found that the Magic Mirror was probably inspired by [the Talking Mirror in Lohr
am Main](http://spessartmuseum.de/seiten/schneewittchen_engl.html). That's it,
that's the story.
## License
`lohr` is distributed under the terms of both the MIT license and the Apache
License (Version 2.0).
See [LICENSE-APACHE](LICENSE-APACHE) and [LICENSE-MIT](LICENSE-MIT) for details.

View file

@ -1,103 +0,0 @@
#+title: lohr
=lohr= is a Git mirroring tool.
I created it to solve a simple problem I had: I host my own git server at
[[https://git.alarsyo.net]], but want to mirror my public projects to GitHub /
GitLab, for backup and visibility purposes.
GitLab has a mirroring setting, but it doesn't allow for multiple mirrors, as
far as I know. I also wanted my instance to be the single source of truth.
** How it works
Gitea is setup to send webhooks to my =lohr= server on every push update. When
=lohr= receives a push, it clones the concerned repository, or updates it if
already cloned. Then it pushes the update to *all remotes listed* in the [[file:.lohr][.lohr]]
file at the repo root.
*** Destructive
This is a very destructive process: anything removed from the single source of
truth is effectively removed from any mirror as well.
** Setup
*** Quickstart
Setting up =lohr= should be quite simple:
1. Create a =Rocket.toml= file and [[https://rocket.rs/v0.4/guide/configuration/][add your configuration]].
2. Run =lohr=:
#+begin_src sh
$ cargo run # or `cargo run --release` for production usage
#+end_src
3. Configure your favorite git server to send a webhook to =lohr='s address on
every push event.
I used [[https://docs.gitea.io/en-us/webhooks/][Gitea's webhooks format]], but I *think* they're similar to GitHub and
GitLab's webhooks, so these should work too! (If they don't, *please* file an
issue!)
4. Add a =.lohr= file containing the remotes you want to mirror this repo to:
#+begin_example
git@github.com:you/your_repo
#+end_example
and push it. That's it! =lohr= is mirroring your repo now.
*** Configuration
**** Home directory
=lohr= needs a place to clone repos and store its data. By default, it's the
current directory, but you can set the =LOHR_HOME= environment variable to
customize it.
**** Extra remote configuration
=lohr= looks for a =lohr-config.yaml= file in its =LOHR_HOME= directory. This
file takes the following format:
#+begin_src yaml
default_remotes:
- "git@github:user"
- "git@gitlab:user"
additional_remotes:
- "git@git.sr.ht:~user"
#+end_src
- ~default_remotes~ is a list of remotes to use if no ~.lohr~ file is found in a
repository.
- ~additional_remotes~ is a list of remotes to add in any case, whether the
original set of remotes is set via ~default_remotes~ or via a =.lohr= file.
Both settings take as input a list of "stems", i.e. incomplete remote addresses,
to which the repo's name will be appended (so for example, if my
~default_remotes~ contains ~git@github.com:alarsyo~, and a push event webhook
is received for repository =git@gitlab.com:some/long/path/repo_name=, then the
mirror destination will be =git@github.com:alarsyo/repo_name=.
** Contributing
I accept patches anywhere! Feel free to [[https://github.com/alarsyo/lohr/pulls][open a GitHub Pull Request]], [[https://gitlab.com/alarsyo/lohr/-/merge_requests][a GitLab
Merge Request]], or [[https://lists.sr.ht/~alarsyo/lohr-dev][send me a patch by email]]!
** Why lohr?
I was looking for a cool name, and thought about the Magic Mirror in Snow White.
Some *[[https://en.wikipedia.org/wiki/Magic_Mirror_(Snow_White)][furious wikipedia searching]]* later, I found that the Magic Mirror was
probably inspired by [[http://spessartmuseum.de/seiten/schneewittchen_engl.html][the Talking Mirror in Lohr am Main]]. That's it, that's the
story.
** License
=lohr= is distributed under the terms of both the MIT license and the Apache
License (Version 2.0).
See [[file:LICENSE-APACHE][LICENSE-APACHE]] and [[file:LICENSE-MIT][LICENSE-MIT]] for details.

81
flake.lock generated
View file

@ -3,11 +3,11 @@
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1606424373,
"narHash": "sha256-oq8d4//CJOrVj+EcOaSXvMebvuTkmBJuT5tzlfewUnQ=",
"lastModified": 1733328505,
"narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "99f1c2157fba4bfe6211a321fd0ee43199025dbf",
"rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
"type": "github"
},
"original": {
@ -17,69 +17,35 @@
}
},
"flake-utils": {
"locked": {
"lastModified": 1614513358,
"narHash": "sha256-LakhOx3S1dRjnh0b5Dg3mbZyH0ToC9I8Y2wKSkBaTzU=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5466c5bbece17adaab2d82fae80b46e807611bf3",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"mozillapkgs": {
"flake": false,
"locked": {
"lastModified": 1603906276,
"narHash": "sha256-RsNPnEKd7BcogwkqhaV5kI/HuNC4flH/OQCC/4W5y/8=",
"owner": "mozilla",
"repo": "nixpkgs-mozilla",
"rev": "8c007b60731c07dd7a052cce508de3bb1ae849b4",
"type": "github"
},
"original": {
"owner": "mozilla",
"repo": "nixpkgs-mozilla",
"type": "github"
}
},
"naersk": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
"systems": "systems"
},
"locked": {
"lastModified": 1614785451,
"narHash": "sha256-TPw8kQvr2UNCuvndtY+EjyXp6Q5GEW2l9UafXXh1XmI=",
"owner": "nmattia",
"repo": "naersk",
"rev": "e0fe990b478a66178a58c69cf53daec0478ca6f9",
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "nmattia",
"ref": "master",
"repo": "naersk",
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1616670887,
"narHash": "sha256-wn+l9qJfR5sj5Gq4DheJHAcBDfOs9K2p9seW2f35xzs=",
"lastModified": 1740162160,
"narHash": "sha256-SSYxFhqCOb3aiPb6MmN68yEzBIltfom8IgRz7phHscM=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "c0e881852006b132236cbf0301bd1939bb50867e",
"rev": "11415c7ae8539d6292f2928317ee7a8410b28bb9",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"ref": "nixos-24.11",
"repo": "nixpkgs",
"type": "github"
}
@ -88,10 +54,23 @@
"inputs": {
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"mozillapkgs": "mozillapkgs",
"naersk": "naersk",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",

View file

@ -1,14 +1,6 @@
{
inputs = {
naersk = {
url = "github:nmattia/naersk/master";
inputs.nixpkgs.follows = "nixpkgs";
};
mozillapkgs = {
url = "github:mozilla/nixpkgs-mozilla";
flake = false;
};
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
flake-utils.url = "github:numtide/flake-utils";
flake-compat = {
url = "github:edolstra/flake-compat";
@ -16,27 +8,26 @@
};
};
outputs = { self, naersk, mozillapkgs, nixpkgs, flake-utils, ... }:
outputs = { self, nixpkgs, flake-utils, ... }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs { inherit system; };
mozilla = pkgs.callPackage (mozillapkgs + "/package-set.nix") { };
rustNightly = (mozilla.rustChannelOf {
date = "2021-03-29";
channel = "nightly";
sha256 = "sha256-Y94CnslybZgiZlNVV6Cg0TUPV2OeDXakPev1kqdt9Kk=";
}).rust;
naersk-lib = pkgs.callPackage naersk {
cargo = rustNightly;
rustc = rustNightly;
};
in
{
defaultPackage = naersk-lib.buildPackage {
src = ./.;
defaultPackage = pkgs.rustPlatform.buildRustPackage {
pname = "lohr";
version = "0.4.6";
src = ./.;
cargoHash = "sha256-EUhyrhPe+mUgMmm4o+bxRIiSNReJRfw+/O1fPr8r7lo=";
meta = with pkgs.lib; {
description = "A Git mirroring tool";
homepage = "https://github.com/alarsyo/lohr";
license = with licenses; [ mit asl20 ];
platforms = platforms.unix;
};
};
defaultApp = flake-utils.lib.mkApp {
@ -45,11 +36,14 @@
devShell = pkgs.mkShell {
buildInputs = with pkgs; [
cargo
clippy
nixpkgs-fmt
pre-commit
rustPackages.clippy
rustNightly
rustc
rustfmt
rust-analyzer
];
RUST_SRC_PATH = pkgs.rustPlatform.rustLibSrc;

View file

@ -1 +0,0 @@
nightly

2
rust-toolchain.toml Normal file
View file

@ -0,0 +1,2 @@
[toolchain]
channel = "stable"

View file

@ -2,9 +2,8 @@ use serde::Deserialize;
#[derive(Clone, Deserialize)]
pub(crate) struct Repository {
pub(crate) name: String,
pub(crate) full_name: String,
pub(crate) clone_url: String,
pub(crate) ssh_url: String,
}
#[derive(Deserialize)]

View file

@ -38,7 +38,7 @@ impl Job {
let output = Command::new("git")
.arg("clone")
.arg("--mirror")
.arg(&self.repo.clone_url)
.arg(&self.repo.ssh_url)
.arg(format!("{}", self.local_path.as_ref().unwrap().display()))
.output()?;
@ -125,7 +125,13 @@ impl Job {
let output = String::from_utf8(output.stdout)?;
Ok(Some(output.lines().map(String::from).collect()))
Ok(Some(
output
.lines()
.map(String::from)
.filter(|s| !s.is_empty())
.collect(),
))
}
fn get_remotes(&self, config: &GlobalSettings) -> anyhow::Result<Vec<RepoUrl>> {

View file

@ -1,18 +1,16 @@
#![feature(proc_macro_hygiene, decl_macro)]
use std::env;
use std::fs::File;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::{
mpsc::{channel, Receiver, Sender},
Mutex,
};
use std::thread;
use anyhow::Context;
use clap::{crate_version, App, Arg};
use log::{error, info};
use rocket::{http::Status, post, routes, State};
use rocket_contrib::json::Json;
use log::error;
mod gitea;
use gitea::GiteaWebHook;
@ -23,11 +21,29 @@ use job::Job;
mod settings;
use settings::GlobalSettings;
mod signature;
use signature::SignedJson;
struct JobSender(Mutex<Sender<Job>>);
struct Secret(String);
#[post("/", data = "<payload>")]
fn gitea_webhook(payload: Json<GiteaWebHook>, sender: State<JobSender>) -> Status {
// TODO: validate Gitea signature
fn gitea_webhook(
payload: SignedJson<GiteaWebHook>,
sender: &State<JobSender>,
config: &State<GlobalSettings>,
) -> Status {
if config
.blacklist
.iter()
.any(|re| re.is_match(&payload.repository.full_name))
{
info!(
"Ignoring webhook for repo {} which is blacklisted",
payload.repository.full_name
);
return Status::Ok;
}
{
let sender = sender.0.lock().unwrap();
@ -48,34 +64,70 @@ fn repo_updater(rx: Receiver<Job>, homedir: PathBuf, config: GlobalSettings) {
}
}
fn parse_config(mut path: PathBuf) -> anyhow::Result<GlobalSettings> {
path.push("lohr-config");
path.set_extension("yaml");
let config = if let Ok(file) = File::open(path.as_path()) {
serde_yaml::from_reader(file)?
} else {
Default::default()
fn parse_config(home: &Path, flags: &clap::ArgMatches) -> anyhow::Result<GlobalSettings> {
// prioritize CLI flag, then env var
let config_path = flags.value_of("config").map(PathBuf::from);
let config_path = config_path.or_else(|| env::var("LOHR_CONFIG").map(PathBuf::from).ok());
let file = match config_path {
Some(config_path) => File::open(&config_path).with_context(|| {
format!(
"could not open provided configuration file at {}",
config_path.display()
)
})?,
None => {
// check if file exists in lohr home
let config_path = home.join("lohr-config.yaml");
if !config_path.is_file() {
return Ok(Default::default());
}
File::open(config_path).context("failed to open configuration file in LOHR_HOME")?
}
};
Ok(config)
serde_yaml::from_reader(file).context("could not parse configuration file")
}
fn main() -> anyhow::Result<()> {
#[rocket::main]
async fn main() -> anyhow::Result<()> {
let matches = App::new("lohr")
.version(crate_version!())
.about("Git mirroring daemon")
.arg(
Arg::with_name("config")
.short("c")
.long("config")
.value_name("FILE")
.help("Use a custom config file")
.takes_value(true),
)
.get_matches();
let (sender, receiver) = channel();
let homedir = env::var("LOHR_HOME").unwrap_or_else(|_| "./".to_string());
let homedir: PathBuf = homedir.into();
let homedir = homedir.canonicalize().expect("LOHR_HOME isn't valid!");
let config = parse_config(homedir.clone())?;
let secret = env::var("LOHR_SECRET")
.expect("please provide a secret, otherwise anyone can send you a malicious webhook");
let config = parse_config(&homedir, &matches)?;
let config_state = config.clone();
thread::spawn(move || {
repo_updater(receiver, homedir, config);
});
rocket::ignite()
rocket::build()
.mount("/", routes![gitea_webhook])
.manage(JobSender(Mutex::new(sender)))
.launch();
.manage(Secret(secret))
.manage(config_state)
.launch()
.await?;
Ok(())
}

View file

@ -2,7 +2,7 @@ use serde::Deserialize;
pub(crate) type RepoUrl = String; // FIXME: probably needs a better type than this
#[derive(Default, Deserialize)]
#[derive(Clone, Default, Deserialize)]
pub(crate) struct GlobalSettings {
/// List of remote stems to use when no `.lohr` file is found
#[serde(default)]
@ -10,4 +10,8 @@ pub(crate) struct GlobalSettings {
/// List of remote stems to use for every repository
#[serde(default)]
pub additional_remotes: Vec<RepoUrl>,
/// List of regexes, if a repository's name matches any of the, it is not mirrored by `lohr`
/// even if it contains a `.lorh` file.
#[serde(with = "serde_regex", default)]
pub blacklist: Vec<regex::Regex>,
}

114
src/signature.rs Normal file
View file

@ -0,0 +1,114 @@
use std::{
io,
ops::{Deref, DerefMut},
};
use rocket::{
data::{ByteUnit, FromData, Outcome},
http::ContentType,
State,
};
use rocket::{http::Status, request::local_cache};
use rocket::{Data, Request};
use anyhow::{anyhow, Context};
use serde::Deserialize;
use crate::Secret;
const X_GITEA_SIGNATURE: &str = "X-Gitea-Signature";
fn validate_signature(secret: &str, signature: &str, data: &str) -> bool {
use hmac::{Hmac, Mac, NewMac};
use sha2::Sha256;
type HmacSha256 = Hmac<Sha256>;
let mut mac = HmacSha256::new_varkey(secret.as_bytes()).expect("this should never fail");
mac.update(data.as_bytes());
match hex::decode(signature) {
Ok(bytes) => mac.verify(&bytes).is_ok(),
Err(_) => false,
}
}
pub struct SignedJson<T>(pub T);
impl<T> Deref for SignedJson<T> {
type Target = T;
fn deref(&self) -> &T {
&self.0
}
}
impl<T> DerefMut for SignedJson<T> {
fn deref_mut(&mut self) -> &mut T {
&mut self.0
}
}
const LIMIT: ByteUnit = ByteUnit::Mebibyte(1);
impl<'r, T: Deserialize<'r>> SignedJson<T> {
fn from_str(s: &'r str) -> anyhow::Result<Self> {
serde_json::from_str(s)
.map(SignedJson)
.context("could not parse json")
}
}
// This is a one to one implementation of request_contrib::Json's FromData, but with HMAC
// validation.
//
// Tracking issue for chaining Data guards to avoid this:
// https://github.com/SergioBenitez/Rocket/issues/775
#[rocket::async_trait]
impl<'r, T> FromData<'r> for SignedJson<T>
where
T: Deserialize<'r>,
{
type Error = anyhow::Error;
async fn from_data(request: &'r Request<'_>, data: Data<'r>) -> Outcome<'r, Self> {
let json_ct = ContentType::new("application", "json");
if request.content_type() != Some(&json_ct) {
return Outcome::Error((Status::BadRequest, anyhow!("wrong content type")));
}
let signatures = request.headers().get(X_GITEA_SIGNATURE).collect::<Vec<_>>();
if signatures.len() != 1 {
return Outcome::Error((
Status::BadRequest,
anyhow!("request header needs exactly one signature"),
));
}
let size_limit = request.limits().get("json").unwrap_or(LIMIT);
let content = match data.open(size_limit).into_string().await {
Ok(s) if s.is_complete() => s.into_inner(),
Ok(_) => {
let eof = io::ErrorKind::UnexpectedEof;
return Outcome::Error((
Status::PayloadTooLarge,
io::Error::new(eof, "data limit exceeded").into(),
));
}
Err(e) => return Outcome::Error((Status::BadRequest, e.into())),
};
let signature = signatures[0];
let secret = request.guard::<&State<Secret>>().await.unwrap();
if !validate_signature(&secret.0, signature, &content) {
return Outcome::Error((Status::BadRequest, anyhow!("couldn't verify signature")));
}
match Self::from_str(local_cache!(request, content)) {
Ok(content) => Outcome::Success(content),
Err(e) => Outcome::Error((Status::BadRequest, e)),
}
}
}