From 6aed7f31ea37853b189f605a3c3fe9b3a6d16094 Mon Sep 17 00:00:00 2001 From: ngn Date: Tue, 9 Jan 2024 22:45:45 +0300 Subject: [PATCH] Added matter-mirror tool --- Makefile | 2 + README.md | 1 + matter-mirror/.gitignore | 1 + matter-mirror/README.md | 14 +++ matter-mirror/main.py | 246 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 264 insertions(+) create mode 100644 matter-mirror/.gitignore create mode 100644 matter-mirror/README.md create mode 100644 matter-mirror/main.py diff --git a/Makefile b/Makefile index d12033f..369c53e 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,12 @@ prefix = /usr install: install -v -m755 matter-setup/main.sh $(DESTDIR)$(prefix)/bin/matter-setup + install -v -m755 matter-mirror/main.py $(DESTDIR)$(prefix)/bin/matter-mirror install -v -m755 matter-chroot/main.sh $(DESTDIR)$(prefix)/bin/matter-chroot uninstall: rm -v $(DESTDIR)$(prefix)/bin/matter-setup + rm -v $(DESTDIR)$(prefix)/bin/matter-mirror rm -v $(DESTDIR)$(prefix)/bin/matter-chroot .PHONY: install uninstall diff --git a/README.md b/README.md index e00933c..276c9f9 100644 --- a/README.md +++ b/README.md @@ -12,4 +12,5 @@ make install ### Usage All tools/scripts contains usage information in README files: - [matter-chroot](matter-chroot/README.md) +- [matter-mirror](matter-mirror/README.md) - [matter-setup](matter-setup/README.md) diff --git a/matter-mirror/.gitignore b/matter-mirror/.gitignore new file mode 100644 index 0000000..65e3ba2 --- /dev/null +++ b/matter-mirror/.gitignore @@ -0,0 +1 @@ +test/ diff --git a/matter-mirror/README.md b/matter-mirror/README.md new file mode 100644 index 0000000..c4919a8 --- /dev/null +++ b/matter-mirror/README.md @@ -0,0 +1,14 @@ +# matter-mirror +A simple python script to create and manage MatterLinux mirrors. +**Does not support FTP!** + +### Usage +To use the `matter-mirror` script, pass in a URL and a download (output) +directory. For example: +```bash +mkdir base +matter-mirror -u https://pkgs.matterlinux.xyz/base -o base +``` +This will download the repo at `https://pkgs.matterlinux.xyz/base` to the +`base` directory. Running it again, script will only download outdated or +new packages. diff --git a/matter-mirror/main.py b/matter-mirror/main.py new file mode 100644 index 0000000..57fbfba --- /dev/null +++ b/matter-mirror/main.py @@ -0,0 +1,246 @@ +#!/bin/python3 + +from os import path, chdir, listdir, remove +from typing import List +import requests as req +import logging as log +import configparser +import argparse +import tarfile +import hashlib + +class BadResponse(Exception): + def __init__(self, msg: str) -> None: + super().__init__(f"Bad response: {msg}") + +class Pkg: + def __init__(self, name: str, version: str, sum: str) -> None: + self.archive = f"{name}_{version}.mpf" + self.sig = f"{name}_{version}.mpf.sig" + + self.version = version + self.name = name + self.sum = sum + + def remove_old(self) -> None: + files = listdir(".") + for f in files: + if f.startswith(f"{self.name}_"): + remove(f) + +class Repo: + def __init__(self, uri: str, out: str) -> None: + self.pkgs: List[Pkg] = [] + self.author: str + self.name: str + self.pub: str + self.uri = uri + self.out = out + + def join_url(self, pth: str) -> str: + if self.uri.endswith("/") and not pth.startswith("/"): + return self.uri+pth + elif self.uri.endswith("/") and pth.startswith("/"): + return self.uri[:-1]+pth + elif not self.uri.endswith("/") and pth.startswith("/"): + return self.uri+pth + + return self.uri+"/"+pth + + def get_repo(self) -> None: + repourl = self.join_url("repo") + res = req.get(repourl) + + if res.status_code != 200: + raise BadResponse(f"{res.status_code} - {repourl}") + + cfg = configparser.ConfigParser() + cfg.read_string(res.content.decode("utf-8")) + + for k in cfg.keys(): + if k == "DEFAULT": + continue + self.name = k + + self.pub = cfg[self.name]["pub"] + self.author = cfg[self.name]["author"] + + f = open("repo", "wb") + f.write(res.content) + f.close() + + def process_pkgs(self, pkgs: str) -> None: + cfg = configparser.ConfigParser() + cfg.read_string(pkgs) + + for k in cfg.keys(): + try: + ver = cfg[k]["version"] + sum = cfg[k]["sum"] + except: + continue + self.pkgs.append(Pkg(k, ver, sum)) + + def check_pkg(self, pkg: Pkg) -> bool: + # true -> package is in the outdir and its up-to-date + # false -> its not ^ + if not path.exists(pkg.archive) or not path.exists(pkg.sig): + return False + + fhash = hashlib.sha256() + f = open(pkg.archive, "rb") + + while chunk := f.read(8192): + fhash.update(chunk) + + f.close() + if pkg.sum != fhash.hexdigest(): + return False + + return True + + def check_pkgs(self) -> None: + pkgcl = [] + for p in self.pkgs: + pkgcl.append(p) + + for p in self.pkgs: + if self.check_pkg(p): + pkgcl.remove(p) + + self.pkgs = pkgcl + + def get_pkglist(self) -> None: + arcname = f"{self.name}.tar.gz" + pkgsurl = self.join_url(arcname) + res = req.get(pkgsurl) + + if res.status_code != 200: + raise BadResponse(f"{res.status_code} - {pkgsurl}") + + f = open(arcname, "wb") + f.write(res.content) + f.close() + + t = tarfile.open(arcname) + for m in t.getmembers(): + if m.name != "pkgs": + continue + + f = t.extractfile(m) + if f == None: continue + self.process_pkgs(f.read().decode("utf-8")) + f.close() + t.close() + + def download_pkg(self, p: Pkg) -> bool: + p.remove_old() + + arcurl = self.join_url(p.archive) + sigurl = self.join_url(p.sig) + + arcres = req.get(arcurl) + sigres = req.get(sigurl) + + if arcres.status_code != 200: + raise BadResponse(f"{arcres.status_code} - {arcurl}") + + if sigres.status_code != 200: + raise BadResponse(f"{sigres.status_code} - {sigurl}") + + arcf = open(p.archive, "wb") + arcf.write(arcres.content) + arcf.close() + + sigf = open(p.sig, "wb") + sigf.write(sigres.content) + sigf.close() + + return True + +if __name__ == "__main__": + log.basicConfig( + format="[%(levelname)s] [%(asctime)s]: %(message)s", + datefmt="%H:%M:%S", + level=log.INFO + ) + + parser = argparse.ArgumentParser( + prog="matter-mirror", + description="Create and manage MatterLinux mirrors", + epilog="Part of matter-tools | https://git.matterlinux.xyz/matter/matter-tools") + + parser.add_argument("-u", help="Repo URI", required=True, dest="uri") + parser.add_argument("-o", help="Download directory", required=True, dest="out") + args = parser.parse_args() + + if not args.uri.startswith("http://") and not args.uri.startswith("https://"): + log.error(f"Bad URI: {args.uri}") + exit(1) + + if not path.exists(args.out): + log.error(f"Out directory not found: {args.out}") + exit(1) + + if not path.isdir(args.out): + log.error(f"Out directory is not a directory: {args.out}") + exit(1) + + try: + chdir(args.out) + except Exception as e: + log.error(f"Cannot change dir: {args.out}") + exit(1) + + try: + repo = Repo(args.uri, args.out) + repo.get_repo() + except Exception as e: + log.error(e) + exit(1) + + log.info(f"Got repo file => {repo.name}:{repo.author}:{repo.pub}") + log.info("Downloading package list") + + try: + repo.get_pkglist() + except Exception as e: + log.error(e) + exit(1) + + all = len(repo.pkgs) + if all == 0: + log.error("Got no valid packages!") + exit(1) + + log.info(f"Got total of {all} packages") + + try: + repo.check_pkgs() + except Exception as e: + log.error(e) + exit(1) + + old = len(repo.pkgs) + if old == 0: + log.info("All packages are up-to-date!") + exit() + + print(f" Up-to-date packages: {all-old} ({int(100*(all-old)/all)}%)") + print(f" New packages: {old} ({int(100*old/all)}%)") + + resc = 0 + for p in repo.pkgs: + try: + log.info(f"({repo.pkgs.index(p)+1}/{len(repo.pkgs)}) Downloading {p.name}") + try: + repo.download_pkg(p) + except KeyboardInterrupt: + log.error("Stopping downloads") + exit(1) + resc += 1 + except Exception as e: + log.error(f"Download failed: {e}") + continue + + log.info(f"Downloaded {resc} out of {old} packages ({int(100*resc/old)}%)")