From a556ed08aa135feb17e5d17214306c696307e83c Mon Sep 17 00:00:00 2001 From: ngn Date: Tue, 13 Aug 2024 03:01:00 +0300 Subject: [PATCH] update: make things work with the new package system --- .gitignore | 4 +- Makefile | 9 +++ README.md | 47 ++++++++++--- compose.yml | 12 ---- lib/config.go | 33 +++++++++ lib/package.go | 86 +++++++++++++++++++++++ lib/pool.go | 143 +++++++++++++++++++++++++++++++++++++ lib/repo.go | 163 ------------------------------------------- lib/util.go | 134 +++++++++++++---------------------- log/log.go | 23 ++++++ main.go | 162 +++++++++++++++++++----------------------- routes/index.go | 75 ++++++++++++++++++++ routes/package.go | 8 +++ templates/index.html | 30 ++++---- 14 files changed, 556 insertions(+), 373 deletions(-) create mode 100644 Makefile delete mode 100644 compose.yml create mode 100644 lib/config.go create mode 100644 lib/package.go create mode 100644 lib/pool.go delete mode 100644 lib/repo.go create mode 100644 log/log.go create mode 100644 routes/index.go create mode 100644 routes/package.go diff --git a/.gitignore b/.gitignore index 5610643..0ce5de6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +docker-compose.yml +compose.yml +config.json tracker -cfg.json diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..80f08df --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +all: tracker + +tracker: */*.go *.go + go build -o $@ + +format: + gofmt -s -w . + +.PHONY: format diff --git a/README.md b/README.md index 1bb6e3c..9c512b3 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,60 @@ -# tracker | MatterLinux Package Tracker -Soruce code of MatterLinux's package tracker, located at +# tracker | MatterLinux package tracker +Soruce code of MatterLinux's package tracker, located at [tracker.matterlinux.xyz](https://tracker.matterlinux.xyz) ### Configuration -Tracker can be configured to track different repos. Configuration -is stored in the `cfg.json` file. Here is the configuration for tracking +Tracker can be configured to track different repos. Configuration +is stored in the `config.json` file. Here is the configuration for tracking official MatterLinux 24 repos: -``` +```json { - "repos": [ + "pools": [ { + "display": "base (stable)", "name": "base", + "dir": "/srv/pools/base", "branch": "main", "source": "https://git.matterlinux.xyz/Matter/base", - "url": "https://24.matterlinux.xyz/base" + "url": "mptp://stable.matterlinux.xyz/base" }, { + "display": "desktop (stable)", "name": "desktop", + "dir": "/srv/pools/desktop", "branch": "main", "source": "https://git.matterlinux.xyz/Matter/desktop", - "url": "https://24.matterlinux.xyz/desktop" + "url": "mptp://stable.matterlinux.xyz/desktop" + }, + { + "display": "server (stable)", + "name": "server", + "dir": "/srv/pools/server", + "branch": "main", + "source": "https://git.matterlinux.xyz/Matter/server", + "url": "mptp://stable.matterlinux.xyz/server" } ] } ``` ### Deployment -Web server can be built and deployed with docker: +Web server can be built and deployed with docker compose using the following +configuration file: +```yaml +version: "3" + +services: + tracker: + image: mattertracker + restart: unless-stopped + build: + context: ./ + ports: + - "127.0.0.1:9877:9877" + volumes: + - "./config.json:/app/config.json:ro" ``` -docker build --tag mattertracker . +After saving the configuration file, you can build and run the docker container: +```bash docker-compose up -d ``` diff --git a/compose.yml b/compose.yml deleted file mode 100644 index f549e6f..0000000 --- a/compose.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3" - -services: - tracker: - image: mattertracker - restart: unless-stopped - build: - context: ./ - ports: - - "127.0.0.1:9877:9877" - volumes: - - "./cfg.json:/app/cfg.json" diff --git a/lib/config.go b/lib/config.go new file mode 100644 index 0000000..56a5175 --- /dev/null +++ b/lib/config.go @@ -0,0 +1,33 @@ +package lib + +import ( + "encoding/json" + "os" +) + +type Config struct { + Pools []Pool `json:"pools"` +} + +func (c *Config) Load(list *[]Package, file string) error { + var ( + content []byte + err error + ) + + if content, err = os.ReadFile(file); err != nil { + return err + } + + if err = json.Unmarshal(content, c); err != nil { + return err + } + + for _, p := range c.Pools { + if err = p.Load(list); err != nil { + return err + } + } + + return nil +} diff --git a/lib/package.go b/lib/package.go new file mode 100644 index 0000000..2d18f21 --- /dev/null +++ b/lib/package.go @@ -0,0 +1,86 @@ +package lib + +import ( + "fmt" + "io" + "net/url" + "strings" + + "github.com/bigkevmcd/go-configparser" +) + +type Package struct { + Name string `json:"name"` + Pool *Pool `json:"-"` + Version string `json:"version"` + Depends []string `json:"depends"` + Size string `json:"size"` + Desc string `json:"desc"` +} + +func (p *Package) URL() string { + if nil == p.Pool { + return "" + } + url, _ := url.JoinPath(p.Pool.Source, "src/branch/"+p.Pool.Branch+"/src", p.Name) + return url +} + +func (p *Package) DependsToStr() string { + var depends string = "" + for _, d := range p.Depends { + depends += fmt.Sprintf("%s ", d) + } + return depends +} + +func (p *Package) Load(r io.Reader) error { + var ( + err error + size int64 + depends string = "" + section string = "DEFAULT" + ) + + parser := configparser.New() + if err = parser.ParseReader(r); err != nil { + return err + } + + for _, s := range parser.Sections() { + if s == "DEFAULT" { + continue + } + section = s + break + } + + if section == "DEFAULT" { + return fmt.Errorf("DATA does not contain any sections") + } + + p.Name = section + + if p.Version, err = parser.Get(section, "version"); err != nil { + return err + } + + if size, err = parser.GetInt64(section, "size"); err != nil { + return err + } + p.Size = SizeFromBytes(size) + + if p.Desc, err = parser.Get(section, "desc"); err != nil { + return err + } + + depends, _ = parser.Get(section, "depends") + + if depends == "" { + p.Depends = []string{} + } else { + p.Depends = strings.Split(depends, ",") + } + + return nil +} diff --git a/lib/pool.go b/lib/pool.go new file mode 100644 index 0000000..3ffe96e --- /dev/null +++ b/lib/pool.go @@ -0,0 +1,143 @@ +package lib + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "fmt" + "io" + "os" + "path" + + "github.com/bigkevmcd/go-configparser" +) + +type Pool struct { + Maintainer string `json:"-"` + Pubkey string `json:"-"` + Size string `json:"-"` + + Display string `json:"display"` + Branch string `json:"branch"` + Source string `json:"source"` + Name string `json:"name"` + URL string `json:"url"` + Dir string `json:"dir"` +} + +func (p *Pool) Load(list *[]Package) error { + var err error + + if p.Dir == "" { + return fmt.Errorf("pool directory is not specified") + } + + if err = p.LoadInfo(); err != nil { + return err + } + + if err = p.LoadList(list); err != nil { + return err + } + + return nil +} + +func (p *Pool) LoadList(list *[]Package) error { + var ( + list_path string + list_file *os.File + gzip_reader io.Reader + header *tar.Header + err error + ) + + list_path = path.Join(p.Dir, "LIST") + + if list_file, err = os.Open(list_path); err != nil { + return err + } + defer list_file.Close() + + if gzip_reader, err = gzip.NewReader(bufio.NewReader(list_file)); err != nil { + return err + } + + reader := tar.NewReader(gzip_reader) + + for header, err = reader.Next(); err == nil; header, err = reader.Next() { + if header.Typeflag != tar.TypeReg { + continue + } + + if path.Base(header.Name) != "DATA" { + return fmt.Errorf("LIST archive contains an unknown file") + } + + var pkg Package + pkg.Pool = p + + if err = pkg.Load(reader); err != nil { + return err + } + + *list = append(*list, pkg) + } + + return nil +} + +func (p *Pool) LoadInfo() error { + var ( + info_path string + info_file *os.File + section string + size int64 + err error + ) + + info_path = path.Join(p.Dir, "INFO") + + if info_file, err = os.Open(info_path); err != nil { + return err + } + + parser := configparser.New() + + if err = parser.ParseReader(bufio.NewReader(info_file)); err != nil { + return err + } + + section = "DEFAULT" + + for _, s := range parser.Sections() { + if s == "DEFAULT" { + continue + } + section = s + break + } + + if section == "DEFAULT" { + return fmt.Errorf("DATA does not contain any sections") + } + + if p.Name != section { + return fmt.Errorf("pool name (\"%s\") doesn't match with \"%s\"", p.Name, section) + } + + if p.Maintainer, err = parser.Get(p.Name, "maintainer"); err != nil { + return err + } + + if size, err = parser.GetInt64(section, "size"); err != nil { + return err + } + p.Size = SizeFromBytes(size) + + if p.Pubkey, err = parser.Get(section, "pubkey"); err != nil { + return err + } + + return nil +} diff --git a/lib/repo.go b/lib/repo.go deleted file mode 100644 index 0003bfb..0000000 --- a/lib/repo.go +++ /dev/null @@ -1,163 +0,0 @@ -package lib - -import ( - "bytes" - "encoding/json" - "errors" - "log" - "net/url" - "os" - "strconv" - "strings" - - "github.com/bigkevmcd/go-configparser" - "github.com/gofiber/fiber/v2" -) - -type Config struct { - Repos []Repo `json:"repos"` -} - -var Repos []Repo -type Repo struct { - Source string `json:"source"` - Branch string `json:"branch"` - Name string `json:"name"` - URL string `json:"url"` -} - -var Packages []Package -type Package struct { - Name string `json:"name"` - Repo string `json:"repo"` - Desc string `json:"desc"` - Size string `json:"size"` - Deps []string `json:"depends"` - URL string `json:"url"` - Ver string `json:"version"` -} - -func (p Package) StrDeps() string { - return ListToStr(p.Deps) -} - -func LoadPackgae(repo *configparser.ConfigParser, s string) (Package, error) { - var ( - pkg Package - err error - ) - pkg.Name = s - - pkg.Ver, err = repo.Get(s, "version") - if err != nil { - return pkg, err - } - - pkg.Desc, err = repo.Get(s, "desc") - if err != nil { - return pkg, err - } - - ssize, err := repo.Get(s, "size") - if err != nil { - return pkg, err - } - - size, err := strconv.Atoi(ssize) - if err != nil { - return pkg, err - } - pkg.Size = SizeFromBytes(size) - - deps, err := repo.Get(s, "depends") - if err != nil { - return pkg, err - } - - if deps == "" { - pkg.Deps = []string{} - }else { - pkg.Deps = strings.Split(deps, "\n") - } - - return pkg, nil -} - -func GetRepo(r Repo) ([]Package, error){ - var pkgs []Package - - furl, err := url.JoinPath(r.URL, "list") - if err != nil { - return pkgs, err - } - - agent := fiber.Get(furl) - code, body, errs := agent.Bytes() - if len(errs) > 0 { - return pkgs, errors.New("Request failed") - } - - if code != 200 { - return pkgs, errors.New("Bad response") - } - - list, err := ExtractFile(body, "pkgs") - if err != nil { - return pkgs, err - } - - repo := configparser.New() - repo.ParseReader(bytes.NewReader([]byte(list))) - - for _, s := range repo.Sections() { - if s == "DEFAULT" { - continue - } - - pkg, err := LoadPackgae(repo, s) - if err != nil { - log.Printf("Error loading %s: %s", s, err) - continue - } - - pkg.Repo = r.Name - pkg.URL, err = url.JoinPath( - r.Source, "src/branch/"+r.Branch+"/src", pkg.Name, "pkg.sh") - pkgs = append(pkgs, pkg) - } - - return pkgs, nil -} - -func LoadRepos() { - Repos = []Repo{} - - data, err := os.ReadFile("cfg.json") - if err != nil { - log.Printf("Failed to read the configuration") - } - - var cfg Config - err = json.Unmarshal(data, &cfg) - if err != nil { - log.Printf("Failed to parse the configuration: %s", err) - } - - Repos = cfg.Repos -} - -func LoadAllPkgs() { - LoadRepos() - Packages = []Package{} - - for _, r := range Repos { - pkgs, err := GetRepo(r) - if err != nil { - log.Printf("Error loading %s: %s", r.Name, err.Error()) - continue - } - Packages = append(Packages, pkgs...) - } - - log.Printf("Loaded total %d packages", len(Packages)) -} diff --git a/lib/util.go b/lib/util.go index 7be9e4d..7488f0e 100644 --- a/lib/util.go +++ b/lib/util.go @@ -1,12 +1,7 @@ package lib import ( - "archive/tar" - "bytes" - "compress/gzip" - "errors" "fmt" - "io" "strings" "time" @@ -14,99 +9,68 @@ import ( ) func ListToStr(l []string) string { - res := "" - for _, e := range l { - res += e+" " - } - return res + res := "" + for _, e := range l { + res += e + " " + } + return res } -func RenderError(c *fiber.Ctx, code int) error{ - var msg string = "Server Error" - c.Status(code) +func RenderError(c *fiber.Ctx, code int) error { + var msg string = "Server Error" + c.Status(code) - switch code { - case 404: - msg = "Not Found" - } + switch code { + case 404: + msg = "Not Found" + } - return c.Render("error", fiber.Map{ - "msg": msg, - }) + return c.Render("error", fiber.Map{ + "msg": msg, + }) } -func SizeFromBytes(size int) string { - if(size > 1024*1024*1024) { - return fmt.Sprintf("%dGB", (size/1024/1024/1024)) - }else if(size > 1024*1024) { - return fmt.Sprintf("%dMB", (size/1024/1024)) - }else if(size > 1024) { - return fmt.Sprintf("%dKB", (size/1024)) - } - return fmt.Sprintf("%dB", size) +func SizeFromBytes(size int64) string { + if size > 1024*1024*1024 { + return fmt.Sprintf("%dGB", (size / 1024 / 1024 / 1024)) + } else if size > 1024*1024 { + return fmt.Sprintf("%dMB", (size / 1024 / 1024)) + } else if size > 1024 { + return fmt.Sprintf("%dKB", (size / 1024)) + } + return fmt.Sprintf("%dB", size) } -func ExtractFile(raw []byte, file string) (string, error){ - stream := bytes.NewReader(raw) - ustream, err := gzip.NewReader(stream) - if err != nil { - return "", err - } +func TimePassed(t time.Time) string { + diff := time.Since(t) + res := fmt.Sprintf( + "%ds ago", + int(diff.Seconds()), + ) - reader := tar.NewReader(ustream) - var header *tar.Header - - for header, err = reader.Next(); err == nil; header, err = reader.Next() { - if header.Typeflag != tar.TypeReg { - return "", errors.New("Found invalid entry in archive") - } + if diff.Minutes() > 1 { + res = fmt.Sprintf( + "%dm and %ds ago", + int(diff.Minutes()), int(diff.Seconds())-(int(diff.Minutes())*60), + ) + } - if header.Name != file { - continue - } + if diff.Hours() > 1 { + res = fmt.Sprintf("%dh and %dm ago", + int(diff.Hours()), + int(diff.Minutes())-(int(diff.Hours())*60), + ) + } - buf := new(strings.Builder) - _, err := io.Copy(buf, reader) - if err != nil { - return "", errors.New("Failed to extract file to memory") - } - - return buf.String(), nil - } - - return "", errors.New("File not found") + return res } -func GetTimePassed(t time.Time) string { - diff := time.Since(t) - res := fmt.Sprintf( - "%ds ago", - int(diff.Seconds()), - ) +func SanitizeXSS(s string) string { + var bad []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"} - if diff.Minutes() > 1 { - res = fmt.Sprintf( - "%dm and %ds ago", - int(diff.Minutes()), int(diff.Seconds())-(int(diff.Minutes())*60), - ) - } + for _, c := range bad { + s = strings.ReplaceAll(s, c, "") + } - if diff.Hours() > 1 { - res = fmt.Sprintf("%dh and %dm ago", - int(diff.Hours()), - int(diff.Minutes())-(int(diff.Hours())*60), - ) - } - - return res -} - -func CleanString(s string) string{ - var badchars []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"} - - for _, c := range badchars { - s = strings.ReplaceAll(s, c, "") - } - - return s + return s } diff --git a/log/log.go b/log/log.go new file mode 100644 index 0000000..e0eb92f --- /dev/null +++ b/log/log.go @@ -0,0 +1,23 @@ +package log + +import ( + "fmt" + "time" +) + +func Log(p string, f string, args ...interface{}) { + now := time.Now() + nstr := now.Format("[02/01/06 15:04:05]") + + fmt.Printf("%s -%s- ", nstr, p) + fmt.Printf(f, args...) + fmt.Println() +} + +func Info(f string, args ...interface{}) { + Log("INFO", f, args...) +} + +func Error(f string, args ...interface{}) { + Log("ERROR", f, args...) +} diff --git a/main.go b/main.go index 55fe711..119c1d5 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ /* - * tracker | MatterLinux Package Tracker + * tracker | MatterLinux package tracker * MatterLinux 2023-2024 (https://matterlinux.xyz) * This program is free software: you can redistribute it and/or modify @@ -21,107 +21,91 @@ package main import ( - "log" - "strings" "time" "git.matterlinux.xyz/matter/tracker/lib" + "git.matterlinux.xyz/matter/tracker/log" + "git.matterlinux.xyz/matter/tracker/routes" "github.com/gofiber/fiber/v2" "github.com/gofiber/template/html/v2" ) -var lastupdate time.Time -var updatetick = time.NewTicker(time.Hour) -var stopchan = make(chan struct{}) - -func UpdateLoop() { - UpdatePackages() - - for { - select { - case <- updatetick.C: - UpdatePackages() - case <- stopchan: - updatetick.Stop() - return - } - } +type tracker struct { + Channel chan struct{} + Config lib.Config + Last time.Time + List []lib.Package + Tick time.Ticker } -func UpdatePackages() { - lib.LoadAllPkgs() - lastupdate = time.Now() +func (t *tracker) Reload() error { + t.List = []lib.Package{} + err := t.Config.Load(&t.List, "config.json") + t.Last = time.Now() + return err } -func GETIndex(c *fiber.Ctx) error { - repo := c.Query("r") - name := c.Query("n") - exact := c.Query("e") - isjson := c.Query("j") - - if repo == "" && name == "" { - if isjson == "1" { - return c.JSON(lib.Packages) - } - return c.Render("index", fiber.Map{ - "last": lib.GetTimePassed(lastupdate), - "repos": lib.Repos, - "pkgs": lib.Packages, - }) - } +func (t *tracker) Loop() { + var err error - name = lib.CleanString(name) - var res []lib.Package + if err = t.Reload(); err != nil { + log.Error("Failed to update packages: %s", err.Error()) + } - for _, p := range lib.Packages { - if(repo != "all" && p.Repo != repo){ - continue - } - - if(exact == ""){ - if(strings.Contains(p.Name, name)){ - res = append(res, p) - } - }else { - if (p.Name == name) { - res = append(res, p) - } - } - } - - if isjson == "1" { - return c.JSON(res) - } - - return c.Render("index", fiber.Map{ - "search": name, - "last": lib.GetTimePassed(lastupdate), - "repos": lib.Repos, - "pkgs": res, - }) + for { + select { + case <-t.Tick.C: + if err = t.Reload(); err != nil { + log.Error("Failed to update packages: %s", err.Error()) + } + case <-t.Channel: + t.Tick.Stop() + return + } + } } -func main(){ - log.SetFlags(log.Ltime | log.Lshortfile) - - engine := html.New("./templates", ".html") - app := fiber.New(fiber.Config{ - DisableStartupMessage: true, - Views: engine, - }) - - app.Static("/", "./public") - app.Get("/", GETIndex) - - app.Get("*", func(c *fiber.Ctx) error { - return lib.RenderError(c, 404) - }) - - go UpdateLoop() - log.Println("Starting MatterLinux Package Tracker on port 9877") - err := app.Listen(":9877") - if err != nil { - log.Printf("Error starting server: %s", err) - } - close(stopchan) +func (t *tracker) Stop() { + close(t.Channel) +} + +func main() { + var ( + tracker tracker + engine *html.Engine + app *fiber.App + ) + + engine = html.New("./templates", ".html") + app = fiber.New(fiber.Config{ + DisableStartupMessage: true, + Views: engine, + }) + + app.Static("/", "./public") + + app.All("*", func(c *fiber.Ctx) error { + c.Locals("config", &tracker.Config) + c.Locals("list", &tracker.List) + c.Locals("last", &tracker.Last) + return c.Next() + }) + + app.Get("/", routes.GET_index) + app.Get("/p/:name", routes.GET_package) + + app.Get("*", func(c *fiber.Ctx) error { + return lib.RenderError(c, 404) + }) + + go tracker.Loop() + + log.Info("Starting MatterLinux package tracker on port 9877") + + err := app.Listen(":9877") + if err != nil { + log.Error("Error starting server: %s", err.Error()) + } + + tracker.Stop() } diff --git a/routes/index.go b/routes/index.go new file mode 100644 index 0000000..b3538a6 --- /dev/null +++ b/routes/index.go @@ -0,0 +1,75 @@ +package routes + +import ( + "strings" + "time" + + "git.matterlinux.xyz/matter/tracker/lib" + "github.com/gofiber/fiber/v2" +) + +func GET_index(c *fiber.Ctx) error { + var ( + result []lib.Package + list *[]lib.Package + config *lib.Config + pools_str string + pools []string + last *time.Time + ) + + result = []lib.Package{} + pools = []string{} + + config = c.Locals("config").(*lib.Config) + list = c.Locals("list").(*[]lib.Package) + last = c.Locals("last").(*time.Time) + + is_exact := c.Query("exact") == "1" + is_json := c.Query("json") == "1" + + if pools_str = c.Query("pools"); pools_str != "" { + pools = strings.Split(pools_str, ",") + } + query := c.Query("q") + + for _, pkg := range *list { + found := false + + if is_exact && query != "" && pkg.Name != query { + continue + } + + if !is_exact && query != "" && !strings.Contains(pkg.Name, query) { + continue + } + + if len(pools) != 0 { + for _, p := range pools { + if p == pkg.Pool.Name { + found = true + break + } + } + } else { + found = true + } + + if !found { + continue + } + + result = append(result, pkg) + } + + if is_json { + return c.JSON(result) + } + + return c.Render("index", &fiber.Map{ + "query": query, + "list": result, + "last": lib.TimePassed(*last), + "pools": config.Pools, + }) +} diff --git a/routes/package.go b/routes/package.go new file mode 100644 index 0000000..f045e6a --- /dev/null +++ b/routes/package.go @@ -0,0 +1,8 @@ +package routes + +import "github.com/gofiber/fiber/v2" + +func GET_package(c *fiber.Ctx) error { + //name := c.Params("name") + return c.Render("package", &fiber.Map{}) +} diff --git a/templates/index.html b/templates/index.html index 221b7f7..d5739a9 100644 --- a/templates/index.html +++ b/templates/index.html @@ -11,20 +11,20 @@
- {{if .search}} - + {{if .query}} + {{else}} - + {{end}} - + + {{range .pools}} + {{end}}
-

Listing {{len .pkgs}} packages

+

Listing {{len .list}} packages

Last updated: {{.last}}

@@ -32,20 +32,24 @@ - + - {{range .pkgs}} + {{range .list}} - - + {{if .Pool}} + + {{else}} + + {{end}} + - + {{end}}
NameRepoPool Version Size Description Dependencies
{{.Name}}{{.Repo}}{{.Ver }}{{.Pool.Display}}{{.Version }} {{.Size}} {{.Desc}}{{.StrDeps}}{{.DependsToStr}}