update: make things work with the new package system

This commit is contained in:
ngn 2024-08-13 03:01:00 +03:00
parent 528df304e9
commit a556ed08aa
14 changed files with 556 additions and 373 deletions

4
.gitignore vendored
View File

@ -1,2 +1,4 @@
docker-compose.yml
compose.yml
config.json
tracker tracker
cfg.json

9
Makefile Normal file
View File

@ -0,0 +1,9 @@
all: tracker
tracker: */*.go *.go
go build -o $@
format:
gofmt -s -w .
.PHONY: format

View File

@ -1,33 +1,60 @@
# tracker | MatterLinux Package Tracker # tracker | MatterLinux package tracker
Soruce code of MatterLinux's package tracker, located at Soruce code of MatterLinux's package tracker, located at
[tracker.matterlinux.xyz](https://tracker.matterlinux.xyz) [tracker.matterlinux.xyz](https://tracker.matterlinux.xyz)
### Configuration ### Configuration
Tracker can be configured to track different repos. Configuration Tracker can be configured to track different repos. Configuration
is stored in the `cfg.json` file. Here is the configuration for tracking is stored in the `config.json` file. Here is the configuration for tracking
official MatterLinux 24 repos: official MatterLinux 24 repos:
``` ```json
{ {
"repos": [ "pools": [
{ {
"display": "base (stable)",
"name": "base", "name": "base",
"dir": "/srv/pools/base",
"branch": "main", "branch": "main",
"source": "https://git.matterlinux.xyz/Matter/base", "source": "https://git.matterlinux.xyz/Matter/base",
"url": "https://24.matterlinux.xyz/base" "url": "mptp://stable.matterlinux.xyz/base"
}, },
{ {
"display": "desktop (stable)",
"name": "desktop", "name": "desktop",
"dir": "/srv/pools/desktop",
"branch": "main", "branch": "main",
"source": "https://git.matterlinux.xyz/Matter/desktop", "source": "https://git.matterlinux.xyz/Matter/desktop",
"url": "https://24.matterlinux.xyz/desktop" "url": "mptp://stable.matterlinux.xyz/desktop"
},
{
"display": "server (stable)",
"name": "server",
"dir": "/srv/pools/server",
"branch": "main",
"source": "https://git.matterlinux.xyz/Matter/server",
"url": "mptp://stable.matterlinux.xyz/server"
} }
] ]
} }
``` ```
### Deployment ### Deployment
Web server can be built and deployed with docker: Web server can be built and deployed with docker compose using the following
configuration file:
```yaml
version: "3"
services:
tracker:
image: mattertracker
restart: unless-stopped
build:
context: ./
ports:
- "127.0.0.1:9877:9877"
volumes:
- "./config.json:/app/config.json:ro"
``` ```
docker build --tag mattertracker . After saving the configuration file, you can build and run the docker container:
```bash
docker-compose up -d docker-compose up -d
``` ```

View File

@ -1,12 +0,0 @@
version: "3"
services:
tracker:
image: mattertracker
restart: unless-stopped
build:
context: ./
ports:
- "127.0.0.1:9877:9877"
volumes:
- "./cfg.json:/app/cfg.json"

33
lib/config.go Normal file
View File

@ -0,0 +1,33 @@
package lib
import (
"encoding/json"
"os"
)
type Config struct {
Pools []Pool `json:"pools"`
}
func (c *Config) Load(list *[]Package, file string) error {
var (
content []byte
err error
)
if content, err = os.ReadFile(file); err != nil {
return err
}
if err = json.Unmarshal(content, c); err != nil {
return err
}
for _, p := range c.Pools {
if err = p.Load(list); err != nil {
return err
}
}
return nil
}

86
lib/package.go Normal file
View File

@ -0,0 +1,86 @@
package lib
import (
"fmt"
"io"
"net/url"
"strings"
"github.com/bigkevmcd/go-configparser"
)
type Package struct {
Name string `json:"name"`
Pool *Pool `json:"-"`
Version string `json:"version"`
Depends []string `json:"depends"`
Size string `json:"size"`
Desc string `json:"desc"`
}
func (p *Package) URL() string {
if nil == p.Pool {
return ""
}
url, _ := url.JoinPath(p.Pool.Source, "src/branch/"+p.Pool.Branch+"/src", p.Name)
return url
}
func (p *Package) DependsToStr() string {
var depends string = ""
for _, d := range p.Depends {
depends += fmt.Sprintf("%s ", d)
}
return depends
}
func (p *Package) Load(r io.Reader) error {
var (
err error
size int64
depends string = ""
section string = "DEFAULT"
)
parser := configparser.New()
if err = parser.ParseReader(r); err != nil {
return err
}
for _, s := range parser.Sections() {
if s == "DEFAULT" {
continue
}
section = s
break
}
if section == "DEFAULT" {
return fmt.Errorf("DATA does not contain any sections")
}
p.Name = section
if p.Version, err = parser.Get(section, "version"); err != nil {
return err
}
if size, err = parser.GetInt64(section, "size"); err != nil {
return err
}
p.Size = SizeFromBytes(size)
if p.Desc, err = parser.Get(section, "desc"); err != nil {
return err
}
depends, _ = parser.Get(section, "depends")
if depends == "" {
p.Depends = []string{}
} else {
p.Depends = strings.Split(depends, ",")
}
return nil
}

143
lib/pool.go Normal file
View File

@ -0,0 +1,143 @@
package lib
import (
"archive/tar"
"bufio"
"compress/gzip"
"fmt"
"io"
"os"
"path"
"github.com/bigkevmcd/go-configparser"
)
type Pool struct {
Maintainer string `json:"-"`
Pubkey string `json:"-"`
Size string `json:"-"`
Display string `json:"display"`
Branch string `json:"branch"`
Source string `json:"source"`
Name string `json:"name"`
URL string `json:"url"`
Dir string `json:"dir"`
}
func (p *Pool) Load(list *[]Package) error {
var err error
if p.Dir == "" {
return fmt.Errorf("pool directory is not specified")
}
if err = p.LoadInfo(); err != nil {
return err
}
if err = p.LoadList(list); err != nil {
return err
}
return nil
}
func (p *Pool) LoadList(list *[]Package) error {
var (
list_path string
list_file *os.File
gzip_reader io.Reader
header *tar.Header
err error
)
list_path = path.Join(p.Dir, "LIST")
if list_file, err = os.Open(list_path); err != nil {
return err
}
defer list_file.Close()
if gzip_reader, err = gzip.NewReader(bufio.NewReader(list_file)); err != nil {
return err
}
reader := tar.NewReader(gzip_reader)
for header, err = reader.Next(); err == nil; header, err = reader.Next() {
if header.Typeflag != tar.TypeReg {
continue
}
if path.Base(header.Name) != "DATA" {
return fmt.Errorf("LIST archive contains an unknown file")
}
var pkg Package
pkg.Pool = p
if err = pkg.Load(reader); err != nil {
return err
}
*list = append(*list, pkg)
}
return nil
}
func (p *Pool) LoadInfo() error {
var (
info_path string
info_file *os.File
section string
size int64
err error
)
info_path = path.Join(p.Dir, "INFO")
if info_file, err = os.Open(info_path); err != nil {
return err
}
parser := configparser.New()
if err = parser.ParseReader(bufio.NewReader(info_file)); err != nil {
return err
}
section = "DEFAULT"
for _, s := range parser.Sections() {
if s == "DEFAULT" {
continue
}
section = s
break
}
if section == "DEFAULT" {
return fmt.Errorf("DATA does not contain any sections")
}
if p.Name != section {
return fmt.Errorf("pool name (\"%s\") doesn't match with \"%s\"", p.Name, section)
}
if p.Maintainer, err = parser.Get(p.Name, "maintainer"); err != nil {
return err
}
if size, err = parser.GetInt64(section, "size"); err != nil {
return err
}
p.Size = SizeFromBytes(size)
if p.Pubkey, err = parser.Get(section, "pubkey"); err != nil {
return err
}
return nil
}

View File

@ -1,163 +0,0 @@
package lib
import (
"bytes"
"encoding/json"
"errors"
"log"
"net/url"
"os"
"strconv"
"strings"
"github.com/bigkevmcd/go-configparser"
"github.com/gofiber/fiber/v2"
)
type Config struct {
Repos []Repo `json:"repos"`
}
var Repos []Repo
type Repo struct {
Source string `json:"source"`
Branch string `json:"branch"`
Name string `json:"name"`
URL string `json:"url"`
}
var Packages []Package
type Package struct {
Name string `json:"name"`
Repo string `json:"repo"`
Desc string `json:"desc"`
Size string `json:"size"`
Deps []string `json:"depends"`
URL string `json:"url"`
Ver string `json:"version"`
}
func (p Package) StrDeps() string {
return ListToStr(p.Deps)
}
func LoadPackgae(repo *configparser.ConfigParser, s string) (Package, error) {
var (
pkg Package
err error
)
pkg.Name = s
pkg.Ver, err = repo.Get(s, "version")
if err != nil {
return pkg, err
}
pkg.Desc, err = repo.Get(s, "desc")
if err != nil {
return pkg, err
}
ssize, err := repo.Get(s, "size")
if err != nil {
return pkg, err
}
size, err := strconv.Atoi(ssize)
if err != nil {
return pkg, err
}
pkg.Size = SizeFromBytes(size)
deps, err := repo.Get(s, "depends")
if err != nil {
return pkg, err
}
if deps == "" {
pkg.Deps = []string{}
}else {
pkg.Deps = strings.Split(deps, "\n")
}
return pkg, nil
}
func GetRepo(r Repo) ([]Package, error){
var pkgs []Package
furl, err := url.JoinPath(r.URL, "list")
if err != nil {
return pkgs, err
}
agent := fiber.Get(furl)
code, body, errs := agent.Bytes()
if len(errs) > 0 {
return pkgs, errors.New("Request failed")
}
if code != 200 {
return pkgs, errors.New("Bad response")
}
list, err := ExtractFile(body, "pkgs")
if err != nil {
return pkgs, err
}
repo := configparser.New()
repo.ParseReader(bytes.NewReader([]byte(list)))
for _, s := range repo.Sections() {
if s == "DEFAULT" {
continue
}
pkg, err := LoadPackgae(repo, s)
if err != nil {
log.Printf("Error loading %s: %s", s, err)
continue
}
pkg.Repo = r.Name
pkg.URL, err = url.JoinPath(
r.Source, "src/branch/"+r.Branch+"/src", pkg.Name, "pkg.sh")
pkgs = append(pkgs, pkg)
}
return pkgs, nil
}
func LoadRepos() {
Repos = []Repo{}
data, err := os.ReadFile("cfg.json")
if err != nil {
log.Printf("Failed to read the configuration")
}
var cfg Config
err = json.Unmarshal(data, &cfg)
if err != nil {
log.Printf("Failed to parse the configuration: %s", err)
}
Repos = cfg.Repos
}
func LoadAllPkgs() {
LoadRepos()
Packages = []Package{}
for _, r := range Repos {
pkgs, err := GetRepo(r)
if err != nil {
log.Printf("Error loading %s: %s", r.Name, err.Error())
continue
}
Packages = append(Packages, pkgs...)
}
log.Printf("Loaded total %d packages", len(Packages))
}

View File

@ -1,12 +1,7 @@
package lib package lib
import ( import (
"archive/tar"
"bytes"
"compress/gzip"
"errors"
"fmt" "fmt"
"io"
"strings" "strings"
"time" "time"
@ -16,12 +11,12 @@ import (
func ListToStr(l []string) string { func ListToStr(l []string) string {
res := "" res := ""
for _, e := range l { for _, e := range l {
res += e+" " res += e + " "
} }
return res return res
} }
func RenderError(c *fiber.Ctx, code int) error{ func RenderError(c *fiber.Ctx, code int) error {
var msg string = "Server Error" var msg string = "Server Error"
c.Status(code) c.Status(code)
@ -35,49 +30,18 @@ func RenderError(c *fiber.Ctx, code int) error{
}) })
} }
func SizeFromBytes(size int) string { func SizeFromBytes(size int64) string {
if(size > 1024*1024*1024) { if size > 1024*1024*1024 {
return fmt.Sprintf("%dGB", (size/1024/1024/1024)) return fmt.Sprintf("%dGB", (size / 1024 / 1024 / 1024))
}else if(size > 1024*1024) { } else if size > 1024*1024 {
return fmt.Sprintf("%dMB", (size/1024/1024)) return fmt.Sprintf("%dMB", (size / 1024 / 1024))
}else if(size > 1024) { } else if size > 1024 {
return fmt.Sprintf("%dKB", (size/1024)) return fmt.Sprintf("%dKB", (size / 1024))
} }
return fmt.Sprintf("%dB", size) return fmt.Sprintf("%dB", size)
} }
func ExtractFile(raw []byte, file string) (string, error){ func TimePassed(t time.Time) string {
stream := bytes.NewReader(raw)
ustream, err := gzip.NewReader(stream)
if err != nil {
return "", err
}
reader := tar.NewReader(ustream)
var header *tar.Header
for header, err = reader.Next(); err == nil; header, err = reader.Next() {
if header.Typeflag != tar.TypeReg {
return "", errors.New("Found invalid entry in archive")
}
if header.Name != file {
continue
}
buf := new(strings.Builder)
_, err := io.Copy(buf, reader)
if err != nil {
return "", errors.New("Failed to extract file to memory")
}
return buf.String(), nil
}
return "", errors.New("File not found")
}
func GetTimePassed(t time.Time) string {
diff := time.Since(t) diff := time.Since(t)
res := fmt.Sprintf( res := fmt.Sprintf(
"%ds ago", "%ds ago",
@ -101,10 +65,10 @@ func GetTimePassed(t time.Time) string {
return res return res
} }
func CleanString(s string) string{ func SanitizeXSS(s string) string {
var badchars []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"} var bad []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"}
for _, c := range badchars { for _, c := range bad {
s = strings.ReplaceAll(s, c, "") s = strings.ReplaceAll(s, c, "")
} }

23
log/log.go Normal file
View File

@ -0,0 +1,23 @@
package log
import (
"fmt"
"time"
)
func Log(p string, f string, args ...interface{}) {
now := time.Now()
nstr := now.Format("[02/01/06 15:04:05]")
fmt.Printf("%s -%s- ", nstr, p)
fmt.Printf(f, args...)
fmt.Println()
}
func Info(f string, args ...interface{}) {
Log("INFO", f, args...)
}
func Error(f string, args ...interface{}) {
Log("ERROR", f, args...)
}

128
main.go
View File

@ -1,6 +1,6 @@
/* /*
* tracker | MatterLinux Package Tracker * tracker | MatterLinux package tracker
* MatterLinux 2023-2024 (https://matterlinux.xyz) * MatterLinux 2023-2024 (https://matterlinux.xyz)
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
@ -21,107 +21,91 @@
package main package main
import ( import (
"log"
"strings"
"time" "time"
"git.matterlinux.xyz/matter/tracker/lib" "git.matterlinux.xyz/matter/tracker/lib"
"git.matterlinux.xyz/matter/tracker/log"
"git.matterlinux.xyz/matter/tracker/routes"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"
"github.com/gofiber/template/html/v2" "github.com/gofiber/template/html/v2"
) )
var lastupdate time.Time type tracker struct {
var updatetick = time.NewTicker(time.Hour) Channel chan struct{}
var stopchan = make(chan struct{}) Config lib.Config
Last time.Time
List []lib.Package
Tick time.Ticker
}
func UpdateLoop() { func (t *tracker) Reload() error {
UpdatePackages() t.List = []lib.Package{}
err := t.Config.Load(&t.List, "config.json")
t.Last = time.Now()
return err
}
func (t *tracker) Loop() {
var err error
if err = t.Reload(); err != nil {
log.Error("Failed to update packages: %s", err.Error())
}
for { for {
select { select {
case <- updatetick.C: case <-t.Tick.C:
UpdatePackages() if err = t.Reload(); err != nil {
case <- stopchan: log.Error("Failed to update packages: %s", err.Error())
updatetick.Stop() }
case <-t.Channel:
t.Tick.Stop()
return return
} }
} }
} }
func UpdatePackages() { func (t *tracker) Stop() {
lib.LoadAllPkgs() close(t.Channel)
lastupdate = time.Now()
} }
func GETIndex(c *fiber.Ctx) error { func main() {
repo := c.Query("r") var (
name := c.Query("n") tracker tracker
exact := c.Query("e") engine *html.Engine
isjson := c.Query("j") app *fiber.App
)
if repo == "" && name == "" { engine = html.New("./templates", ".html")
if isjson == "1" { app = fiber.New(fiber.Config{
return c.JSON(lib.Packages)
}
return c.Render("index", fiber.Map{
"last": lib.GetTimePassed(lastupdate),
"repos": lib.Repos,
"pkgs": lib.Packages,
})
}
name = lib.CleanString(name)
var res []lib.Package
for _, p := range lib.Packages {
if(repo != "all" && p.Repo != repo){
continue
}
if(exact == ""){
if(strings.Contains(p.Name, name)){
res = append(res, p)
}
}else {
if (p.Name == name) {
res = append(res, p)
}
}
}
if isjson == "1" {
return c.JSON(res)
}
return c.Render("index", fiber.Map{
"search": name,
"last": lib.GetTimePassed(lastupdate),
"repos": lib.Repos,
"pkgs": res,
})
}
func main(){
log.SetFlags(log.Ltime | log.Lshortfile)
engine := html.New("./templates", ".html")
app := fiber.New(fiber.Config{
DisableStartupMessage: true, DisableStartupMessage: true,
Views: engine, Views: engine,
}) })
app.Static("/", "./public") app.Static("/", "./public")
app.Get("/", GETIndex)
app.All("*", func(c *fiber.Ctx) error {
c.Locals("config", &tracker.Config)
c.Locals("list", &tracker.List)
c.Locals("last", &tracker.Last)
return c.Next()
})
app.Get("/", routes.GET_index)
app.Get("/p/:name", routes.GET_package)
app.Get("*", func(c *fiber.Ctx) error { app.Get("*", func(c *fiber.Ctx) error {
return lib.RenderError(c, 404) return lib.RenderError(c, 404)
}) })
go UpdateLoop() go tracker.Loop()
log.Println("Starting MatterLinux Package Tracker on port 9877")
log.Info("Starting MatterLinux package tracker on port 9877")
err := app.Listen(":9877") err := app.Listen(":9877")
if err != nil { if err != nil {
log.Printf("Error starting server: %s", err) log.Error("Error starting server: %s", err.Error())
} }
close(stopchan)
tracker.Stop()
} }

75
routes/index.go Normal file
View File

@ -0,0 +1,75 @@
package routes
import (
"strings"
"time"
"git.matterlinux.xyz/matter/tracker/lib"
"github.com/gofiber/fiber/v2"
)
func GET_index(c *fiber.Ctx) error {
var (
result []lib.Package
list *[]lib.Package
config *lib.Config
pools_str string
pools []string
last *time.Time
)
result = []lib.Package{}
pools = []string{}
config = c.Locals("config").(*lib.Config)
list = c.Locals("list").(*[]lib.Package)
last = c.Locals("last").(*time.Time)
is_exact := c.Query("exact") == "1"
is_json := c.Query("json") == "1"
if pools_str = c.Query("pools"); pools_str != "" {
pools = strings.Split(pools_str, ",")
}
query := c.Query("q")
for _, pkg := range *list {
found := false
if is_exact && query != "" && pkg.Name != query {
continue
}
if !is_exact && query != "" && !strings.Contains(pkg.Name, query) {
continue
}
if len(pools) != 0 {
for _, p := range pools {
if p == pkg.Pool.Name {
found = true
break
}
}
} else {
found = true
}
if !found {
continue
}
result = append(result, pkg)
}
if is_json {
return c.JSON(result)
}
return c.Render("index", &fiber.Map{
"query": query,
"list": result,
"last": lib.TimePassed(*last),
"pools": config.Pools,
})
}

8
routes/package.go Normal file
View File

@ -0,0 +1,8 @@
package routes
import "github.com/gofiber/fiber/v2"
func GET_package(c *fiber.Ctx) error {
//name := c.Params("name")
return c.Render("package", &fiber.Map{})
}

View File

@ -11,20 +11,20 @@
<main> <main>
<div class="search"> <div class="search">
<form action="/" method="GET"> <form action="/" method="GET">
{{if .search}} {{if .query}}
<input placeholder="Hit enter to search" type="text" name="n" value="{{.search}}"> <input placeholder="Hit enter to search" type="text" name="q" value="{{.query}}">
{{else}} {{else}}
<input placeholder="Hit enter to search" type="text" name="n" autofocus> <input placeholder="Hit enter to search" type="text" name="q" autofocus>
{{end}} {{end}}
<select name="r"> <select name="pools">
<option value="all">Select repo</option> <option value="">Select pool</option>
{{range .repos}} {{range .pools}}
<option value="{{.Name}}">{{.Name}}</option> <option value="{{.Name}}">{{.Display}}</option>
{{end}} {{end}}
</select> </select>
</form> </form>
<div class="status"> <div class="status">
<p>Listing {{len .pkgs}} packages</p> <p>Listing {{len .list}} packages</p>
<p>Last updated: {{.last}}</p> <p>Last updated: {{.last}}</p>
</div> </div>
</div> </div>
@ -32,20 +32,24 @@
<table class="pkgs"> <table class="pkgs">
<tr> <tr>
<th>Name</th> <th>Name</th>
<th>Repo</th> <th>Pool</th>
<th>Version</th> <th>Version</th>
<th>Size</th> <th>Size</th>
<th>Description</th> <th>Description</th>
<th>Dependencies</th> <th>Dependencies</th>
</tr> </tr>
{{range .pkgs}} {{range .list}}
<tr> <tr>
<td><a href="{{.URL}}">{{.Name}}</a></td> <td><a href="{{.URL}}">{{.Name}}</a></td>
<td>{{.Repo}}</td> {{if .Pool}}
<td>{{.Ver }}</td> <td>{{.Pool.Display}}</td>
{{else}}
<td></td>
{{end}}
<td>{{.Version }}</td>
<td>{{.Size}}</td> <td>{{.Size}}</td>
<td>{{.Desc}}</td> <td>{{.Desc}}</td>
<td>{{.StrDeps}}</td> <td>{{.DependsToStr}}</td>
</tr> </tr>
{{end}} {{end}}
</table> </table>