update: make things work with the new package system

This commit is contained in:
ngn 2024-08-13 03:01:00 +03:00
parent 528df304e9
commit a556ed08aa
14 changed files with 556 additions and 373 deletions

4
.gitignore vendored
View File

@ -1,2 +1,4 @@
docker-compose.yml
compose.yml
config.json
tracker
cfg.json

9
Makefile Normal file
View File

@ -0,0 +1,9 @@
all: tracker
tracker: */*.go *.go
go build -o $@
format:
gofmt -s -w .
.PHONY: format

View File

@ -1,33 +1,60 @@
# tracker | MatterLinux Package Tracker
# tracker | MatterLinux package tracker
Soruce code of MatterLinux's package tracker, located at
[tracker.matterlinux.xyz](https://tracker.matterlinux.xyz)
### Configuration
Tracker can be configured to track different repos. Configuration
is stored in the `cfg.json` file. Here is the configuration for tracking
is stored in the `config.json` file. Here is the configuration for tracking
official MatterLinux 24 repos:
```
```json
{
"repos": [
"pools": [
{
"display": "base (stable)",
"name": "base",
"dir": "/srv/pools/base",
"branch": "main",
"source": "https://git.matterlinux.xyz/Matter/base",
"url": "https://24.matterlinux.xyz/base"
"url": "mptp://stable.matterlinux.xyz/base"
},
{
"display": "desktop (stable)",
"name": "desktop",
"dir": "/srv/pools/desktop",
"branch": "main",
"source": "https://git.matterlinux.xyz/Matter/desktop",
"url": "https://24.matterlinux.xyz/desktop"
"url": "mptp://stable.matterlinux.xyz/desktop"
},
{
"display": "server (stable)",
"name": "server",
"dir": "/srv/pools/server",
"branch": "main",
"source": "https://git.matterlinux.xyz/Matter/server",
"url": "mptp://stable.matterlinux.xyz/server"
}
]
}
```
### Deployment
Web server can be built and deployed with docker:
Web server can be built and deployed with docker compose using the following
configuration file:
```yaml
version: "3"
services:
tracker:
image: mattertracker
restart: unless-stopped
build:
context: ./
ports:
- "127.0.0.1:9877:9877"
volumes:
- "./config.json:/app/config.json:ro"
```
docker build --tag mattertracker .
After saving the configuration file, you can build and run the docker container:
```bash
docker-compose up -d
```

View File

@ -1,12 +0,0 @@
version: "3"
services:
tracker:
image: mattertracker
restart: unless-stopped
build:
context: ./
ports:
- "127.0.0.1:9877:9877"
volumes:
- "./cfg.json:/app/cfg.json"

33
lib/config.go Normal file
View File

@ -0,0 +1,33 @@
package lib
import (
"encoding/json"
"os"
)
type Config struct {
Pools []Pool `json:"pools"`
}
func (c *Config) Load(list *[]Package, file string) error {
var (
content []byte
err error
)
if content, err = os.ReadFile(file); err != nil {
return err
}
if err = json.Unmarshal(content, c); err != nil {
return err
}
for _, p := range c.Pools {
if err = p.Load(list); err != nil {
return err
}
}
return nil
}

86
lib/package.go Normal file
View File

@ -0,0 +1,86 @@
package lib
import (
"fmt"
"io"
"net/url"
"strings"
"github.com/bigkevmcd/go-configparser"
)
type Package struct {
Name string `json:"name"`
Pool *Pool `json:"-"`
Version string `json:"version"`
Depends []string `json:"depends"`
Size string `json:"size"`
Desc string `json:"desc"`
}
func (p *Package) URL() string {
if nil == p.Pool {
return ""
}
url, _ := url.JoinPath(p.Pool.Source, "src/branch/"+p.Pool.Branch+"/src", p.Name)
return url
}
func (p *Package) DependsToStr() string {
var depends string = ""
for _, d := range p.Depends {
depends += fmt.Sprintf("%s ", d)
}
return depends
}
func (p *Package) Load(r io.Reader) error {
var (
err error
size int64
depends string = ""
section string = "DEFAULT"
)
parser := configparser.New()
if err = parser.ParseReader(r); err != nil {
return err
}
for _, s := range parser.Sections() {
if s == "DEFAULT" {
continue
}
section = s
break
}
if section == "DEFAULT" {
return fmt.Errorf("DATA does not contain any sections")
}
p.Name = section
if p.Version, err = parser.Get(section, "version"); err != nil {
return err
}
if size, err = parser.GetInt64(section, "size"); err != nil {
return err
}
p.Size = SizeFromBytes(size)
if p.Desc, err = parser.Get(section, "desc"); err != nil {
return err
}
depends, _ = parser.Get(section, "depends")
if depends == "" {
p.Depends = []string{}
} else {
p.Depends = strings.Split(depends, ",")
}
return nil
}

143
lib/pool.go Normal file
View File

@ -0,0 +1,143 @@
package lib
import (
"archive/tar"
"bufio"
"compress/gzip"
"fmt"
"io"
"os"
"path"
"github.com/bigkevmcd/go-configparser"
)
type Pool struct {
Maintainer string `json:"-"`
Pubkey string `json:"-"`
Size string `json:"-"`
Display string `json:"display"`
Branch string `json:"branch"`
Source string `json:"source"`
Name string `json:"name"`
URL string `json:"url"`
Dir string `json:"dir"`
}
func (p *Pool) Load(list *[]Package) error {
var err error
if p.Dir == "" {
return fmt.Errorf("pool directory is not specified")
}
if err = p.LoadInfo(); err != nil {
return err
}
if err = p.LoadList(list); err != nil {
return err
}
return nil
}
func (p *Pool) LoadList(list *[]Package) error {
var (
list_path string
list_file *os.File
gzip_reader io.Reader
header *tar.Header
err error
)
list_path = path.Join(p.Dir, "LIST")
if list_file, err = os.Open(list_path); err != nil {
return err
}
defer list_file.Close()
if gzip_reader, err = gzip.NewReader(bufio.NewReader(list_file)); err != nil {
return err
}
reader := tar.NewReader(gzip_reader)
for header, err = reader.Next(); err == nil; header, err = reader.Next() {
if header.Typeflag != tar.TypeReg {
continue
}
if path.Base(header.Name) != "DATA" {
return fmt.Errorf("LIST archive contains an unknown file")
}
var pkg Package
pkg.Pool = p
if err = pkg.Load(reader); err != nil {
return err
}
*list = append(*list, pkg)
}
return nil
}
func (p *Pool) LoadInfo() error {
var (
info_path string
info_file *os.File
section string
size int64
err error
)
info_path = path.Join(p.Dir, "INFO")
if info_file, err = os.Open(info_path); err != nil {
return err
}
parser := configparser.New()
if err = parser.ParseReader(bufio.NewReader(info_file)); err != nil {
return err
}
section = "DEFAULT"
for _, s := range parser.Sections() {
if s == "DEFAULT" {
continue
}
section = s
break
}
if section == "DEFAULT" {
return fmt.Errorf("DATA does not contain any sections")
}
if p.Name != section {
return fmt.Errorf("pool name (\"%s\") doesn't match with \"%s\"", p.Name, section)
}
if p.Maintainer, err = parser.Get(p.Name, "maintainer"); err != nil {
return err
}
if size, err = parser.GetInt64(section, "size"); err != nil {
return err
}
p.Size = SizeFromBytes(size)
if p.Pubkey, err = parser.Get(section, "pubkey"); err != nil {
return err
}
return nil
}

View File

@ -1,163 +0,0 @@
package lib
import (
"bytes"
"encoding/json"
"errors"
"log"
"net/url"
"os"
"strconv"
"strings"
"github.com/bigkevmcd/go-configparser"
"github.com/gofiber/fiber/v2"
)
type Config struct {
Repos []Repo `json:"repos"`
}
var Repos []Repo
type Repo struct {
Source string `json:"source"`
Branch string `json:"branch"`
Name string `json:"name"`
URL string `json:"url"`
}
var Packages []Package
type Package struct {
Name string `json:"name"`
Repo string `json:"repo"`
Desc string `json:"desc"`
Size string `json:"size"`
Deps []string `json:"depends"`
URL string `json:"url"`
Ver string `json:"version"`
}
func (p Package) StrDeps() string {
return ListToStr(p.Deps)
}
func LoadPackgae(repo *configparser.ConfigParser, s string) (Package, error) {
var (
pkg Package
err error
)
pkg.Name = s
pkg.Ver, err = repo.Get(s, "version")
if err != nil {
return pkg, err
}
pkg.Desc, err = repo.Get(s, "desc")
if err != nil {
return pkg, err
}
ssize, err := repo.Get(s, "size")
if err != nil {
return pkg, err
}
size, err := strconv.Atoi(ssize)
if err != nil {
return pkg, err
}
pkg.Size = SizeFromBytes(size)
deps, err := repo.Get(s, "depends")
if err != nil {
return pkg, err
}
if deps == "" {
pkg.Deps = []string{}
}else {
pkg.Deps = strings.Split(deps, "\n")
}
return pkg, nil
}
func GetRepo(r Repo) ([]Package, error){
var pkgs []Package
furl, err := url.JoinPath(r.URL, "list")
if err != nil {
return pkgs, err
}
agent := fiber.Get(furl)
code, body, errs := agent.Bytes()
if len(errs) > 0 {
return pkgs, errors.New("Request failed")
}
if code != 200 {
return pkgs, errors.New("Bad response")
}
list, err := ExtractFile(body, "pkgs")
if err != nil {
return pkgs, err
}
repo := configparser.New()
repo.ParseReader(bytes.NewReader([]byte(list)))
for _, s := range repo.Sections() {
if s == "DEFAULT" {
continue
}
pkg, err := LoadPackgae(repo, s)
if err != nil {
log.Printf("Error loading %s: %s", s, err)
continue
}
pkg.Repo = r.Name
pkg.URL, err = url.JoinPath(
r.Source, "src/branch/"+r.Branch+"/src", pkg.Name, "pkg.sh")
pkgs = append(pkgs, pkg)
}
return pkgs, nil
}
func LoadRepos() {
Repos = []Repo{}
data, err := os.ReadFile("cfg.json")
if err != nil {
log.Printf("Failed to read the configuration")
}
var cfg Config
err = json.Unmarshal(data, &cfg)
if err != nil {
log.Printf("Failed to parse the configuration: %s", err)
}
Repos = cfg.Repos
}
func LoadAllPkgs() {
LoadRepos()
Packages = []Package{}
for _, r := range Repos {
pkgs, err := GetRepo(r)
if err != nil {
log.Printf("Error loading %s: %s", r.Name, err.Error())
continue
}
Packages = append(Packages, pkgs...)
}
log.Printf("Loaded total %d packages", len(Packages))
}

View File

@ -1,12 +1,7 @@
package lib
import (
"archive/tar"
"bytes"
"compress/gzip"
"errors"
"fmt"
"io"
"strings"
"time"
@ -14,99 +9,68 @@ import (
)
func ListToStr(l []string) string {
res := ""
for _, e := range l {
res += e+" "
}
return res
res := ""
for _, e := range l {
res += e + " "
}
return res
}
func RenderError(c *fiber.Ctx, code int) error{
var msg string = "Server Error"
c.Status(code)
func RenderError(c *fiber.Ctx, code int) error {
var msg string = "Server Error"
c.Status(code)
switch code {
case 404:
msg = "Not Found"
}
switch code {
case 404:
msg = "Not Found"
}
return c.Render("error", fiber.Map{
"msg": msg,
})
return c.Render("error", fiber.Map{
"msg": msg,
})
}
func SizeFromBytes(size int) string {
if(size > 1024*1024*1024) {
return fmt.Sprintf("%dGB", (size/1024/1024/1024))
}else if(size > 1024*1024) {
return fmt.Sprintf("%dMB", (size/1024/1024))
}else if(size > 1024) {
return fmt.Sprintf("%dKB", (size/1024))
}
return fmt.Sprintf("%dB", size)
func SizeFromBytes(size int64) string {
if size > 1024*1024*1024 {
return fmt.Sprintf("%dGB", (size / 1024 / 1024 / 1024))
} else if size > 1024*1024 {
return fmt.Sprintf("%dMB", (size / 1024 / 1024))
} else if size > 1024 {
return fmt.Sprintf("%dKB", (size / 1024))
}
return fmt.Sprintf("%dB", size)
}
func ExtractFile(raw []byte, file string) (string, error){
stream := bytes.NewReader(raw)
ustream, err := gzip.NewReader(stream)
if err != nil {
return "", err
}
func TimePassed(t time.Time) string {
diff := time.Since(t)
res := fmt.Sprintf(
"%ds ago",
int(diff.Seconds()),
)
reader := tar.NewReader(ustream)
var header *tar.Header
if diff.Minutes() > 1 {
res = fmt.Sprintf(
"%dm and %ds ago",
int(diff.Minutes()), int(diff.Seconds())-(int(diff.Minutes())*60),
)
}
for header, err = reader.Next(); err == nil; header, err = reader.Next() {
if header.Typeflag != tar.TypeReg {
return "", errors.New("Found invalid entry in archive")
}
if diff.Hours() > 1 {
res = fmt.Sprintf("%dh and %dm ago",
int(diff.Hours()),
int(diff.Minutes())-(int(diff.Hours())*60),
)
}
if header.Name != file {
continue
}
buf := new(strings.Builder)
_, err := io.Copy(buf, reader)
if err != nil {
return "", errors.New("Failed to extract file to memory")
}
return buf.String(), nil
}
return "", errors.New("File not found")
return res
}
func GetTimePassed(t time.Time) string {
diff := time.Since(t)
res := fmt.Sprintf(
"%ds ago",
int(diff.Seconds()),
)
func SanitizeXSS(s string) string {
var bad []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"}
if diff.Minutes() > 1 {
res = fmt.Sprintf(
"%dm and %ds ago",
int(diff.Minutes()), int(diff.Seconds())-(int(diff.Minutes())*60),
)
}
for _, c := range bad {
s = strings.ReplaceAll(s, c, "")
}
if diff.Hours() > 1 {
res = fmt.Sprintf("%dh and %dm ago",
int(diff.Hours()),
int(diff.Minutes())-(int(diff.Hours())*60),
)
}
return res
}
func CleanString(s string) string{
var badchars []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"}
for _, c := range badchars {
s = strings.ReplaceAll(s, c, "")
}
return s
return s
}

23
log/log.go Normal file
View File

@ -0,0 +1,23 @@
package log
import (
"fmt"
"time"
)
func Log(p string, f string, args ...interface{}) {
now := time.Now()
nstr := now.Format("[02/01/06 15:04:05]")
fmt.Printf("%s -%s- ", nstr, p)
fmt.Printf(f, args...)
fmt.Println()
}
func Info(f string, args ...interface{}) {
Log("INFO", f, args...)
}
func Error(f string, args ...interface{}) {
Log("ERROR", f, args...)
}

162
main.go
View File

@ -1,6 +1,6 @@
/*
* tracker | MatterLinux Package Tracker
* tracker | MatterLinux package tracker
* MatterLinux 2023-2024 (https://matterlinux.xyz)
* This program is free software: you can redistribute it and/or modify
@ -21,107 +21,91 @@
package main
import (
"log"
"strings"
"time"
"git.matterlinux.xyz/matter/tracker/lib"
"git.matterlinux.xyz/matter/tracker/log"
"git.matterlinux.xyz/matter/tracker/routes"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/template/html/v2"
)
var lastupdate time.Time
var updatetick = time.NewTicker(time.Hour)
var stopchan = make(chan struct{})
func UpdateLoop() {
UpdatePackages()
for {
select {
case <- updatetick.C:
UpdatePackages()
case <- stopchan:
updatetick.Stop()
return
}
}
type tracker struct {
Channel chan struct{}
Config lib.Config
Last time.Time
List []lib.Package
Tick time.Ticker
}
func UpdatePackages() {
lib.LoadAllPkgs()
lastupdate = time.Now()
func (t *tracker) Reload() error {
t.List = []lib.Package{}
err := t.Config.Load(&t.List, "config.json")
t.Last = time.Now()
return err
}
func GETIndex(c *fiber.Ctx) error {
repo := c.Query("r")
name := c.Query("n")
exact := c.Query("e")
isjson := c.Query("j")
func (t *tracker) Loop() {
var err error
if repo == "" && name == "" {
if isjson == "1" {
return c.JSON(lib.Packages)
}
return c.Render("index", fiber.Map{
"last": lib.GetTimePassed(lastupdate),
"repos": lib.Repos,
"pkgs": lib.Packages,
})
}
if err = t.Reload(); err != nil {
log.Error("Failed to update packages: %s", err.Error())
}
name = lib.CleanString(name)
var res []lib.Package
for _, p := range lib.Packages {
if(repo != "all" && p.Repo != repo){
continue
}
if(exact == ""){
if(strings.Contains(p.Name, name)){
res = append(res, p)
}
}else {
if (p.Name == name) {
res = append(res, p)
}
}
}
if isjson == "1" {
return c.JSON(res)
}
return c.Render("index", fiber.Map{
"search": name,
"last": lib.GetTimePassed(lastupdate),
"repos": lib.Repos,
"pkgs": res,
})
for {
select {
case <-t.Tick.C:
if err = t.Reload(); err != nil {
log.Error("Failed to update packages: %s", err.Error())
}
case <-t.Channel:
t.Tick.Stop()
return
}
}
}
func main(){
log.SetFlags(log.Ltime | log.Lshortfile)
engine := html.New("./templates", ".html")
app := fiber.New(fiber.Config{
DisableStartupMessage: true,
Views: engine,
})
app.Static("/", "./public")
app.Get("/", GETIndex)
app.Get("*", func(c *fiber.Ctx) error {
return lib.RenderError(c, 404)
})
go UpdateLoop()
log.Println("Starting MatterLinux Package Tracker on port 9877")
err := app.Listen(":9877")
if err != nil {
log.Printf("Error starting server: %s", err)
}
close(stopchan)
func (t *tracker) Stop() {
close(t.Channel)
}
func main() {
var (
tracker tracker
engine *html.Engine
app *fiber.App
)
engine = html.New("./templates", ".html")
app = fiber.New(fiber.Config{
DisableStartupMessage: true,
Views: engine,
})
app.Static("/", "./public")
app.All("*", func(c *fiber.Ctx) error {
c.Locals("config", &tracker.Config)
c.Locals("list", &tracker.List)
c.Locals("last", &tracker.Last)
return c.Next()
})
app.Get("/", routes.GET_index)
app.Get("/p/:name", routes.GET_package)
app.Get("*", func(c *fiber.Ctx) error {
return lib.RenderError(c, 404)
})
go tracker.Loop()
log.Info("Starting MatterLinux package tracker on port 9877")
err := app.Listen(":9877")
if err != nil {
log.Error("Error starting server: %s", err.Error())
}
tracker.Stop()
}

75
routes/index.go Normal file
View File

@ -0,0 +1,75 @@
package routes
import (
"strings"
"time"
"git.matterlinux.xyz/matter/tracker/lib"
"github.com/gofiber/fiber/v2"
)
func GET_index(c *fiber.Ctx) error {
var (
result []lib.Package
list *[]lib.Package
config *lib.Config
pools_str string
pools []string
last *time.Time
)
result = []lib.Package{}
pools = []string{}
config = c.Locals("config").(*lib.Config)
list = c.Locals("list").(*[]lib.Package)
last = c.Locals("last").(*time.Time)
is_exact := c.Query("exact") == "1"
is_json := c.Query("json") == "1"
if pools_str = c.Query("pools"); pools_str != "" {
pools = strings.Split(pools_str, ",")
}
query := c.Query("q")
for _, pkg := range *list {
found := false
if is_exact && query != "" && pkg.Name != query {
continue
}
if !is_exact && query != "" && !strings.Contains(pkg.Name, query) {
continue
}
if len(pools) != 0 {
for _, p := range pools {
if p == pkg.Pool.Name {
found = true
break
}
}
} else {
found = true
}
if !found {
continue
}
result = append(result, pkg)
}
if is_json {
return c.JSON(result)
}
return c.Render("index", &fiber.Map{
"query": query,
"list": result,
"last": lib.TimePassed(*last),
"pools": config.Pools,
})
}

8
routes/package.go Normal file
View File

@ -0,0 +1,8 @@
package routes
import "github.com/gofiber/fiber/v2"
func GET_package(c *fiber.Ctx) error {
//name := c.Params("name")
return c.Render("package", &fiber.Map{})
}

View File

@ -11,20 +11,20 @@
<main>
<div class="search">
<form action="/" method="GET">
{{if .search}}
<input placeholder="Hit enter to search" type="text" name="n" value="{{.search}}">
{{if .query}}
<input placeholder="Hit enter to search" type="text" name="q" value="{{.query}}">
{{else}}
<input placeholder="Hit enter to search" type="text" name="n" autofocus>
<input placeholder="Hit enter to search" type="text" name="q" autofocus>
{{end}}
<select name="r">
<option value="all">Select repo</option>
{{range .repos}}
<option value="{{.Name}}">{{.Name}}</option>
<select name="pools">
<option value="">Select pool</option>
{{range .pools}}
<option value="{{.Name}}">{{.Display}}</option>
{{end}}
</select>
</form>
<div class="status">
<p>Listing {{len .pkgs}} packages</p>
<p>Listing {{len .list}} packages</p>
<p>Last updated: {{.last}}</p>
</div>
</div>
@ -32,20 +32,24 @@
<table class="pkgs">
<tr>
<th>Name</th>
<th>Repo</th>
<th>Pool</th>
<th>Version</th>
<th>Size</th>
<th>Description</th>
<th>Dependencies</th>
</tr>
{{range .pkgs}}
{{range .list}}
<tr>
<td><a href="{{.URL}}">{{.Name}}</a></td>
<td>{{.Repo}}</td>
<td>{{.Ver }}</td>
{{if .Pool}}
<td>{{.Pool.Display}}</td>
{{else}}
<td></td>
{{end}}
<td>{{.Version }}</td>
<td>{{.Size}}</td>
<td>{{.Desc}}</td>
<td>{{.StrDeps}}</td>
<td>{{.DependsToStr}}</td>
</tr>
{{end}}
</table>