update: make things work with the new package system

.gitignore

@@ -1,2 +1,4 @@
+docker-compose.yml
+compose.yml
+config.json
 tracker
-cfg.json

Makefile

@@ -0,0 +1,9 @@
+all: tracker
+
+tracker: */*.go *.go
+	go build -o $@
+
+format:
+	gofmt -s -w .
+
+.PHONY: format

README.md

@@ -1,33 +1,60 @@
-# tracker | MatterLinux Package Tracker 
+# tracker | MatterLinux package tracker
 Soruce code of MatterLinux's package tracker, located at
 [tracker.matterlinux.xyz](https://tracker.matterlinux.xyz)
 
 ### Configuration
 Tracker can be configured to track different repos. Configuration
-is stored in the `cfg.json` file. Here is the configuration for tracking
+is stored in the `config.json` file. Here is the configuration for tracking
 official MatterLinux 24 repos:
-```
+```json
 {
-  "repos": [ 
+  "pools": [
     {
+      "display": "base (stable)",
       "name": "base",
+      "dir": "/srv/pools/base",
       "branch": "main",
       "source": "https://git.matterlinux.xyz/Matter/base",
-      "url": "https://24.matterlinux.xyz/base"
+      "url": "mptp://stable.matterlinux.xyz/base"
     },
     {
+      "display": "desktop (stable)",
       "name": "desktop",
+      "dir": "/srv/pools/desktop",
       "branch": "main",
       "source": "https://git.matterlinux.xyz/Matter/desktop",
-      "url": "https://24.matterlinux.xyz/desktop"
+      "url": "mptp://stable.matterlinux.xyz/desktop"
+    },
+    {
+      "display": "server (stable)",
+      "name": "server",
+      "dir": "/srv/pools/server",
+      "branch": "main",
+      "source": "https://git.matterlinux.xyz/Matter/server",
+      "url": "mptp://stable.matterlinux.xyz/server"
     }
   ]
 }
 ```
 
 ### Deployment
-Web server can be built and deployed with docker:
+Web server can be built and deployed with docker compose using the following
+configuration file:
+```yaml
+version: "3"
+
+services:
+  tracker:
+    image: mattertracker
+    restart: unless-stopped
+    build:
+      context: ./
+    ports:
+      - "127.0.0.1:9877:9877"
+    volumes:
+      - "./config.json:/app/config.json:ro"
 ```
-docker build --tag mattertracker .
+After saving the configuration file, you can build and run the docker container:
+```bash
 docker-compose up -d
 ```

compose.yml

@@ -1,12 +0,0 @@
-version: "3"
-
-services:
-  tracker:
-    image: mattertracker
-    restart: unless-stopped
-    build:
-      context: ./
-    ports:
-      - "127.0.0.1:9877:9877"
-    volumes:
-      - "./cfg.json:/app/cfg.json"

lib/config.go

@@ -0,0 +1,33 @@
+package lib
+
+import (
+	"encoding/json"
+	"os"
+)
+
+type Config struct {
+	Pools []Pool `json:"pools"`
+}
+
+func (c *Config) Load(list *[]Package, file string) error {
+	var (
+		content []byte
+		err     error
+	)
+
+	if content, err = os.ReadFile(file); err != nil {
+		return err
+	}
+
+	if err = json.Unmarshal(content, c); err != nil {
+		return err
+	}
+
+	for _, p := range c.Pools {
+		if err = p.Load(list); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

lib/package.go

@@ -0,0 +1,86 @@
+package lib
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"strings"
+
+	"github.com/bigkevmcd/go-configparser"
+)
+
+type Package struct {
+	Name    string   `json:"name"`
+	Pool    *Pool    `json:"-"`
+	Version string   `json:"version"`
+	Depends []string `json:"depends"`
+	Size    string   `json:"size"`
+	Desc    string   `json:"desc"`
+}
+
+func (p *Package) URL() string {
+	if nil == p.Pool {
+		return ""
+	}
+	url, _ := url.JoinPath(p.Pool.Source, "src/branch/"+p.Pool.Branch+"/src", p.Name)
+	return url
+}
+
+func (p *Package) DependsToStr() string {
+	var depends string = ""
+	for _, d := range p.Depends {
+		depends += fmt.Sprintf("%s ", d)
+	}
+	return depends
+}
+
+func (p *Package) Load(r io.Reader) error {
+	var (
+		err     error
+		size    int64
+		depends string = ""
+		section string = "DEFAULT"
+	)
+
+	parser := configparser.New()
+	if err = parser.ParseReader(r); err != nil {
+		return err
+	}
+
+	for _, s := range parser.Sections() {
+		if s == "DEFAULT" {
+			continue
+		}
+		section = s
+		break
+	}
+
+	if section == "DEFAULT" {
+		return fmt.Errorf("DATA does not contain any sections")
+	}
+
+	p.Name = section
+
+	if p.Version, err = parser.Get(section, "version"); err != nil {
+		return err
+	}
+
+	if size, err = parser.GetInt64(section, "size"); err != nil {
+		return err
+	}
+	p.Size = SizeFromBytes(size)
+
+	if p.Desc, err = parser.Get(section, "desc"); err != nil {
+		return err
+	}
+
+	depends, _ = parser.Get(section, "depends")
+
+	if depends == "" {
+		p.Depends = []string{}
+	} else {
+		p.Depends = strings.Split(depends, ",")
+	}
+
+	return nil
+}

lib/pool.go

@@ -0,0 +1,143 @@
+package lib
+
+import (
+	"archive/tar"
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	"github.com/bigkevmcd/go-configparser"
+)
+
+type Pool struct {
+	Maintainer string `json:"-"`
+	Pubkey     string `json:"-"`
+	Size       string `json:"-"`
+
+	Display string `json:"display"`
+	Branch  string `json:"branch"`
+	Source  string `json:"source"`
+	Name    string `json:"name"`
+	URL     string `json:"url"`
+	Dir     string `json:"dir"`
+}
+
+func (p *Pool) Load(list *[]Package) error {
+	var err error
+
+	if p.Dir == "" {
+		return fmt.Errorf("pool directory is not specified")
+	}
+
+	if err = p.LoadInfo(); err != nil {
+		return err
+	}
+
+	if err = p.LoadList(list); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *Pool) LoadList(list *[]Package) error {
+	var (
+		list_path   string
+		list_file   *os.File
+		gzip_reader io.Reader
+		header      *tar.Header
+		err         error
+	)
+
+	list_path = path.Join(p.Dir, "LIST")
+
+	if list_file, err = os.Open(list_path); err != nil {
+		return err
+	}
+	defer list_file.Close()
+
+	if gzip_reader, err = gzip.NewReader(bufio.NewReader(list_file)); err != nil {
+		return err
+	}
+
+	reader := tar.NewReader(gzip_reader)
+
+	for header, err = reader.Next(); err == nil; header, err = reader.Next() {
+		if header.Typeflag != tar.TypeReg {
+			continue
+		}
+
+		if path.Base(header.Name) != "DATA" {
+			return fmt.Errorf("LIST archive contains an unknown file")
+		}
+
+		var pkg Package
+		pkg.Pool = p
+
+		if err = pkg.Load(reader); err != nil {
+			return err
+		}
+
+		*list = append(*list, pkg)
+	}
+
+	return nil
+}
+
+func (p *Pool) LoadInfo() error {
+	var (
+		info_path string
+		info_file *os.File
+		section   string
+		size      int64
+		err       error
+	)
+
+	info_path = path.Join(p.Dir, "INFO")
+
+	if info_file, err = os.Open(info_path); err != nil {
+		return err
+	}
+
+	parser := configparser.New()
+
+	if err = parser.ParseReader(bufio.NewReader(info_file)); err != nil {
+		return err
+	}
+
+	section = "DEFAULT"
+
+	for _, s := range parser.Sections() {
+		if s == "DEFAULT" {
+			continue
+		}
+		section = s
+		break
+	}
+
+	if section == "DEFAULT" {
+		return fmt.Errorf("DATA does not contain any sections")
+	}
+
+	if p.Name != section {
+		return fmt.Errorf("pool name (\"%s\") doesn't match with \"%s\"", p.Name, section)
+	}
+
+	if p.Maintainer, err = parser.Get(p.Name, "maintainer"); err != nil {
+		return err
+	}
+
+	if size, err = parser.GetInt64(section, "size"); err != nil {
+		return err
+	}
+	p.Size = SizeFromBytes(size)
+
+	if p.Pubkey, err = parser.Get(section, "pubkey"); err != nil {
+		return err
+	}
+
+	return nil
+}

lib/repo.go

@@ -1,163 +0,0 @@
-package lib
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"log"
-	"net/url"
-	"os"
-	"strconv"
-	"strings"
-
-	"github.com/bigkevmcd/go-configparser"
-	"github.com/gofiber/fiber/v2"
-)
-
-type Config struct {
-  Repos   []Repo  `json:"repos"`
-}
-
-var Repos []Repo
-type Repo struct {
-  Source  string  `json:"source"`
-  Branch  string  `json:"branch"`
-  Name    string  `json:"name"`
-  URL     string  `json:"url"`
-}
-
-var Packages []Package
-type Package struct {
-  Name  string   `json:"name"`
-  Repo  string   `json:"repo"`
-  Desc  string   `json:"desc"`
-  Size  string   `json:"size"`
-  Deps  []string `json:"depends"`
-  URL   string   `json:"url"`
-  Ver   string   `json:"version"`
-}
-
-func (p Package) StrDeps() string {
-  return ListToStr(p.Deps)
-}
-
-func LoadPackgae(repo *configparser.ConfigParser, s string) (Package, error) {
-  var (
-    pkg Package 
-    err error
-  ) 
-  pkg.Name = s 
-
-  pkg.Ver, err = repo.Get(s, "version")
-  if err != nil {
-    return pkg, err
-  }
-
-  pkg.Desc, err = repo.Get(s, "desc")
-  if err != nil {
-    return pkg, err
-  }
-
-  ssize, err := repo.Get(s, "size")
-  if err != nil {
-    return pkg, err
-  }
-
-  size, err := strconv.Atoi(ssize)
-  if err != nil {
-    return pkg, err
-  }
-  pkg.Size = SizeFromBytes(size)
-
-  deps, err := repo.Get(s, "depends")
-  if err != nil {
-    return pkg, err
-  }
-
-  if deps == "" {
-    pkg.Deps = []string{}
-  }else {
-    pkg.Deps = strings.Split(deps, "\n")
-  }
-
-  return pkg, nil
-}
-
-func GetRepo(r Repo) ([]Package, error){
-  var pkgs []Package
-
-  furl, err := url.JoinPath(r.URL, "list")
-  if err != nil {
-    return pkgs, err
-  }
-
-  agent := fiber.Get(furl)
-  code, body, errs := agent.Bytes()
-  if len(errs) > 0 {
-    return pkgs, errors.New("Request failed")
-  }
-
-  if code != 200 {
-    return pkgs, errors.New("Bad response")
-  }
-
-  list, err := ExtractFile(body, "pkgs")
-  if err != nil {
-    return pkgs, err
-  }
-
-  repo := configparser.New()
-  repo.ParseReader(bytes.NewReader([]byte(list)))
-  
-  for _, s := range repo.Sections() {
-    if s == "DEFAULT" {
-      continue
-    }
-
-    pkg, err := LoadPackgae(repo, s)
-    if err != nil {
-      log.Printf("Error loading %s: %s", s, err)
-      continue
-    }
-
-    pkg.Repo = r.Name
-    pkg.URL, err = url.JoinPath(
-      r.Source, "src/branch/"+r.Branch+"/src",  pkg.Name, "pkg.sh")
-    pkgs = append(pkgs, pkg)
-  }
-  
-  return pkgs, nil
-}
-
-func LoadRepos() {
-  Repos = []Repo{}
-
-  data, err := os.ReadFile("cfg.json")
-  if err != nil {
-    log.Printf("Failed to read the configuration")
-  }
-
-  var cfg Config
-  err = json.Unmarshal(data, &cfg)
-  if err != nil {
-    log.Printf("Failed to parse the configuration: %s", err)
-  }
-
-  Repos = cfg.Repos
-}
-
-func LoadAllPkgs() {
-  LoadRepos()
-  Packages = []Package{}
-
-  for _, r := range Repos {
-    pkgs, err := GetRepo(r)
-    if err != nil {
-      log.Printf("Error loading %s: %s", r.Name, err.Error())
-      continue
-    }
-    Packages = append(Packages, pkgs...)
-  }
-
-  log.Printf("Loaded total %d packages", len(Packages))
-}

lib/util.go

@@ -1,12 +1,7 @@
 package lib
 
 import (
-	"archive/tar"
-	"bytes"
-	"compress/gzip"
-	"errors"
 	"fmt"
-	"io"
 	"strings"
 	"time"
 
@@ -35,49 +30,18 @@ func RenderError(c *fiber.Ctx, code int) error{
 	})
 }
 
-func SizeFromBytes(size int) string {
-  if(size > 1024*1024*1024) {
+func SizeFromBytes(size int64) string {
+	if size > 1024*1024*1024 {
 		return fmt.Sprintf("%dGB", (size / 1024 / 1024 / 1024))
-  }else if(size > 1024*1024) {
+	} else if size > 1024*1024 {
 		return fmt.Sprintf("%dMB", (size / 1024 / 1024))
-  }else if(size > 1024) { 
+	} else if size > 1024 {
 		return fmt.Sprintf("%dKB", (size / 1024))
 	}
 	return fmt.Sprintf("%dB", size)
 }
 
-func ExtractFile(raw []byte, file string) (string, error){
-  stream := bytes.NewReader(raw) 
-  ustream, err := gzip.NewReader(stream)
-  if err != nil {
-    return "", err
-  }
-
-  reader := tar.NewReader(ustream)
-  var header *tar.Header
-    
-  for header, err = reader.Next(); err == nil; header, err = reader.Next() {
-    if header.Typeflag != tar.TypeReg {
-      return "", errors.New("Found invalid entry in archive")
-    }
-
-    if header.Name != file {
-      continue
-    }
-
-    buf := new(strings.Builder)
-    _, err := io.Copy(buf, reader)
-    if err != nil {
-      return "", errors.New("Failed to extract file to memory")
-    }
-
-    return buf.String(), nil
-  }
-
-  return "", errors.New("File not found")
-}
-
-func GetTimePassed(t time.Time) string {
+func TimePassed(t time.Time) string {
 	diff := time.Since(t)
 	res := fmt.Sprintf(
 		"%ds ago",
@@ -101,10 +65,10 @@ func GetTimePassed(t time.Time) string {
 	return res
 }
 
-func CleanString(s string) string{
-  var badchars []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"}
+func SanitizeXSS(s string) string {
+	var bad []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"}
 
-  for _, c := range badchars {
+	for _, c := range bad {
 		s = strings.ReplaceAll(s, c, "")
 	}
 

log/log.go

@@ -0,0 +1,23 @@
+package log
+
+import (
+	"fmt"
+	"time"
+)
+
+func Log(p string, f string, args ...interface{}) {
+	now := time.Now()
+	nstr := now.Format("[02/01/06 15:04:05]")
+
+	fmt.Printf("%s -%s- ", nstr, p)
+	fmt.Printf(f, args...)
+	fmt.Println()
+}
+
+func Info(f string, args ...interface{}) {
+	Log("INFO", f, args...)
+}
+
+func Error(f string, args ...interface{}) {
+	Log("ERROR", f, args...)
+}

main.go

@@ -1,6 +1,6 @@
 /*
 
- * tracker | MatterLinux Package Tracker
+ * tracker | MatterLinux package tracker
  * MatterLinux 2023-2024 (https://matterlinux.xyz)
 
  * This program is free software: you can redistribute it and/or modify
@@ -21,107 +21,91 @@
 package main
 
 import (
-	"log"
-	"strings"
 	"time"
 
 	"git.matterlinux.xyz/matter/tracker/lib"
+	"git.matterlinux.xyz/matter/tracker/log"
+	"git.matterlinux.xyz/matter/tracker/routes"
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/template/html/v2"
 )
 
-var lastupdate time.Time 
-var updatetick = time.NewTicker(time.Hour)
-var stopchan = make(chan struct{})
+type tracker struct {
+	Channel chan struct{}
+	Config  lib.Config
+	Last    time.Time
+	List    []lib.Package
+	Tick    time.Ticker
+}
 
-func UpdateLoop() {
-  UpdatePackages()
+func (t *tracker) Reload() error {
+	t.List = []lib.Package{}
+	err := t.Config.Load(&t.List, "config.json")
+	t.Last = time.Now()
+	return err
+}
+
+func (t *tracker) Loop() {
+	var err error
+
+	if err = t.Reload(); err != nil {
+		log.Error("Failed to update packages: %s", err.Error())
+	}
 
 	for {
 		select {
-      case <- updatetick.C:
-        UpdatePackages()
-      case <- stopchan:
-        updatetick.Stop()
+		case <-t.Tick.C:
+			if err = t.Reload(); err != nil {
+				log.Error("Failed to update packages: %s", err.Error())
+			}
+		case <-t.Channel:
+			t.Tick.Stop()
 			return
 		}
 	}
 }
 
-func UpdatePackages() {
-  lib.LoadAllPkgs()
-  lastupdate = time.Now() 
-}
-
-func GETIndex(c *fiber.Ctx) error {
-  repo := c.Query("r")
-  name := c.Query("n")
-  exact := c.Query("e")
-  isjson := c.Query("j")
-  
-  if repo == "" && name == "" {
-    if isjson == "1" {
-      return c.JSON(lib.Packages)
-    }
-    return c.Render("index", fiber.Map{
-      "last": lib.GetTimePassed(lastupdate),
-      "repos": lib.Repos,
-      "pkgs": lib.Packages,
-    })
-  }
-
-  name = lib.CleanString(name) 
-  var res []lib.Package
-
-  for _, p := range lib.Packages {
-    if(repo != "all" && p.Repo != repo){
-      continue
-    }
-
-    if(exact == ""){
-      if(strings.Contains(p.Name, name)){
-        res = append(res, p)
-      }
-    }else {
-      if (p.Name == name) {
-        res = append(res, p)
-      }
-    }
-  }
-
-  if isjson == "1" {
-    return c.JSON(res)
-  }
-  
-  return c.Render("index", fiber.Map{
-    "search": name,
-    "last": lib.GetTimePassed(lastupdate),
-    "repos": lib.Repos,
-    "pkgs": res,
-  })
+func (t *tracker) Stop() {
+	close(t.Channel)
 }
 
 func main() {
-  log.SetFlags(log.Ltime | log.Lshortfile)
+	var (
+		tracker tracker
+		engine  *html.Engine
+		app     *fiber.App
+	)
 
-  engine := html.New("./templates", ".html")
-  app := fiber.New(fiber.Config{
+	engine = html.New("./templates", ".html")
+	app = fiber.New(fiber.Config{
 		DisableStartupMessage: true,
 		Views:                 engine,
 	})
 
 	app.Static("/", "./public")
-  app.Get("/", GETIndex)
+
+	app.All("*", func(c *fiber.Ctx) error {
+		c.Locals("config", &tracker.Config)
+		c.Locals("list", &tracker.List)
+		c.Locals("last", &tracker.Last)
+		return c.Next()
+	})
+
+	app.Get("/", routes.GET_index)
+	app.Get("/p/:name", routes.GET_package)
 
 	app.Get("*", func(c *fiber.Ctx) error {
 		return lib.RenderError(c, 404)
 	})
 
-  go UpdateLoop()
-  log.Println("Starting MatterLinux Package Tracker on port 9877")
+	go tracker.Loop()
+
+	log.Info("Starting MatterLinux package tracker on port 9877")
+
 	err := app.Listen(":9877")
 	if err != nil {
-    log.Printf("Error starting server: %s", err)
+		log.Error("Error starting server: %s", err.Error())
 	}
-  close(stopchan)
+
+	tracker.Stop()
 }

routes/index.go

@@ -0,0 +1,75 @@
+package routes
+
+import (
+	"strings"
+	"time"
+
+	"git.matterlinux.xyz/matter/tracker/lib"
+	"github.com/gofiber/fiber/v2"
+)
+
+func GET_index(c *fiber.Ctx) error {
+	var (
+		result    []lib.Package
+		list      *[]lib.Package
+		config    *lib.Config
+		pools_str string
+		pools     []string
+		last      *time.Time
+	)
+
+	result = []lib.Package{}
+	pools = []string{}
+
+	config = c.Locals("config").(*lib.Config)
+	list = c.Locals("list").(*[]lib.Package)
+	last = c.Locals("last").(*time.Time)
+
+	is_exact := c.Query("exact") == "1"
+	is_json := c.Query("json") == "1"
+
+	if pools_str = c.Query("pools"); pools_str != "" {
+		pools = strings.Split(pools_str, ",")
+	}
+	query := c.Query("q")
+
+	for _, pkg := range *list {
+		found := false
+
+		if is_exact && query != "" && pkg.Name != query {
+			continue
+		}
+
+		if !is_exact && query != "" && !strings.Contains(pkg.Name, query) {
+			continue
+		}
+
+		if len(pools) != 0 {
+			for _, p := range pools {
+				if p == pkg.Pool.Name {
+					found = true
+					break
+				}
+			}
+		} else {
+			found = true
+		}
+
+		if !found {
+			continue
+		}
+
+		result = append(result, pkg)
+	}
+
+	if is_json {
+		return c.JSON(result)
+	}
+
+	return c.Render("index", &fiber.Map{
+		"query": query,
+		"list":  result,
+		"last":  lib.TimePassed(*last),
+		"pools": config.Pools,
+	})
+}

routes/package.go

@@ -0,0 +1,8 @@
+package routes
+
+import "github.com/gofiber/fiber/v2"
+
+func GET_package(c *fiber.Ctx) error {
+	//name := c.Params("name")
+	return c.Render("package", &fiber.Map{})
+}

templates/index.html

@@ -11,20 +11,20 @@
     <main>
       <div class="search">
         <form action="/" method="GET">
-          {{if .search}}
-            <input placeholder="Hit enter to search" type="text" name="n" value="{{.search}}">
+          {{if .query}}
+            <input placeholder="Hit enter to search" type="text" name="q" value="{{.query}}">
           {{else}}
-            <input placeholder="Hit enter to search" type="text" name="n" autofocus>
+            <input placeholder="Hit enter to search" type="text" name="q" autofocus>
           {{end}}
-          <select name="r">
-            <option value="all">Select repo</option>
-            {{range .repos}}
-              <option value="{{.Name}}">{{.Name}}</option>
+          <select name="pools">
+            <option value="">Select pool</option>
+            {{range .pools}}
+              <option value="{{.Name}}">{{.Display}}</option>
             {{end}}
           </select>
         </form>
         <div class="status">
-          <p>Listing {{len .pkgs}} packages</p>
+          <p>Listing {{len .list}} packages</p>
           <p>Last updated: {{.last}}</p>
         </div>
       </div>
@@ -32,20 +32,24 @@
       <table class="pkgs">
         <tr>
           <th>Name</th>
-          <th>Repo</th>
+          <th>Pool</th>
           <th>Version</th>
           <th>Size</th>
           <th>Description</th>
           <th>Dependencies</th>
         </tr>
-        {{range .pkgs}}
+        {{range .list}}
           <tr>
             <td><a href="{{.URL}}">{{.Name}}</a></td>
-            <td>{{.Repo}}</td>
-            <td>{{.Ver }}</td>
+            {{if .Pool}}
+              <td>{{.Pool.Display}}</td>
+            {{else}}
+              <td></td>
+            {{end}}
+            <td>{{.Version }}</td>
             <td>{{.Size}}</td>
             <td>{{.Desc}}</td>
-            <td>{{.StrDeps}}</td>
+            <td>{{.DependsToStr}}</td>
           </tr>
         {{end}}
       </table>