Compare commits

...

10 Commits

19 changed files with 990 additions and 442 deletions

4
.gitignore vendored
View File

@ -1,2 +1,4 @@
docker-compose.yml
compose.yml
config.json
tracker
cfg.json

View File

@ -1,12 +1,15 @@
FROM golang:1.21.6
FROM golang:1.22.5
WORKDIR /app
COPY *.go ./
COPY *.mod ./
COPY *.sum ./
COPY lib ./lib
COPY log ./log
COPY public ./public
COPY routes ./routes
COPY templates ./templates
RUN go build .

13
Makefile Normal file
View File

@ -0,0 +1,13 @@
all: tracker
tracker: */*.go *.go
go build -o $@
format:
gofmt -s -w .
update:
go get -u
go mod tidy
.PHONY: format update

View File

@ -1,33 +1,60 @@
# tracker | MatterLinux Package Tracker
# tracker | MatterLinux package tracker
Soruce code of MatterLinux's package tracker, located at
[tracker.matterlinux.xyz](https://tracker.matterlinux.xyz)
### Configuration
Tracker can be configured to track different repos. Configuration
is stored in the `cfg.json` file. Here is the configuration for tracking
is stored in the `config.json` file. Here is the configuration for tracking
official MatterLinux 24 repos:
```
```json
{
"repos": [
"pools": [
{
"display": "base (stable)",
"name": "base",
"dir": "/srv/pools/base",
"branch": "main",
"source": "https://git.matterlinux.xyz/Matter/base",
"url": "https://24.matterlinux.xyz/base"
"url": "mptp://stable.matterlinux.xyz/base"
},
{
"display": "desktop (stable)",
"name": "desktop",
"dir": "/srv/pools/desktop",
"branch": "main",
"source": "https://git.matterlinux.xyz/Matter/desktop",
"url": "https://24.matterlinux.xyz/desktop"
"url": "mptp://stable.matterlinux.xyz/desktop"
},
{
"display": "server (stable)",
"name": "server",
"dir": "/srv/pools/server",
"branch": "main",
"source": "https://git.matterlinux.xyz/Matter/server",
"url": "mptp://stable.matterlinux.xyz/server"
}
]
}
```
### Deployment
Web server can be built and deployed with docker:
Web server can be built and deployed with docker compose using the following
configuration file:
```yaml
version: "3"
services:
tracker:
image: mattertracker
restart: unless-stopped
build:
context: ./
ports:
- "127.0.0.1:9877:9877"
volumes:
- "./config.json:/app/config.json:ro"
```
docker build --tag mattertracker .
After saving the configuration file, you can build and run the docker container:
```bash
docker-compose up -d
```

View File

@ -1,10 +0,0 @@
version: "3"
services:
tracker:
image: mattertracker
restart: unless-stopped
ports:
- "127.0.0.1:9877:9877"
volumes:
- "./cfg.json:/app/cfg.json"

24
go.mod
View File

@ -1,25 +1,25 @@
module git.matterlinux.xyz/matter/tracker
go 1.21.6
go 1.22.5
require (
github.com/bigkevmcd/go-configparser v0.0.0-20230427073640-c6b631f70126
github.com/gofiber/fiber/v2 v2.52.0
github.com/gofiber/template/html/v2 v2.1.0
github.com/bigkevmcd/go-configparser v0.0.0-20240808124832-fc81059ea0bd
github.com/gofiber/fiber/v2 v2.52.5
github.com/gofiber/template/html/v2 v2.1.2
)
require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/gofiber/template v1.8.2 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/gofiber/template v1.8.3 // indirect
github.com/gofiber/utils v1.1.0 // indirect
github.com/google/uuid v1.5.0 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.51.0 // indirect
github.com/valyala/fasthttp v1.55.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/sys v0.24.0 // indirect
)

47
go.sum
View File

@ -1,21 +1,21 @@
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/bigkevmcd/go-configparser v0.0.0-20230427073640-c6b631f70126 h1:uru++pUKoS/yYU3Ohq9VItZdK/cT7FFJH/UUjOlxc+s=
github.com/bigkevmcd/go-configparser v0.0.0-20230427073640-c6b631f70126/go.mod h1:zqqfbfnDeSdRs1WihmMjSbhb2Ptw8Jbus831xoqiIec=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/bigkevmcd/go-configparser v0.0.0-20240808124832-fc81059ea0bd h1:MsTk4yo6KVYdulsDscuH4AwiZN1CyuCJAg59EWE7HPQ=
github.com/bigkevmcd/go-configparser v0.0.0-20240808124832-fc81059ea0bd/go.mod h1:vzEQfW+A1T+AMJmTIX+SXNLNECHOM7GEinHhw0IjykI=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofiber/fiber/v2 v2.52.0 h1:S+qXi7y+/Pgvqq4DrSmREGiFwtB7Bu6+QFLuIHYw/UE=
github.com/gofiber/fiber/v2 v2.52.0/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
github.com/gofiber/template v1.8.2 h1:PIv9s/7Uq6m+Fm2MDNd20pAFFKt5wWs7ZBd8iV9pWwk=
github.com/gofiber/template v1.8.2/go.mod h1:bs/2n0pSNPOkRa5VJ8zTIvedcI/lEYxzV3+YPXdBvq8=
github.com/gofiber/template/html/v2 v2.1.0 h1:FjwzqhhdJpnhyCvav60Z1ytnBqOUr5sGO/aTeob9/ng=
github.com/gofiber/template/html/v2 v2.1.0/go.mod h1:txXsRQN/G7Fr2cqGfr6zhVHgreCfpsBS+9+DJyrddJc=
github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
github.com/gofiber/template v1.8.3 h1:hzHdvMwMo/T2kouz2pPCA0zGiLCeMnoGsQZBTSYgZxc=
github.com/gofiber/template v1.8.3/go.mod h1:bs/2n0pSNPOkRa5VJ8zTIvedcI/lEYxzV3+YPXdBvq8=
github.com/gofiber/template/html/v2 v2.1.2 h1:wkK/mYJ3nIhongTkG3t0QgV4ADdgOYJYVSAF2AHnh8Y=
github.com/gofiber/template/html/v2 v2.1.2/go.mod h1:E98Z/FzvpaSib06aWEgYk6GXNf3ctoyaJH8yW5ay5ak=
github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM=
github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0=
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
@ -25,24 +25,25 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

33
lib/config.go Normal file
View File

@ -0,0 +1,33 @@
package lib
import (
"encoding/json"
"os"
)
type Config struct {
Pools []Pool `json:"pools"`
}
func (c *Config) Load(list *[]Package, file string) error {
var (
content []byte
err error
)
if content, err = os.ReadFile(file); err != nil {
return err
}
if err = json.Unmarshal(content, c); err != nil {
return err
}
for _, p := range c.Pools {
if err = p.Load(list); err != nil {
return err
}
}
return nil
}

135
lib/package.go Normal file
View File

@ -0,0 +1,135 @@
package lib
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"fmt"
"io"
"net/url"
"os"
"path"
"git.matterlinux.xyz/matter/tracker/log"
"github.com/bigkevmcd/go-configparser"
)
type Package struct {
Name string `json:"name"`
Pool *Pool `json:"-"`
Version string `json:"version"`
Depends []string `json:"depends"`
Size string `json:"size"`
Desc string `json:"desc"`
Archive string `json:"archive"`
}
func (p *Package) Files() []string {
var (
gzip_reader io.Reader
header *tar.Header
result []string
file *os.File
err error
)
if file, err = os.Open(p.Archive); err != nil {
log.Error("Failed to open %s", p.Archive)
return result
}
defer file.Close()
if gzip_reader, err = gzip.NewReader(bufio.NewReader(file)); err != nil {
log.Error("Failed to create reader for %s", p.Archive)
return result
}
reader := tar.NewReader(gzip_reader)
for header, err = reader.Next(); err == nil; header, err = reader.Next() {
if header.Typeflag != tar.TypeReg {
continue
}
if path.Base(header.Name) != "files.tar.gz" {
continue
}
if result, err = GetFiles(reader); err == nil {
break
}
log.Error("Failed to get file list for %s: %s", p.Archive, err.Error())
return []string{}
}
return result
}
func (p *Package) URL() string {
if nil == p.Pool {
return ""
}
url, _ := url.JoinPath(p.Pool.Source, "src/branch/"+p.Pool.Branch+"/src", p.Name)
return url
}
func (p *Package) DependsToStr() string {
var depends string = ""
for _, d := range p.Depends {
depends += fmt.Sprintf("%s ", d)
}
return depends
}
func (p *Package) Load(r io.Reader) error {
var (
err error
size int64
section string = "DEFAULT"
buffer []byte
)
if buffer, err = io.ReadAll(r); err != nil {
return err
}
if p.Depends, err = GetMultiple("depends", bytes.NewReader(buffer)); err != nil {
return err
}
parser := configparser.New()
if err = parser.ParseReader(bytes.NewReader(buffer)); err != nil {
return err
}
for _, s := range parser.Sections() {
if s == "DEFAULT" {
continue
}
section = s
break
}
if section == "DEFAULT" {
return fmt.Errorf("DATA does not contain any sections")
}
p.Name = section
if p.Version, err = parser.Get(section, "version"); err != nil {
return err
}
if size, err = parser.GetInt64(section, "size"); err != nil {
return err
}
p.Size = SizeFromBytes(size)
if p.Desc, err = parser.Get(section, "desc"); err != nil {
return err
}
return nil
}

162
lib/pool.go Normal file
View File

@ -0,0 +1,162 @@
package lib
import (
"archive/tar"
"bufio"
"compress/gzip"
"fmt"
"io"
"os"
"path"
"strings"
"unicode"
"github.com/bigkevmcd/go-configparser"
)
type Pool struct {
Maintainer string `json:"-"`
Pubkey string `json:"-"`
Size string `json:"-"`
Display string `json:"display"`
Branch string `json:"branch"`
Source string `json:"source"`
Name string `json:"name"`
URL string `json:"url"`
Dir string `json:"dir"`
}
func (p *Pool) ID() string {
var res string = ""
for _, c := range p.Display {
if unicode.IsDigit(c) || unicode.IsLetter(c) {
res += string(c)
continue
}
res += "-"
}
return strings.TrimSuffix(res, "-")
}
func (p *Pool) Load(list *[]Package) error {
var err error
if p.Dir == "" {
return fmt.Errorf("pool directory is not specified")
}
if err = p.LoadInfo(); err != nil {
return err
}
if err = p.LoadList(list); err != nil {
return err
}
return nil
}
func (p *Pool) LoadList(list *[]Package) error {
var (
list_path string
list_file *os.File
gzip_reader io.Reader
header *tar.Header
err error
)
list_path = path.Join(p.Dir, "LIST")
if list_file, err = os.Open(list_path); err != nil {
return err
}
defer list_file.Close()
if gzip_reader, err = gzip.NewReader(bufio.NewReader(list_file)); err != nil {
return err
}
reader := tar.NewReader(gzip_reader)
for header, err = reader.Next(); err == nil; header, err = reader.Next() {
if header.Typeflag != tar.TypeReg {
continue
}
if path.Base(header.Name) != "DATA" {
return fmt.Errorf("LIST archive contains an unknown file")
}
var pkg Package
if err = pkg.Load(reader); err != nil {
return err
}
pkg.Archive = path.Join(p.Dir, fmt.Sprintf("%s_%s.mpf", pkg.Name, pkg.Version))
pkg.Pool = p
*list = append(*list, pkg)
}
return nil
}
func (p *Pool) LoadInfo() error {
var (
info_path string
info_file *os.File
section string
size int64
err error
)
info_path = path.Join(p.Dir, "INFO")
if info_file, err = os.Open(info_path); err != nil {
return err
}
parser := configparser.New()
if err = parser.ParseReader(bufio.NewReader(info_file)); err != nil {
return err
}
section = "DEFAULT"
for _, s := range parser.Sections() {
if s == "DEFAULT" {
continue
}
section = s
break
}
if section == "DEFAULT" {
return fmt.Errorf("DATA does not contain any sections")
}
if p.Name != section {
return fmt.Errorf("pool name (\"%s\") doesn't match with \"%s\"", p.Name, section)
}
if p.Maintainer, err = parser.Get(p.Name, "maintainer"); err != nil {
return err
}
if size, err = parser.GetInt64(section, "size"); err != nil {
return err
}
p.Size = SizeFromBytes(size)
if p.Pubkey, err = parser.Get(section, "pubkey"); err != nil {
return err
}
return nil
}

View File

@ -1,163 +0,0 @@
package lib
import (
"bytes"
"encoding/json"
"errors"
"log"
"net/url"
"os"
"strconv"
"strings"
"github.com/bigkevmcd/go-configparser"
"github.com/gofiber/fiber/v2"
)
type Config struct {
Repos []Repo `json:"repos"`
}
var Repos []Repo
type Repo struct {
Source string `json:"source"`
Branch string `json:"branch"`
Name string `json:"name"`
URL string `json:"url"`
}
var Packages []Package
type Package struct {
Name string `json:"name"`
Repo string `json:"repo"`
Desc string `json:"desc"`
Size string `json:"size"`
Deps []string `json:"depends"`
URL string `json:"url"`
Ver string `json:"version"`
}
func (p Package) StrDeps() string {
return ListToStr(p.Deps)
}
func LoadPackgae(repo *configparser.ConfigParser, s string) (Package, error) {
var (
pkg Package
err error
)
pkg.Name = s
pkg.Ver, err = repo.Get(s, "version")
if err != nil {
return pkg, err
}
pkg.Desc, err = repo.Get(s, "desc")
if err != nil {
return pkg, err
}
ssize, err := repo.Get(s, "size")
if err != nil {
return pkg, err
}
size, err := strconv.Atoi(ssize)
if err != nil {
return pkg, err
}
pkg.Size = SizeFromBytes(size)
deps, err := repo.Get(s, "depends")
if err != nil {
return pkg, err
}
if deps == "" {
pkg.Deps = []string{}
}else {
pkg.Deps = strings.Split(deps, "\n")
}
return pkg, nil
}
func GetRepo(r Repo) ([]Package, error){
var pkgs []Package
furl, err := url.JoinPath(r.URL, "list")
if err != nil {
return pkgs, err
}
agent := fiber.Get(furl)
code, body, errs := agent.Bytes()
if len(errs) > 0 {
return pkgs, errors.New("Request failed")
}
if code != 200 {
return pkgs, errors.New("Bad response")
}
list, err := ExtractFile(body, "pkgs")
if err != nil {
return pkgs, err
}
repo := configparser.New()
repo.ParseReader(bytes.NewReader([]byte(list)))
for _, s := range repo.Sections() {
if s == "DEFAULT" {
continue
}
pkg, err := LoadPackgae(repo, s)
if err != nil {
log.Printf("Error loading %s: %s", s, err)
continue
}
pkg.Repo = r.Name
pkg.URL, err = url.JoinPath(
r.Source, "src/branch/"+r.Branch+"/src", pkg.Name, "pkg.sh")
pkgs = append(pkgs, pkg)
}
return pkgs, nil
}
func LoadRepos() {
Repos = []Repo{}
data, err := os.ReadFile("cfg.json")
if err != nil {
log.Printf("Failed to read the configuration")
}
var cfg Config
err = json.Unmarshal(data, &cfg)
if err != nil {
log.Printf("Failed to parse the configuration: %s", err)
}
Repos = cfg.Repos
}
func LoadAllPkgs() {
LoadRepos()
Packages = []Package{}
for _, r := range Repos {
pkgs, err := GetRepo(r)
if err != nil {
log.Printf("Error loading %s: %s", r.Name, err.Error())
continue
}
Packages = append(Packages, pkgs...)
}
log.Printf("Loaded total %d packages", len(Packages))
}

View File

@ -2,9 +2,8 @@ package lib
import (
"archive/tar"
"bytes"
"bufio"
"compress/gzip"
"errors"
"fmt"
"io"
"strings"
@ -13,15 +12,67 @@ import (
"github.com/gofiber/fiber/v2"
)
// configparser can't get multiple keys, so heres a function to manually extract them
func GetMultiple(k string, r io.Reader) ([]string, error) {
var (
res []string
value string
scanner *bufio.Scanner
)
scanner = bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
value = strings.TrimPrefix(line, fmt.Sprintf("%s = ", k))
value = strings.TrimPrefix(value, fmt.Sprintf("%s =", k))
value = strings.TrimPrefix(value, fmt.Sprintf("%s= ", k))
value = strings.TrimPrefix(value, fmt.Sprintf("%s=", k))
if line == value {
continue
}
res = append(res, value)
}
return res, nil
}
func GetFiles(r io.Reader) ([]string, error) {
var (
gzip_reader io.Reader
header *tar.Header
result []string
err error
)
if gzip_reader, err = gzip.NewReader(r); err != nil {
return result, err
}
reader := tar.NewReader(gzip_reader)
for header, err = reader.Next(); err == nil; header, err = reader.Next() {
if header.Typeflag != tar.TypeReg {
continue
}
result = append(result, header.Name)
}
return result, nil
}
func ListToStr(l []string) string {
res := ""
for _, e := range l {
res += e+" "
res += e + " "
}
return res
}
func RenderError(c *fiber.Ctx, code int) error{
func RenderError(c *fiber.Ctx, code int) error {
var msg string = "Server Error"
c.Status(code)
@ -35,49 +86,18 @@ func RenderError(c *fiber.Ctx, code int) error{
})
}
func SizeFromBytes(size int) string {
if(size > 1024*1024*1024) {
return fmt.Sprintf("%dGB", (size/1024/1024/1024))
}else if(size > 1024*1024) {
return fmt.Sprintf("%dMB", (size/1024/1024))
}else if(size > 1024) {
return fmt.Sprintf("%dKB", (size/1024))
func SizeFromBytes(size int64) string {
if size > 1024*1024*1024 {
return fmt.Sprintf("%dGB", (size / 1024 / 1024 / 1024))
} else if size > 1024*1024 {
return fmt.Sprintf("%dMB", (size / 1024 / 1024))
} else if size > 1024 {
return fmt.Sprintf("%dKB", (size / 1024))
}
return fmt.Sprintf("%dB", size)
}
func ExtractFile(raw []byte, file string) (string, error){
stream := bytes.NewReader(raw)
ustream, err := gzip.NewReader(stream)
if err != nil {
return "", err
}
reader := tar.NewReader(ustream)
var header *tar.Header
for header, err = reader.Next(); err == nil; header, err = reader.Next() {
if header.Typeflag != tar.TypeReg {
return "", errors.New("Found invalid entry in archive")
}
if header.Name != file {
continue
}
buf := new(strings.Builder)
_, err := io.Copy(buf, reader)
if err != nil {
return "", errors.New("Failed to extract file to memory")
}
return buf.String(), nil
}
return "", errors.New("File not found")
}
func GetTimePassed(t time.Time) string {
func TimePassed(t time.Time) string {
diff := time.Since(t)
res := fmt.Sprintf(
"%ds ago",
@ -101,10 +121,10 @@ func GetTimePassed(t time.Time) string {
return res
}
func CleanString(s string) string{
var badchars []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"}
func SanitizeXSS(s string) string {
var bad []string = []string{"~", "'", "\"", "/", "<", ">", "?", "=", "#", "(", ")", "{", "}", "*", "!", "`", "[", "]"}
for _, c := range badchars {
for _, c := range bad {
s = strings.ReplaceAll(s, c, "")
}

23
log/log.go Normal file
View File

@ -0,0 +1,23 @@
package log
import (
"fmt"
"time"
)
func Log(p string, f string, args ...interface{}) {
now := time.Now()
nstr := now.Format("[02/01/06 15:04:05]")
fmt.Printf("%s -%s- ", nstr, p)
fmt.Printf(f, args...)
fmt.Println()
}
func Info(f string, args ...interface{}) {
Log("INFO", f, args...)
}
func Error(f string, args ...interface{}) {
Log("ERROR", f, args...)
}

128
main.go
View File

@ -1,6 +1,6 @@
/*
* tracker | MatterLinux Package Tracker
* tracker | MatterLinux package tracker
* MatterLinux 2023-2024 (https://matterlinux.xyz)
* This program is free software: you can redistribute it and/or modify
@ -21,107 +21,91 @@
package main
import (
"log"
"strings"
"time"
"git.matterlinux.xyz/matter/tracker/lib"
"git.matterlinux.xyz/matter/tracker/log"
"git.matterlinux.xyz/matter/tracker/routes"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/template/html/v2"
)
var lastupdate time.Time
var updatetick = time.NewTicker(time.Hour)
var stopchan = make(chan struct{})
type tracker struct {
Channel chan struct{}
Config lib.Config
Last time.Time
List []lib.Package
Tick time.Ticker
}
func UpdateLoop() {
UpdatePackages()
func (t *tracker) Reload() error {
t.List = []lib.Package{}
err := t.Config.Load(&t.List, "config.json")
t.Last = time.Now()
return err
}
func (t *tracker) Loop() {
var err error
if err = t.Reload(); err != nil {
log.Error("Failed to update packages: %s", err.Error())
}
for {
select {
case <- updatetick.C:
UpdatePackages()
case <- stopchan:
updatetick.Stop()
case <-t.Tick.C:
if err = t.Reload(); err != nil {
log.Error("Failed to update packages: %s", err.Error())
}
case <-t.Channel:
t.Tick.Stop()
return
}
}
}
func UpdatePackages() {
lib.LoadAllPkgs()
lastupdate = time.Now()
func (t *tracker) Stop() {
close(t.Channel)
}
func GETIndex(c *fiber.Ctx) error {
repo := c.Query("r")
name := c.Query("n")
exact := c.Query("e")
isjson := c.Query("j")
func main() {
var (
tracker tracker
engine *html.Engine
app *fiber.App
)
if repo == "" && name == "" {
if isjson == "1" {
return c.JSON(lib.Packages)
}
return c.Render("index", fiber.Map{
"last": lib.GetTimePassed(lastupdate),
"repos": lib.Repos,
"pkgs": lib.Packages,
})
}
name = lib.CleanString(name)
var res []lib.Package
for _, p := range lib.Packages {
if(repo != "all" && p.Repo != repo){
continue
}
if(exact == ""){
if(strings.Contains(p.Name, name)){
res = append(res, p)
}
}else {
if (p.Name == name) {
res = append(res, p)
}
}
}
if isjson == "1" {
return c.JSON(res)
}
return c.Render("index", fiber.Map{
"search": name,
"last": lib.GetTimePassed(lastupdate),
"repos": lib.Repos,
"pkgs": res,
})
}
func main(){
log.SetFlags(log.Ltime | log.Lshortfile)
engine := html.New("./templates", ".html")
app := fiber.New(fiber.Config{
engine = html.New("./templates", ".html")
app = fiber.New(fiber.Config{
DisableStartupMessage: true,
Views: engine,
})
app.Static("/", "./public")
app.Get("/", GETIndex)
app.All("*", func(c *fiber.Ctx) error {
c.Locals("config", &tracker.Config)
c.Locals("list", &tracker.List)
c.Locals("last", &tracker.Last)
return c.Next()
})
app.Get("/", routes.GET_index)
app.Get("/p/:name/:version", routes.GET_package)
app.Get("*", func(c *fiber.Ctx) error {
return lib.RenderError(c, 404)
})
go UpdateLoop()
log.Println("Starting MatterLinux Package Tracker on port 9877")
go tracker.Loop()
log.Info("Starting MatterLinux package tracker on port 9877")
err := app.Listen(":9877")
if err != nil {
log.Printf("Error starting server: %s", err)
log.Error("Error starting server: %s", err.Error())
}
close(stopchan)
tracker.Stop()
}

View File

@ -5,43 +5,160 @@ main {
padding: 40px 10% 0% 10%;
}
.search {
display: flex;
color: var(--bright-main);
background: var(--dark-second);
border: solid 1px var(--bright-main);
flex-direction: column;
padding: 30px;
gap: 10px;
.package-main {
padding: 40px 20% 0% 20%;
}
.status {
.package {
border-collapse: collapse;
width: 100%;
}
.package tr {
font-size: 15px;
}
.package th {
padding: 10px;
border: 1px solid var(--bright-third);
background: var(--dark-second);
color: var(--bright-main);
vertical-align: top;
text-align: left;
font-weight: 400;
font-size: 16px;
flex-grow: 4;
}
.package .files {
display: flex;
flex-direction: column;
gap: 3px;
}
.package th a {
color: var(--bright-second);
}
.package th a:hover {
color: var(--bright-main);
}
.package tr th:first-child {
font-weight: 900;
width: 20%;
}
.package tr th:last-child {
font-weight: 400;
font-style: italic;
}
.package-name th {
background: var(--dark-third);
border: 1px solid var(--bright-second);
}
.detail {
display: flex;
flex-direction: row;
align-items: center;
justify-content: space-between;
margin-bottom: 18px;
}
form {
flex-direction: row;
justify-content: space-around;
.detail h3 {
font-size: 30px;
color: var(--bright-main);
}
.detail .links {
display: flex;
flex-direction: row;
}
.detail .links a {
border: solid 1px var(--bright-third);
background: var(--dark-second);
color: var(--bright-second);
font-size: 16px;
padding: 7px 10px;
}
.detail .links a:hover {
border: solid 1px var(--bright-second);
color: var(--bright-main);
}
.checkbox {
display: flex;
flex-direction: row;
align-items: center;
gap: 5px;
}
form input {
flex-grow: 4;
form {
display: flex;
flex-direction: row;
color: var(--bright-main);
background: var(--dark-second);
border: solid 1px var(--bright-second);
padding: 30px;
gap: 10px;
}
form .search {
width: 100%;
display: flex;
flex-direction: column;
gap: 10px;
flex: 0 0 80%;
}
@media only screen and (max-width: 1500px) {
form .search {
flex: 0 0 70%;
}
}
@media only screen and (max-width: 1000px) {
form .search {
flex: 0 0 60%;
}
}
form .search input {
font-size: 15px;
color: white;
background: var(--dark-main);
padding: 10px;
outline: none;
flex-grow: 4;
border: solid 1px var(--bright-third);
}
form select {
form .search .status{
color: var(--bright-main);
display: flex;
flex-direction: row;
justify-content: space-between;
}
@media only screen and (max-width: 900px) {
form .search .status{
flex-direction: column;
align-items: left;
gap: 5px;
}
}
form .options {
width: 100%;
display: flex;
flex-direction: column;
gap: 10px;
}
form .options select {
appearance: none;
flex-grow: 1;
font-size: 15px;
@ -52,7 +169,7 @@ form select {
cursor: pointer;
display: grid;
border: none;
width: 10%;
width: 100%;
border: solid 1px var(--bright-third);
margin: 0;
@ -77,17 +194,17 @@ form select {
background-repeat: no-repeat;
}
table {
.list {
margin-top: 20px;
border-collapse: collapse;
width: 100%;
}
table tr {
.list tr {
font-size: 15px;
}
table th, table td {
.list th, .list td {
padding: 10px;
border: 1px solid var(--bright-second);
background: var(--dark-second);
@ -96,15 +213,19 @@ table th, table td {
flex-grow: 4;
}
table td a {
.list td a {
color: var(--bright-second);
}
table th {
.list td a:hover {
color: var(--bright-main);
}
.list th {
font-weight: 900;
color: var(--bright-main);
}
table td {
.list td {
font-style: italic;
}

75
routes/index.go Normal file
View File

@ -0,0 +1,75 @@
package routes
import (
"strings"
"time"
"git.matterlinux.xyz/matter/tracker/lib"
"github.com/gofiber/fiber/v2"
)
func GET_index(c *fiber.Ctx) error {
var (
result []lib.Package
list *[]lib.Package
config *lib.Config
pools_str string
pools []string
last *time.Time
)
result = []lib.Package{}
pools = []string{}
config = c.Locals("config").(*lib.Config)
list = c.Locals("list").(*[]lib.Package)
last = c.Locals("last").(*time.Time)
is_exact := c.Query("exact") == "1"
is_json := c.Query("json") == "1"
if pools_str = c.Query("pools"); pools_str != "" {
pools = strings.Split(pools_str, ",")
}
query := c.Query("q")
for _, pkg := range *list {
found := false
if is_exact && query != "" && pkg.Name != query {
continue
}
if !is_exact && query != "" && !strings.Contains(pkg.Name, query) {
continue
}
if len(pools) != 0 {
for _, p := range pools {
if p == pkg.Pool.ID() {
found = true
break
}
}
} else {
found = true
}
if !found {
continue
}
result = append(result, pkg)
}
if is_json {
return c.JSON(result)
}
return c.Render("index", &fiber.Map{
"query": query,
"list": result,
"last": lib.TimePassed(*last),
"pools": config.Pools,
})
}

45
routes/package.go Normal file
View File

@ -0,0 +1,45 @@
package routes
import (
"fmt"
"path"
"git.matterlinux.xyz/matter/tracker/lib"
"github.com/gofiber/fiber/v2"
)
func GET_package(c *fiber.Ctx) error {
var (
name string
version string
pool string
list *[]lib.Package
)
is_json := c.Query("json") == "1"
is_download := c.Query("download") == "1"
list = c.Locals("list").(*[]lib.Package)
version = c.Params("version")
name = c.Params("name")
pool = c.Query("p")
for _, pkg := range *list {
if pkg.Name != name || (version != "ANY" && pkg.Version != version) || (pool != "" && pkg.Pool.ID() != pool) {
continue
}
if is_download {
c.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", path.Base(pkg.Archive)))
return c.SendFile(pkg.Archive)
}
if is_json {
return c.JSON(pkg)
}
return c.Render("package", &pkg)
}
return lib.RenderError(c, 404)
}

View File

@ -9,43 +9,56 @@
<body>
{{template "parts/bar" .}}
<main>
<div class="search">
<form action="/" method="GET">
{{if .search}}
<input placeholder="Hit enter to search" type="text" name="n" value="{{.search}}">
{{else}}
<input placeholder="Hit enter to search" type="text" name="n" autofocus>
{{end}}
<select name="r">
<option value="all">Select repo</option>
{{range .repos}}
<option value="{{.Name}}">{{.Name}}</option>
<div class="options">
<select name="pools">
<option value="">Select pool</option>
{{range .pools}}
<option value="{{.ID}}">{{.Display}}</option>
{{end}}
</select>
</form>
<div class="checkbox">
<input type="checkbox" name="exact" value="1">
<label>Only show exact matches</label>
</div>
<div class="checkbox">
<input type="checkbox" name="json" value="1">
<label>Show results as JSON</label>
</div>
</div>
<div class="search">
{{if .query}}
<input placeholder="Hit enter to search" type="text" name="q" value="{{.query}}">
{{else}}
<input placeholder="Hit enter to search" type="text" name="q" autofocus>
{{end}}
<div class="status">
<p>Listing {{len .pkgs}} packages</p>
<p>Listing {{len .list}} packages</p>
<p>Last updated: {{.last}}</p>
</div>
</div>
<table class="pkgs">
</form>
<table class="list">
<tr>
<th>Name</th>
<th>Repo</th>
<th>Pool</th>
<th>Version</th>
<th>Size</th>
<th>Description</th>
<th>Dependencies</th>
</tr>
{{range .pkgs}}
{{range .list}}
<tr>
<td><a href="{{.URL}}">{{.Name}}</a></td>
<td>{{.Repo}}</td>
<td>{{.Ver }}</td>
<td><a href="/p/{{.Name}}/{{.Version}}?p={{.Pool.ID}}">{{.Name}}</a></td>
{{if .Pool}}
<td>{{.Pool.Display}}</td>
{{else}}
<td></td>
{{end}}
<td>{{.Version }}</td>
<td>{{.Size}}</td>
<td>{{.Desc}}</td>
<td>{{.StrDeps}}</td>
<td>{{.DependsToStr}}</td>
</tr>
{{end}}
</table>

64
templates/package.html Normal file
View File

@ -0,0 +1,64 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>MatterLinux | {{.Name}} {{.Version}}</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=1200">
<link href="/style.css" rel="stylesheet">
</head>
<body>
{{template "parts/bar" .}}
<main class="package-main">
<div class="detail">
<h3>Package details</h3>
<div class="links">
<a href="{{.URL}}">View source</a>
<a href="/p/{{.Name}}/{{.Version}}?download=1">Download archive</a>
</div>
</div>
<table class="package">
<tr class="package-name">
<th>Name</th>
<th>{{.Name}}</th>
</tr>
<tr>
<th>Version</th>
<th>{{.Version}}</th>
</tr>
<tr>
<th>Size</th>
<th>{{.Size}}</th>
</tr>
<tr>
<th>Pool</th>
<th>{{.Pool.Display}}</th>
</tr>
<tr>
<th>Description</th>
<th>{{.Desc}}</th>
</tr>
{{if .Depends}}
<tr>
<th>Dependencies</th>
<th>
{{range .Depends}}
<a href="/p/{{.}}/ANY">{{.}}</a>
{{end}}
</th>
</tr>
{{end}}
{{$files := .Files}}
{{if $files}}
<tr>
<th>Files</th>
<th class="files">
{{range $files}}
<p>/{{.}}</p>
{{end}}
</th>
</tr>
{{end}}
</table>
</main>
</body>
</html>