new: mtsc-common, mp-build and mp-pool

This commit is contained in:
ngn
2024-08-11 17:45:54 +03:00
parent e9db1d41a5
commit aa5fbb82d6
26 changed files with 1619 additions and 478 deletions

16
mp-build/Makefile Normal file
View File

@ -0,0 +1,16 @@
SCRIPTS = $(patsubst scripts/%.sh,%,$(wildcard scripts/*.sh))
PREFIX = /usr
install:
@for s in $(SCRIPTS) ; do \
echo "installing script: $$s" ; \
install -m755 "scripts/$$s.sh" $(DESTDIR)/$(PREFIX)/bin/$$s ; \
done
uninstall:
@for s in $(SCRIPTS) ; do \
echo "removing script: $$s" ; \
rm $(DESTDIR)/$(PREFIX)/bin/$$s ; \
done
.PHONY: install uninstall

26
mp-build/README.md Normal file
View File

@ -0,0 +1,26 @@
# mp-build | MatterLinux package build scripts
The package build scripts for MatterLinux package system. These scripts
are used for building and creating packages.
### Usage
To create an package source directory, you can use the `mp-new`
script:
```bash
mp-new <name>_<version>
```
This will create all the required source files, all templated
according to the package name and the version you specify.
After editing the source files, you can build the package by
using the `mp-build` script:
```bash
mp-build /path/to/package
```
When the build is completed, the package archive will be created
inside the `/path/to/package/dist` directory.
You can also specify different build options, to list these
options you can run:
```bash
mp-build --help
```

452
mp-build/scripts/mp-build.sh Executable file
View File

@ -0,0 +1,452 @@
#!/bin/bash
# mp-build | MatterLinux package build script
# MatterLinux 2023-2024 (https://matterlinux.xyz)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#############################
## import common functions ##
#############################
location="$(dirname "${0}")"
location="$(realpath "${location}")"
commonsh="$(echo "${location}" | sed 's/\/bin/\/lib/g')/mtsc-common.sh"
source "${commonsh}"
if [ "${?}" != "0" ]; then
echo "Failed to import mtsc-common"
exit 1
fi
####################
## util functions ##
####################
# extracts the filename from an URL
get_fn_url() {
file=$(echo "${1}" | rev | cut -d "/" -f -1 | rev)
}
# download/copy file, used for obtaning $FILES
get_file() {
if [[ "${1}" == "https://"* || "${1}" == "http://"* || "${1}" == "ftp://"* ]]
then
curl "${1}" --progress-bar -OL
return $?
elif [ -f "${pkgpath}/${1}" ]; then
cp "${pkgpath}/${1}" .
return $?
else
return 1
fi
}
# length based hash check function
check_hash() {
if [[ "$2" == "NOHASH" ]]; then
info "Using NOHASH for $1"
return 0
fi
local len=${#2}
if [[ $len == 128 ]]; then
local hsh=$(sha512sum "$1" | cut -d " " -f 1)
elif [[ $len == 64 ]]; then
local hsh=$(sha256sum "$1" | cut -d " " -f 1)
elif [[ $len == 64 ]]; then
local hsh=$(sha256sum "$1" | cut -d " " -f 1)
elif [[ $len == 40 ]]; then
local hsh=$(sha1sum "$1" | cut -d " " -f 1)
elif [[ $len == 32 ]]; then
local hsh=$(md5sum "$1" | cut -d " " -f 1)
fi
if [[ "$hsh" == "$2" ]]; then
success "Hash matches for $1"
return 0
else
error "Bad hash for $1"
return 1
fi
}
# prints the help info
help_cmd() {
info "MatterLinux package build script"
info "Usage: ${0} <options> [package directory]"
info "Options:"
echo_color " $BOLD--no-depend$RESET: don't check depends"
echo_color " $BOLD--no-stdout$RESET: disable stdout for PACKAGE() function"
echo_color " $BOLD--no-cache$RESET: don't check cache"
echo_color " $BOLD--no-opts$RESET: don't show/list options"
echo_color " $BOLD--cores$RESET: how many cores to use for the build"
echo
info "Licensed under GPLv3, see <https://www.gnu.org/licenses/> for more information"
}
# clean the this directory
clean_dist() {
rm -f "${distpath}/DATA"
rm -f "${distpath}/CHANGES"
rm -f "${distpath}/INSTALL"
rm -f "${distpath}/HASHES"
rm -f "${distpath}/files.tar.gz"
}
# checks/installs a list of depends
check_depends() {
local list=("$@")
if [ "${#list[@]}" == "0" ]; then
info "Got zero depends, skipping depend check"
return 0
fi
if [ "$EUID" -ne 0 ]; then
if type doas > /dev/null; then
DOAS="doas"
elif type sudo > /dev/null; then
DOAS="sudo"
else
error "Failed to find doas or sudo, skipping depend check"
return 0
fi
fi
for dep in "${list[@]}"; do
if [ -z "${depends_str}" ]; then
depends_str="${dep}"
else
depends_str="${depends_str} ${dep}"
fi
done
depends_uniq=$(echo "${depends_str}" | tr ' ' '\n' | sort | uniq)
depends_count=$(echo "${depends_uniq}" | wc -l)
depends_uniq=$(echo "${depends_uniq}" | tr '\n' ' ')
info "Installing ${depends_count} depends"
$DOAS matt install --yes $depends_uniq
return $?
}
#################
## main script ##
#################
OPT_NO_DEPEND=0 # checking depends is ENABLED
OPT_NO_STDOUT=0 # PACKAGE() function output is ENABLED
OPT_NO_CACHE=0 # cache is ENABLED
OPT_NO_OPTS=0 # showing/listing options is ENABLED
OPT_CORES=$(nproc) # use ALL CPU cores
# parses all the options
for arg in "$@"; do
case $arg in
"--help")
help_cmd
exit 0
;;
"--no-depend")
OPT_NO_DEPEND=1 ;;
"--no-stdout")
OPT_NO_STDOUT=1 ;;
"--no-cache")
OPT_NO_CACHE=1 ;;
"--no-opts")
OPT_NO_OPTS=1 ;;
"--cores"*)
OPT_CORES=$(echo "${arg}" | cut -d "=" -f2) ;;
--*)
error "Unknown option: ${arg}"
exit 1 ;;
*)
if [ -z "${TARGET}" ]; then
TARGET="${arg}"
else
error "Unknown argument: ${arg}"
exit 1
fi
;;
esac
done
if [ -z "${TARGET}" ]; then
error "Package directory is not specified, run with \"--help\" for more information"
exit 1
fi
if [ ! -d "${TARGET}" ]; then
error "Package directory \"${TARGET}\" does not exist"
exit 1
fi
if [ $OPT_NO_DEPEND -eq 0 ] && ! command -v matt &> /dev/null; then
error "!!! BUILD ON MATTERLINUX !!!"
error "matt is not installed, please build on a MatterLinux system"
error "Do NOT create bug reports if build fails on non-MatterLinux systems"
info "Auto enabling NO_DEPEND as depend check will fail without matt"
OPT_NO_DEPEND=1
fi
# print the options
if [ $OPT_NO_OPTS -eq 0 ]; then
info "Running mp-build with the options:"
print " $BOLD NO_DEPEND = $(itoyn $OPT_NO_DEPEND)"
print " $BOLD NO_SDTOUT = $(itoyn $OPT_NO_STDOUT)"
print " $BOLD NO_CACHE = $(itoyn $OPT_NO_CACHE)"
print " $BOLD NO_OPTS = $(itoyn $OPT_NO_OPTS)"
print " $BOLD CORES = $OPT_CORES"
fi
cd "${TARGET}"
check_ret "Failed to change directory into \"${TARGET}\""
if [ ! -f "pkg.sh" ]; then
error "Package directory does not contain a package script (pkg.sh)"
exit 1
fi
# source and verify the package script
source "pkg.sh"
check_ret "Failed to source the package script"
check_pkg_vars
check_ret
# check the changes file
if [ ! -f "changes.md" ]; then
warn "Package directory does not contain a changes file (changes.md)"
warn "Creating an empty one, however you should you should create an actual changes file"
echo "# ${VERSION}" > "changes.md"
echo "No changes specified" >> "changes.md"
fi
# setup package directories
pkgpath="$(realpath .)"
cachepath="$(realpath '.cache')"
distpath="$(realpath 'dist')"
rootpath="$(realpath 'root')"
mkdir -p "${rootpath}"
check_ret "Failed to create the root directory"
# check the cache
package_hash=$(md5sum "${pkgpath}/pkg.sh" 2> /dev/null | awk '{print $1}')
changes_hash=$(md5sum "${pkgpath}/changes.md" 2> /dev/null | awk '{print $1}')
if [ $OPT_NO_CACHE -eq 0 ] && [ -f "${cachepath}/last" ]; then
package_cache=$(sed -n '1p' "${cachepath}/last")
changes_cache=$(sed -n '2p' "${cachepath}/last")
if [[ "${package_cache}" == "${package_hash}" ]] && [[ "${changes_cache}" == "${changes_hash}" ]]; then
info "Found build in the cache (add --no-cache if you want to rebuild anyway)"
success "Build was successful"
exit 0
fi
fi
# check all the depends for the build
all_depends+=("${DEPENDS[@]}")
all_depends+=("${BUILD[@]}")
if [ $OPT_NO_DEPEND -eq 0 ]; then
check_depends "${all_depends[@]}"
check_ret "Failed to check all the dependencies"
fi
# make sure file count and the hash count is the same
file_count=${#FILES[@]}
hash_count=${#HASHES[@]}
if [[ $file_count != $hash_count ]]; then
error "There are ${file_count} files but ${hash_count} hashes"
exit 1
fi
# cleanup files from previous builds
info "Cleaning up files from the previous build"
set_indent
for f in "${rootpath}"/*; do
# do not remove the file if its required for build
for ((i = 0; i < $file_count; i++)) do
get_fn_url "${FILES[i]}"
if [[ "$(realpath "${rootpath}/${file}")" == "$(realpath ${f})" ]]; then
isdownload=1
break
fi
done
if [ -z $isdownload ]; then
info "Removing file from previous build: $f"
rm -rf "$f"
fi
unset isdownload
done
unset_indent
cd "${rootpath}"
check_ret "Failed to change directory into the root directory"
info "Obtaining all the files"
set_indent
for ((i = 0; i < $file_count; i++)) do
get_fn_url "${FILES[i]}"
# check if the file is already present
if [ -f "$file" ]; then
check_hash $file ${HASHES[i]} > /dev/null
if [ $? -eq 0 ]; then
success "($(($i+1))/$file_count) Using the file from previous build: $file"
continue
fi
rm -f $file
fi
info "($(($i+1))/$file_count) Obtaining file: $file"
get_file ${FILES[i]}
check_ret "Failed to obtain the file!"
check_hash "${file}" ${HASHES[i]}
check_ret "Verification failed for file: $file"
done
unset_indent
info "Running the build function"
export CFLAGS="-march=x86-64 -mtune=generic -O2"
export ROOTDIR="${rootpath}"
export MAKEFLAGS="-j${OPT_CORES}"
export MAKEOPTS="-j${OPT_CORES}"
export XORG_PREFIX="/usr"
export XORG_CONFIG="--prefix=${XORG_PREFIX} --sysconfdir=/etc \
--localstatedir=/var --disable-static"
SECONDS=0
if [ $OPT_NO_STDOUT -eq 0 ]; then
fakeroot "${location}/mp-wrap" "../pkg.sh"
check_ret "Package PACKAGE() function failed"
else
fakeroot "${location}/mp-wrap" "../pkg.sh" > /dev/null
check_ret "Package PACKAGE() function failed"
fi
unset XORG_CONFIG XORG_PREFIX
unset CFLAGS ROOTDIR
unset MAKEOPTS MAKEFLAGS
if [ "$SECONDS" != "0" ]; then
success "Completed the build in ${SECONDS}s"
else
success "Completed the build in less than a second"
fi
# remove all the $FILES
for ((i = 0; i < $file_count; i++)) do
get_fn_url ${FILES[i]} && rm -f $file
success "Removed file: $file"
done
info "Building the package archive"
set_indent
# cleanup the dist directory
mkdir -p "${distpath}"
clean_dist
# build the files archive
find . -printf "%P\n" | fakeroot tar -czf "${distpath}/files.tar.gz" --no-recursion -T -
check_ret "(1/6) Failed to create the files archive (files.tar.gz)"
success "(1/6) Created the files archive (files.tar.gz)"
# build the DATA file
cat > "${distpath}/DATA" << EOF
[${NAME}]
version = ${VERSION}
desc = ${DESC}
size = $(du -sb "${rootpath}" | awk '{print $1}')
EOF
for dep in "${DEPENDS[@]}"; do
echo "depends = ${dep}" >> "${distpath}/DATA"
done
for keep in "${KEEP[@]}"; do
echo "keep = ${keep}" >> "${distpath}/DATA"
done
success "(2/6) Created the package data file (DATA)"
# create the changes file
cp "${pkgpath}/changes.md" "${distpath}/CHANGES"
check_ret "(3/6) Failed to create the changes file (CHANGES)"
success "(3/6) Created the changes file (CHANGES)"
# create the install script
if type INSTALL &>/dev/null; then
echo "$(type INSTALL | tail -n+2)" > "${distpath}/INSTALL"
check_ret "(4/6) Failed to create the install script (INSTALL)"
fi
success "(4/6) Created the install script (INSTALL)"
# create the HASHES file
find . -type f -exec md5sum {} >> "${distpath}/HASHES" \;
check_ret "(5/6) Failed to create the file hash list (HASHES)"
success "(5/6) Created the file hash list (HASHES)"
# remove previous build archives
for old in "${distpath}/${NAME}_"*.mpf; do
rm -f $old
done
# create the final archive
archive="${NAME}_${VERSION}.mpf"
pushd "${distpath}" > /dev/null
find . -printf "%P\n" | fakeroot tar -czf "${archive}" --no-recursion -T -
check_ret "(6/6) Failed to create the package archive (${archive})"
popd > /dev/null
success "(6/6) Created the package archive (${archive})"
# clean up
cd "${pkgpath}"
info "Cleaning up the dist directory"
clean_dist
info "Cleaning the root directory"
rm -rf "${rootpath}"
# update the cache
unset_indent
info "Updating the package cache"
set_indent
mkdir -p "${cachepath}"
echo "${package_hash}" > "${cachepath}/last"
check_ret "(1/2) Failed to add package script to the cache"
success "(1/2) Added package script to the cache"
echo "${changes_hash}" >> "${cachepath}/last"
check_ret "(2/2) Failed to add changes file to the cache"
success "(2/2) Added changes file to the cache"
unset_indent
success "Build was successful"
exit 0

94
mp-build/scripts/mp-migrate.sh Executable file
View File

@ -0,0 +1,94 @@
#!/bin/bash
# mp-migrate | MatterLinux package migration script
# MatterLinux 2023-2024 (https://matterlinux.xyz)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#############################
## import common functions ##
#############################
location="$(dirname "${0}")"
location="$(realpath "${location}")"
commonsh="$(echo "${location}" | sed 's/\/bin/\/lib/g')/mtsc-common.sh"
source "${commonsh}"
if [ "${?}" != "0" ]; then
echo "Failed to import mtsc-common"
exit 1
fi
#################
## main script ##
#################
if [ -z "$1" ]; then
error "Please specify a package directory"
exit 1
fi
if [ ! -d "$1" ]; then
error "Failed to access to the specified directory"
exit 1
fi
target="$(realpath "${1}")"
pkg_script="${target}/pkg.sh"
install_script="${target}/install.sh"
if [ ! -f "${pkg_script}" ]; then
error "Specified directory does not contain a package script (pkg.sh)"
exit 1
fi
info "Sourcing the package script"
source "${pkg_script}"
if [ -z "${VERSION}" ]; then
error "Required package variable \"\$VERSION\" is not set"
exit 1
fi
info "Auto generating the changes file"
cat > "${target}/changes.md" << EOF
# ${VERSION}
First version
EOF
info "Adding the gitignore file"
cat > "${target}/.gitignore" << EOF
.cache/
dist/
root/
EOF
sed -i 's/build() {/PACKAGE() {/' "${pkg_script}"
check_ret "Failed to update the build function"
if [ -f "${install_script}" ]; then
info "Migrating the install script"
echo >> "${pkg_script}"
echo "INSTALL(){" >> "${pkg_script}"
while read l; do
echo " ${l}" >> "${pkg_script}"
done < "${install_script}"
echo "}" >> "${pkg_script}"
rm "${install_script}"
check_ret "Failed to remove the install script"
fi
success "Migration is completed"

108
mp-build/scripts/mp-new.sh Executable file
View File

@ -0,0 +1,108 @@
#!/bin/bash
# mp-new | MatterLinux package creation script
# MatterLinux 2023-2024 (https://matterlinux.xyz)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#############################
## import common functions ##
#############################
location="$(dirname "${0}")"
location="$(realpath "${location}")"
commonsh="$(echo "${location}" | sed 's/\/bin/\/lib/g')/mtsc-common.sh"
source "${commonsh}"
if [ "${?}" != "0" ]; then
echo "Failed to import mtsc-common"
exit 1
fi
#################
## main script ##
#################
if [ -z "$1" ]; then
error "Please specify a package"
exit 1
fi
pkg=$(basename "${1}")
target=$(dirname "${1}")
case "${1}" in
*_*)
name=$(echo "${pkg}" | cut -d "_" -f1)
version=$(echo "${pkg}" | cut -d "_" -f2)
;;
*)
warn "Package version is not specified, using \"1\" as version instead"
name="${pkg}"
version="1"
;;
esac
if [ -d "${target}/${name}" ]; then
error "There is already a directory for \"${target}/${name}\""
exit 1
fi
info "Creating directory for \"${name}\""
mkdir "${target}/${name}"
check_ret "Failed to create package directory for \"${name}\""
cd "${target}/${name}"
cat > pkg.sh << EOF
# general info
NAME="${name}"
DESC=""
VERSION="${version}"
# required files
FILES=()
HASHES=()
# install and build depends
DEPENDS=()
BUILD=()
PACKAGE(){
tar xf "\${NAME}-\${VERSION}.tar.gz"
cd "\${NAME}-\${VERSION}"
# build commands
cd .. && rm -r "\${NAME}-\${VERSION}"
}
INSTALL(){
# install script
# remove if not requied
}
EOF
cat > changes.md << EOF
# ${version}
First version
EOF
cat > .gitignore << EOF
.cache/
dist/
root/
EOF
success "Created package directory and files for \"${name}\""

56
mp-build/scripts/mp-wrap.sh Executable file
View File

@ -0,0 +1,56 @@
#!/bin/bash
# mp-wrap | MatterLinux package build function wrapper
# MatterLinux 2023-2024 (https://matterlinux.xyz)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#############################
## import common functions ##
#############################
location="$(dirname "${0}")"
location="$(realpath "${location}")"
commonsh="$(echo "${location}" | sed 's/\/bin/\/lib/g')/mtsc-common.sh"
source "${commonsh}"
if [ "${?}" != "0" ]; then
echo "Failed to import mtsc-common"
exit 1
fi
#################
## main script ##
#################
if [ -z "${1}" ]; then
error "Package script not specified"
exit 1
fi
ninja(){
echo "[ninja wrapper] adding MAKEFLAGS"
/usr/bin/ninja $MAKEFLAGS $@
}
echo "running the PACKAGE() function"
set -e
source "${1}"
PACKAGE
ret="$?"
set +e
echo "PACKAGE() returned with code $ret"
exit $ret