mtsc/mp-build/scripts/mp-build.sh

#!/bin/bash

# mp-build | MatterLinux package build script
# MatterLinux 2023-2024 (https://matterlinux.xyz)

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

#############################
## import common functions ##
#############################
location="$(dirname "${0}")"
location="$(realpath "${location}")"
commonsh="$(echo "${location}" | sed 's/\/bin/\/lib/g')/mtsc-common.sh"

source "${commonsh}"

if [ "${?}" != "0" ]; then
  echo "Failed to import mtsc-common"
  exit 1
fi

####################
## util functions ##
####################
# extracts the filename from an URL
get_fn_url() {
  file=$(echo "${1}" | rev | cut -d "/" -f -1 | rev)
}

# download/copy file, used for obtaning $FILES
get_file() {
  if [[ "${1}" == "https://"* || "${1}" == "http://"* || "${1}" == "ftp://"* ]]
  then
    if [ $OPT_INSECURE -eq 1 ]; then
      curl "${1}" --insecure --progress-bar -OL
    else
      curl "${1}" --progress-bar -OL
    fi
    return $?
  elif [ -f "${pkgpath}/${1}" ]; then
    cp "${pkgpath}/${1}" .
    return $?
  else
    return 1
  fi
}

# length based hash check function
check_hash() {
  if [[ "$2" == "NOHASH" ]]; then
    info "Using NOHASH for $1"
    return 0
  fi

  local len=${#2}
  if [[ $len == 128 ]]; then
    local hsh=$(sha512sum "$1" | cut -d " " -f 1)
  elif [[ $len == 64 ]]; then
    local hsh=$(sha256sum "$1" | cut -d " " -f 1)
  elif [[ $len == 64 ]]; then
    local hsh=$(sha256sum "$1" | cut -d " " -f 1)
  elif [[ $len == 40 ]]; then
    local hsh=$(sha1sum "$1" | cut -d " " -f 1)
  elif [[ $len == 32 ]]; then
    local hsh=$(md5sum "$1" | cut -d " " -f 1)
  fi

  if [[ "$hsh" == "$2" ]]; then
    success "Hash matches for $1"
    return 0
  else
    error "Bad hash for $1"
    return 1
  fi
}

# prints the help info
help_cmd() {
  info  "MatterLinux package build script (mtsc ${MTSC_VERSION})" # sourced from mtsc-common
  info  "Usage: ${0} <options> [package directory]"
  info  "Options:"
  echo_color "   $BOLD--no-depend$RESET:  don't check depends"
  echo_color "   $BOLD--no-stdout$RESET:  disable stdout for PACKAGE() function"
  echo_color "   $BOLD--no-cache$RESET:   don't check cache"
  echo_color "   $BOLD--insecure$RESET:   allow insecure curl downloads"
  echo_color "   $BOLD--no-opts$RESET:    don't show/list options"
  echo_color "   $BOLD--cores$RESET:      how many cores to use for the build"
  echo_color "   $BOLD--out$RESET:        directory for the output archive"
  echo
  info  "Licensed under GPLv3, see <https://www.gnu.org/licenses/> for more information"
}

# clean the this directory
clean_dist() {
  rm -f "${distpath}/DATA"
  rm -f "${distpath}/CHANGES"
  rm -f "${distpath}/INSTALL"
  rm -f "${distpath}/HASHES"
  rm -f "${distpath}/files.tar.gz"
}

# install a list of packages with matt 
matt_install(){
  if [ "$EUID" -ne 0 ]; then
    if type doas > /dev/null; then
      DOAS="doas"
    elif type sudo > /dev/null; then
      DOAS="sudo"
    else
      error "Failed to find doas or sudo, cannot run the install command"
      return 1
    fi
  fi
  
  $DOAS matt install --yes --skip --ignore-none $@
  return $?
}

# checks and installs a list of depends
check_depends() {
  local list=("$@")

  if [ "${#list[@]}" == "0" ]; then
    info "Got zero depends, skipping depend check"
    return 0
  fi

  for dep in "${list[@]}"; do
    if [ -z "${depends_str}" ]; then
      depends_str="${dep}"
    else
      depends_str="${depends_str} ${dep}"
    fi
	done

  depends_uniq="$(echo "${depends_str}" | tr ' ' '\n' | sort | uniq)"
  depends_count="$(echo "${depends_uniq}" | wc -l)"
  depends_uniq="$(echo "${depends_uniq}" | tr '\n' ' ')"

  info "Installing ${depends_count} depends"
  matt_install $depends_uniq
  return $?
}

#################
## main script ##
#################
OPT_NO_DEPEND=0     # checking depends is ENABLED
OPT_NO_STDOUT=0     # PACKAGE() function output is ENABLED
OPT_NO_CACHE=0      # cache is ENABLED
OPT_INSECURE=0      # insecure curl downloads are DISABLED
OPT_NO_OPTS=0       # showing/listing options is ENABLED
OPT_CORES=$(nproc)  # use ALL CPU cores
OPT_OUT="DEFAULT"   # use the package dist directory for output

# parses all the options
for arg in "$@"; do
  case $arg in
    "--help")
      help_cmd
      exit 0
      ;;
    "--no-depend")
      OPT_NO_DEPEND=1 ;;
    "--no-stdout")
      OPT_NO_STDOUT=1 ;;
    "--no-cache")
      OPT_NO_CACHE=1 ;;
    "--insecure")
      OPT_INSECURE=1 ;;
    "--no-opts")
      OPT_NO_OPTS=1 ;;
    "--cores"*)
      OPT_CORES="$(echo "${arg}" | cut -d '=' -f2)" ;;
    "--out"*)
      OPT_OUT="$(echo "${arg}" | cut -d '=' -f2)"
      OPT_OUT="$(echo "${OPT_OUT}" | sed "s/'//g")"
      OPT_OUT="$(echo "${OPT_OUT}" | sed 's/"//g')"
      if [ ! -d "${OPT_OUT}" ]; then
        error "Failed to access to the output directory (${OPT_OUT})"
        exit 1
      fi ;;
    --*)
      error "Unknown option: ${arg}"
      exit 1 ;;
    *)
      if [ -z "${TARGET}" ]; then
        TARGET="${arg}"
      else
        error "Unknown argument: ${arg}"
        exit 1
      fi
      ;;
  esac
done

if [ -z "${TARGET}" ]; then
  error "Package directory is not specified, run with \"--help\" for more information"
  exit 1
fi

if [ ! -d "${TARGET}" ]; then
  error "Package directory \"${TARGET}\" does not exist"
  exit 1
fi

if [ $OPT_NO_DEPEND -eq 0 ] && ! command -v matt &> /dev/null; then
  error "BUILD ON MATTERLINUX!"
  set_indent
  warn  "matt is not installed, please build on a MatterLinux system"
  warn  "Do NOT create bug reports if build fails on non-MatterLinux systems"
  info  "Auto enabling NO_DEPEND as depend check will fail without matt"
  unset_indent
  OPT_NO_DEPEND=1
fi

# print the options
if [ $OPT_NO_OPTS -eq 0 ]; then
  info    "Running mp-build with the options:"
  print   " $BOLD NO_DEPEND  = $(itoyn $OPT_NO_DEPEND)"
  print   " $BOLD NO_SDTOUT  = $(itoyn $OPT_NO_STDOUT)"
  print   " $BOLD NO_CACHE   = $(itoyn $OPT_NO_CACHE)"
  print   " $BOLD INSECURE   = $(itoyn $OPT_INSECURE)"
  print   " $BOLD NO_OPTS    = $(itoyn $OPT_NO_OPTS)"
  print   " $BOLD CORES      = ${OPT_CORES}"
  print   " $BOLD OUT        = ${OPT_OUT}"
fi

cd "${TARGET}"
check_ret "Failed to change directory into \"${TARGET}\""

if [ ! -f "pkg.sh" ]; then
  error "Package directory does not contain a package script (pkg.sh)"
  exit 1
fi

# source and verify the package script
source "pkg.sh"
check_ret "Failed to source the package script"

check_pkg_vars
check_ret

# check the changes file
if [ ! -f "changes.md" ]; then
  warn "Package directory does not contain a changes file (changes.md)"
  warn "Creating an empty one, however you should you should create an actual changes file"
  echo "# ${VERSION}" > "changes.md"
  echo "No changes specified" >> "changes.md"
fi

# setup package directories
pkgpath="$(realpath .)"
cachepath="$(realpath '.cache')"
distpath="$(realpath 'dist')"
if [ "${OPT_OUT}" == "DEFAULT" ]; then
  outpath="${distpath}"
else
  outpath="${OPT_OUT}"
fi
rootpath="$(realpath 'root')"

mkdir -p "${rootpath}"
check_ret "Failed to create the root directory"

# check the cache
package_hash=$(md5sum "${pkgpath}/pkg.sh" 2> /dev/null | awk '{print $1}')
changes_hash=$(md5sum "${pkgpath}/changes.md" 2> /dev/null | awk '{print $1}')

if [ $OPT_NO_CACHE -eq 0 ] && [ -f "${cachepath}/last" ]; then
  package_cache=$(sed -n '1p' "${cachepath}/last")
  changes_cache=$(sed -n '2p' "${cachepath}/last")

  if [ -f "${outpath}/${NAME}_${VERSION}.mpf" ] && [[ "${package_cache}" == "${package_hash}" ]] && [[ "${changes_cache}" == "${changes_hash}" ]]; then
    info "Found build in the cache (add --no-cache if you want to rebuild anyway)"
    success "Build was successful"
    exit 0
  fi
fi

# check all the depends for the build
all_depends+=("${DEPENDS[@]}")
all_depends+=("${BUILD[@]}")

if [ $OPT_NO_DEPEND -eq 0 ]; then
  info "Checking all the dependencies"
  set_indent

  is_essential_installed="$(matt info build-essential --grep | grep INSTALLED)"
  is_essential_installed="$(echo "${is_essential_installed}" | cut -d: -f2)"

  if [ "${is_essential_installed}" == "0" ]; then
    info "Installing build-essential (required for every build)"
    matt_install build-essential
    check_ret "Failed to install build-essential package"
  fi

  check_depends "${all_depends[@]}"
  check_ret "Failed to check all the dependencies"
  unset_indent
fi

# make sure file count and the hash count is the same
file_count=${#FILES[@]}
hash_count=${#HASHES[@]}

if [[ $file_count != $hash_count ]]; then
  error "There are ${file_count} files but ${hash_count} hashes"
  exit 1
fi

# cleanup files from previous builds
info "Cleaning up files from the previous build"
set_indent

for f in "${rootpath}"/*; do
  # do not remove the file if its required for build
  for ((i = 0; i < $file_count; i++)) do
    get_fn_url "${FILES[i]}"
    if [[ "$(realpath "${rootpath}/${file}")" == "$(realpath ${f})" ]]; then
      isdownload=1
      break
    fi
  done

  if [ -z $isdownload ]; then
    info "Removing file from previous build: $f"
    rm -rf "$f"
  fi

  unset isdownload
done

unset_indent

cd "${rootpath}"
check_ret "Failed to change directory into the root directory"

info "Obtaining all the files"
set_indent

for ((i = 0; i < $file_count; i++)) do
  get_fn_url "${FILES[i]}"

  # check if the file is already present
  if [ -f "$file" ]; then
    check_hash $file ${HASHES[i]} > /dev/null
    if [ $? -eq 0 ]; then
      success "($(($i+1))/$file_count) Using the file from previous build: $file"
      continue
    fi
    rm -f $file
  fi

  info "($(($i+1))/$file_count) Obtaining file: $file"
  get_file ${FILES[i]}
  check_ret "Failed to obtain the file!"

  check_hash "${file}" ${HASHES[i]}
  check_ret "Verification failed for file: $file"
done

unset_indent

info "Running the build function"
export CC="gcc"
export CXX="g++"
export CFLAGS="-march=x86-64 -mtune=generic -O2"
export CPPFLAGS="-march=x86-64 -mtune=generic -O2"
export CXXFLAGS="-march=x86-64 -mtune=generic -O2"
export ROOTDIR="${rootpath}"
export MAKEFLAGS="-j${OPT_CORES}"
export MAKEOPTS="-j${OPT_CORES}"
export XORG_PREFIX="/usr"
export XORG_CONFIG="--prefix=${XORG_PREFIX} --sysconfdir=/etc \
      --localstatedir=/var --disable-static"

SECONDS=0

if [ $OPT_NO_STDOUT -eq 0 ]; then
  fakeroot "${location}/mp-wrap" "../pkg.sh"
  check_ret "Package PACKAGE() function failed"
else
  fakeroot "${location}/mp-wrap" "../pkg.sh" > /dev/null
  check_ret "Package PACKAGE() function failed"
fi

unset XORG_CONFIG XORG_PREFIX
unset MAKEOPTS MAKEFLAGS CXX
unset CPPFLAGS CXXFLAGS
unset CFLAGS ROOTDIR CC

if [ "$SECONDS" != "0" ]; then
  success "Completed the build in ${SECONDS}s"
else
  success "Completed the build in less than a second"
fi

# remove all the $FILES
for ((i = 0; i < $file_count; i++)) do
  get_fn_url ${FILES[i]} && rm -f $file
  success "Removed file: $file"
done

info "Building the package archive"
set_indent

# cleanup the dist directory
mkdir -p "${distpath}"
clean_dist

# build the files archive
find . -printf "%P\n" | fakeroot tar -czf "${distpath}/files.tar.gz" --no-recursion -T -
check_ret "(1/6) Failed to create the files archive (files.tar.gz)"
success "(1/6) Created the files archive (files.tar.gz)"

# build the DATA file
cat > "${distpath}/DATA" << EOF
[${NAME}]
version = ${VERSION}
desc = ${DESC}
size = $(du -sb "${rootpath}" | awk '{print $1}')
EOF

for dep in "${DEPENDS[@]}"; do
  echo "depends = ${dep}" >> "${distpath}/DATA"
done

for keep in "${KEEP[@]}"; do
  echo "keep = ${keep}" >> "${distpath}/DATA"
done

success "(2/6) Created the package data file (DATA)"

# create the changes file
cp "${pkgpath}/changes.md" "${distpath}/CHANGES"
check_ret "(3/6) Failed to create the changes file (CHANGES)"
success "(3/6) Created the changes file (CHANGES)"

# create the install script
if type INSTALL &>/dev/null; then
  echo "$(type INSTALL | head -n-1 | sed '1,3d' | sed 's/    //')" > "${distpath}/INSTALL"
  check_ret "(4/6) Failed to create the install script (INSTALL)"
fi
success "(4/6) Created the install script (INSTALL)"

# create the HASHES file
find . -type f -exec md5sum {} >> "${distpath}/HASHES" \;
check_ret "(5/6) Failed to create the file hash list (HASHES)"
success "(5/6) Created the file hash list (HASHES)"

# remove previous build archives
for old in "${distpath}/${NAME}_"*.mpf; do
  rm -f $old
done

# create the final archive
archive="${NAME}_${VERSION}.mpf"
pushd "${distpath}" > /dev/null
  find . -printf "%P\n" | fakeroot tar -czf "${archive}" --no-recursion -T -
  check_ret "(6/6) Failed to create the package archive (${archive})"
popd > /dev/null
success "(6/6) Created the package archive (${archive})"

# clean up
cd "${pkgpath}"

info "Cleaning up the dist directory"
clean_dist

info "Cleaning the root directory"
rm -rf "${rootpath}"

# move archive to out directory
if [ "$(realpath "${distpath}")" != "$(realpath "${outpath}")" ]; then
  mv "${distpath}/${archive}" "${outpath}" 2> /dev/null
  check_ret "Failed to move the archive to the output directory"
fi

# update the cache
unset_indent
info "Updating the package cache"
set_indent

mkdir -p "${cachepath}"

echo "${package_hash}" > "${cachepath}/last"
check_ret "(1/2) Failed to add package script to the cache"
success "(1/2) Added package script to the cache"

echo "${changes_hash}" >> "${cachepath}/last"
check_ret "(2/2) Failed to add changes file to the cache"
success "(2/2) Added changes file to the cache"

unset_indent

success "Build was successful"
exit 0