Commit 3c532d29 authored by AlmAck's avatar AlmAck

let's use the chakra-gitlab-ci-scripts repository to retrieve the scripts

with this method we can modify the scripts and run again the build process without generating a new commit.

the user inizialization is now part of chakralinux/makepkg
[skip-ci]
parent 3308b817
Pipeline #73 skipped
#!/usr/bin/env bash
SCRIPT_DIR=$( dirname $( readlink -e $0 ) )
source "$SCRIPT_DIR/../.build-lib/ci-library.sh"
_do list_packages
# sort repository packages based on (inter-)dependency hierarchy
# - it is assumed that the repository contains only compatible package
# versions, hence version information is ignored. If this assumption
# does not hold, makepkg will fail later on
_do sort_packages_by_dependency
_do build_packages
#!/usr/bin/env bash
SCRIPT_DIR=$( dirname $( readlink -e $0 ) )
source "$SCRIPT_DIR/../.build-lib/ci-library.sh"
source "$SCRIPT_DIR/../.build-lib/ci-deploy-library.sh"
# get the list of packages to upload
_do list_upload_packages
if [ -z "$UPLOAD_LIST" ]; then
_log success "Done nothing to upload!"
else
# rsync upload
_do upload_files $(get_repository)
# akbm to add to the repository
_do update_remote_db
_log success "Done uploading!"
fi
#!/usr/bin/env bash
SCRIPT_DIR=$( dirname $( readlink -e $0 ) )
source "$SCRIPT_DIR/../.build-lib/ci-library.sh"
_do list_packages
# `gpg --recv-key` requires write access to the current user's home directory!
_do gpg --recv-key $(get_validpgpkeys)
\ No newline at end of file
#!/usr/bin/env bash
SCRIPT_DIR=$( dirname $( readlink -e $0 ) )
source "$SCRIPT_DIR/../.build-lib/ci-library.sh"
# prepare the build environment
_log command "Setting up locale.conf..."
_do touch "/etc/locale.conf"
_do sh -c "echo 'LANG=C' >> '/etc/locale.conf'"
_do sh -c "echo 'LC_MESSAGES=C' >> '/etc/locale.conf'"
# create a local makepkg settings
_log command "Setting up makepkg.conf..."
echo "source /etc/makepkg.conf
#-- Make Flags: change this for DistCC/SMP systems
MAKEFLAGS=\"-j$(($(nproc)+1))\"
#-- Packager: name/email of the person or organization building packages
PACKAGER=\"$GITLAB_USER_ID <$GITLAB_USER_EMAIL>\"" > /home/builder/makepkg.conf
# patch current makepkg with chakra custom mods
_log command "Patching makepkg..."
_do wget https://code.chakralinux.org/tools/chakrabuildsystem/raw/master/chakra/bin/makepkg
_do cp makepkg /usr/bin/makepkg
# determine the the repository where we should build against (stable|testing|staging)
repo_name=$(get_repository)
_log command "Enabling [$repo_name] pacman.conf..."
_do wget "https://code.chakralinux.org/tools/chakrabuildsystem/raw/master/conf/$repo_name.conf"
_do cp $repo_name.conf /etc/pacman.conf
# resync database and update with latest changes
_do pacman -Syyu --noconfirm
ci-library.sh is based upon
https://github.com/Alexpux/MINGW-packages/blob/master/ci-library.sh
licensed under the "3-clause BSD" license found below.
Modifications are licensed under the Apache License, Version 2.0 (the "License")
- see LICENSE file in the repository root.
"3-clause BSD" license:
Copyright (c) 2013, Алексей
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
* Neither the name of the {organization} nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ci-depoly-library depends on ci-library
UPLOAD_LIST=()
# Execute command and stop execution if the command fails
function _do_deploy() {
CMD=$@
_log command "$CMD"
$CMD || { _log failure "FAILED: $CMD"; _unlock_repo; exit 1; }
return $?
}
function _do_akbm() {
local output
CMD=$@
_log command "$CMD"
output=$($CMD)
if ! [[ "$output" == *"::SUCCESS::"* ]]; then
_log failure "FAILED: $CMD output: $output"
_unlock_repo
exit 1
fi
return $?
}
# performs a remote lock over a repository
function _lock_repo() {
_do_akbm ssh $SSH_USER@$DEPLOY_SERVER -p $SSH_PORT "akbm --repo-name $DEPLOY_REPO --arch x86_64 --lock"
}
# performs a remote unlock over a repository
function _unlock_repo() {
_do_akbm ssh $SSH_USER@$DEPLOY_SERVER -p $SSH_PORT "akbm --repo-name $DEPLOY_REPO --arch x86_64 --unlock"
}
function _set_deploy_repo() {
if [[ "$1" == "stable" ]]; then
# convert stable to the real repository name
# DEPLOY_REPO=$CI_PROJECT_NAME
# force uploading to testing
DEPLOY_REPO="testing"
else
DEPLOY_REPO=$1
fi
}
# checks whether $1 contains a globbing pattern
# returns: '$1' or '*$1*'
function _glob() {
if [[ "$1" =~ [][*?] ]]; then
echo -n "$1"
else
echo -n "*$1*"
fi
}
# takes a list of find patterns and transform it into arguments to find
function _find_args() {
local -a fargs=("-name" "$(_glob $1)"); shift
while [[ $# -gt 0 ]]; do
fargs+=("-o" "-name" "$(_glob $1)"); shift
done
echo "${fargs[@]}"
}
function list_upload_packages() {
UPLOAD_LIST+=( $(set -o noglob; find $REPODIR -name '*.pkg.tar.*' \( $(_find_args ${args[@]}) \) -print) )
}
function _upload_files() {
local -a files=( $* ) # files to upload
rsync -rltoDvh \
--omit-dir-times \
--numeric-ids \
--progress \
--delay-updates \
-e "ssh -p $SSH_PORT" \
"${files[@]}" $SSH_USER@$DEPLOY_SERVER:/srv/www/rsync.chakralinux.org/packages/$DEPLOY_REPO/x86_64/
}
function upload_files() {
_do _set_deploy_repo $1
_log build_step "Start uploading to $DEPLOY_REPO the following packages: ${UPLOAD_LIST[@]}"
_do _lock_repo
_do_deploy _upload_files "${UPLOAD_LIST[@]}"
_do _unlock_repo
_log success "rsync upload done"
}
function update_remote_db() {
_log build_step "Start importing pkgs with akbm"
local -a file_names=( ${UPLOAD_LIST[@]##*/} )
_do _lock_repo
# we can no tuse _do in this case, because we have to parse the output to know if was successfully executed
_do_akbm ssh "$SSH_USER@$DEPLOY_SERVER" -p "$SSH_PORT" "akbm --repo-name $DEPLOY_REPO --arch x86_64 --repo-add" "${file_names[@]}"
_do _unlock_repo
}
#_ensure-var "DEPLOY_REPO"
_ensure-var "DEPLOY_SERVER"
_ensure-var "SSH_USER"
_ensure-var "SSH_PORT"
# Based upon
# https://github.com/Alexpux/MINGW-packages/blob/master/ci-library.sh
# and https://github.com/episource/archlinux-overlay
PACKAGES=()
# Print a colored log message
function _log() {
local type="${1}"
shift
local msg="${@}"
local normal='\e[0m'
local red='\e[1;31m'
local green='\e[1;32m'
local yellow='\e[1;33m'
local cyan='\e[1;36m'
case "${type}" in
failure) echo -e "$red$msg$normal" ;;
success) echo -e "$green$msg$normal" ;;
build_step) echo -e "$green$msg$normal" ;;
command) echo -e "$cyan$msg$normal" ;;
message) echo -e "$msg" ;;
esac
}
# Execute command and stop execution if the command fails
function _do() {
CMD=$@
_log command "$CMD"
$CMD || { _log failure "FAILED: $CMD"; exit 1; }
return $?
}
# Ensure that the given environment variable has been defined and is not empty
function _ensure-var() {
local -n VARNAME=$1
if [[ -z ${VARNAME+x} ]]; then
_log failure "Environment variable $1 not defined."
exit 1
fi
}
# Package provides another (ignoring version constraints)
function _package_provides() {
local package="${1}"
local another_without_version="${2%%[<>=]*}"
local pkgname provides
_package_info "${package}" pkgname provides
for pkg_name in "${pkgname[@]}"; do [[ "${pkg_name}" = "${another_without_version}" ]] && return 0; done
for provided in "${provides[@]}"; do [[ "${provided}" = "${another_without_version}" ]] && return 0; done
return 1
}
# Get package information
function _package_info() {
local package="${1}"
local properties=("${@:2}")
for property in "${properties[@]}"; do
local -n nameref_property="${property}"
nameref_property=($(
source "${package}/PKGBUILD"
declare -n nameref_property="${property}"
echo "${nameref_property[@]}"))
done
}
# Add package to build after required dependencies
function _build_add() {
local package="${1}"
local depends makedepends
for sorted_package in "${sorted_packages[@]}"; do
[[ "${sorted_package}" = "${package}" ]] && return 0
done
_package_info "${package}" depends makedepends
for dependency in "${depends[@]}" "${makedepends[@]}"; do
for unsorted_package in "${PACKAGES[@]}"; do
[[ "${package}" = "${unsorted_package}" ]] && continue
_package_provides "${unsorted_package}" "${dependency}" && _build_add "${unsorted_package}"
done
done
sorted_packages+=("${package}")
}
# Convert lines to array
_as_list() {
local -n nameref_list="${1}"
local filter="${2}"
local strip="${3}"
local lines="${4}"
local result=1
nameref_list=()
while IFS= read -r line; do
test -z "${line}" && continue
result=0
[[ "${line}" = ${filter} ]] && nameref_list+=("${line/${strip}/}")
done <<< "${lines}"
return "${result}"
}
# Changes since master or from head
function _list_changes() {
local list_name="${1}"
local filter="${2}"
local strip="${3}"
local git_options=("${@:4}")
#_as_list "${list_name}" "${filter}" "${strip}" "$(git log "${git_options[@]}" master.. | sort -u)" ||
#_as_list "${list_name}" "${filter}" "${strip}" "$(git log "${git_options[@]}" HEAD^.. | sort -u)"
_as_list "${list_name}" "${filter}" "${strip}" "$(git diff-tree "${git_options[@]}" HEAD)"
}
# Added commits
function list_commits() {
_list_changes commits '*' '#*::' --pretty=format:'%ai::[%h] %s'
}
# Changed recipes
function list_packages() {
local _packages
local _orders
_list_changes _packages '*/PKGBUILD' '%/PKGBUILD' --no-commit-id --pretty=format: --name-only -r || return 1
for _package in "${_packages[@]}"; do
PACKAGES+=("${_package}")
done
# check if there are some .order file
_list_changes _orders '*.order' '%' --no-commit-id --pretty=format: --name-only -r || return 1
for _order in "${_orders[@]}"; do
exec 3<$_order
while read -u3 _line; do
[[ $_line =~ ^[:blank:]*$ ]] && continue
local comment_re="^[:blank:]*#"
[[ $_line =~ $comment_re ]] && continue
PACKAGES+=("${_line}")
done
done
return 0
}
# extracts all 'validpgpkeys' from the PKGBUILDs
# extracts all 'validpgpkeys' listed in the PKGBUILDs belonging to $PACKAGES
function get_validpgpkeys() {
_VALIDPGPKEYS=()
for p in "${PACKAGES[@]}"; do
local validpgpkeys=()
_package_info "$p" validpgpkeys
_VALIDPGPKEYS+=$validpgpkeys
done
echo "${_VALIDPGPKEYS[@]}"
}
# Sort packages by dependency
# reorders $PACKAGES such that dependencies are built first
function sort_packages_by_dependency() {
local sorted_packages=()
for p in "${PACKAGES[@]}"; do
_build_add "${p}"
done
PACKAGES=("${sorted_packages[@]}")
}
# determine the repository to build against
function get_repository() {
# save the current branch as the default repository
local current_repo=$CI_COMMIT_REF_NAME
if [ "$current_repo" == "master" ]; then
current_repo="stable"
fi
# verify if a manual override is present in the git comment
GIT_COMMIT_MESSAGE=$(git rev-list --format=%B --max-count=1 HEAD)
# extract the text between brackets, ex. [stable], [testing]
tokens=$(echo $GIT_COMMIT_MESSAGE | cut -d "[" -f2 | cut -d "]" -f1)
for token in $tokens;
do
case "$token" in
"stable")
current_repo="stable"
;;
"testing")
current_repo="testing"
;;
"staging" | "unstable")
current_repo="staging"
;;
*)
;;
esac
done
echo "$current_repo"
}
# Build all packages defined in array variable PACKAGES
# builds all $PACKAGES in the given order
function build_packages() {
_log build_step "Start building packages: ${PACKAGES[@]}"
_do mkdir -p "$REPODIR"
for p in "${PACKAGES[@]}"; do
cd $p
_log command "Building pkg: $p"
PKGEXT=".pkg.tar.xz" PKGDEST="$REPODIR" \
_do makepkg --noconfirm --noprogressbar --nosign --syncdeps --cleanbuild
cd - > /dev/null
# resync to update local repo
#_do pacman -Sy --noconfirm
done
_log success "Done building packages!"
}
_ensure-var "REPODIR"
#_ensure-var "REPONAME"
......@@ -8,52 +8,19 @@ stages:
- sign
- deploy
before_script:
# The repository build script is run by user 'builder'. The build scripts needs
# to install dependencies using pacman. This requires root permissions.
# (Note: the build script itself can't be run as root, as makepkg would
# complain)
- useradd -m -G users -s /bin/bash builder
- "echo 'builder ALL=(ALL) NOPASSWD: /usr/bin/pacman' >> /etc/sudoers"
#- echo -en "builder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/10-builder
# SSH setup
# Run ssh-agent (inside the build environment)
- eval $(ssh-agent -s)
# Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
- ssh-add <(echo "$SSH_PRIVATE_KEY")
# For Docker builds disable host key checking. Be aware that by adding that
# you are suspectible to man-in-the-middle attacks.
# WARNING: Use this only with the Docker executor, if you use it with shell
# you will overwrite your user's SSH config.
#- mkdir -p ~/.ssh
#- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
# In order to properly check the server's host key, assuming you created the
# SSH_SERVER_HOSTKEYS variable previously, uncomment the following two lines
# instead.
- mkdir -p ~/.ssh
- '[[ -f /.dockerenv ]] && echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts'
- echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa && chmod 700 ~/.ssh
# GPG setup
- gpg -v --batch --import <(echo "$GPG_PRIVATE_KEY")
- echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf
- gpg-connect-agent reloadagent /bye
# import the gitlab-ci scripts
- git clone --depth 1 https://code.chakralinux.org/tools/chakra-gitlab-ci-scripts _gitlab-ci-scripts
# setup basic environment settings, as root
- .build-bin/prepare.sh
- _gitlab-ci-scripts/prepare.sh
# import pgp keys flagged as valid
- sudo -u builder -E -H .build-bin/import-validpgpkeys.sh
- sudo -u builder -E -H _gitlab-ci-scripts/import-validpgpkeys.sh
build_repo:
stage: package
script:
# the build script can't be run as root, as makepkg would complain...
- sudo -u builder -E -H .build-bin/build.sh
- sudo -u builder -E -H _gitlab-ci-scripts/build.sh
artifacts:
# expire artifacts per default - the gitlab web frontend can be used to keep
......@@ -90,7 +57,7 @@ deploy_repo:
SSH_PORT: "$SSH_PORT"
environment: staging-repo
script:
- .build-bin/deploy.sh
- _gitlab-ci-scripts/deploy.sh
when: manual
tags:
- rsync
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment