ci/cd打包

.gitlab-ci.yml

+# This file is a template, and might need editing before it works on your project.
+# see https://docs.gitlab.com/ee/ci/yaml/README.html for all available options
+
+# you can delete this line if you're not using Docker
+before_script:
+  - echo "before script."
+  - echo "everything is ok."
+
+after_script:
+  - echo "hello world"
+  - echo "everything is ok."
+
+variables:
+  BUILD_DIR: /build/
+  PACKAGE_NAME: armbian-build
+  GIT_DEPTH: "1"
+
+stages:
+  - build
+  - notify_failure
+
+build_job:
+  stage: build
+  script:
+    - echo "Building the package ${PACKAGE_NAME}"
+    - echo "Building paramaters:"
+    - echo "${BUILD_PACKAGE_JSON}" | base64 -d; echo 
+    - python3 gitlab-ci.py --signal=package --event "${BUILD_PACKAGE_EVENT}" --build-directory "${BUILD_DIR}" --project-id "${CI_PROJECT_ID}" --pipeline-id "${CI_PIPELINE_ID}" "$(echo "${BUILD_PACKAGE_JSON}" | base64 -d)"
+    - python3 gitlab-ci.py --signal=notify_success --build-directory "${BUILD_DIR}" --project-id "${CI_PROJECT_ID}" --pipeline-id "${CI_PIPELINE_ID}"
+    - rm -f ".result/${CI_PROJECT_ID}-${CI_PIPELINE_ID}.json"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "trigger"'
+  tags:
+    - armbian-build-runner
+  cache:
+    key: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - .result/
+
+failure_notify_job:
+  script:
+    - python3 gitlab-ci.py --signal=notify_failure --project-id "${CI_PROJECT_ID}" --pipeline-id "${CI_PIPELINE_ID}"
+  stage: notify_failure
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "trigger"'
+      when: on_failure
+  cache:
+    key: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - .result/

gitlab-ci.py

+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+import os
+import json
+import subprocess
+import argparse
+import urllib.request
+
+
+OPENAPI_LOGIN_URL = "https://openapi.linkfog.cn/sys/openapi/login"
+OPENAPI_PACKAGE_RESULT_CALLBACK_URL = "https://open.dianxinai.com/api/v2/channelPkg/buildQueryResult"
+
+
+def make_argument_parser():
+    parser = argparse.ArgumentParser(prog='deploy', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        'json_argv',
+        nargs='?',
+        default=None,
+        help='json format argv'
+    )
+    parser.add_argument(
+        '--build-directory',
+        default='/build',
+        help='build directory'
+    )
+    parser.add_argument(
+        '--vendor',
+        default='AB',
+        help='vendor'
+    )
+    parser.add_argument(
+        '--orgcode',
+        default='A03',
+        help='organization code'
+    )
+    parser.add_argument(
+        '--channel-id',
+        default='CBV9UHAA0320220905143123262',
+        help='channel id'
+    )
+    parser.add_argument(
+        '--package-type',
+        default='owner',
+        help='package type: owner or other'
+    )
+    parser.add_argument(
+        '--signal',
+        default='package',
+        help='signal: pakage, notify_failure, notify_success'
+    )
+    parser.add_argument(
+        '--pipeline-id',
+        default='',
+        help='pipeline id'
+    )
+    parser.add_argument(
+        '--project-id',
+        default='',
+        help='project id'
+    )
+    parser.add_argument(
+        '--url-tag',
+        default='',
+        help='url tag'
+    )
+    parser.add_argument(
+        '--event',
+        default='',
+        help='package event'
+    )
+    return parser
+
+
+def package(options):
+    argv = {}
+    if options.json_argv:
+        try:
+            argv = json.loads(options.json_argv)
+        except Exception:
+            os._exit(1)
+
+    argv['build_directory'] = argv.get('build_directory', options.build_directory)
+    argv['vendor'] = argv.get('vendor', options.vendor)
+    argv['orgcode'] = argv.get('orgcode', options.orgcode)
+    argv['channel_id'] = argv.get('channel_id', options.channel_id)
+    argv['package_type'] = argv.get('package_type', options.package_type)
+    argv['project_id'] = argv.get('project_id', options.project_id)
+    argv['pipeline_id'] = argv.get('pipeline_id', options.pipeline_id)
+    argv['url_tag'] = argv.get('url_tag', options.url_tag)
+    package_script = './pack-ci.sh'
+    subprocess.run([
+        package_script,
+        '--build-directory', argv['build_directory'],
+        '--vendor', argv['vendor'],
+        '--orgcode', argv['orgcode'],
+        '--channel-id', argv['channel_id'],
+        '--package-type', argv['package_type'],
+        '--project-id', argv['project_id'],
+        '--pipeline-id', argv['pipeline_id'],
+        '--url-tag', argv['url_tag'],
+    ])
+
+
+def __login_openapi():
+    data = {
+        "username": "open_admin",
+        "password": "open_admin123"
+    }
+    encoded_data = json.dumps(data).encode('utf-8')
+    req = urllib.request.Request(OPENAPI_LOGIN_URL, data=encoded_data, method='POST')
+    req.add_header('Content-Type', 'application/json')
+    try:
+        with urllib.request.urlopen(req) as response:
+            return json.loads(response.read())
+    except urllib.error.URLError as e:
+        print(e.reason)
+        os._exit(3)
+
+
+def notify_success(pipeline_id, project_id):
+    data = {
+        "pipelineId": pipeline_id,
+        "projectId": project_id,
+        "code": 0,
+        "msg": "success",
+    }
+    with open('.result/{}-{}.json'.format(project_id, pipeline_id), 'r') as f:
+        data.update(json.loads(f.read()))
+
+    encoded_data = json.dumps(data).encode('utf-8')
+    print(encoded_data)
+    req = urllib.request.Request(OPENAPI_PACKAGE_RESULT_CALLBACK_URL, data=encoded_data, method='POST')
+    req.add_header('Content-Type', 'application/json')
+    try:
+        with urllib.request.urlopen(req, timeout=10) as response:
+            print(response.read())
+    except urllib.error.URLError as e:
+        print(e.reason)
+        os._exit(2)
+
+
+def notify_failure(pipeline_id, project_id):
+    data = {
+        "pipelineId": pipeline_id,
+        "projectId": project_id,
+        "code": 1,
+        "msg": "failure"
+    }
+
+    encoded_data = json.dumps(data).encode('utf-8')
+    print(encoded_data)
+    req = urllib.request.Request(OPENAPI_PACKAGE_RESULT_CALLBACK_URL, data=encoded_data, method='POST')
+    req.add_header('Content-Type', 'application/json')
+    try:
+        with urllib.request.urlopen(req, timeout=10) as response:
+            print(response.read())
+    except urllib.error.URLError as e:
+        print(e.reason)
+        os._exit(2)
+
+
+def main():
+    parser = make_argument_parser()
+    options = parser.parse_args()
+
+    if options.signal == 'package':
+        package(options)
+    elif options.signal == 'notify_success':
+        notify_success(options.pipeline_id, options.project_id)
+    elif options.signal == 'notify_failure':
+        notify_failure(options.pipeline_id, options.project_id)
+
+
+if __name__ == '__main__':
+    main()

pack-ci.sh

+#!/bin/bash
+usage() {
+    echo "Usage: $0 --build-directory [build directory, default: /build]
+                    --vendor <vendor>
+                    --orgcode <organization code>
+                    --channel-id <channel id>
+                    --package-type [packtype: owner or other; default: owner]
+                    --project-id <project id>
+                    --pipeline-id <pipeline id>
+                    --url-tag <url tag>
+                    --help"
+    exit 1
+}
+
+BUILD_DIR="./image_build"
+VENDOR=""
+ORGCODE=""
+CHANNELID=""
+PACK_TYPE="owner"
+PROJECT_ID=""
+PIPELINE_ID=""
+URL_TAG=""
+
+CHROOT_TYPE="androidrom"
+DEPLOY="elf"
+
+export PATH=${PATH}:$(pwd)/tools/
+PARSED_ARUGMENTS=$(getopt -a -n "$0" -o 'x' --long help,build-directory:,vendor:,orgcode:,channel-id:,package-type:,project-id:,pipeline-id:,url-tag: -- "$@")
+eval set -- "${PARSED_ARUGMENTS}"
+while :
+do
+    case "$1" in
+        --build-directory) BUILD_DIR="$2"; shift 2;;
+        --vendor) VENDOR="$2"; shift 2;;
+        --orgcode) ORGCODE="$2"; shift 2;;
+        --channel-id) CHANNELID="$2"; shift 2;;
+        --package-type) PACK_TYPE="$2"; shift 2;;
+        --project-id) PROJECT_ID="$2"; shift 2;;
+        --pipeline-id) PIPELINE_ID="$2"; shift 2;;
+        --url-tag) URL_TAG="$2"; shift 2;;
+        --help) usage ;;
+        --) shift; break;;
+        *) usage ;;
+    esac
+done
+
+build_dir=$(mktemp -d "$BUILD_DIR"/tmp.XXXXX)
+packge_dir=$(mktemp -d "$BUILD_DIR"/tmp.XXXXX)
+boot_dir=$(mktemp -d "$BUILD_DIR"/tmp.XXXXX)
+
+build_image() {
+    local img="$1"
+    local android_sdk
+    local system_mount_point
+    local system_path
+
+    systemfile="$build_dir/system.PARTITION"
+    systemfile_ext4="$build_dir/system.PARTITION.ext4"
+
+    aml_image_v2_packer -d "$img" "$build_dir"
+    if file "$systemfile" | grep "Android sparse image"
+    then
+        simg2img "$systemfile" "$systemfile_ext4"
+        loopdev=$(losetup -P -f --show "$systemfile_ext4")
+    else
+        loopdev=$(losetup -P -f --show "$systemfile")
+    fi
+
+    system_mount_point="$build_dir/system"
+    mkdir "$system_mount_point"
+    mount "$loopdev" "$system_mount_point"
+
+    if [[ -d "$system_mount_point/system" ]]
+    then
+        system_path="$system_mount_point/system"
+    else
+        system_path="$system_mount_point"
+    fi
+
+    touch "$system_path/.androidrom"
+
+    rm -f "$system_path/hugep-all.zip"
+    rm -f "$system_path/bin/hugep-all.zip"
+
+    android_sdk="$(grep "build\.version\.sdk" "$system_path"/build.prop | awk -F'=' '{print $2}')"
+    android_sdk="${android_sdk//$'\r'}"
+    android_sdk="${android_sdk//$'\n'}"
+    if [[ "$android_sdk" -ge 28 ]]
+    then
+        find ./services/ -type f -name "*.rc" -exec install -t "$system_path"/etc/init/ {} \;
+    else
+        pushd "$boot_dir"
+        abootimg -x "$build_dir/boot.PARTITION" ramdisk
+        mkdir initrd_dir
+        pushd initrd_dir
+        gunzip -c ../initrd.img | cpio -i
+        popd
+        cp initrd_dir/init.rc .
+        rm -rf initrd_dir
+        popd
+        cp "$boot_dir"/init.rc "$packge_dir"/init.rc
+        rm -rf "$boot_dir"
+        if ! find_autostart -script_name preinstall.sh -initrc "$packge_dir"/init.rc -root_path "$build_dir" > "$packge_dir"/find_autostart.log 2>/dev/null
+        then
+            find_autostart -script_name recovery.sh -initrc "$packge_dir"/init.rc -root_path "$build_dir" > "$packge_dir"/find_autostart.log 2>/dev/null
+        fi
+        local auto_start_script
+        auto_start_script="$(cat "$packge_dir"/find_autostart.log)"
+        if [[ -z "$auto_start_script" ]] && [[ -f "$system_path"/bin/startsoftdetector.sh ]]
+        then
+            # TODO: startsoftdetector.sh 会阻塞在某一步，要把 starthugep.sh 加在中间部分
+            auto_start_script="/system/bin/startsoftdetector.sh"
+        fi
+        # NOTE: 返回的脚本路径里一定以 /system/ 开头, 所以下边拼路径时没有加上 /system
+        if [[ -n "$auto_start_script" ]]
+        then
+            local _script_path
+            _script_path="$system_path/$(echo "$auto_start_script" | sed '|^/system||')"
+            if ! grep -q "#!/system/bin/sh" "$_script_path"
+            then
+                echo "#!/system/bin/sh" >> "$_script_path"
+            fi
+            echo " " >> "$_script_path"
+            echo "/system/bin/starthugep.sh > /dev/null 2>&1 &" >> "$_script_path"
+            chmod 0755 "$_script_path"
+            chcon u:object_r:drmserver_exec:s0 "$_script_path"
+        fi
+    fi
+
+    cp -f "$build_dir/hugep-all.zip" "$system_path"/bin/hugep-all.zip
+
+    find ./services/ -type f ! -name  "*.rc" ! -name "*.zip" -exec install -m 0777 -t "$system_path"/bin/ {} \;
+    cp -f services/manager_bin_file.zip "$system_path"/bin/manager_bin_file.zip
+
+    for file in $(find ./services/ -type f ! -name "*.rc" ! -name "*.zip")
+    do
+        chcon u:object_r:drmserver_exec:s0 "$system_path/bin/$(basename "$file")"
+    done
+
+    umount "$loopdev"
+    losetup -d "$loopdev"
+    rm -rf "$system_mount_point"
+    sync
+
+    if [[ -f "$systemfile_ext4" ]]
+    then
+        rm -f "$systemfile"
+        img2simg "$systemfile_ext4" "$systemfile"
+        rm -f "$systemfile_ext4"
+    fi
+    aml_image_v2_packer -r "/$build_dir/image.cfg" "$build_dir" "${packge_dir}/$(basename "$img")"
+    rm -rf "$build_dir"
+}
+
+generate_vendor_confg() {
+    local yaml_file_src
+    local yaml_file
+
+    yaml_file="$build_dir"/vendor.yaml
+    yaml_file_src="$build_dir"/vendor.yaml.src
+    cp vendor.yaml.src "$yaml_file_src"
+    sed -i'' "s#__VENDOR__#$VENDOR#g"  "$yaml_file_src"
+    sed -i'' "s#__ORGCODE__#$ORGCODE#g"  "$yaml_file_src"
+    sed -i'' "s#__CHROOT_TYPE__#$CHROOT_TYPE#g"  "$yaml_file_src"
+    sed -i'' "s#__DEPLOY__#$DEPLOY#g"  "$yaml_file_src"
+    sed -i'' "s#__CHANNEL_ID__#$CHANNELID#g"  "$yaml_file_src"
+    sed -i'' "s#__PACK_TYPE__#$PACK_TYPE#g"  "$yaml_file_src"
+    aio -m enc -i "$yaml_file_src" -o "$yaml_file"
+}
+
+update_hugepall_zipfile() {
+    local yaml_file
+    local hugep_zip_file
+    local hugep_zip_file_origin
+
+    yaml_file="$build_dir"/vendor.yaml
+    hugep_zip_file="$build_dir/hugep-all.zip"
+    hugep_zip_file_origin="./services/hugep-all.zip"
+
+    cp "$hugep_zip_file_origin" "$hugep_zip_file"
+
+    zip -u -j "$hugep_zip_file" "$yaml_file"
+}
+
+upload() {
+    local date_i
+    local outfile
+    local outfile_gz
+    local uri
+    local random_num
+    local -A result
+
+    outfile="${packge_dir}/$(basename "$IMAGE_FILE")"
+    outfile_gz="${outfile}.gz"
+    date_i="$(date +%Y-%m-%d-%s)"
+
+    zstd --ultra --format=gzip "$outfile" -o "$outfile_gz"
+    random_num=$(< /dev/urandom tr -dC 0-9a-z | head -c 8)
+    [[ -z "$URL_TAG" ]] && URL_TAG="${date_i}-${random_num}"
+    uri="product/terminal/chroot_androidrom/""$ORGCODE/${date_i}-${URL_TAG}/$(basename "$outfile_gz")"
+    obsutil cp "$outfile_gz" obs://fae-cdn.linkfog.cn/"$uri"
+
+    result["url"]="https://fae-cdn.linkfog.cn/""$uri"
+    result["md5"]="$(md5sum "${outfile}" | awk '{print $1}')"
+    mkdir -p .result
+    jq -n --arg url "${result[url]}" --arg md5 "${result[md5]}" '{$url, $md5}' > ".result/${PROJECT_ID}-${PIPELINE_ID}.json"
+    echo ".result/${PROJECT_ID}-${PIPELINE_ID}.json"
+    cat ".result/${PROJECT_ID}-${PIPELINE_ID}.json"
+
+    rm -f "$outfile" "$outfile_gz"
+    rm -rf "$packge_dir"
+}
+
+IMAGE_FILE="/baseimages/$VENDOR-$ORGCODE.img"
+generate_vendor_confg
+update_hugepall_zipfile
+build_image "$IMAGE_FILE"
+upload