From 43a5cf26fe3749e9b9a5de68b94f3350325314f4 Mon Sep 17 00:00:00 2001 From: vidma Date: Thu, 16 Sep 2021 09:31:45 +0300 Subject: [PATCH 1/4] Build system: Manual gh action build *_bl.hex uploadable to FC Add a manual Github Action which can build *_bl.hex files uploadable to Flight Controller. The action can be used on custom branches, and thus very useful for live testing of PRs on real FC before merging them to master on a manual selection of FC models (thus rather fast build). The build firmwave *_bl.hex is uploaded to Github and can be easily downloaded. Changes: - read_string_from_filepath - make history optional for now (missing size cmd) - fix utf-8 decode - upload built files --- .github/workflows/manual_firmware_build.yml | 72 ++++++++++++ Tools/scripts/build_binaries.py | 115 +++++++++++++++++--- 2 files changed, 170 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/manual_firmware_build.yml diff --git a/.github/workflows/manual_firmware_build.yml b/.github/workflows/manual_firmware_build.yml new file mode 100644 index 00000000000000..0653674d3713e0 --- /dev/null +++ b/.github/workflows/manual_firmware_build.yml @@ -0,0 +1,72 @@ +name: Manual Firmware build (produce downloadable *_with_bl.hex ready to upload to FC) + +on: + workflow_dispatch: + inputs: + boards: + description: 'Boards to build (see `Tools/scripts/board_list.py` for a full list)' + required: true + default: 'MatekF405-Wing,MatekF405-CAN,MatekF765-Wing,MambaF405v2' + projects: + description: 'Boards to build (e.g. `arducopter,arduplane,rover,antennatracker,ardusub,AP_Periph` )' + required: true + default: 'arduplane' + +concurrency: + group: ci-${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: 'macos-latest' + + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Install Prerequisites + shell: bash + run: | + Tools/environment_install/install-prereqs-mac.sh -y + source ~/.bash_profile + # Put ccache into github cache for faster build + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + run: | + NOW=$(date -u +"%F-%T") + echo "::set-output name=timestamp::${NOW}" + - name: ccache cache files + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ${{github.workflow}}-ccache-manual-${{steps.ccache_cache_timestamp.outputs.timestamp}} + restore-keys: ${{github.workflow}}-ccache-manual # restore ccache from either previous build on this branch or on master + - name: setup ccache + run: | + mkdir -p ~/.ccache + echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf + echo "compression = true" >> ~/.ccache/ccache.conf + echo "compression_level = 6" >> ~/.ccache/ccache.conf + echo "max_size = 400M" >> ~/.ccache/ccache.conf + ccache -s + ccache -z + - name: manual build for ${{ github.event.inputs.boards }} + env: + BUILDLOGS: /tmp/buildlogs + BUILD_BINARIES_PATH: /tmp/ardu-build-dir + shell: bash + run: | + source ~/.bash_profile + PATH="/github/home/.local/bin:$PATH" + mkdir -p $BUILDLOGS/binaries + mkdir -p $BUILD_BINARIES_PATH + # latest tag would use newer compiled, but it's not installed here (linked)... + ./Tools/scripts/build_binaries.py --tags beta --boards ${{ github.event.inputs.boards }} --projects ${{ github.event.inputs.projects }} --skip-history yes --require-checkout no + tar -cvf /tmp/buildlogs/built_binaries.tar /tmp/buildlogs/binaries/Plane/ + ccache -s + ccache -z + - name: 'Upload' + uses: actions/upload-artifact@v2 + with: + name: arduplane build files tar of ${{ github.event.inputs.boards }} + path: /tmp/buildlogs/built_binaries.tar diff --git a/Tools/scripts/build_binaries.py b/Tools/scripts/build_binaries.py index 01553f4123b9ef..e95c9124ab0307 100755 --- a/Tools/scripts/build_binaries.py +++ b/Tools/scripts/build_binaries.py @@ -62,9 +62,32 @@ def get_required_compiler(tag, board): return None +ARDUCOPTER = "arducopter" +ARDUPLANE = "arduplane" +ROVER = "rover" +ANTENNATRACKER = "antennatracker" +ARDUSUB = "ardusub" +AP_PERIPH = "AP_Periph" + +ALL_PROJECTS = [ + ARDUCOPTER, + ARDUPLANE, + ROVER, + ANTENNATRACKER, + ARDUSUB, + AP_PERIPH +] + + class build_binaries(object): - def __init__(self, tags): + def __init__(self, tags, boards=None, projects=ALL_PROJECTS, require_checkout=True): self.tags = tags + self.require_checkout = require_checkout + if boards: + self.selected_boards = boards + else: + self.selected_boards = [] # build all boards + self.projects = projects self.dirty = False binaries_history_filepath = os.path.join(self.buildlogs_dirpath(), "build_binaries_history.sqlite") @@ -285,7 +308,7 @@ def skip_build(self, buildtag, builddir): def write_string_to_filepath(self, string, filepath): '''writes the entirety of string to filepath''' - with open(filepath, "w") as x: + with open(filepath, "w") as x: # FIXME: encoding='utf-8' ? x.write(string) def version_h_path(self, src): @@ -301,7 +324,7 @@ def addfwversion_gitversion(self, destdir, src): gitversion_content = gitlog versionfile = self.version_h_path(src) if os.path.exists(versionfile): - content = self.read_string_from_filepath(versionfile) + content = self.read_string_from_filepath(versionfile).decode('utf-8') match = re.search('define.THISFIRMWARE "([^"]+)"', content) if match is None: self.progress("Failed to retrieve THISFIRMWARE from version.h") @@ -323,7 +346,8 @@ def addfwversion_firmwareversiontxt(self, destdir, src): ss = r".*define +FIRMWARE_VERSION[ ]+(?P\d+)[ ]*,[ ]*" \ r"(?P\d+)[ ]*,[ ]*(?P\d+)[ ]*,[ ]*" \ r"(?P[A-Z_]+)[ ]*" - content = self.read_string_from_filepath(versionfile) + # FIXME: content returned by read_string_from_filepath is binary! + content = self.read_string_from_filepath(versionfile).decode('utf-8') match = re.search(ss, content) if match is None: self.progress("Failed to retrieve FIRMWARE_VERSION from version.h") @@ -342,7 +366,11 @@ def addfwversion_firmwareversiontxt(self, destdir, src): def addfwversion(self, destdir, src): '''write version information into destdir''' self.addfwversion_gitversion(destdir, src) - self.addfwversion_firmwareversiontxt(destdir, src) + try: + self.addfwversion_firmwareversiontxt(destdir, src) + except: + import traceback + traceback.print_stack() def read_string_from_filepath(self, filepath): '''returns content of filepath as a string''' @@ -394,7 +422,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, '''build vehicle binaries''' self.progress("Building %s %s binaries (cwd=%s)" % (vehicle, tag, os.getcwd())) - + boards = self.filter_selected_boards(boards) board_count = len(boards) count = 0 for board in sorted(boards, key=str.lower): @@ -410,7 +438,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, framesuffix = "" else: framesuffix = "-%s" % frame - if not self.checkout(vehicle, tag, board, frame, submodule_update=False): + if self.require_checkout and not self.checkout(vehicle, tag, board, frame, submodule_update=False): msg = ("Failed checkout of %s %s %s %s" % (vehicle, board, tag, frame,)) self.progress(msg) @@ -509,12 +537,17 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, self.copyit(path, ddir, tag, vehicle) except Exception as e: self.progress("Failed to copy %s to %s: %s" % (path, ddir, str(e))) + import traceback + traceback.print_stack() # why is touching this important? -pb20170816 self.touch_filepath(os.path.join(self.binaries, vehicle_binaries_subdir, tag)) # record some history about this build - self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build) + try: + self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build) + except: + pass self.checkout(vehicle, "latest") @@ -522,6 +555,11 @@ def common_boards(self): '''returns list of boards common to all vehicles''' return AUTOBUILD_BOARDS + def filter_selected_boards(self, boards): + no_selected_boards = len(self.selected_boards) == 0 + return [b for b in boards + if no_selected_boards or (b in self.selected_boards)] + def AP_Periph_boards(self): return AP_PERIPH_BOARDS @@ -660,7 +698,7 @@ def run(self): now = datetime.datetime.now() self.progress(now) - if not self.dirty: + if not self.dirty and self.require_checkout: self.run_git(["checkout", "-f", "master"]) githash = self.run_git(["rev-parse", "HEAD"]) githash = githash.rstrip() @@ -684,15 +722,24 @@ def run(self): self.buildroot = os.path.join(os.environ.get("TMPDIR"), "binaries.build") + for tag in self.tags: t0 = time.time() - self.build_arducopter(tag) - self.build_arduplane(tag) - self.build_rover(tag) - self.build_antennatracker(tag) - self.build_ardusub(tag) - self.build_AP_Periph(tag) - self.history.record_run(githash, tag, t0, time.time()-t0) + possible_builds = [ + (ARDUCOPTER, lambda: self.build_arducopter(tag)), + (ARDUPLANE, lambda: self.build_arduplane(tag)), + (ROVER, lambda: self.build_rover(tag)), + (ANTENNATRACKER, lambda: self.build_antennatracker(tag)), + (ARDUSUB, lambda: self.build_ardusub(tag)), + (AP_PERIPH, lambda: self.build_AP_Periph(tag)) + ] + for p, build_fn in possible_builds: + if p in self.projects: + build_fn() + try: + self.history.record_run(githash, tag, t0, time.time()-t0) + except: + print('history failed') if os.path.exists(self.tmpdir): shutil.rmtree(self.tmpdir) @@ -704,11 +751,38 @@ def run(self): sys.exit(len(self.error_strings)) +def flatten_comma_opts(opts): + """ + allow multiple options to be passed separated by comma, e.g `arduplane,arducoper` + (useful for manual build workflow) + + >>> flatten_comma_opts(['a', 'b']) + ['a', 'b'] + >>> flatten_comma_opts(['a', 'b,c,d']) + ['a', 'b', 'c', 'd'] + """ + return [extracted_opt + for opt_value in opts + for extracted_opt in opt_value.replace(' ', '').split(',') + if extracted_opt != ''] + + +def filter_valid_projects(projects): + return [p for p in projects if p in ALL_PROJECTS] + if __name__ == '__main__': parser = optparse.OptionParser("build_binaries.py") parser.add_option("", "--tags", action="append", type="string", default=[], help="tags to build") + parser.add_option("", "--boards", action="append", type="string", + default=[], help="boards to build") + parser.add_option("", "--projects", action="append", type="string", + default=[], help="projects to build") + parser.add_option("", "--skip-history", type="string", + default=[], help="skip recording of build history?") + parser.add_option("", "--require-checkout", type="string", + default='no', help="shall we do git checkout?") # FIXME! cmd_opts, cmd_args = parser.parse_args() tags = cmd_opts.tags @@ -716,5 +790,12 @@ def run(self): # FIXME: wedge this defaulting into parser somehow tags = ["stable", "beta", "latest"] - bb = build_binaries(tags) + boards = flatten_comma_opts(cmd_opts.boards) + projects = flatten_comma_opts(cmd_opts.projects) + require_checkout = cmd_opts.require_checkout == "yes" + if len(projects) == 0: + projects = ALL_PROJECTS + projects = filter_valid_projects(projects) + + bb = build_binaries(tags, boards=boards, projects=projects, require_checkout=require_checkout) bb.run() From f60e4a2c4c59839d343e1f07714c8390cc2ab802 Mon Sep 17 00:00:00 2001 From: vidmantas zemleris Date: Mon, 3 Jan 2022 11:35:45 +0200 Subject: [PATCH 2/4] cleanup, should still work --- .github/workflows/manual_firmware_build.yml | 11 ++++++----- Tools/scripts/build_binaries.py | 15 +++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manual_firmware_build.yml b/.github/workflows/manual_firmware_build.yml index 0653674d3713e0..315695ad0332ea 100644 --- a/.github/workflows/manual_firmware_build.yml +++ b/.github/workflows/manual_firmware_build.yml @@ -8,7 +8,7 @@ on: required: true default: 'MatekF405-Wing,MatekF405-CAN,MatekF765-Wing,MambaF405v2' projects: - description: 'Boards to build (e.g. `arducopter,arduplane,rover,antennatracker,ardusub,AP_Periph` )' + description: 'Projects to build (e.g. `arducopter,arduplane,rover,antennatracker,ardusub,AP_Periph` )' required: true default: 'arduplane' @@ -60,13 +60,14 @@ jobs: PATH="/github/home/.local/bin:$PATH" mkdir -p $BUILDLOGS/binaries mkdir -p $BUILD_BINARIES_PATH - # latest tag would use newer compiled, but it's not installed here (linked)... + # P.S.: 'latest' tags would use newer compiler, but it's not installed or linked in current build environment ./Tools/scripts/build_binaries.py --tags beta --boards ${{ github.event.inputs.boards }} --projects ${{ github.event.inputs.projects }} --skip-history yes --require-checkout no - tar -cvf /tmp/buildlogs/built_binaries.tar /tmp/buildlogs/binaries/Plane/ + # FIXME: we probably want to upload other projects too? + tar -cvf $BUILDLOGS/built_arduplane_binaries.tar $BUILDLOGS/binaries/Plane/ ccache -s ccache -z - - name: 'Upload' + - name: 'Upload build artefacts to Github' uses: actions/upload-artifact@v2 with: name: arduplane build files tar of ${{ github.event.inputs.boards }} - path: /tmp/buildlogs/built_binaries.tar + path: $BUILDLOGS/built_arduplane_binaries.tar diff --git a/Tools/scripts/build_binaries.py b/Tools/scripts/build_binaries.py index e95c9124ab0307..a409d9d6d4f3bb 100755 --- a/Tools/scripts/build_binaries.py +++ b/Tools/scripts/build_binaries.py @@ -80,7 +80,7 @@ def get_required_compiler(tag, board): class build_binaries(object): - def __init__(self, tags, boards=None, projects=ALL_PROJECTS, require_checkout=True): + def __init__(self, tags, boards=None, projects=ALL_PROJECTS, require_checkout=True, skip_history=False): self.tags = tags self.require_checkout = require_checkout if boards: @@ -504,6 +504,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, # record some history about this build t1 = time.time() time_taken_to_build = t1-t0 + # FIXME: do we need to handle skip_history here too? self.history.record_build(githash, tag, vehicle, board, frame, None, t0, time_taken_to_build) continue @@ -543,11 +544,12 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, self.touch_filepath(os.path.join(self.binaries, vehicle_binaries_subdir, tag)) - # record some history about this build + # record some history about this build if needed try: self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build) except: - pass + if not skip_history: + raise self.checkout(vehicle, "latest") @@ -780,9 +782,9 @@ def filter_valid_projects(projects): parser.add_option("", "--projects", action="append", type="string", default=[], help="projects to build") parser.add_option("", "--skip-history", type="string", - default=[], help="skip recording of build history?") + default='no', help="skip recording of build history?") parser.add_option("", "--require-checkout", type="string", - default='no', help="shall we do git checkout?") # FIXME! + default='yes', help="shall we do git checkout?") cmd_opts, cmd_args = parser.parse_args() tags = cmd_opts.tags @@ -793,9 +795,10 @@ def filter_valid_projects(projects): boards = flatten_comma_opts(cmd_opts.boards) projects = flatten_comma_opts(cmd_opts.projects) require_checkout = cmd_opts.require_checkout == "yes" + skip_history = cmd_opts.skip_history == "yes" if len(projects) == 0: projects = ALL_PROJECTS projects = filter_valid_projects(projects) - bb = build_binaries(tags, boards=boards, projects=projects, require_checkout=require_checkout) + bb = build_binaries(tags, boards=boards, projects=projects, require_checkout=require_checkout, skip_history=skip_history) bb.run() From d785ba557217e076c9fe4deb68dae8109567ca20 Mon Sep 17 00:00:00 2001 From: vidmantas zemleris Date: Mon, 3 Jan 2022 11:44:42 +0200 Subject: [PATCH 3/4] cleanup --- Tools/scripts/build_binaries.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Tools/scripts/build_binaries.py b/Tools/scripts/build_binaries.py index a409d9d6d4f3bb..8f8ccf4d88ac8e 100755 --- a/Tools/scripts/build_binaries.py +++ b/Tools/scripts/build_binaries.py @@ -80,13 +80,10 @@ def get_required_compiler(tag, board): class build_binaries(object): - def __init__(self, tags, boards=None, projects=ALL_PROJECTS, require_checkout=True, skip_history=False): + def __init__(self, tags, selected_boards=[], projects=ALL_PROJECTS, require_checkout=True, skip_history=False): self.tags = tags self.require_checkout = require_checkout - if boards: - self.selected_boards = boards - else: - self.selected_boards = [] # build all boards + self.selected_boards = selected_boards # P.S. build all boards when selected_boards=[] self.projects = projects self.dirty = False binaries_history_filepath = os.path.join(self.buildlogs_dirpath(), @@ -366,11 +363,13 @@ def addfwversion_firmwareversiontxt(self, destdir, src): def addfwversion(self, destdir, src): '''write version information into destdir''' self.addfwversion_gitversion(destdir, src) + # FIXME: this seem to sometimes result in utf-8/encoding issues. at least add traceback for now try: self.addfwversion_firmwareversiontxt(destdir, src) except: import traceback traceback.print_stack() + raise def read_string_from_filepath(self, filepath): '''returns content of filepath as a string''' @@ -793,12 +792,10 @@ def filter_valid_projects(projects): tags = ["stable", "beta", "latest"] boards = flatten_comma_opts(cmd_opts.boards) - projects = flatten_comma_opts(cmd_opts.projects) + projects = flatten_comma_opts(cmd_opts.projects) or ALL_PROJECTS require_checkout = cmd_opts.require_checkout == "yes" skip_history = cmd_opts.skip_history == "yes" - if len(projects) == 0: - projects = ALL_PROJECTS projects = filter_valid_projects(projects) - bb = build_binaries(tags, boards=boards, projects=projects, require_checkout=require_checkout, skip_history=skip_history) + bb = build_binaries(tags, selected_boards=boards, projects=projects, require_checkout=require_checkout, skip_history=skip_history) bb.run() From 602b2da34b002771bfe9c7ec55f74d80e0be6cda Mon Sep 17 00:00:00 2001 From: vidmantas zemleris Date: Mon, 3 Jan 2022 11:46:45 +0200 Subject: [PATCH 4/4] debug --- .github/workflows/manual_firmware_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/manual_firmware_build.yml b/.github/workflows/manual_firmware_build.yml index 315695ad0332ea..4976c43d608e0e 100644 --- a/.github/workflows/manual_firmware_build.yml +++ b/.github/workflows/manual_firmware_build.yml @@ -63,6 +63,8 @@ jobs: # P.S.: 'latest' tags would use newer compiler, but it's not installed or linked in current build environment ./Tools/scripts/build_binaries.py --tags beta --boards ${{ github.event.inputs.boards }} --projects ${{ github.event.inputs.projects }} --skip-history yes --require-checkout no # FIXME: we probably want to upload other projects too? + echo "listing files in BUILDLOGS/binaries/" + ls -lh $BUILDLOGS/binaries tar -cvf $BUILDLOGS/built_arduplane_binaries.tar $BUILDLOGS/binaries/Plane/ ccache -s ccache -z