diff --git a/.github/workflows/manual_firmware_build.yml b/.github/workflows/manual_firmware_build.yml new file mode 100644 index 00000000000000..4976c43d608e0e --- /dev/null +++ b/.github/workflows/manual_firmware_build.yml @@ -0,0 +1,75 @@ +name: Manual Firmware build (produce downloadable *_with_bl.hex ready to upload to FC) + +on: + workflow_dispatch: + inputs: + boards: + description: 'Boards to build (see `Tools/scripts/board_list.py` for a full list)' + required: true + default: 'MatekF405-Wing,MatekF405-CAN,MatekF765-Wing,MambaF405v2' + projects: + description: 'Projects to build (e.g. `arducopter,arduplane,rover,antennatracker,ardusub,AP_Periph` )' + required: true + default: 'arduplane' + +concurrency: + group: ci-${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: 'macos-latest' + + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Install Prerequisites + shell: bash + run: | + Tools/environment_install/install-prereqs-mac.sh -y + source ~/.bash_profile + # Put ccache into github cache for faster build + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + run: | + NOW=$(date -u +"%F-%T") + echo "::set-output name=timestamp::${NOW}" + - name: ccache cache files + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ${{github.workflow}}-ccache-manual-${{steps.ccache_cache_timestamp.outputs.timestamp}} + restore-keys: ${{github.workflow}}-ccache-manual # restore ccache from either previous build on this branch or on master + - name: setup ccache + run: | + mkdir -p ~/.ccache + echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf + echo "compression = true" >> ~/.ccache/ccache.conf + echo "compression_level = 6" >> ~/.ccache/ccache.conf + echo "max_size = 400M" >> ~/.ccache/ccache.conf + ccache -s + ccache -z + - name: manual build for ${{ github.event.inputs.boards }} + env: + BUILDLOGS: /tmp/buildlogs + BUILD_BINARIES_PATH: /tmp/ardu-build-dir + shell: bash + run: | + source ~/.bash_profile + PATH="/github/home/.local/bin:$PATH" + mkdir -p $BUILDLOGS/binaries + mkdir -p $BUILD_BINARIES_PATH + # P.S.: 'latest' tags would use newer compiler, but it's not installed or linked in current build environment + ./Tools/scripts/build_binaries.py --tags beta --boards ${{ github.event.inputs.boards }} --projects ${{ github.event.inputs.projects }} --skip-history yes --require-checkout no + # FIXME: we probably want to upload other projects too? + echo "listing files in BUILDLOGS/binaries/" + ls -lh $BUILDLOGS/binaries + tar -cvf $BUILDLOGS/built_arduplane_binaries.tar $BUILDLOGS/binaries/Plane/ + ccache -s + ccache -z + - name: 'Upload build artefacts to Github' + uses: actions/upload-artifact@v2 + with: + name: arduplane build files tar of ${{ github.event.inputs.boards }} + path: $BUILDLOGS/built_arduplane_binaries.tar diff --git a/Tools/scripts/build_binaries.py b/Tools/scripts/build_binaries.py index 01553f4123b9ef..8f8ccf4d88ac8e 100755 --- a/Tools/scripts/build_binaries.py +++ b/Tools/scripts/build_binaries.py @@ -62,9 +62,29 @@ def get_required_compiler(tag, board): return None +ARDUCOPTER = "arducopter" +ARDUPLANE = "arduplane" +ROVER = "rover" +ANTENNATRACKER = "antennatracker" +ARDUSUB = "ardusub" +AP_PERIPH = "AP_Periph" + +ALL_PROJECTS = [ + ARDUCOPTER, + ARDUPLANE, + ROVER, + ANTENNATRACKER, + ARDUSUB, + AP_PERIPH +] + + class build_binaries(object): - def __init__(self, tags): + def __init__(self, tags, selected_boards=[], projects=ALL_PROJECTS, require_checkout=True, skip_history=False): self.tags = tags + self.require_checkout = require_checkout + self.selected_boards = selected_boards # P.S. build all boards when selected_boards=[] + self.projects = projects self.dirty = False binaries_history_filepath = os.path.join(self.buildlogs_dirpath(), "build_binaries_history.sqlite") @@ -285,7 +305,7 @@ def skip_build(self, buildtag, builddir): def write_string_to_filepath(self, string, filepath): '''writes the entirety of string to filepath''' - with open(filepath, "w") as x: + with open(filepath, "w") as x: # FIXME: encoding='utf-8' ? x.write(string) def version_h_path(self, src): @@ -301,7 +321,7 @@ def addfwversion_gitversion(self, destdir, src): gitversion_content = gitlog versionfile = self.version_h_path(src) if os.path.exists(versionfile): - content = self.read_string_from_filepath(versionfile) + content = self.read_string_from_filepath(versionfile).decode('utf-8') match = re.search('define.THISFIRMWARE "([^"]+)"', content) if match is None: self.progress("Failed to retrieve THISFIRMWARE from version.h") @@ -323,7 +343,8 @@ def addfwversion_firmwareversiontxt(self, destdir, src): ss = r".*define +FIRMWARE_VERSION[ ]+(?P\d+)[ ]*,[ ]*" \ r"(?P\d+)[ ]*,[ ]*(?P\d+)[ ]*,[ ]*" \ r"(?P[A-Z_]+)[ ]*" - content = self.read_string_from_filepath(versionfile) + # FIXME: content returned by read_string_from_filepath is binary! + content = self.read_string_from_filepath(versionfile).decode('utf-8') match = re.search(ss, content) if match is None: self.progress("Failed to retrieve FIRMWARE_VERSION from version.h") @@ -342,7 +363,13 @@ def addfwversion_firmwareversiontxt(self, destdir, src): def addfwversion(self, destdir, src): '''write version information into destdir''' self.addfwversion_gitversion(destdir, src) - self.addfwversion_firmwareversiontxt(destdir, src) + # FIXME: this seem to sometimes result in utf-8/encoding issues. at least add traceback for now + try: + self.addfwversion_firmwareversiontxt(destdir, src) + except: + import traceback + traceback.print_stack() + raise def read_string_from_filepath(self, filepath): '''returns content of filepath as a string''' @@ -394,7 +421,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, '''build vehicle binaries''' self.progress("Building %s %s binaries (cwd=%s)" % (vehicle, tag, os.getcwd())) - + boards = self.filter_selected_boards(boards) board_count = len(boards) count = 0 for board in sorted(boards, key=str.lower): @@ -410,7 +437,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, framesuffix = "" else: framesuffix = "-%s" % frame - if not self.checkout(vehicle, tag, board, frame, submodule_update=False): + if self.require_checkout and not self.checkout(vehicle, tag, board, frame, submodule_update=False): msg = ("Failed checkout of %s %s %s %s" % (vehicle, board, tag, frame,)) self.progress(msg) @@ -476,6 +503,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, # record some history about this build t1 = time.time() time_taken_to_build = t1-t0 + # FIXME: do we need to handle skip_history here too? self.history.record_build(githash, tag, vehicle, board, frame, None, t0, time_taken_to_build) continue @@ -509,12 +537,18 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir, self.copyit(path, ddir, tag, vehicle) except Exception as e: self.progress("Failed to copy %s to %s: %s" % (path, ddir, str(e))) + import traceback + traceback.print_stack() # why is touching this important? -pb20170816 self.touch_filepath(os.path.join(self.binaries, vehicle_binaries_subdir, tag)) - # record some history about this build - self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build) + # record some history about this build if needed + try: + self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build) + except: + if not skip_history: + raise self.checkout(vehicle, "latest") @@ -522,6 +556,11 @@ def common_boards(self): '''returns list of boards common to all vehicles''' return AUTOBUILD_BOARDS + def filter_selected_boards(self, boards): + no_selected_boards = len(self.selected_boards) == 0 + return [b for b in boards + if no_selected_boards or (b in self.selected_boards)] + def AP_Periph_boards(self): return AP_PERIPH_BOARDS @@ -660,7 +699,7 @@ def run(self): now = datetime.datetime.now() self.progress(now) - if not self.dirty: + if not self.dirty and self.require_checkout: self.run_git(["checkout", "-f", "master"]) githash = self.run_git(["rev-parse", "HEAD"]) githash = githash.rstrip() @@ -684,15 +723,24 @@ def run(self): self.buildroot = os.path.join(os.environ.get("TMPDIR"), "binaries.build") + for tag in self.tags: t0 = time.time() - self.build_arducopter(tag) - self.build_arduplane(tag) - self.build_rover(tag) - self.build_antennatracker(tag) - self.build_ardusub(tag) - self.build_AP_Periph(tag) - self.history.record_run(githash, tag, t0, time.time()-t0) + possible_builds = [ + (ARDUCOPTER, lambda: self.build_arducopter(tag)), + (ARDUPLANE, lambda: self.build_arduplane(tag)), + (ROVER, lambda: self.build_rover(tag)), + (ANTENNATRACKER, lambda: self.build_antennatracker(tag)), + (ARDUSUB, lambda: self.build_ardusub(tag)), + (AP_PERIPH, lambda: self.build_AP_Periph(tag)) + ] + for p, build_fn in possible_builds: + if p in self.projects: + build_fn() + try: + self.history.record_run(githash, tag, t0, time.time()-t0) + except: + print('history failed') if os.path.exists(self.tmpdir): shutil.rmtree(self.tmpdir) @@ -704,11 +752,38 @@ def run(self): sys.exit(len(self.error_strings)) +def flatten_comma_opts(opts): + """ + allow multiple options to be passed separated by comma, e.g `arduplane,arducoper` + (useful for manual build workflow) + + >>> flatten_comma_opts(['a', 'b']) + ['a', 'b'] + >>> flatten_comma_opts(['a', 'b,c,d']) + ['a', 'b', 'c', 'd'] + """ + return [extracted_opt + for opt_value in opts + for extracted_opt in opt_value.replace(' ', '').split(',') + if extracted_opt != ''] + + +def filter_valid_projects(projects): + return [p for p in projects if p in ALL_PROJECTS] + if __name__ == '__main__': parser = optparse.OptionParser("build_binaries.py") parser.add_option("", "--tags", action="append", type="string", default=[], help="tags to build") + parser.add_option("", "--boards", action="append", type="string", + default=[], help="boards to build") + parser.add_option("", "--projects", action="append", type="string", + default=[], help="projects to build") + parser.add_option("", "--skip-history", type="string", + default='no', help="skip recording of build history?") + parser.add_option("", "--require-checkout", type="string", + default='yes', help="shall we do git checkout?") cmd_opts, cmd_args = parser.parse_args() tags = cmd_opts.tags @@ -716,5 +791,11 @@ def run(self): # FIXME: wedge this defaulting into parser somehow tags = ["stable", "beta", "latest"] - bb = build_binaries(tags) + boards = flatten_comma_opts(cmd_opts.boards) + projects = flatten_comma_opts(cmd_opts.projects) or ALL_PROJECTS + require_checkout = cmd_opts.require_checkout == "yes" + skip_history = cmd_opts.skip_history == "yes" + projects = filter_valid_projects(projects) + + bb = build_binaries(tags, selected_boards=boards, projects=projects, require_checkout=require_checkout, skip_history=skip_history) bb.run()