Manual gh action builds for *_bl.hex uploadable to FC

.github/workflows/manual_firmware_build.yml

@@ -0,0 +1,75 @@
+name: Manual Firmware build (produce downloadable *_with_bl.hex ready to upload to FC)
+
+on:
+  workflow_dispatch:
+    inputs:
+      boards:
+        description: 'Boards to build (see `Tools/scripts/board_list.py` for a full list)'
+        required: true
+        default: 'MatekF405-Wing,MatekF405-CAN,MatekF765-Wing,MambaF405v2'
+      projects:
+        description: 'Projects to build (e.g. `arducopter,arduplane,rover,antennatracker,ardusub,AP_Periph` )'
+        required: true
+        default: 'arduplane'
+
+concurrency:
+  group: ci-${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: 'macos-latest'
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Install Prerequisites
+        shell: bash
+        run: |
+          Tools/environment_install/install-prereqs-mac.sh -y
+          source ~/.bash_profile
+      # Put ccache into github cache for faster build
+      - name: Prepare ccache timestamp
+        id: ccache_cache_timestamp
+        run: |
+          NOW=$(date -u +"%F-%T")
+          echo "::set-output name=timestamp::${NOW}"
+      - name: ccache cache files
+        uses: actions/cache@v2
+        with:
+          path: ~/.ccache
+          key: ${{github.workflow}}-ccache-manual-${{steps.ccache_cache_timestamp.outputs.timestamp}}
+          restore-keys: ${{github.workflow}}-ccache-manual # restore ccache from either previous build on this branch or on master
+      - name: setup ccache
+        run: |
+          mkdir -p ~/.ccache
+          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
+          echo "compression = true" >> ~/.ccache/ccache.conf
+          echo "compression_level = 6" >> ~/.ccache/ccache.conf
+          echo "max_size = 400M" >> ~/.ccache/ccache.conf
+          ccache -s
+          ccache -z
+      - name: manual build for ${{ github.event.inputs.boards }}
+        env:
+          BUILDLOGS: /tmp/buildlogs
+          BUILD_BINARIES_PATH: /tmp/ardu-build-dir
+        shell: bash
+        run: |
+          source ~/.bash_profile
+          PATH="/github/home/.local/bin:$PATH"
+          mkdir -p $BUILDLOGS/binaries
+          mkdir -p $BUILD_BINARIES_PATH
+          # P.S.: 'latest' tags would use newer compiler, but it's not installed or linked in current build environment
+          ./Tools/scripts/build_binaries.py --tags beta --boards ${{ github.event.inputs.boards }} --projects ${{ github.event.inputs.projects }} --skip-history yes --require-checkout no
+          # FIXME: we probably want to upload other projects too?
+          echo "listing files in BUILDLOGS/binaries/"
+          ls -lh $BUILDLOGS/binaries
+          tar -cvf $BUILDLOGS/built_arduplane_binaries.tar $BUILDLOGS/binaries/Plane/
+          ccache -s
+          ccache -z
+      - name: 'Upload build artefacts to Github'
+        uses: actions/upload-artifact@v2
+        with:
+          name: arduplane build files tar of ${{ github.event.inputs.boards }}
+          path: $BUILDLOGS/built_arduplane_binaries.tar

Tools/scripts/build_binaries.py

@@ -62,9 +62,29 @@ def get_required_compiler(tag, board):
     return None
 
 
+ARDUCOPTER = "arducopter"
+ARDUPLANE = "arduplane"
+ROVER = "rover"
+ANTENNATRACKER = "antennatracker"
+ARDUSUB = "ardusub"
+AP_PERIPH = "AP_Periph"
+
+ALL_PROJECTS = [
+    ARDUCOPTER,
+    ARDUPLANE,
+    ROVER,
+    ANTENNATRACKER,
+    ARDUSUB,
+    AP_PERIPH
+]
+
+
 class build_binaries(object):
-    def __init__(self, tags):
+    def __init__(self, tags, selected_boards=[], projects=ALL_PROJECTS, require_checkout=True, skip_history=False):
         self.tags = tags
+        self.require_checkout = require_checkout
+        self.selected_boards = selected_boards # P.S. build all boards when selected_boards=[]
+        self.projects = projects
         self.dirty = False
         binaries_history_filepath = os.path.join(self.buildlogs_dirpath(),
                                                  "build_binaries_history.sqlite")
@@ -285,7 +305,7 @@ def skip_build(self, buildtag, builddir):
 
     def write_string_to_filepath(self, string, filepath):
         '''writes the entirety of string to filepath'''
-        with open(filepath, "w") as x:
+        with open(filepath, "w") as x:  # FIXME: encoding='utf-8' ?
             x.write(string)
 
     def version_h_path(self, src):
@@ -301,7 +321,7 @@ def addfwversion_gitversion(self, destdir, src):
         gitversion_content = gitlog
         versionfile = self.version_h_path(src)
         if os.path.exists(versionfile):
-            content = self.read_string_from_filepath(versionfile)
+            content = self.read_string_from_filepath(versionfile).decode('utf-8')
             match = re.search('define.THISFIRMWARE "([^"]+)"', content)
             if match is None:
                 self.progress("Failed to retrieve THISFIRMWARE from version.h")
@@ -323,7 +343,8 @@ def addfwversion_firmwareversiontxt(self, destdir, src):
         ss = r".*define +FIRMWARE_VERSION[	 ]+(?P<major>\d+)[ ]*,[ 	]*" \
              r"(?P<minor>\d+)[ ]*,[	 ]*(?P<point>\d+)[ ]*,[	 ]*" \
              r"(?P<type>[A-Z_]+)[	 ]*"
-        content = self.read_string_from_filepath(versionfile)
+        # FIXME: content returned by read_string_from_filepath is binary!    
+        content = self.read_string_from_filepath(versionfile).decode('utf-8')
         match = re.search(ss, content)
         if match is None:
             self.progress("Failed to retrieve FIRMWARE_VERSION from version.h")
@@ -342,7 +363,13 @@ def addfwversion_firmwareversiontxt(self, destdir, src):
     def addfwversion(self, destdir, src):
         '''write version information into destdir'''
         self.addfwversion_gitversion(destdir, src)
-        self.addfwversion_firmwareversiontxt(destdir, src)
+        # FIXME: this seem to sometimes result in utf-8/encoding issues. at least add traceback for now
+        try:
+            self.addfwversion_firmwareversiontxt(destdir, src)
+        except:
+            import traceback
+            traceback.print_stack()
+            raise
 
     def read_string_from_filepath(self, filepath):
         '''returns content of filepath as a string'''
@@ -394,7 +421,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir,
         '''build vehicle binaries'''
         self.progress("Building %s %s binaries (cwd=%s)" %
                       (vehicle, tag, os.getcwd()))
-
+        boards = self.filter_selected_boards(boards)
         board_count = len(boards)
         count = 0
         for board in sorted(boards, key=str.lower):
@@ -410,7 +437,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir,
                     framesuffix = ""
                 else:
                     framesuffix = "-%s" % frame
-                if not self.checkout(vehicle, tag, board, frame, submodule_update=False):
+                if self.require_checkout and not self.checkout(vehicle, tag, board, frame, submodule_update=False):
                     msg = ("Failed checkout of %s %s %s %s" %
                            (vehicle, board, tag, frame,))
                     self.progress(msg)
@@ -476,6 +503,7 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir,
                     # record some history about this build
                     t1 = time.time()
                     time_taken_to_build = t1-t0
+                    # FIXME: do we need to handle skip_history here too?
                     self.history.record_build(githash, tag, vehicle, board, frame, None, t0, time_taken_to_build)
                     continue
 
@@ -509,19 +537,30 @@ def build_vehicle(self, tag, vehicle, boards, vehicle_binaries_subdir,
                         self.copyit(path, ddir, tag, vehicle)
                     except Exception as e:
                         self.progress("Failed to copy %s to %s: %s" % (path, ddir, str(e)))
+                        import traceback
+                        traceback.print_stack()
                 # why is touching this important? -pb20170816
                 self.touch_filepath(os.path.join(self.binaries,
                                                  vehicle_binaries_subdir, tag))
 
-                # record some history about this build
-                self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build)
+                # record some history about this build if needed
+                try:
+                    self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build)
+                except:
+                    if not skip_history:
+                        raise
 
         self.checkout(vehicle, "latest")
 
     def common_boards(self):
         '''returns list of boards common to all vehicles'''
         return AUTOBUILD_BOARDS
 
+    def filter_selected_boards(self, boards):
+        no_selected_boards = len(self.selected_boards) == 0
+        return [b for b in boards
+                if no_selected_boards or (b in self.selected_boards)]
+
     def AP_Periph_boards(self):
         return AP_PERIPH_BOARDS
 
@@ -660,7 +699,7 @@ def run(self):
         now = datetime.datetime.now()
         self.progress(now)
 
-        if not self.dirty:
+        if not self.dirty and self.require_checkout:
             self.run_git(["checkout", "-f", "master"])
         githash = self.run_git(["rev-parse", "HEAD"])
         githash = githash.rstrip()
@@ -684,15 +723,24 @@ def run(self):
         self.buildroot = os.path.join(os.environ.get("TMPDIR"),
                                       "binaries.build")
 
+
         for tag in self.tags:
             t0 = time.time()
-            self.build_arducopter(tag)
-            self.build_arduplane(tag)
-            self.build_rover(tag)
-            self.build_antennatracker(tag)
-            self.build_ardusub(tag)
-            self.build_AP_Periph(tag)
-            self.history.record_run(githash, tag, t0, time.time()-t0)
+            possible_builds = [
+                (ARDUCOPTER, lambda: self.build_arducopter(tag)),
+                (ARDUPLANE, lambda: self.build_arduplane(tag)),
+                (ROVER, lambda: self.build_rover(tag)),
+                (ANTENNATRACKER, lambda: self.build_antennatracker(tag)),
+                (ARDUSUB, lambda: self.build_ardusub(tag)),
+                (AP_PERIPH, lambda: self.build_AP_Periph(tag))
+            ]
+            for p, build_fn in possible_builds:
+                if p in self.projects:
+                    build_fn()
+            try:
+                self.history.record_run(githash, tag, t0, time.time()-t0)
+            except:
+                print('history failed')
 
         if os.path.exists(self.tmpdir):
             shutil.rmtree(self.tmpdir)
@@ -704,17 +752,50 @@ def run(self):
         sys.exit(len(self.error_strings))
 
 
+def flatten_comma_opts(opts):
+    """
+    allow multiple options to be passed separated by comma, e.g `arduplane,arducoper`
+    (useful for manual build workflow)
+
+    >>> flatten_comma_opts(['a', 'b'])
+    ['a', 'b']
+    >>> flatten_comma_opts(['a', 'b,c,d'])
+    ['a', 'b', 'c', 'd']
+    """
+    return [extracted_opt
+            for opt_value in opts
+            for extracted_opt in opt_value.replace(' ', '').split(',')
+            if extracted_opt != '']
+
+
+def filter_valid_projects(projects):
+    return [p for p in projects if p in ALL_PROJECTS]
+
 if __name__ == '__main__':
     parser = optparse.OptionParser("build_binaries.py")
 
     parser.add_option("", "--tags", action="append", type="string",
                       default=[], help="tags to build")
+    parser.add_option("", "--boards", action="append", type="string",
+                      default=[], help="boards to build")
+    parser.add_option("", "--projects", action="append", type="string",
+                      default=[], help="projects to build")
+    parser.add_option("", "--skip-history", type="string",
+                      default='no', help="skip recording of build history?")
+    parser.add_option("", "--require-checkout", type="string",
+                      default='yes', help="shall we do git checkout?")
     cmd_opts, cmd_args = parser.parse_args()
 
     tags = cmd_opts.tags
     if len(tags) == 0:
         # FIXME: wedge this defaulting into parser somehow
         tags = ["stable", "beta", "latest"]
 
-    bb = build_binaries(tags)
+    boards = flatten_comma_opts(cmd_opts.boards)
+    projects = flatten_comma_opts(cmd_opts.projects) or ALL_PROJECTS
+    require_checkout = cmd_opts.require_checkout == "yes"
+    skip_history = cmd_opts.skip_history == "yes"
+    projects = filter_valid_projects(projects)
+
+    bb = build_binaries(tags, selected_boards=boards, projects=projects, require_checkout=require_checkout, skip_history=skip_history)
     bb.run()