diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca867087d..35038e1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,13 +77,10 @@ jobs: wget https://github.com/decompals/WiBo/releases/download/0.2.4/wibo && chmod +x wibo && sudo cp wibo /usr/bin/ - name: Run backend tests run: |- - mkdir -p "${WINEPREFIX}" - wineboot --init cd backend poetry run python3 manage.py test env: SYSTEM_ENV: GITHUB_WORKFLOW - WINEPREFIX: /tmp/wine - name: Migrate backend run: cd backend && poetry run python3 ./manage.py migrate @@ -125,10 +122,7 @@ jobs: - uses: actions/checkout@v2 - name: Build decompme_backend image run: |- - docker build backend \ - -t decompme_backend \ - --build-arg ENABLE_WII_GC_SUPPORT=YES \ - --build-arg ENABLE_PS1_SUPPORT=YES + docker build backend -t decompme_backend - name: Run tests run: |- mkdir -p sandbox && chmod 777 sandbox @@ -140,7 +134,6 @@ jobs: --security-opt seccomp=unconfined \ --entrypoint /bin/bash \ -e COMPILER_BASE_PATH=/compilers \ - -e WINEPREFIX=/tmp/wine \ -e LOCAL_FILE_DIR=/local_files \ decompme_backend -c 'cd /decomp.me/backend && \ poetry install && \ diff --git a/backend/Dockerfile b/backend/Dockerfile index 39b8a574e..28c2d52fa 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -34,20 +34,25 @@ RUN git clone "https://github.com/google/nsjail" --recursive --branch 3.1 /nsjai FROM base AS build RUN apt-get -y update && apt-get install -y \ + binutils-aarch64-linux-gnu \ binutils-mips-linux-gnu \ binutils-powerpc-linux-gnu \ - binutils-aarch64-linux-gnu \ curl \ + dos2unix \ gcc-mips-linux-gnu \ + libc6-dev-i386 \ libnl-route-3-200 \ libprotobuf-dev \ + libtinfo5 \ netcat \ unzip \ wget \ - libtinfo5 \ - libc6-dev-i386 \ && rm -rf /var/lib/apt/lists/* +RUN dpkg --add-architecture i386 && apt-get update \ + && apt-get install -y --no-install-recommends -o APT::Immediate-Configure=false \ + libstdc++6:i386 + RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | \ POETRY_VERSION=1.1.13 POETRY_HOME=/etc/poetry python3.9 - @@ -55,23 +60,6 @@ COPY --from=nsjail /nsjail/nsjail /bin/nsjail COPY --from=ghcr.io/decompals/wibo:0.2.4 /usr/local/sbin/wibo /usr/bin/ -# windows compilers need i386 wine -ARG ENABLE_NDS_SUPPORT -ARG ENABLE_PS1_SUPPORT -ARG ENABLE_WII_GC_SUPPORT -RUN if [ "${ENABLE_NDS_SUPPORT}" = "YES" ] || \ - [ "${ENABLE_PS1_SUPPORT}" = "YES" ] || \ - [ "${ENABLE_WII_GC_SUPPORT}" = "YES" ]; then \ - dpkg --add-architecture i386 && apt-get update && \ - apt-get install -y -o APT::Immediate-Configure=false \ - wine; \ - fi - -# ps1 specifics -RUN if [ "${ENABLE_PS1_SUPPORT}" = "YES" ] ; then \ - apt-get install -y dos2unix; \ - fi - # gc/wii specifics COPY --from=devkitpro/devkitppc:20210726 /opt/devkitpro/devkitPPC/bin/powerpc* /usr/bin/ @@ -83,14 +71,17 @@ COPY compilers/download.py /compilers/ ARG ENABLE_GBA_SUPPORT ARG ENABLE_N64_SUPPORT +ARG ENABLE_NDS_SUPPORT +ARG ENABLE_PS1_SUPPORT ARG ENABLE_SWITCH_SUPPORT +ARG ENABLE_WII_GC_SUPPORT ENV ENABLE_GBA_SUPPORT=${ENABLE_GBA_SUPPORT} -ENV ENABLE_WII_GC_SUPPORT=${ENABLE_WII_GC_SUPPORT} ENV ENABLE_N64_SUPPORT=${ENABLE_N64_SUPPORT} ENV ENABLE_NDS_SUPPORT=${ENABLE_NDS_SUPPORT} ENV ENABLE_PS1_SUPPORT=${ENABLE_PS1_SUPPORT} ENV ENABLE_SWITCH_SUPPORT=${ENABLE_SWITCH_SUPPORT} +ENV ENABLE_WII_GC_SUPPORT=${ENABLE_WII_GC_SUPPORT} RUN python3.9 -m pip install requests tqdm \ && python3.9 /compilers/download.py \ @@ -100,25 +91,14 @@ RUN mkdir -p /etc/fonts WORKDIR /backend -ENV WINEPREFIX=/tmp/wine - # create a non-root user & /sandbox with correct ownership RUN useradd --create-home user \ && mkdir -p /sandbox \ - && chown -R user:user /sandbox \ - && mkdir -p "${WINEPREFIX}" \ - && chown user:user "${WINEPREFIX}" + && chown -R user:user /sandbox # switch to non-root user USER user -# initialize wine files to /home/user/.wine -RUN if [ "${ENABLE_PS1_SUPPORT}" = "YES" ] || \ - [ "${ENABLE_WII_GC_SUPPORT}" = "YES" ] || \ - [ "${ENABLE_NDS_SUPPORT}" = "YES" ]; then \ - wineboot --init; \ - fi - ENV PATH="$PATH:/etc/poetry/bin" ENTRYPOINT ["/backend/docker_entrypoint.sh"] diff --git a/backend/compilers/download.py b/backend/compilers/download.py index 5b01ede62..4daa8801f 100755 --- a/backend/compilers/download.py +++ b/backend/compilers/download.py @@ -528,7 +528,7 @@ def download_wii_gc(): if not compiler_dir.exists(): shutil.move(COMPILERS_DIR / group_id / ver, compiler_dir) - # Rename dll to uppercase - WSL is case sensitive without wine + # Rename dll to uppercase lowercase_lmgr = compiler_dir / "lmgr326b.dll" if lowercase_lmgr.exists(): shutil.move(lowercase_lmgr, compiler_dir / "LMGR326B.dll") diff --git a/backend/coreapp/compiler_wrapper.py b/backend/coreapp/compiler_wrapper.py index 1ab17a9ca..dfd314ef4 100644 --- a/backend/coreapp/compiler_wrapper.py +++ b/backend/coreapp/compiler_wrapper.py @@ -40,13 +40,6 @@ def lru_cache(maxsize: int = 128, typed: bool = False) -> Callable[[F], F]: else: PATH = os.environ["PATH"] -WINE: str -if "microsoft" in uname().release.lower() and not settings.USE_SANDBOX_JAIL: - logger.info("WSL detected & nsjail disabled: wine not required.") - WINE = "" -else: - WINE = "wine" - WIBO: str if "microsoft" in uname().release.lower() and not settings.USE_SANDBOX_JAIL: logger.info("WSL detected & nsjail disabled: wibo not required.") diff --git a/backend/coreapp/compilers.py b/backend/coreapp/compilers.py index d3fac41d8..dfa070bfc 100644 --- a/backend/coreapp/compilers.py +++ b/backend/coreapp/compilers.py @@ -45,7 +45,6 @@ class Compiler: is_gcc: ClassVar[bool] = False is_ido: ClassVar[bool] = False is_mwcc: ClassVar[bool] = False - needs_wine = False @property def path(self) -> Path: @@ -184,8 +183,7 @@ def preset_from_name(name: str) -> Optional[Preset]: cc='mips-linux-gnu-cpp -Wall -lang-c -gstabs "$INPUT" | "${COMPILER_DIR}"/cc1 -mips1 -mcpu=3000 $COMPILER_FLAGS | mips-linux-gnu-as -march=r3000 -mtune=r3000 -no-pad-sections -O1 -o "$OUTPUT"', ) -PSYQ_CC = 'cpp -P "$INPUT" | unix2dos | ${WINE} ${COMPILER_DIR}/CC1PSX.EXE -quiet ${COMPILER_FLAGS} -o "$OUTPUT".s && ${WINE} ${COMPILER_DIR}/ASPSX.EXE -quiet "$OUTPUT".s -o "$OUTPUT".obj && ${COMPILER_DIR}/psyq-obj-parser "$OUTPUT".obj -o "$OUTPUT"' - +PSYQ_CC = 'cpp -P "$INPUT" | unix2dos | ${WIBO} ${COMPILER_DIR}/CC1PSX.EXE -quiet ${COMPILER_FLAGS} -o "$OUTPUT".s && ${WIBO} ${COMPILER_DIR}/ASPSX.EXE -quiet "$OUTPUT".s -o "$OUTPUT".obj && ${COMPILER_DIR}/psyq-obj-parser "$OUTPUT".obj -o "$OUTPUT"' PSYQ40 = GCCPS1Compiler( id="psyq4.0", platform=PS1, @@ -245,11 +243,11 @@ def preset_from_name(name: str) -> Optional[Preset]: GCC272SN = GCCCompiler( id="gcc2.7.2sn", platform=N64, - cc='cpp -P "$INPUT" | ${WINE} "${COMPILER_DIR}"/cc1n64.exe -quiet -G0 -mcpu=vr4300 -mips3 -mhard-float -meb ${COMPILER_FLAGS} -o "$OUTPUT".s && ${WINE} "${COMPILER_DIR}"/asn64.exe -q -G0 "$OUTPUT".s -o "$OUTPUT".obj && "${COMPILER_DIR}"/psyq-obj-parser "$OUTPUT".obj -o "$OUTPUT" -b -n', + cc='cpp -P "$INPUT" | ${WIBO} "${COMPILER_DIR}"/cc1n64.exe -quiet -G0 -mcpu=vr4300 -mips3 -mhard-float -meb ${COMPILER_FLAGS} -o "$OUTPUT".s && ${WIBO} "${COMPILER_DIR}"/asn64.exe -q -G0 "$OUTPUT".s -o "$OUTPUT".obj && "${COMPILER_DIR}"/psyq-obj-parser "$OUTPUT".obj -o "$OUTPUT" -b -n', ) # MACOS9 -MWCPPC_CC = 'printf "%s" "${COMPILER_FLAGS}" | xargs -x -- ${WINE} "${COMPILER_DIR}/MWCPPC.exe" -o object.o "${INPUT}" && printf "%s" "-dis -h -module ".${FUNCTION}" -nonames -nodata" | xargs -x -- ${WINE} "${COMPILER_DIR}/MWLinkPPC.exe" "${OUTPUT}" > "${OUTPUT}.s" && python3 ${COMPILER_DIR}/convert_gas_syntax.py "${OUTPUT}.s" ".${FUNCTION}" > "${OUTPUT}_new.s" && powerpc-linux-gnu-as "${OUTPUT}_new.s" -o "${OUTPUT}"' +MWCPPC_CC = 'printf "%s" "${COMPILER_FLAGS}" | xargs -x -- ${WIBO} "${COMPILER_DIR}/MWCPPC.exe" -o object.o "${INPUT}" && printf "%s" "-dis -h -module ".${FUNCTION}" -nonames -nodata" | xargs -x -- ${WIBO} "${COMPILER_DIR}/MWLinkPPC.exe" "${OUTPUT}" > "${OUTPUT}.s" && python3 ${COMPILER_DIR}/convert_gas_syntax.py "${OUTPUT}.s" ".${FUNCTION}" > "${OUTPUT}_new.s" && powerpc-linux-gnu-as "${OUTPUT}_new.s" -o "${OUTPUT}"' MWCPPC_23 = MWCCCompiler( id="mwcppc_23", @@ -314,7 +312,7 @@ def preset_from_name(name: str) -> Optional[Preset]: # GC_WII # Thanks to Gordon Davisson for the xargs trick: # https://superuser.com/questions/1529226/get-bash-to-respect-quotes-when-word-splitting-subshell-output/1529316#1529316 -MWCCEPPC_CC = 'printf "%s" "${COMPILER_FLAGS}" | xargs -x -- ${WINE} "${COMPILER_DIR}/mwcceppc.exe" -c -proc gekko -nostdinc -stderr -o "${OUTPUT}" "${INPUT}"' +MWCCEPPC_CC = 'printf "%s" "${COMPILER_FLAGS}" | xargs -x -- ${WIBO} "${COMPILER_DIR}/mwcceppc.exe" -c -proc gekko -nostdinc -stderr -o "${OUTPUT}" "${INPUT}"' MWCC_233_144 = MWCCCompiler( id="mwcc_233_144", @@ -336,7 +334,7 @@ def preset_from_name(name: str) -> Optional[Preset]: MWCC_233_163E = MWCCCompiler( id="mwcc_233_163e", platform=GC_WII, - cc='${WINE} "${COMPILER_DIR}/mwcceppc.125.exe" -c -proc gekko -nostdinc -stderr ${COMPILER_FLAGS} -o "${OUTPUT}.1" "${INPUT}" && ${WINE} "${COMPILER_DIR}/mwcceppc.exe" -c -proc gekko -nostdinc -stderr ${COMPILER_FLAGS} -o "${OUTPUT}.2" "${INPUT}" && python3 "${COMPILER_DIR}/frank.py" "${OUTPUT}.1" "${OUTPUT}.2" "${OUTPUT}"', + cc='${WIBO} "${COMPILER_DIR}/mwcceppc.125.exe" -c -proc gekko -nostdinc -stderr ${COMPILER_FLAGS} -o "${OUTPUT}.1" "${INPUT}" && ${WIBO} "${COMPILER_DIR}/mwcceppc.exe" -c -proc gekko -nostdinc -stderr ${COMPILER_FLAGS} -o "${OUTPUT}.2" "${INPUT}" && python3 "${COMPILER_DIR}/frank.py" "${OUTPUT}.1" "${OUTPUT}.2" "${OUTPUT}"', ) MWCC_242_81 = MWCCCompiler( @@ -412,7 +410,7 @@ def preset_from_name(name: str) -> Optional[Preset]: ) # NDS_ARM9 -MWCCARM_CC = '${WINE} "${COMPILER_DIR}/mwccarm.exe" -c -proc arm946e -nostdinc -stderr ${COMPILER_FLAGS} -o "${OUTPUT}" "${INPUT}"' +MWCCARM_CC = '${WIBO} "${COMPILER_DIR}/mwccarm.exe" -c -proc arm946e -nostdinc -stderr ${COMPILER_FLAGS} -o "${OUTPUT}" "${INPUT}"' MWCC_20_72 = MWCCCompiler( id="mwcc_20_72", diff --git a/backend/coreapp/sandbox.py b/backend/coreapp/sandbox.py index c6e141aa9..9dccf3035 100644 --- a/backend/coreapp/sandbox.py +++ b/backend/coreapp/sandbox.py @@ -46,10 +46,8 @@ def sandbox_command(self, mounts: List[Path], env: Dict[str, str]) -> List[str]: return [] settings.SANDBOX_CHROOT_PATH.mkdir(parents=True, exist_ok=True) - settings.WINEPREFIX.mkdir(parents=True, exist_ok=True) assert ":" not in str(self.path) - assert ":" not in str(settings.WINEPREFIX) # fmt: off wrapper = [ str(settings.SANDBOX_NSJAIL_BIN_PATH), @@ -69,7 +67,7 @@ def sandbox_command(self, mounts: List[Path], env: Dict[str, str]) -> List[str]: "--env", "PATH=/usr/bin:/bin", "--cwd", "/tmp", "--rlimit_fsize", "soft", - "--rlimit_nofile", "soft", + "--rlimit_nofile", "soft", "--rlimit_cpu", "30", # seconds "--time_limit", "30", # seconds # the following are settings that can be removed once we are done with wine diff --git a/backend/decompme/settings.py b/backend/decompme/settings.py index 9b124eed1..f49ce5c3b 100644 --- a/backend/decompme/settings.py +++ b/backend/decompme/settings.py @@ -29,7 +29,6 @@ GITHUB_CLIENT_SECRET=(str, ""), COMPILER_BASE_PATH=(str, BASE_DIR / "compilers"), COMPILATION_CACHE_SIZE=(int, 100), - WINEPREFIX=(str, "/tmp/wine"), ) for stem in [".env.local", ".env"]: @@ -186,5 +185,3 @@ GITHUB_CLIENT_SECRET = env("GITHUB_CLIENT_SECRET", str) COMPILATION_CACHE_SIZE = env("COMPILATION_CACHE_SIZE", int) - -WINEPREFIX = Path(env("WINEPREFIX")) diff --git a/docker-compose.yaml b/docker-compose.yaml index bcf64b16f..bcaa8ad1a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,10 +15,9 @@ services: args: ENABLE_GBA_SUPPORT: "YES" ENABLE_N64_SUPPORT: "YES" - # dont install wine32 etc by default - ENABLE_NDS_SUPPORT: "NO" - ENABLE_PS1_SUPPORT: "NO" - ENABLE_WII_GC_SUPPORT: "NO" + ENABLE_NDS_SUPPORT: "YES" + ENABLE_PS1_SUPPORT: "YES" + ENABLE_WII_GC_SUPPORT: "YES" # dont install clang by default ENABLE_SWITCH_SUPPORT: "NO" cap_drop: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 99402ce67..dac2e38e3 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -78,7 +78,6 @@ yarn dev ### Optional steps -- [Configure wine for Windows compiler on Linux](WINE.md) - [Set up GitHub authentication](GITHUB.md) - [Install nsjail to run the compiler sandbox](SANDBOX.md) - [Configure an nginx reverse proxy](NGINX.md) diff --git a/docs/WINE.md b/docs/WINE.md deleted file mode 100644 index 4cfe86b83..000000000 --- a/docs/WINE.md +++ /dev/null @@ -1,10 +0,0 @@ -### Wine setup (for local development, running Windows compilers) -- Create a wineprefix dir -```shell -WINEPREFIX=$HOME/.wine WINEARCH=win32 wineboot --init -``` - -- Add the WINEPREFIX setting to your .local.env file in the root of the repo -```shell -echo "WINEPREFIX=$HOME/.wine" >> .local.env -```