From 18d8aba20f4e55986d665b0a79f93ed9517c0662 Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Sun, 23 Jan 2022 22:19:25 -0600 Subject: [PATCH] feat: support WORLD files in compressed tar format (#1297) --- .github/workflows/discord.yml | 2 +- Dockerfile | 3 +- README.md | 12 +-- scripts/start-setupWorld | 32 +++++--- scripts/start-utils | 15 ++++ tests/.gitignore | 1 + ...ompose.yml => docker-compose.disabled.yml} | 0 ...ompose.yml => docker-compose.disabled.yml} | 0 .../generic-packs/docker-compose.yml | 4 +- tests/setuponlytests/generic-packs/verify.sh | 1 + tests/setuponlytests/test.sh | 69 ++++++++++-------- .../world_from_tgz/docker-compose.yml | 14 ++++ tests/setuponlytests/world_from_tgz/verify.sh | 1 + .../worlds/world-for-testing.tgz | Bin 0 -> 10240 bytes .../world_from_zip/docker-compose.yml | 14 ++++ tests/setuponlytests/world_from_zip/verify.sh | 1 + .../worlds/world-for-testing.zip | Bin 0 -> 116 bytes tests/test.sh | 19 ++--- 18 files changed, 123 insertions(+), 65 deletions(-) create mode 100644 tests/.gitignore rename tests/setuponlytests/forgeapimods_file/{docker-compose.yml => docker-compose.disabled.yml} (100%) rename tests/setuponlytests/forgeapimods_projectids/{docker-compose.yml => docker-compose.disabled.yml} (100%) create mode 100644 tests/setuponlytests/generic-packs/verify.sh create mode 100644 tests/setuponlytests/world_from_tgz/docker-compose.yml create mode 100644 tests/setuponlytests/world_from_tgz/verify.sh create mode 100644 tests/setuponlytests/world_from_tgz/worlds/world-for-testing.tgz create mode 100644 tests/setuponlytests/world_from_zip/docker-compose.yml create mode 100644 tests/setuponlytests/world_from_zip/verify.sh create mode 100644 tests/setuponlytests/world_from_zip/worlds/world-for-testing.zip diff --git a/.github/workflows/discord.yml b/.github/workflows/discord.yml index e4e6b42..495b6e4 100644 --- a/.github/workflows/discord.yml +++ b/.github/workflows/discord.yml @@ -2,7 +2,7 @@ name: discord on: workflow_run: - workflows: ["ContinuousIntegration", "PullRequest", "Build and Publish", "Build and publish multiarch" ] + workflows: ["ContinuousIntegration", "Build and Publish", "Build and publish multiarch" ] types: - completed diff --git a/Dockerfile b/Dockerfile index c9133e3..836ee44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive \ apt-get install -y \ imagemagick \ + file \ gosu \ sudo \ net-tools \ @@ -60,7 +61,7 @@ RUN easy-add --var os=${TARGETOS} --var arch=${TARGETARCH}${TARGETVARIANT} \ --var version=0.1.1 --var app=maven-metadata-release --file {{.app}} \ --from https://github.com/itzg/{{.app}}/releases/download/{{.version}}/{{.app}}_{{.version}}_{{.os}}_{{.arch}}.tar.gz -ARG MC_HELPER_VERSION=1.11.0 +ARG MC_HELPER_VERSION=1.16.0 ARG MC_HELPER_BASE_URL=https://github.com/itzg/mc-image-helper/releases/download/v${MC_HELPER_VERSION} RUN curl -fsSL ${MC_HELPER_BASE_URL}/mc-image-helper-${MC_HELPER_VERSION}.tgz \ | tar -C /usr/share -zxf - \ diff --git a/README.md b/README.md index 21faed0..05f36fa 100644 --- a/README.md +++ b/README.md @@ -829,23 +829,19 @@ before unpacking new content from the MODPACK or MODS. ### Downloadable world -Instead of mounting the `/data` volume, you can instead specify the URL of a ZIP file containing an archived world. It will be searched for a file `level.dat` and the containing subdirectory moved to the directory named by `$LEVEL`. This means that most of the archived Minecraft worlds downloadable from the Internet will already be in the correct format. +Instead of mounting the `/data` volume, you can instead specify the URL of a ZIP or compressed TAR file containing an archived world. It will be searched for a file `level.dat` and the containing subdirectory moved to the directory named by `$LEVEL`. This means that most of the archived Minecraft worlds downloadable from the Internet will already be in the correct format. docker run -d -e WORLD=http://www.example.com/worlds/MySave.zip ... -**NOTE:** This URL must be accessible from inside the container. Therefore, -you should use an IP address or a globally resolvable FQDN, or else the -name of a linked container. +**NOTE:** This URL must be accessible from inside the container. Therefore, you should use an IP address or a globally resolvable FQDN, or else the name of a linked container. **NOTE:** If the archive contains more than one `level.dat`, then the one to select can be picked with `WORLD_INDEX`, which defaults to 1. ### Cloning world from a container path -The `WORLD` option can also be used to reference a directory or zip file that will be used as a source to clone or unzip the world directory. +The `WORLD` option can also be used to reference a directory, zip file, or compressed tar file that will be used as a source to clone or extract the world directory. -For example, the following would initially clone the world's content -from `/worlds/basic`. Also notice in the example that you can use a -read-only volume attachment to ensure the clone source remains pristine. +For example, the following would initially clone the world's content from `/worlds/basic`. Also notice in the example that you should use a read-only volume attachment to ensure the clone source remains pristine. ``` docker run ... -v $HOME/worlds:/worlds:ro -e WORLD=/worlds/basic diff --git a/scripts/start-setupWorld b/scripts/start-setupWorld index db14635..dc3901e 100755 --- a/scripts/start-setupWorld +++ b/scripts/start-setupWorld @@ -5,7 +5,7 @@ set -e isDebugging && set -x -if [ $TYPE = "CURSEFORGE" ]; then +if [ "$TYPE" = "CURSEFORGE" ]; then worldDest=$FTB_DIR/${LEVEL:-world} else worldDest=/data/${LEVEL:-world} @@ -19,19 +19,23 @@ if [[ "$WORLD" ]] && ( isTrue "${FORCE_WORLD_COPY}" || [ ! -d "$worldDest" ] ); "${worldDest}_the_end" fi - if isURL $WORLD; then - curl -fsSL "$WORLD" -o /tmp/world.zip - zipSrc=/tmp/world.zip - elif [[ "$WORLD" =~ .*\.zip ]]; then - zipSrc="$WORLD" + if isURL "$WORLD"; then + log "Downloading world from $WORLD" + if ! get -o /tmp/world.bin "$WORLD"; then + log "ERROR: failed to download world from $WORLD" + exit 1 + fi + WORLD=/tmp/world.bin fi - if [[ "$zipSrc" ]]; then - log "Unzipping world" + if [ -f "$WORLD" ]; then + log "Extracting world" # Stage contents so that the correct subdirectory can be picked off mkdir -p /tmp/world-data - (cd /tmp/world-data && unzip -o -q "$zipSrc") + if ! extract "$WORLD" /tmp/world-data; then + exit 1 + fi if [ "$FAMILY" = "SPIGOT" ]; then baseDirs=$(find /tmp/world-data -name "level.dat" -not -path "*_nether*" -not -path "*_the_end*" -exec dirname "{}" \;) @@ -39,6 +43,11 @@ if [[ "$WORLD" ]] && ( isTrue "${FORCE_WORLD_COPY}" || [ ! -d "$worldDest" ] ); baseDirs=$(find /tmp/world-data -name "level.dat" -exec dirname "{}" \;) fi + if ! [[ $baseDirs ]]; then + log "ERROR world content is not valid since level.dat could not be found" + exit 2 + fi + count=$(echo "$baseDirs" | wc -l) if [[ $count -gt 1 ]]; then baseDir="$(echo "$baseDirs" | sed -n ${WORLD_INDEX:-1}p)" @@ -56,9 +65,12 @@ if [[ "$WORLD" ]] && ( isTrue "${FORCE_WORLD_COPY}" || [ ! -d "$worldDest" ] ); [ -d "${baseDir}_nether" ] && rsync --remove-source-files --recursive --delete "${baseDir}_nether/" "${worldDest}_nether" [ -d "${baseDir}_the_end" ] && rsync --remove-source-files --recursive --delete "${baseDir}_the_end/" "${worldDest}_the_end" fi - else + elif [ -d "$WORLD" ]; then log "Cloning world directory from $WORLD ..." rsync --recursive --delete "${WORLD%/}"/ "$worldDest" + else + log "ERROR: world file/directory $WORLD is missing" + exit 1 fi if [ "$FAMILY" = "SPIGOT" ]; then diff --git a/scripts/start-utils b/scripts/start-utils index 0861e24..a2da3d5 100755 --- a/scripts/start-utils +++ b/scripts/start-utils @@ -188,4 +188,19 @@ function isType() { fi done return 1 +} + +function extract() { + src=${1?} + destDir=${2?} + + type=$(file -b --mime-type "${src}") + if [[ $type == application/zip ]]; then + unzip -q -d "${destDir}" "${src}" + elif [[ $type == application/x-tar ]]; then + tar -C "${destDir}" -xf "${src}" + else + log "ERROR: unsupported archive type: $type" + return 1 + fi } \ No newline at end of file diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..adbb97d --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +data/ \ No newline at end of file diff --git a/tests/setuponlytests/forgeapimods_file/docker-compose.yml b/tests/setuponlytests/forgeapimods_file/docker-compose.disabled.yml similarity index 100% rename from tests/setuponlytests/forgeapimods_file/docker-compose.yml rename to tests/setuponlytests/forgeapimods_file/docker-compose.disabled.yml diff --git a/tests/setuponlytests/forgeapimods_projectids/docker-compose.yml b/tests/setuponlytests/forgeapimods_projectids/docker-compose.disabled.yml similarity index 100% rename from tests/setuponlytests/forgeapimods_projectids/docker-compose.yml rename to tests/setuponlytests/forgeapimods_projectids/docker-compose.disabled.yml diff --git a/tests/setuponlytests/generic-packs/docker-compose.yml b/tests/setuponlytests/generic-packs/docker-compose.yml index 46dd834..071c430 100644 --- a/tests/setuponlytests/generic-packs/docker-compose.yml +++ b/tests/setuponlytests/generic-packs/docker-compose.yml @@ -9,6 +9,4 @@ services: GENERIC_PACKS: https://github.com/itzg/mc-image-helper/releases/download/v1.9.5/mc-image-helper-1.9.5.zip,/packs/testing.zip volumes: - ./packs:/packs - - data:/data -volumes: - data: {} + - ./data:/data diff --git a/tests/setuponlytests/generic-packs/verify.sh b/tests/setuponlytests/generic-packs/verify.sh new file mode 100644 index 0000000..15166fe --- /dev/null +++ b/tests/setuponlytests/generic-packs/verify.sh @@ -0,0 +1 @@ +mc-image-helper assert fileExists one.txt mods/two.txt \ No newline at end of file diff --git a/tests/setuponlytests/test.sh b/tests/setuponlytests/test.sh index 169cf0b..8c8cb38 100644 --- a/tests/setuponlytests/test.sh +++ b/tests/setuponlytests/test.sh @@ -1,43 +1,50 @@ #!/bin/bash +set -euo pipefail +IFS=$'\n\t' # go to script root directory cd "$(dirname "$0")" || exit 1 -# compose down function for reuse -down() { - docker-compose down -v --remove-orphans -} - -checkandExitOnFailure(){ - failed=$1 - # docker-compose logs outputs messages from the specified container - if $failed; then - docker-compose logs mc - down - cd .. - exit 2 - fi -} - -# tests that only run the setup files for things like downloads and configuration. +# tests that only run the setup files for things like downloads and configuration. setupOnlyMinecraftTest(){ folder=$1 cd "$folder" - failed=false - # run the monitor to validate the Minecraft image is healthy - docker-compose --log-level ERROR up --quiet-pull --exit-code-from mc 2>/dev/null || failed=true - echo "${folder} Result: failed=$failed" - checkandExitOnFailure $failed - down + result=0 + + if ! logs=$(docker compose run --quiet-pull mc 2>&1); then + echo "${folder} setup FAILED" + echo ":::::::::::: LOGS :::::::::::::::: +$logs +:::::::::::::::::::::::::::::::::: +" + result=1 + elif [ -f verify.sh ]; then + if ! docker run --rm --entrypoint bash -v "${PWD}/data":/data -v "${PWD}/verify.sh":/verify "${IMAGE_TO_TEST:-itzg/minecraft-server}" /verify; then + echo "${folder} verify FAILED" + result=1 + else + echo "${folder} verify PASS" + fi + else + echo "${folder} PASS" + fi + + docker compose down -v --remove-orphans cd .. + + return $result } # go through each folder in setuponly and test setups -FOLDERS=$(ls) -for folder in $FOLDERS; do - # If folder is a directory - if [ -d "$folder" ]; then - echo "Starting Tests on ${folder}" - setupOnlyMinecraftTest $folder - fi -done +if (( $# > 0 )); then + for folder in "$@"; do + echo "Starting Tests in ${folder}" + setupOnlyMinecraftTest "$folder" + done +else + readarray -t folders < <(find . -maxdepth 2 -mindepth 2 -name docker-compose.yml -printf '%h\n') + for folder in "${folders[@]}"; do + echo "Starting Tests in ${folder}" + setupOnlyMinecraftTest "$folder" + done +fi diff --git a/tests/setuponlytests/world_from_tgz/docker-compose.yml b/tests/setuponlytests/world_from_tgz/docker-compose.yml new file mode 100644 index 0000000..530b1ed --- /dev/null +++ b/tests/setuponlytests/world_from_tgz/docker-compose.yml @@ -0,0 +1,14 @@ +version: "3" + +services: + mc: + restart: "no" + image: ${IMAGE_TO_TEST:-itzg/minecraft-server} + environment: + EULA: "TRUE" + SETUP_ONLY: "TRUE" + VERSION: ${MINECRAFT_VERSION:-LATEST} + WORLD: /worlds/world-for-testing.tgz + volumes: + - ./worlds:/worlds:ro + - ./data:/data diff --git a/tests/setuponlytests/world_from_tgz/verify.sh b/tests/setuponlytests/world_from_tgz/verify.sh new file mode 100644 index 0000000..b2138d0 --- /dev/null +++ b/tests/setuponlytests/world_from_tgz/verify.sh @@ -0,0 +1 @@ +mc-image-helper assert fileExists world/level.dat \ No newline at end of file diff --git a/tests/setuponlytests/world_from_tgz/worlds/world-for-testing.tgz b/tests/setuponlytests/world_from_tgz/worlds/world-for-testing.tgz new file mode 100644 index 0000000000000000000000000000000000000000..6f51bb81d4a4e45070316b58ecd347cf24211c33 GIT binary patch literal 10240 zcmeIuJqmy@42Izxr6-WYsyUBfbtwq`onM_49Gtqn-;$D$5}urnl&hiNq+IlhyUSI~ zTYcAibVsZ)kFisYQPixnT7s54>G^qIw|)shrZkV^)Be2v=AQ)`2q1s}0tg_000Iag pfB*srAb