Adding MODS_FORGEAPI to auto-download by project ids (#1275)

Co-authored-by: christopher blodgett <christopher.blodgett@gmail.com>
Co-authored-by: Geoff Bourne <itzgeoff@gmail.com>

.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: ContinuousIntegration
+
+on:
+  push:
+    branches:
+      - 'dev*'
+      - '!master'
+      - '!java*'
+      - '!multi*'
+    paths-ignore:
+      - "*.md"
+      - "docs/**"
+      - "examples/**"
+
+env:
+  IMAGE_TO_TEST: ${{ secrets.IMAGE_ORG }}/minecraft-server:test-${{ github.repository_owner }}-${{ github.run_id }}
+  MODS_FORGEAPI_KEY: ${{ secrets.MODS_FORGEAPI_KEY }}
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2.4.0
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2.7.0
+        with:
+          context: .
+          platforms: linux/amd64
+          tags: ${{ env.IMAGE_TO_TEST }}
+          load: true
+          cache-from: type=gha
+
+      - name: Run Setup Only Tests
+        run: sh tests/setuponlytests/test.sh
+
+      # - name: Run Full Minecraft Service Tests
+      #   run: |
+      #     tests/fulltests/test.sh          

.github/workflows/discord.yml

@@ -0,0 +1,27 @@
+name: discord
+
+on:
+  workflow_run:
+    workflows: ["ContinuousIntegration", "PullRequest", "Build and Publish", "Build and publish multiarch" ]
+    types:
+      - completed
+
+env:
+  DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+  GITHUB_WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}
+
+jobs:
+  discord:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: on-success
+        if: ${{ github.event.workflow_run.conclusion == 'success' }}
+        uses: Ilshidur/action-discord@master
+        with:
+          args: "Github repo: ${{ github.repository }}\n- Branch: ${{ github.ref }}\n- [Link: to Actions](<${{ env.GITHUB_WORKFLOW_URL }}>)\n- Status: 🎉  ${{ github.event.workflow_run.conclusion }}  🍏"
+      - name: on-failure
+        if: ${{ github.event.workflow_run.conclusion == 'failure' }}
+        uses: Ilshidur/action-discord@master
+        with:
+          args: "Github repo: ${{ github.repository }}\n- Branch: ${{ github.ref }}\n- [Link: to Actions](<${{ env.GITHUB_WORKFLOW_URL }}>)\n- Status: 🤔  ${{ github.event.workflow_run.conclusion }}  💣💥"

.github/workflows/issue-label.yml

@@ -0,0 +1,17 @@
+on:
+  issues:
+    types: [labeled]
+
+env:
+  DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+
+jobs:
+  labelNotify:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: notify
+        if: github.event.label.name == 'enhancement' || github.event.label.name == 'bug'
+        uses: Ilshidur/action-discord@master
+        with:
+          args: "[${{ github.event.issue.title }}](<${{ github.event.issue.html_url }}>) added `${{ github.event.label.name }}` label"

.github/workflows/pr.yml

@@ -1,4 +1,4 @@
-name: Validate PR
+name: PullRequest
 
 on:
   pull_request:
@@ -11,6 +11,7 @@ on:
 
 env:
   IMAGE_TO_TEST: itzg/minecraft-server:test-${{ github.repository_owner }}-${{ github.run_id }}
+  MODS_FORGEAPI_KEY: ${{ secrets.MODS_FORGEAPI_KEY }}
 
 jobs:
   test:

.gitignore

@@ -2,5 +2,4 @@
 /data/
 /.idea/
 *.iml
-*.zip
 /gh-md-toc

README.md

@@ -745,6 +745,47 @@ You may also download or copy over individual mods using the `MODS` environment
 
   docker run -d -e MODS=https://www.example.com/mods/mod1.jar,/plugins/common,/plugins/special/mod2.jar ...
 
+### **ForgeAPI** usage to use non-version specific projects
+
+**NOTE:** This potentially could lead to unexpected behavior if the Mod recieves an update with unexpected behavior.
+
+This is more complicated because you will be pulling/using the latest mod for the release of your game. To get started make sure you have a [CursedForge API Key](https://docs.curseforge.com/#getting-started). Then use the environmental parameters in your docker build.
+
+Parameters to use the ForgeAPI:
+
+* `MODS_FORGEAPI_KEY` - Required
+* `MODS_FORGEAPI_FILE` - Required or use MODS_FORGEAPI_PROJECTIDS (Overrides MODS_FORGEAPI_PROJECTIDS)
+* `MODS_FORGEAPI_PROJECTIDS` - Required or use MODS_FORGEAPI_FILE
+* `MODS_FORGEAPI_RELEASES` - Default is release, Options: [Release|Beta|Alpha]
+* `REMOVE_OLD_FORGEAPI_MODS` - Default is False
+* `REMOVE_OLD_DATAPACKS_DEPTH` - Default is 1
+* `REMOVE_OLD_DATAPACKS_INCLUDE` - Default is *.jar
+
+Example of expected forge api project ids, releases, and key: 
+
+```yaml
+  MODS_FORGEAPI_PROJECTIDS: 306612,256717
+  MODS_FORGEAPI_RELEASES: Release
+  MODS_FORGEAPI_KEY: $WRX...
+```
+
+Example of expected ForgeAPI file format: **Note**: name is currently unused, but can be used to document each entry.
+
+```json
+[
+  {
+      "name": "fabric api",
+      "projectId": "306612",
+      "releaseType": "release"
+  },
+  {
+      "name": "fabric voice mod",
+      "projectId": "416089",
+      "releaseType": "beta"
+  }
+]
+```
+
 ### Generic pack file
 
 To install all of the server content (jars, mods, plugins, configs, etc) from a zip file, such as a CurseForge modpack that is missing a server start script, then set `GENERIC_PACK` to the container path of the zip file. That, combined with `TYPE`, allows for custom content along with container managed server download and install.  

scripts/start-setupDatapack

@@ -74,4 +74,4 @@ elif [[ "$DATAPACKS_FILE" ]]; then
   fi
 fi
 
-exec "${SCRIPTS:-/}start-setupModpack" "$@"
+exec "${SCRIPTS:-/}start-setupForgeApiMods" "$@"

scripts/start-setupForgeApiMods

@@ -0,0 +1,177 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+: "${MODS_FORGEAPI_KEY:=}"
+: "${REMOVE_OLD_FORGEAPI_MODS:=false}"
+: "${MODS_FORGEAPI_PROJECTIDS:=}"
+: "${MODS_FORGEAPI_FILE:=}"
+: "${MODS_FORGEAPI_RELEASES:=RELEASE}"
+: "${REMOVE_OLD_MODS_DEPTH:=1} "
+: "${REMOVE_OLD_MODS_INCLUDE:=*.jar}"
+
+#  FORGEAPI_BASE_URL used in manifest downloads below
+FORGEAPI_BASE_URL=${FORGEAPI_BASE_URL:-https://api.curseforge.com/v1}
+RELEASE_NUMBER_FILTER=1
+MINECRAFT_GAME_ID=432
+out_dir=/data/mods
+
+# shellcheck source=start-utils
+. "${SCRIPTS:-/}start-utils"
+isDebugging && set -x
+
+# Remove old mods/plugins
+if isTrue "${REMOVE_OLD_FORGEAPI_MODS}" && [ -z "${MODS_FORGEAPI_FILE}" ]; then
+  removeOldMods /data/mods
+fi
+
+ensureModKey(){
+  if [ -z "$MODS_FORGEAPI_KEY" ]; then
+    log "ERROR: MODS_FORGEAPI_KEY REQUIRED to Connect to FORGE API, you supplied: ${MODS_FORGEAPI_KEY}"
+    exit 2
+  fi
+}
+
+# Set the global release type per the text.
+# NOTE: downcasing release type for comparing types.
+updateReleaseNumber(){
+  releaseType=$1
+  if [ "release" = "${releaseType,,}" ]; then
+    RELEASE_NUMBER_FILTER=1
+  elif [ "beta" = "${releaseType,,}" ]; then
+    RELEASE_NUMBER_FILTER=2
+  elif [ "alpha" = "${releaseType,,}" ]; then
+    RELEASE_NUMBER_FILTER=3
+  fi
+}
+
+retrieveVersionTypeNumber(){
+  VERSION_NAME="Minecraft ${VANILLA_VERSION%.*}"
+  minecraft_types=$(curl -X GET -s \
+    "${FORGEAPI_BASE_URL}/games/${MINECRAFT_GAME_ID}/version-types" \
+    -H 'Accept: application/json' -H 'x-api-key: '${MODS_FORGEAPI_KEY}'')
+
+  if [ ! "$minecraft_types" ]; then
+    log "ERROR: unable to retrieve version types for ${VERSION_NAME} from ForgeAPI"
+    exit 2
+  fi
+
+  TYPE_ID=$(jq -n "$minecraft_types" | jq --arg VERSION_NAME "$VERSION_NAME" -jc '
+      .data[] | select(.name==$VERSION_NAME) | .id')
+
+  if [ ! "$TYPE_ID" ]; then
+    log "ERROR: unable to retrieve version types for ${VERSION_NAME} from ForgeAPI"
+    exit 2
+  fi
+}
+
+modFileByProjectID(){
+  project_id=$(echo $1 | tr -d '"')
+  project_id_release_type=$2
+  unset PROJECT_FILE
+
+  # if Type id isn't defined use minecraft version to go get it.
+  if [ ! "$TYPE_ID" ]; then
+    retrieveVersionTypeNumber
+  fi
+
+  # JQ is struggling with larger page sizes so having to pagination for mods with a lot of releases
+  pageSize=42
+  index=0
+  total_count=1
+
+  while [ $index -lt $total_count ]; do
+    project_files=$(curl -X GET -s \
+      "${FORGEAPI_BASE_URL}/mods/${project_id}/files?gameVersionTypeId=${TYPE_ID}&index=${index}&pageSize=${pageSize}" \
+      -H 'Accept: application/json' -H 'x-api-key: '${MODS_FORGEAPI_KEY}'')
+
+    if [ ! "$project_files" ]; then
+      log "ERROR: unable to retrieve any project id files for ${project_id} from ForgeAPI"
+      exit 2
+    fi
+    # Use project files to grab out the total count of mods.
+    total_count=$(jq -n "$project_files" | jq -c '.pagination.totalCount' )
+
+    # Checking for a individual release type input, if not use global
+    if [ $project_id_release_type ]; then
+      updateReleaseNumber $project_id_release_type
+    else
+      updateReleaseNumber $MODS_FORGEAPI_RELEASES
+    fi
+    # grabs the highest ID of the releaseTypes selected.
+    # Default is 1 for Release, Beta is 2, and Alpha is 3. Using less than we can validate highest release.
+    current_project_file=$(jq -n "$project_files" | jq --arg RELEASE_FILTER "$RELEASE_NUMBER_FILTER" -jc '
+      .data | sort_by(.id) | reverse | map(select(.releaseType<=($RELEASE_FILTER|tonumber))) | .[0]')
+
+    # Logic to grab the latest release over the entire pagination
+    if [ ! "$PROJECT_FILE" ]; then
+      PROJECT_FILE=$current_project_file
+    elif [ "$current_project_file" ]; then
+      current_project_file_id=$(jq -n "$current_project_file" | jq -jc '.id' )
+      PROJECT_FILE_ID=$(jq -n "$PROJECT_FILE" | jq -jc '.id' )
+      if (( current_project_file_id > PROJECT_FILE_ID )); then
+        PROJECT_FILE=$current_project_file
+      fi
+    fi
+    
+    # check to see if we have gone to far or lost our index and exit with an error
+    if [ -z "$index" ] || [ -z "$total_count" ] || [ $index -ge $total_count ]; then
+      log "ERROR: Unable to retrieve any files for ${project_id} from ForgeAPI also Validate files have release type associated with no. ${RELEASE_NUMBER_FILTER}"
+      exit 2
+    fi
+    # Increment start index to new set.
+    index=$(($index + $pageSize))
+  done
+}
+
+downloadModPackfromModFile() {
+  if [ ! "$PROJECT_FILE" ]; then
+    log "ERROR: Project File not found from the ForgeAPI"
+    exit 2
+  fi
+
+  # grabs needed values from our json return
+  file_name=$(jq -n "$PROJECT_FILE" | jq -jc '.fileName' )
+  download_url=$(jq -n "$PROJECT_FILE" | jq -jc '.downloadUrl' )
+  
+  # trys to make the output directory incase it doesnt exist.
+  mkdir -p "$out_dir"
+  echo "Downloading ${download_url}"
+  if ! get -o "${out_dir}/${file_name}" $download_url ; then 
+    log "ERROR: failed to download from ${download_url}"
+    exit 2
+  fi
+}
+
+# Use forge api json file to filter and download the correct mods
+if [ "$MODS_FORGEAPI_FILE" ] && [ -z "$MODS_FORGEAPI_PROJECTIDS" ]; then
+  ensureModKey
+  if [ ! -f "$MODS_FORGEAPI_FILE" ]; then
+    log "ERROR: given MODS_FORGEAPI_FILE file does not exist"
+    exit 2
+  fi
+  MODS_FORGEAPI_PROJECTIDS=$(jq --raw-output '[.[] | .projectId] | join(",")' $MODS_FORGEAPI_FILE)
+  if [ ! "$MODS_FORGEAPI_PROJECTIDS" ]; then
+    log "ERROR: unable to retrieve packs from $MODS_FORGEAPI_FILE"
+    exit 2
+  fi
+
+  # Needs loop here to look up release types befor calling download.
+  for project_id in ${MODS_FORGEAPI_PROJECTIDS//,/ }; do
+    current_release_type=$(jq --arg PROJECT_ID "$project_id" -jc '
+      .[] | select(.projectId==$PROJECT_ID) | .releaseType' "$MODS_FORGEAPI_FILE")
+    modFileByProjectID $project_id $current_release_type
+    downloadModPackfromModFile    
+  done
+fi
+
+# Use only project ids and global release data.
+if [ "$MODS_FORGEAPI_PROJECTIDS" ] && [ -z "$MODS_FORGEAPI_FILE" ]; then
+  ensureModKey
+  for project_id in ${MODS_FORGEAPI_PROJECTIDS//,/ }; do
+    modFileByProjectID $project_id 
+    downloadModPackfromModFile    
+  done
+fi
+
+exec "${SCRIPTS:-/}start-setupModpack" "$@"

tests/fulltests/test.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# go to script root directory
+cd "$(dirname "$0")" || exit 1
+
+# compose down function for reuse
+down() {
+  docker-compose down -v --remove-orphans
+}
+
+checkandExitOnFailure(){
+  failed=$1
+  # docker-compose logs outputs messages from the specified container
+  if $failed; then
+    docker-compose logs mc
+    down
+    cd ..
+    exit 2
+  fi
+}
+
+# tests to completely spin up Minecraft and use the monitor to validate the service is running.
+fullMinecraftUpTest(){
+  folder=$1
+  cd "$folder"
+  failed=false
+  # run the monitor to validate the Minecraft image is healthy
+  docker-compose run monitor || failed=true
+  echo "${folder} Result: failed=$failed"
+  checkandExitOnFailure $failed
+  down
+  cd ..
+}
+
+# go through each folder in fulltests and run fullbuilds
+FOLDERS=$(ls)
+for folder in $FOLDERS; do
+    # If folder is a directory
+    if [ -d "$folder" ]; then
+      echo "Starting Tests on ${folder}"
+      fullMinecraftUpTest $folder
+    fi
+done

tests/setuponlytests/forgeapimods_file/docker-compose.yml

@@ -0,0 +1,17 @@
+version: "3"
+
+services:
+  mc:
+    restart: "no"
+    image: ${IMAGE_TO_TEST:-itzg/minecraft-server}
+    environment:
+      EULA: "TRUE"
+      SETUP_ONLY: "TRUE"
+      VERSION: ${MINECRAFT_VERSION:-LATEST}
+      MODS_FORGEAPI_FILE: /config/example.json
+      # Key is defined in .github/workflows/pr.yml and ci.yml
+      # This should be coming from github secrets.
+      MODS_FORGEAPI_KEY: ${MODS_FORGEAPI_KEY}
+      REMOVE_OLD_FORGEAPI_MODS: "TRUE"
+    volumes:
+      - ./example.json:/config/example.json:ro

tests/setuponlytests/forgeapimods_file/example.json

@@ -0,0 +1,11 @@
+[{
+        "name": "fabric api",
+        "projectId": "306612",
+        "releaseType": "release"
+    },
+    {
+        "name": "Fabric Voice Mod",
+        "projectId": "416089",
+        "releaseType": "beta"
+    }
+]

tests/setuponlytests/forgeapimods_projectids/docker-compose.yml

@@ -0,0 +1,15 @@
+version: "3"
+
+services:
+  mc:
+    restart: "no"
+    image: ${IMAGE_TO_TEST:-itzg/minecraft-server}
+    environment:
+      EULA: "TRUE"
+      SETUP_ONLY: "TRUE"
+      VERSION: ${MINECRAFT_VERSION:-LATEST}
+      MODS_FORGEAPI_PROJECTIDS: 306612,416089
+      # Allows for Beta releases of 416089 the Fabric Voice Mod 
+      MODS_FORGEAPI_RELEASES: BETA
+      MODS_FORGEAPI_KEY: ${MODS_FORGEAPI_KEY}
+      REMOVE_OLD_FORGEAPI_MODS: "TRUE"

tests/setuponlytests/test.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# go to script root directory
+cd "$(dirname "$0")" || exit 1
+
+# compose down function for reuse
+down() {
+  docker-compose down -v --remove-orphans
+}
+
+checkandExitOnFailure(){
+  failed=$1
+  # docker-compose logs outputs messages from the specified container
+  if $failed; then
+    docker-compose logs mc
+    down
+    cd ..
+    exit 2
+  fi
+}
+
+# tests that only run the setup files for things like downloads and configuration. 
+setupOnlyMinecraftTest(){
+  folder=$1
+  cd "$folder"
+  failed=false
+  # run the monitor to validate the Minecraft image is healthy
+  docker-compose --log-level ERROR up --quiet-pull --exit-code-from mc 2>/dev/null || failed=true
+  echo "${folder} Result: failed=$failed"
+  checkandExitOnFailure $failed
+  down
+  cd ..
+}
+
+# go through each folder in setuponly and test setups
+FOLDERS=$(ls)
+for folder in $FOLDERS; do
+    # If folder is a directory
+    if [ -d "$folder" ]; then
+      echo "Starting Tests on ${folder}"
+      setupOnlyMinecraftTest $folder
+    fi
+done

tests/test.sh

@@ -3,54 +3,16 @@
 # go to script root directory
 cd "$(dirname "$0")" || exit 1
 
-# compose down function for reuse
-down() {
-  docker-compose down -v --remove-orphans
-}
-
-fullMinecraftUpTest(){
-  name=$1
-  failed=false
-  # run the monitor to validate the Minecraft image is healthy
-  docker-compose run monitor || failed=true
-  echo "${name} Result: failed=$failed"
-
-  # docker-compose logs outputs messages from the specified container
-  if $failed; then
-    docker-compose logs mc
-    down
-    exit 2
-  fi
-  down
-}
-
-setupOnlyMinecraftTest(){
-  folder=$1
-  failed=false
-  # run the monitor to validate the Minecraft image is healthy
-  docker-compose --log-level ERROR up --quiet-pull --exit-code-from mc 2>/dev/null || failed=true
-  echo "${folder} Result: failed=$failed"
-
-  # docker-compose logs outputs messages from the specified container
-  if $failed; then
-    docker-compose logs mc
-    down
-    cd ..
-    exit 2
-  fi
-  down
-  cd ..
-}
-
-# run tests on base docker compose and validate mc service with monitor
-fullMinecraftUpTest 'Full Vanilla Test'
-
-# go through each folder to test builds
+# go through top level folders and trigger the tests in the subfolders
 FOLDERS=$(ls)
 for folder in $FOLDERS; do
     # If folder is a directory
     if [ -d "$folder" ]; then
       cd "$folder"
-      setupOnlyMinecraftTest $folder
+      if [ -f "./test.sh" ]; then
+        echo "Starting ${folder} Tests"
+        sh ./test.sh
+      fi
+      cd ..
     fi
 done