Merge branch 'Download-with-more-headers' into 2022-02-02_Theile-base

CHANGELOG.md

@@ -6,6 +6,7 @@
 - New variable `RETURN_LABEL_NAME`. If given the value `1`, like `RETURN_LABEL_NAME=1` then Installomator only returns the name of the label. It makes for a better user friendly message for displaying in DEPNotify if that is integrated.
 - Changed logic if `IGNORE_APP_STORE_APPS=yes`. Before this version a label like `microsoftonedrive` that was installed from App Store, and that we want to replace with the “ordinary” version, Installomator would still use `updateTool`, even though `IGNORE_APP_STORE_APPS=yes`. So we would have to have `INSTALL=force` in order to have the app replaced, as `updateTool` would be used. But now if `IGNORE_APP_STORE_APPS=yes` then `updateTool` will be not set, and the App Store app will be replaced. BUT if the installed software was not from App Store, then `updateTool` will not be used, and it would be a kind of a forced install (in the example of `microsoftonedrive`), except if the version is the same (where installation is skipped).
 - Added variable `SYSTEMOWNER` that is used when copying files when installing. Default `0` is to change owner of the app to the current user on the Mac, like this user was installing this app themselves. When using `1` we will put “root:wheel” on the app, which can be useful for shared machines.
+- Added option `curlOptions` to the labels. It can be filled with extra headers need for downloading the specific software. It needs to be an array, like `curlOptions=( )`. See “mocha”-software-labels. 
 
 ## v8.0
 

README.md

@@ -317,6 +317,12 @@ The URL from which to download the archive.
 The URL can be generated by a series of commands, for example when you need to parse an xml file for the latest URL. (See `bbedit`, `desktoppr`, or `omnigraffle` for examples.)
 Sometimes version differs between Intel and Apple Silicon versions. (See `brave`,  `obsidian`, `omnidisksweeper`, or `notion`).
 
+- `curlOptions`: (array, optional)
+Options to the `curl` command, needed for `curl` to be able to download the software.
+Usually used for adding extra headers that some servers need in order to serve the file.
+`curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" )`
+(See “mocha”-labels, for examples on labels, and `buildLabel.sh` for header-examples.)
+
 - `appNewVersion` (optional, but recommended):
 Version of the downloaded software.
 If given, it will be compared to installed version, to see if download is different.

fragments/disabled/mochakeyboard.sh

@@ -1,7 +0,0 @@
-mochakeyboard)
-    name="Mocha Keyboard"
-    type="appInDmgInZip"
-    downloadURL="https://mochasoft.dk/mochakeyboard.dmg.zip"
-    appNewVersion=""
-    expectedTeamID="RR9F5EPNVW"
-    ;;

fragments/disabled/mochatelnet.sh

@@ -1,7 +0,0 @@
-mochatelnet)
-    name="Telnet"
-    type="appInDmgInZip"
-    downloadURL="https://mochasoft.dk/telnet.dmg.zip"
-    appNewVersion=""
-    expectedTeamID="RR9F5EPNVW"
-    ;;

fragments/disabled/mochatn3270.sh

@@ -1,7 +0,0 @@
-mochatn3270)
-    name="TN3270"
-    type="appInDmgInZip"
-    downloadURL="https://mochasoft.dk/tn3270.dmg.zip"
-    appNewVersion=""
-    expectedTeamID="RR9F5EPNVW"
-    ;;

fragments/disabled/mochatn3812.sh

@@ -1,7 +0,0 @@
-mochatn3812)
-    name="TN3812"
-    type="appInDmgInZip"
-    downloadURL="https://mochasoft.dk/tn3812.dmg.zip"
-    appNewVersion=""
-    expectedTeamID="Frydendal"
-    ;;

fragments/disabled/mochatn5250.sh

@@ -1,7 +0,0 @@
-mochatn5250)
-    name="TN5250"
-    type="appInDmgInZip"
-    downloadURL="https://mochasoft.dk/tn5250.dmg.zip"
-    appNewVersion=""
-    expectedTeamID="RR9F5EPNVW"
-    ;;

fragments/header.sh

@@ -156,6 +156,12 @@ REOPEN="yes"
 #   URL to download the dmg.
 #   Can be generated with a series of commands (see BBEdit for an example).
 #
+# - curlOptions: (array, optional)
+#   Options to the curl command, needed for curl to be able to download the software.
+#   Usually used for adding extra headers that some servers need in order to serve the file.
+#   curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" )
+#   (See “mocha”-labels, for examples on labels, and buildLabel.sh for header-examples.)
+#
 # - appNewVersion: (optional)
 #   Version of the downloaded software.
 #   If given, it will be compared to the installed version, to see if the download is different.

fragments/labels/diskspace.sh

@@ -0,0 +1,8 @@
+diskspace)
+    name="diskspace"
+    type="pkg"
+    packageID="com.scriptingosx.diskspace"
+    downloadURL="$(downloadURLFromGit scriptingosx diskspace)"
+    appNewVersion="$(versionFromGit scriptingosx diskspace)"
+    expectedTeamID="JME5BW3F3R"
+    ;;

fragments/labels/evernote.sh

@@ -1,7 +1,7 @@
 evernote)
     name="Evernote"
     type="dmg"
-    downloadURL=$(curl -fs -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15)" "https://evernote.com/download" | grep -i ".dmg" | cut -d '"' -f2)
+    downloadURL=$(curl -fs -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15)" "https://evernote.com/download" | grep -i ".dmg" | grep -ioe "href.*" | cut -d '"' -f2)
     appNewVersion=$( echo "${downloadURL}" | sed -E 's/.*\/[a-zA-Z]*-([0-9.]*)-.*/\1/g' )
     expectedTeamID="Q79WDW8YH9"
     appName="Evernote.app"

fragments/labels/golang.sh

@@ -1,10 +1,9 @@
 golang)
-    # credit: Søren Theilgaard (@theilgaard)
     name="GoLang"
     type="pkg"
     packageID="org.golang.go"
-    downloadURL="$(curl -fsIL "https://golang.org$(curl -fs "https://golang.org/dl/" | grep -i "downloadBox" | grep "pkg" | tr '"' '\n' | grep "pkg")" | grep -i "^location" | awk '{print $2}' | tr -d '\r\n')"
+    downloadURL="https://go.dev$(curl -fs "https://go.dev/dl/" | grep -i "downloadBox" | grep "pkg" | tr '"' '\n' | grep "pkg")"
-    appNewVersion="$( echo "${downloadURL}" | sed -E 's/.*\/(go[0-9.]*)\..*/\1/g' )" # Version includes letters "go"
+    appNewVersion="$( echo "${downloadURL}" | sed -E 's/.*\/(go[0-9.]*)\..*/\1/g' )" # Version includes letters "go" in the beginning
     expectedTeamID="EQHXZ8M8AV"
     blockingProcesses=( NONE )
     ;;

fragments/labels/logitechoptions.sh

@@ -1,8 +1,8 @@
 logitechoptions)
     name="Logitech Options"
     type="pkgInZip"
-    downloadURL=$(curl -fs https://support.logi.com/api/v2/help_center/en-us/articles.json | tr "," "\n" | grep -A 10 "macOS" | grep -oie "https.*/.*/options.*\.zip")
+    downloadURL=$(curl -fs https://support.logi.com/api/v2/help_center/en-us/articles.json | tr "," "\n" | grep -A 10 "macOS" | grep -oie "https.*/.*/options/.*\.zip")
-    appNewVersion=$(curl -fs https://support.logi.com/api/v2/help_center/en-us/articles.json | tr "," "\n" | grep -A 10 "macOS" | grep -B 5 -ie "https.*/.*/options.*\.zip" | grep "Software Version" | sed 's/\\u[0-9a-z][0-9a-z][0-9a-z][0-9a-z]//g' | grep -ioe "Software Version.*[0-9.]*" | tr "/" "\n" | grep -oe "[0-9.]*" | head -1)
+    appNewVersion=$(curl -fs https://support.logi.com/api/v2/help_center/en-us/articles.json | tr "," "\n" | grep -A 10 "macOS" | grep -B 5 -ie "https.*/.*/options/.*\.zip" | grep "Software Version" | sed 's/\\u[0-9a-z][0-9a-z][0-9a-z][0-9a-z]//g' | grep -ioe "Software Version.*[0-9.]*" | tr "/" "\n" | grep -oe "[0-9.]*" | head -1)
     #pkgName="LogiMgr Installer "*".app/Contents/Resources/LogiMgr.pkg"
     pkgName=LogiMgr.pkg
     expectedTeamID="QED4VVPZWA"

fragments/labels/mochakeyboard.sh

@@ -0,0 +1,16 @@
+mochakeyboard)
+    name="Mocha Keyboard"
+    type="appInDmgInZip"
+    downloadURL="https://mochasoft.dk/mochakeyboard.dmg.zip"
+    curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15"
+-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+-H "accept-encoding: gzip, deflate, br"
+-H "accept-language: en-US,en;q=0.9"
+-H "sec-fetch-dest: document"
+-H "sec-fetch-mode: navigate"
+-H "sec-fetch-user: ?1"
+-H "sec-gpc: 1"
+-H "upgrade-insecure-requests: 1" )
+    appNewVersion=""
+    expectedTeamID="RR9F5EPNVW"
+    ;;

fragments/labels/mochatelnet.sh

@@ -0,0 +1,16 @@
+mochatelnet)
+    name="Telnet"
+    type="appInDmgInZip"
+    downloadURL="https://mochasoft.dk/telnet.dmg.zip"
+    curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15"
+-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+-H "accept-encoding: gzip, deflate, br"
+-H "accept-language: en-US,en;q=0.9"
+-H "sec-fetch-dest: document"
+-H "sec-fetch-mode: navigate"
+-H "sec-fetch-user: ?1"
+-H "sec-gpc: 1"
+-H "upgrade-insecure-requests: 1" )
+    appNewVersion=""
+    expectedTeamID="RR9F5EPNVW"
+    ;;

fragments/labels/mochatn3270.sh

@@ -0,0 +1,16 @@
+mochatn3270)
+    name="TN3270"
+    type="appInDmgInZip"
+    downloadURL="https://mochasoft.dk/tn3270.dmg.zip"
+    curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15"
+-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+-H "accept-encoding: gzip, deflate, br"
+-H "accept-language: en-US,en;q=0.9"
+-H "sec-fetch-dest: document"
+-H "sec-fetch-mode: navigate"
+-H "sec-fetch-user: ?1"
+-H "sec-gpc: 1"
+-H "upgrade-insecure-requests: 1" )
+    appNewVersion=""
+    expectedTeamID="RR9F5EPNVW"
+    ;;

fragments/labels/mochatn3812.sh

@@ -0,0 +1,16 @@
+mochatn3812)
+    name="TN3812"
+    type="appInDmgInZip"
+    downloadURL="https://mochasoft.dk/tn3812.dmg.zip"
+    curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15"
+-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+-H "accept-encoding: gzip, deflate, br"
+-H "accept-language: en-US,en;q=0.9"
+-H "sec-fetch-dest: document"
+-H "sec-fetch-mode: navigate"
+-H "sec-fetch-user: ?1"
+-H "sec-gpc: 1"
+-H "upgrade-insecure-requests: 1" )
+    appNewVersion=""
+    expectedTeamID="Frydendal"
+    ;;

fragments/labels/mochatn5250.sh

@@ -0,0 +1,16 @@
+mochatn5250)
+    name="TN5250"
+    type="appInDmgInZip"
+    downloadURL="https://mochasoft.dk/tn5250.dmg.zip"
+    curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15"
+-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+-H "accept-encoding: gzip, deflate, br"
+-H "accept-language: en-US,en;q=0.9"
+-H "sec-fetch-dest: document"
+-H "sec-fetch-mode: navigate"
+-H "sec-fetch-user: ?1"
+-H "sec-gpc: 1"
+-H "upgrade-insecure-requests: 1" )
+    appNewVersion=""
+    expectedTeamID="RR9F5EPNVW"
+    ;;

fragments/main.sh

@@ -168,7 +168,7 @@ fi
 if [ -f "$archiveName" ] && [ "$DEBUG" -eq 1 ]; then
     printlog "$archiveName exists and DEBUG mode 1 enabled, skipping download"
 else
-    # download the dmg
+    # download
     printlog "Downloading $downloadURL to $archiveName"
     if [[ $currentUser != "loginwindow" && $NOTIFY == "all" ]]; then
         printlog "notifying"
@@ -178,7 +178,7 @@ else
             displaynotification "Downloading new $name" "Download in progress …"
         fi
     fi
-    if ! curl --location --fail --silent "$downloadURL" -o "$archiveName"; then
+    if ! curl -fsL ${curlOptions} "$downloadURL" -o "$archiveName"; then
         printlog "error downloading $downloadURL"
         message="$name update/installation failed. This will be logged, so IT can follow up."
         if [[ $currentUser != "loginwindow" && $NOTIFY == "all" ]]; then

utils/buildLabel.sh

@@ -153,32 +153,105 @@ echo "Downloading $downloadURL"
 echo $(basename $downloadURL)
 # First trying to find redirection headers on the download, as those can contain version numbers
 echo "Redirecting to (maybe this can help us with version):\n$(curl -fsIL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "Referrer Policy: strict-origin-when-cross-origin" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" | grep -i "^[location|x\-amz\-meta\-version]*")"
-# Now downloading without extra headers
+# Now downloading without various sets of extra headers
-if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then
+if ! downloadOut1="$( \
-    echo "error downloading $downloadURL using standard headers."
+curl -fL "$downloadURL" --remote-header-name --remote-name \
-    echo "result: $downloadOut"
+-w "%{filename_effective}\n%{url_effective}\n")"
-    echo "Trying all headers…" # that I know of
+then
-    if ! downloadOut="$(curl -fL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "Referrer Policy: strict-origin-when-cross-origin" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then
+    echo "error downloading $downloadURL with no headers."
-        echo "Trying almost all headers…" # that I know of
+    echo "result: $downloadOut1"
-        if ! downloadOut="$(curl -fL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then
+    echo "Trying 1st set of extra headers to download."
-            # we are only here if the download failed
+    if ! downloadOut2="$( \
-            echo "error downloading $downloadURL using two different sets of headers."
+    curl -fL \
-            echo "result: $downloadOut"
+    -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" \
-            # Sometimes a server will give some results to the downloaded output
+    -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \
-            if [[ -n $downloadOut ]]; then
+    -H "accept-encoding: gzip, deflate, br" \
-                echo "Trying output of this…"
+    -H "accept-language: en-US,en;q=0.9" \
-                downloadURL="$(echo $downloadOut | tail -1)"
+    -H "sec-fetch-dest: document" \
-                # Last chance for succes on this download
+    -H "sec-fetch-mode: navigate" \
-                if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then
+    -H "sec-fetch-user: ?1" \
-                    echo "error downloading $downloadURL using previous output."
+    -H "sec-gpc: 1" \
-                    echo "result: $downloadOut"
+    -H "upgrade-insecure-requests: 1" \
+    "$downloadURL" --remote-header-name --remote-name \
+    -w "%{filename_effective}\n%{url_effective}\n")"
+    then
+        echo "error downloading $downloadURL with 1st set of headers."
+        echo "result: $downloadOut2"
+        echo "Trying 2nd set of extra headers to download."
+        if ! downloadOut3="$( \
+        curl -fL \
+        -H "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" \
+        -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \
+        -H "accept-encoding: gzip, deflate, br" \
+        -H "accept-language: en-US,en;q=0.9" \
+        -H "sec-fetch-dest: document" \
+        -H "sec-fetch-mode: navigate" \
+        -H "sec-fetch-site: same-site" \
+        -H "sec-fetch-user: ?1" \
+        -H "sec-gpc: 1" \
+        -H "upgrade-insecure-requests: 1" \
+        "$downloadURL" --remote-header-name --remote-name \
+        -w "%{filename_effective}\n%{url_effective}\n")"
+        then
+            echo "error downloading $downloadURL with 2nd set of headers."
+            echo "result: $downloadOut3"
+            echo "Trying 3rd set of extra headers to download."
+            if ! downloadOut4="$( \
+            curl -fL \
+            -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" \
+            -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \
+            -H "accept-encoding: gzip, deflate, br" \
+            -H "accept-language: en-US,en;q=0.9" \
+            -H "sec-fetch-dest: document" \
+            -H "sec-fetch-mode: navigate" \
+            -H "sec-fetch-user: ?1" \
+            -H "sec-gpc: 1" \
+            -H "upgrade-insecure-requests: 1" \
+            -H "Referrer Policy: strict-origin-when-cross-origin" \
+            "$downloadURL" --remote-header-name --remote-name \
+            -w "%{filename_effective}\n%{url_effective}\n")"
+            then
+                # we are only here if the download failed
+                echo "error downloading $downloadURL with 3rd set of headers."
+                echo "result: $downloadOut4"
+                echo "no more header sets to try"
+                # Sometimes a server will give some results to the downloaded output
+                echo "If any information came out of the previous download attempts, we can try those…"
+                downloadOuts=( "$downloadOut1" "$downloadOut3" "$downloadOut3" "$downloadOut4" )
+                downloadOutCount=${#downloadOuts}
+                for downloadOut in $downloadOuts ; do
+                    if [[ -n $downloadOut ]]; then
+                        echo "Trying output of this…"
+                        downloadURL="$(echo $downloadOut | tail -1)"
+                        # Last chance for succes on this download
+                        if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then
+                            echo "error downloading $downloadURL using previous output."
+                            echo "result: $downloadOut"
+                            ((downloadOutCount--))
+                        else
+                            echo "Success on this download."
+                            succesDownloadOut=$downloadOut
+                            break 2
+                        fi
+                    fi
+                done
+                if [[ $downloadOutCount -eq 0 ]]; then
                     echo "No more tries. Cannot continue."
                     exit 1
                 fi
+            else
+                succesDownloadOut=$downloadOut4
             fi
+        else
+            succesDownloadOut=$downloadOut3
         fi
+    else
+        succesDownloadOut=$downloadOut2
     fi
+else
+    succesDownloadOut=$downloadOut1
 fi
+downloadOut=$succesDownloadOut
 
 # Now we have downloaded the archive, and we need to analyze this
 # The download have returned both {filename_effective} and {url_effective}
@@ -251,7 +324,7 @@ fi
 
 identifier=${name:l} # making lower case
 identifier=${identifier//\%[0-9a-fA-F][0-9a-fA-F]} # removing certain characters
-identifier=${identifier//[,._*@$\(\)\-]} # removing more characters from label name
+identifier=${identifier//[ ,._*@$\(\)\-]} # removing more characters from label name
 echo "identifier: $identifier"
 
 # github-part to figure out if we can find author and repo, to use our github functions for the label

utils/checkLabels.sh

@@ -3,7 +3,7 @@
 export PATH=/usr/bin:/bin:/usr/sbin:/sbin
 
 # Check Installomator labels from fragments
-# 2021 Søren Theilgaard (@theilgaard)
+# 2021-2022 Søren Theilgaard (@theilgaard)
 
 # This script will test labels and check if download link is active, and if version is defined.
 # If labels are written to the script only those will be tested.
@@ -97,132 +97,172 @@ YELLOW='\033[1;33m'
 BLUE='\033[1;34m'
 NC='\033[0m' # No Color
 
-# Labels with the $(arch) call for different versions for Intel and Apple Silicon should be listed here:
+# Has label(s) been given as arguments or not, and list those
-archLabels=( $(grep "\$(arch)" ${labels_dir}/* | awk '{print $1}' | sed -E 's/.*\/([a-z0-9\_-]*)\..*/\1/g'| uniq ) )
+# Figure out which ones of these include "$(arch)" so those will be testet for both i386 and arm64 architectures
-echo "${BLUE}Labels with \"\$(arch)\" call:${NC}\n${archLabels}\n"
-
 if [[ $# -eq 0 ]]; then
     allLabels=( $(grep -h -E '^([a-z0-9\_-]*)(\)|\|\\)$' ${labels_dir}/*.sh | tr -d ')|\\' | sort) )
+    archLabels=( $(grep "\$(arch)" ${labels_dir}/* | awk '{print $1}' | sed -E 's/.*\/([a-z0-9\_-]*)\..*/\1/g'| uniq ) )
 else
     allLabels=( ${=@} )
+    # Check if labels exist
+    for checkLabel in $allLabels; do
+        if [ ! $(ls ${labels_dir}/${checkLabel}.sh 2>/dev/null) ] ; then
+            # Remove label from array
+            allLabels=("${(@)allLabels:#$checkLabel}")
+        fi
+    done
+    # Figure out if labels has "$(arch)" in them
+    archLabels=( $allLabels )
+    for checkLabel in $archLabels; do
+        if [ ! -n "$(grep "\$(arch)" ${labels_dir}/${checkLabel}.sh 2>/dev/null)" ] ; then
+            # Remove label from array
+            archLabels=("${(@)archLabels:#$checkLabel}")
+        fi
+    done
 fi
+
 echo "${BLUE}Total labels:${NC}\n${allLabels}\n"
+echo "${BLUE}Labels with \"\$(arch)\" call:${NC}\n${archLabels}\n"
 
 secondRoundLabels="" # variable for labels with $(arch) call in them
+warningLabels="" # variable for labels with warnings
+errorLabels="" # variable for labels with errors
 countWarning=0
 countError=0
+
+# Loop through the 2 architectures
 for fixedArch in i386 arm64; do
-echo "${BLUE}Architecture: $fixedArch${NC}"
+    echo "${BLUE}Architecture: $fixedArch${NC}"
-echo
+    echo
 
-# Go through all labels
+    # Loop through all labels
-for label in $allLabels; do
+    for label in $allLabels; do
-    echo "########## $label"
+        echo "########## $label"
-    labelWarning=0; labelError=0; expectedExtension=""; URLextension=""
+        labelWarning=0; labelError=0; expectedExtension=""; URLextension=""
-    name=""; type=""; downloadURL=""; appNewVersion=""; expectedTeamID=""; blockingProcesses=""; updateTool=""; updateToolArguments=""; archiveName=""
+        name=""; type=""; downloadURL=""; curlOptions=""; appNewVersion=""; expectedTeamID=""; blockingProcesses=""; updateTool=""; updateToolArguments=""; archiveName=""
         
-    #caseLabel
+        #caseLabel
-    if cat "${labels_dir}/${label}.sh" | grep -v -E '^[a-z0-9\_-]*(\)|\|\\)$' | grep -v ";;" > checkLabelCurrent.sh; then
+        if cat "${labels_dir}/${label}.sh" | grep -v -E '^[a-z0-9\_-]*(\)|\|\\)$' | grep -v ";;" > checkLabelCurrent.sh; then
-        source checkLabelCurrent.sh
+            source checkLabelCurrent.sh
 
-        echo "Name: $name"
+            echo "Name: $name"
-        echo "Download URL: $downloadURL"
+            echo "Download URL: $downloadURL"
-        echo "Type: $type"
+            echo "Type: $type"
-        case $type in
+            case $type in
-            dmg|pkg|zip|tbz)
+                dmg|pkg|zip|tbz)
-                expectedExtension="$type"
+                    expectedExtension="$type"
-                ;;
+                    ;;
-            pkgInDmg)
+                pkgInDmg)
-                expectedExtension="dmg"
+                    expectedExtension="dmg"
-                ;;
+                    ;;
-            *InZip)
+                *InZip)
-                expectedExtension="zip"
+                    expectedExtension="zip"
-                ;;
+                    ;;
-            *)
+                *)
-                echo "Cannot handle type $type"
+                    echo "Cannot handle type $type"
-                ;;
+                    ;;
-        esac
+            esac
-        if [[ "$appNewVersion" == "" ]] ; then
+            if [[ "$appNewVersion" == "" ]] ; then
-            echo "No appNewVersion!"
+                echo "No appNewVersion!"
-        else
-            if [[ $( echo "$appNewVersion" | grep -i "[0-9.]" ) == "" || $appNewVersion == "" ]]; then
-                echo "${RED}-> !! ERROR in appNewVersion${NC}"
-                labelError=1
             else
-                if [[ $appNewVersion != $( echo "$appNewVersion" | sed -E 's/[^0-9]*([0-9.]*)[^0-9]*/\1/g' ) ]]; then
+                if [[ $( echo "$appNewVersion" | grep -i "[0-9.]" ) == "" || $appNewVersion == "" ]]; then
-                    echo "${YELLOW}Warning: Version contain not only numbers and dots.${NC}"
+                    echo "${RED}-> !! ERROR in appNewVersion${NC}"
-                    labelWarning=1
+                    labelError=1
+                else
+                    if [[ $appNewVersion != $( echo "$appNewVersion" | sed -E 's/[^0-9]*([0-9.]*)[^0-9]*/\1/g' ) ]]; then
+                        echo "${YELLOW}Warning: Version contain not only numbers and dots.${NC}"
+                        labelWarning=1
+                    fi
+                    echo "Version: $appNewVersion" ;
                 fi
-                echo "Version: $appNewVersion" ;
             fi
-        fi
+            if curl -sfL ${curlOptions} --output /dev/null -r 0-0 "$downloadURL" ; then
-        if curl -sfL --output /dev/null -r 0-0 "$downloadURL" ; then
+                echo "${GREEN}OK: downloadURL works OK${NC}"
-            echo "${GREEN}OK: downloadURL works OK${NC}"
+                if [[ $(echo "$downloadURL" | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' ) == "${expectedExtension}" ]]; then
-            if [[ $(echo "$downloadURL" | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' ) == "${expectedExtension}" ]]; then
+                    echo "${GREEN}OK: download extension MATCH on ${expectedExtension}${NC}"
-                echo "${GREEN}OK: download extension MATCH on ${expectedExtension}${NC}"
+                else
-            else
+                    if [[ $(echo "$downloadURL" | grep -io "github.com") != "github.com" ]]; then
-                if [[ $(echo "$downloadURL" | grep -io "github.com") != "github.com" ]]; then
+                        URLheader=$( curl -fsIL "$downloadURL" )
-                    URLheader=$( curl -fsIL "$downloadURL" )
+                        if [[ "${URLheader}" != "" ]]; then
-                    if [[ "${URLheader}" != "" ]]; then
+                            URLlocation=$( echo "${URLheader}" | grep -i "^location" )
-                        URLlocation=$( echo "${URLheader}" | grep -i "^location" )
+                            URLfilename=$( echo "${URLheader}" | grep -i "filename=" )
-                        URLfilename=$( echo "${URLheader}" | grep -i "filename=" )
+                            if [[ "${URLlocation}" != "" ]]; then
-                        if [[ "${URLlocation}" != "" ]]; then
+                                URLextension=$( echo "${URLlocation}" | tail -1 | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' | tr -d '\r\n' )
-                            URLextension=$( echo "${URLlocation}" | tail -1 | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' | tr -d '\r\n' )
+                            else
+                                URLextension=$( echo "${URLfilename}" | tail -1 | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' | tr -d '\r\n' )
+                            fi
+                            URLextension=${${URLextension:l}%%\?*}
+                            if [[ "${URLextension}" == "${expectedExtension}" ]]; then
+                                echo "${GREEN}OK: download extension MATCH on ${URLextension}${NC}"
+                            else
+                                echo "${RED}-> !! ERROR in download extension, expected ${expectedExtension}, but got ${URLextension}.${NC}"
+                                labelError=1
+                            fi
                         else
-                            URLextension=$( echo "${URLfilename}" | tail -1 | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' | tr -d '\r\n' )
+                            echo "no header provided from server."
                         fi
-                        URLextension=${${URLextension:l}%%\?*}
+                    else
-                        if [[ "${URLextension}" == "${expectedExtension}" ]]; then
+                        githubPart="$(echo "$downloadURL" | cut -d "/" -f4-6)"
-                            echo "${GREEN}OK: download extension MATCH on ${URLextension}${NC}"
+                        if [[ "$(curl -fsL "$downloadURL" | grep -io "${githubPart}.*\.${expectedExtension}")" != "" ]]; then
+                            echo "${GREEN}OK: download extension MATCH on ${expectedExtension}${NC}"
                         else
-                            echo "${RED}-> !! ERROR in download extension, expected ${expectedExtension}, but got ${URLextension}.${NC}"
+                            echo "${RED}-> !! ERROR in download extension, expected ${expectedExtension}, but it was wrong${NC}"
                             labelError=1
                         fi
-                    else
-                        echo "no header provided from server."
-                    fi
-                else
-                    githubPart="$(echo "$downloadURL" | cut -d "/" -f4-6)"
-                    if [[ "$(curl -fsL "$downloadURL" | grep -io "${githubPart}.*\.${expectedExtension}")" != "" ]]; then
-                        echo "${GREEN}OK: download extension MATCH on ${expectedExtension}${NC}"
-                    else
-                        echo "${RED}-> !! ERROR in download extension, expected ${expectedExtension}, but it was wrong${NC}"
-                        labelError=1
                     fi
                 fi
+            else
+                echo "${RED}-> !! ERROR in downloadURL${NC}"
+                labelError=1
+            fi
+            if [[ $labelWarning != 0 ]]; then; echo "${YELLOW}########## Warning in label: $label${NC}"; ((countWarning++)); warningLabels+=( "$label" ); fi
+            if [[ $labelError != 0 ]]; then
+                echo "${RED}########## ERROR in label: $label${NC}"
+                    echo "Testing using Installomator"
+                    exit_status=$( . $repo_dir/assemble.sh $label DEBUG=2 INSTALL=force IGNORE_APP_STORE_APPS=yes BLOCKING_PROCESS_ACTION=ignore | grep exit | tail -1 | sed -E 's/.*exit code ([0-9]).*/\1/g' )
+                    if [[ ${exit_status} -eq 0 ]] ; then
+                        echo "${GREEN}$label works fine!${NC}"
+                        #errorLabels=("${(@)errorLabels:#$errorLabel}")
+                    else
+                        echo "${RED}$label NOT WORKING!${NC}"
+                        ((countError++))
+                        errorLabels+=( "$label" )
+                    fi
+
+                fi
+
+            if (($archLabels[(Ie)$label])); then
+                secondRoundLabels+=( "$label" )
             fi
         else
-            echo "${RED}-> !! ERROR in downloadURL${NC}"
+            echo "Label: ${label} is not it's own file in Labels-folder. Skipping"
-            labelError=1
-        fi
-        if [[ $labelWarning != 0 ]]; then; echo "${YELLOW}########## Warning in label: $label${NC}"; ((countWarning++)); fi
-        if [[ $labelError != 0 ]]; then; echo "${RED}########## ERROR in label: $label${NC}"; ((countError++)); fi
-
-        if (($archLabels[(Ie)$label])); then
-            secondRoundLabels+=( "$label" )
         fi
+        echo
+    done
+    if [[ $fixedArch == i386 ]] ; then
+        errorLabelsi386=( ${=errorLabels} )
     else
-        echo "Label: ${label} is not it's own file in Labels-folder. Skipping"
+        errorLabelsarm64=( ${=errorLabels} )
     fi
+
+    errorLabels=""
+    allLabels=( ${=secondRoundLabels} )
+    archLabels=()
     echo
 done
-allLabels=( ${=secondRoundLabels} )
-archLabels=()
-echo
-done
 
 rm checkLabelCurrent.sh
 
-#${SELFLOCATION}/Installomator.sh version
+if [[ countWarning -gt 0 ]]; then
-#echo
-
-if [[ countWarning > 0 ]]; then
     echo "${YELLOW}Warnings counted: $countWarning${NC}"
+    echo "${YELLOW}${warningLabels}${NC}"
 else
     echo "${GREEN}No warnings detected!${NC}"
 fi
-if [[ countError > 0 ]]; then
+if [[ countError -gt 0 ]]; then
     echo "${RED}ERRORS counted: $countError${NC}"
+    echo "${RED}i386 : ${errorLabelsi386}${NC}"
+    echo "${RED}arm64: ${errorLabelsarm64}${NC}"
 else
     echo "${GREEN}No errors detected!${NC}"
 fi