diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dd2a79..31e2ce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - New variable `RETURN_LABEL_NAME`. If given the value `1`, like `RETURN_LABEL_NAME=1` then Installomator only returns the name of the label. It makes for a better user friendly message for displaying in DEPNotify if that is integrated. - Changed logic if `IGNORE_APP_STORE_APPS=yes`. Before this version a label like `microsoftonedrive` that was installed from App Store, and that we want to replace with the “ordinary” version, Installomator would still use `updateTool`, even though `IGNORE_APP_STORE_APPS=yes`. So we would have to have `INSTALL=force` in order to have the app replaced, as `updateTool` would be used. But now if `IGNORE_APP_STORE_APPS=yes` then `updateTool` will be not set, and the App Store app will be replaced. BUT if the installed software was not from App Store, then `updateTool` will not be used, and it would be a kind of a forced install (in the example of `microsoftonedrive`), except if the version is the same (where installation is skipped). - Added variable `SYSTEMOWNER` that is used when copying files when installing. Default `0` is to change owner of the app to the current user on the Mac, like this user was installing this app themselves. When using `1` we will put “root:wheel” on the app, which can be useful for shared machines. +- Added option `curlOptions` to the labels. It can be filled with extra headers need for downloading the specific software. It needs to be an array, like `curlOptions=( )`. See “mocha”-software-labels. ## v8.0 diff --git a/README.md b/README.md index 72a5ed5..e76bfa4 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,12 @@ The URL from which to download the archive. The URL can be generated by a series of commands, for example when you need to parse an xml file for the latest URL. (See `bbedit`, `desktoppr`, or `omnigraffle` for examples.) Sometimes version differs between Intel and Apple Silicon versions. (See `brave`, `obsidian`, `omnidisksweeper`, or `notion`). +- `curlOptions`: (array, optional) +Options to the `curl` command, needed for `curl` to be able to download the software. +Usually used for adding extra headers that some servers need in order to serve the file. +`curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" )` +(See “mocha”-labels, for examples on labels, and `buildLabel.sh` for header-examples.) + - `appNewVersion` (optional, but recommended): Version of the downloaded software. If given, it will be compared to installed version, to see if download is different. diff --git a/fragments/disabled/mochakeyboard.sh b/fragments/disabled/mochakeyboard.sh deleted file mode 100644 index 5f742a5..0000000 --- a/fragments/disabled/mochakeyboard.sh +++ /dev/null @@ -1,7 +0,0 @@ -mochakeyboard) - name="Mocha Keyboard" - type="appInDmgInZip" - downloadURL="https://mochasoft.dk/mochakeyboard.dmg.zip" - appNewVersion="" - expectedTeamID="RR9F5EPNVW" - ;; diff --git a/fragments/disabled/mochatelnet.sh b/fragments/disabled/mochatelnet.sh deleted file mode 100644 index 5eba112..0000000 --- a/fragments/disabled/mochatelnet.sh +++ /dev/null @@ -1,7 +0,0 @@ -mochatelnet) - name="Telnet" - type="appInDmgInZip" - downloadURL="https://mochasoft.dk/telnet.dmg.zip" - appNewVersion="" - expectedTeamID="RR9F5EPNVW" - ;; diff --git a/fragments/disabled/mochatn3270.sh b/fragments/disabled/mochatn3270.sh deleted file mode 100644 index 74ce410..0000000 --- a/fragments/disabled/mochatn3270.sh +++ /dev/null @@ -1,7 +0,0 @@ -mochatn3270) - name="TN3270" - type="appInDmgInZip" - downloadURL="https://mochasoft.dk/tn3270.dmg.zip" - appNewVersion="" - expectedTeamID="RR9F5EPNVW" - ;; diff --git a/fragments/disabled/mochatn3812.sh b/fragments/disabled/mochatn3812.sh deleted file mode 100644 index 54f3884..0000000 --- a/fragments/disabled/mochatn3812.sh +++ /dev/null @@ -1,7 +0,0 @@ -mochatn3812) - name="TN3812" - type="appInDmgInZip" - downloadURL="https://mochasoft.dk/tn3812.dmg.zip" - appNewVersion="" - expectedTeamID="Frydendal" - ;; diff --git a/fragments/disabled/mochatn5250.sh b/fragments/disabled/mochatn5250.sh deleted file mode 100644 index ae8ccf2..0000000 --- a/fragments/disabled/mochatn5250.sh +++ /dev/null @@ -1,7 +0,0 @@ -mochatn5250) - name="TN5250" - type="appInDmgInZip" - downloadURL="https://mochasoft.dk/tn5250.dmg.zip" - appNewVersion="" - expectedTeamID="RR9F5EPNVW" - ;; diff --git a/fragments/header.sh b/fragments/header.sh index c1e018d..da0811f 100644 --- a/fragments/header.sh +++ b/fragments/header.sh @@ -156,6 +156,12 @@ REOPEN="yes" # URL to download the dmg. # Can be generated with a series of commands (see BBEdit for an example). # +# - curlOptions: (array, optional) +# Options to the curl command, needed for curl to be able to download the software. +# Usually used for adding extra headers that some servers need in order to serve the file. +# curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" ) +# (See “mocha”-labels, for examples on labels, and buildLabel.sh for header-examples.) +# # - appNewVersion: (optional) # Version of the downloaded software. # If given, it will be compared to the installed version, to see if the download is different. diff --git a/fragments/labels/diskspace.sh b/fragments/labels/diskspace.sh new file mode 100644 index 0000000..ab8f88c --- /dev/null +++ b/fragments/labels/diskspace.sh @@ -0,0 +1,8 @@ +diskspace) + name="diskspace" + type="pkg" + packageID="com.scriptingosx.diskspace" + downloadURL="$(downloadURLFromGit scriptingosx diskspace)" + appNewVersion="$(versionFromGit scriptingosx diskspace)" + expectedTeamID="JME5BW3F3R" + ;; diff --git a/fragments/labels/evernote.sh b/fragments/labels/evernote.sh index 85253e6..e65fe95 100644 --- a/fragments/labels/evernote.sh +++ b/fragments/labels/evernote.sh @@ -1,7 +1,7 @@ evernote) name="Evernote" type="dmg" - downloadURL=$(curl -fs -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15)" "https://evernote.com/download" | grep -i ".dmg" | cut -d '"' -f2) + downloadURL=$(curl -fs -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15)" "https://evernote.com/download" | grep -i ".dmg" | grep -ioe "href.*" | cut -d '"' -f2) appNewVersion=$( echo "${downloadURL}" | sed -E 's/.*\/[a-zA-Z]*-([0-9.]*)-.*/\1/g' ) expectedTeamID="Q79WDW8YH9" appName="Evernote.app" diff --git a/fragments/labels/golang.sh b/fragments/labels/golang.sh index 81b81f9..007c423 100644 --- a/fragments/labels/golang.sh +++ b/fragments/labels/golang.sh @@ -1,10 +1,9 @@ golang) - # credit: Søren Theilgaard (@theilgaard) name="GoLang" type="pkg" packageID="org.golang.go" - downloadURL="$(curl -fsIL "https://golang.org$(curl -fs "https://golang.org/dl/" | grep -i "downloadBox" | grep "pkg" | tr '"' '\n' | grep "pkg")" | grep -i "^location" | awk '{print $2}' | tr -d '\r\n')" - appNewVersion="$( echo "${downloadURL}" | sed -E 's/.*\/(go[0-9.]*)\..*/\1/g' )" # Version includes letters "go" + downloadURL="https://go.dev$(curl -fs "https://go.dev/dl/" | grep -i "downloadBox" | grep "pkg" | tr '"' '\n' | grep "pkg")" + appNewVersion="$( echo "${downloadURL}" | sed -E 's/.*\/(go[0-9.]*)\..*/\1/g' )" # Version includes letters "go" in the beginning expectedTeamID="EQHXZ8M8AV" blockingProcesses=( NONE ) ;; diff --git a/fragments/labels/logitechoptions.sh b/fragments/labels/logitechoptions.sh index 98bdf6b..c3cf225 100644 --- a/fragments/labels/logitechoptions.sh +++ b/fragments/labels/logitechoptions.sh @@ -1,8 +1,8 @@ logitechoptions) name="Logitech Options" type="pkgInZip" - downloadURL=$(curl -fs https://support.logi.com/api/v2/help_center/en-us/articles.json | tr "," "\n" | grep -A 10 "macOS" | grep -oie "https.*/.*/options.*\.zip") - appNewVersion=$(curl -fs https://support.logi.com/api/v2/help_center/en-us/articles.json | tr "," "\n" | grep -A 10 "macOS" | grep -B 5 -ie "https.*/.*/options.*\.zip" | grep "Software Version" | sed 's/\\u[0-9a-z][0-9a-z][0-9a-z][0-9a-z]//g' | grep -ioe "Software Version.*[0-9.]*" | tr "/" "\n" | grep -oe "[0-9.]*" | head -1) + downloadURL=$(curl -fs https://support.logi.com/api/v2/help_center/en-us/articles.json | tr "," "\n" | grep -A 10 "macOS" | grep -oie "https.*/.*/options/.*\.zip") + appNewVersion=$(curl -fs https://support.logi.com/api/v2/help_center/en-us/articles.json | tr "," "\n" | grep -A 10 "macOS" | grep -B 5 -ie "https.*/.*/options/.*\.zip" | grep "Software Version" | sed 's/\\u[0-9a-z][0-9a-z][0-9a-z][0-9a-z]//g' | grep -ioe "Software Version.*[0-9.]*" | tr "/" "\n" | grep -oe "[0-9.]*" | head -1) #pkgName="LogiMgr Installer "*".app/Contents/Resources/LogiMgr.pkg" pkgName=LogiMgr.pkg expectedTeamID="QED4VVPZWA" diff --git a/fragments/labels/mochakeyboard.sh b/fragments/labels/mochakeyboard.sh new file mode 100644 index 0000000..17445e0 --- /dev/null +++ b/fragments/labels/mochakeyboard.sh @@ -0,0 +1,16 @@ +mochakeyboard) + name="Mocha Keyboard" + type="appInDmgInZip" + downloadURL="https://mochasoft.dk/mochakeyboard.dmg.zip" + curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" +-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" +-H "accept-encoding: gzip, deflate, br" +-H "accept-language: en-US,en;q=0.9" +-H "sec-fetch-dest: document" +-H "sec-fetch-mode: navigate" +-H "sec-fetch-user: ?1" +-H "sec-gpc: 1" +-H "upgrade-insecure-requests: 1" ) + appNewVersion="" + expectedTeamID="RR9F5EPNVW" + ;; diff --git a/fragments/labels/mochatelnet.sh b/fragments/labels/mochatelnet.sh new file mode 100644 index 0000000..b34d58f --- /dev/null +++ b/fragments/labels/mochatelnet.sh @@ -0,0 +1,16 @@ +mochatelnet) + name="Telnet" + type="appInDmgInZip" + downloadURL="https://mochasoft.dk/telnet.dmg.zip" + curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" +-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" +-H "accept-encoding: gzip, deflate, br" +-H "accept-language: en-US,en;q=0.9" +-H "sec-fetch-dest: document" +-H "sec-fetch-mode: navigate" +-H "sec-fetch-user: ?1" +-H "sec-gpc: 1" +-H "upgrade-insecure-requests: 1" ) + appNewVersion="" + expectedTeamID="RR9F5EPNVW" + ;; diff --git a/fragments/labels/mochatn3270.sh b/fragments/labels/mochatn3270.sh new file mode 100644 index 0000000..4be205d --- /dev/null +++ b/fragments/labels/mochatn3270.sh @@ -0,0 +1,16 @@ +mochatn3270) + name="TN3270" + type="appInDmgInZip" + downloadURL="https://mochasoft.dk/tn3270.dmg.zip" + curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" +-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" +-H "accept-encoding: gzip, deflate, br" +-H "accept-language: en-US,en;q=0.9" +-H "sec-fetch-dest: document" +-H "sec-fetch-mode: navigate" +-H "sec-fetch-user: ?1" +-H "sec-gpc: 1" +-H "upgrade-insecure-requests: 1" ) + appNewVersion="" + expectedTeamID="RR9F5EPNVW" + ;; diff --git a/fragments/labels/mochatn3812.sh b/fragments/labels/mochatn3812.sh new file mode 100644 index 0000000..e27a33e --- /dev/null +++ b/fragments/labels/mochatn3812.sh @@ -0,0 +1,16 @@ +mochatn3812) + name="TN3812" + type="appInDmgInZip" + downloadURL="https://mochasoft.dk/tn3812.dmg.zip" + curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" +-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" +-H "accept-encoding: gzip, deflate, br" +-H "accept-language: en-US,en;q=0.9" +-H "sec-fetch-dest: document" +-H "sec-fetch-mode: navigate" +-H "sec-fetch-user: ?1" +-H "sec-gpc: 1" +-H "upgrade-insecure-requests: 1" ) + appNewVersion="" + expectedTeamID="Frydendal" + ;; diff --git a/fragments/labels/mochatn5250.sh b/fragments/labels/mochatn5250.sh new file mode 100644 index 0000000..2707120 --- /dev/null +++ b/fragments/labels/mochatn5250.sh @@ -0,0 +1,16 @@ +mochatn5250) + name="TN5250" + type="appInDmgInZip" + downloadURL="https://mochasoft.dk/tn5250.dmg.zip" + curlOptions=( -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" +-H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" +-H "accept-encoding: gzip, deflate, br" +-H "accept-language: en-US,en;q=0.9" +-H "sec-fetch-dest: document" +-H "sec-fetch-mode: navigate" +-H "sec-fetch-user: ?1" +-H "sec-gpc: 1" +-H "upgrade-insecure-requests: 1" ) + appNewVersion="" + expectedTeamID="RR9F5EPNVW" + ;; diff --git a/fragments/main.sh b/fragments/main.sh index cef364c..640c244 100644 --- a/fragments/main.sh +++ b/fragments/main.sh @@ -168,7 +168,7 @@ fi if [ -f "$archiveName" ] && [ "$DEBUG" -eq 1 ]; then printlog "$archiveName exists and DEBUG mode 1 enabled, skipping download" else - # download the dmg + # download printlog "Downloading $downloadURL to $archiveName" if [[ $currentUser != "loginwindow" && $NOTIFY == "all" ]]; then printlog "notifying" @@ -178,7 +178,7 @@ else displaynotification "Downloading new $name" "Download in progress …" fi fi - if ! curl --location --fail --silent "$downloadURL" -o "$archiveName"; then + if ! curl -fsL ${curlOptions} "$downloadURL" -o "$archiveName"; then printlog "error downloading $downloadURL" message="$name update/installation failed. This will be logged, so IT can follow up." if [[ $currentUser != "loginwindow" && $NOTIFY == "all" ]]; then diff --git a/utils/buildLabel.sh b/utils/buildLabel.sh index 8f96e3c..a6de0c1 100755 --- a/utils/buildLabel.sh +++ b/utils/buildLabel.sh @@ -153,32 +153,105 @@ echo "Downloading $downloadURL" echo $(basename $downloadURL) # First trying to find redirection headers on the download, as those can contain version numbers echo "Redirecting to (maybe this can help us with version):\n$(curl -fsIL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "Referrer Policy: strict-origin-when-cross-origin" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" | grep -i "^[location|x\-amz\-meta\-version]*")" -# Now downloading without extra headers -if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then - echo "error downloading $downloadURL using standard headers." - echo "result: $downloadOut" - echo "Trying all headers…" # that I know of - if ! downloadOut="$(curl -fL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "Referrer Policy: strict-origin-when-cross-origin" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then - echo "Trying almost all headers…" # that I know of - if ! downloadOut="$(curl -fL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then - # we are only here if the download failed - echo "error downloading $downloadURL using two different sets of headers." - echo "result: $downloadOut" - # Sometimes a server will give some results to the downloaded output - if [[ -n $downloadOut ]]; then - echo "Trying output of this…" - downloadURL="$(echo $downloadOut | tail -1)" - # Last chance for succes on this download - if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then - echo "error downloading $downloadURL using previous output." - echo "result: $downloadOut" +# Now downloading without various sets of extra headers +if ! downloadOut1="$( \ +curl -fL "$downloadURL" --remote-header-name --remote-name \ +-w "%{filename_effective}\n%{url_effective}\n")" +then + echo "error downloading $downloadURL with no headers." + echo "result: $downloadOut1" + echo "Trying 1st set of extra headers to download." + if ! downloadOut2="$( \ + curl -fL \ + -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + "$downloadURL" --remote-header-name --remote-name \ + -w "%{filename_effective}\n%{url_effective}\n")" + then + echo "error downloading $downloadURL with 1st set of headers." + echo "result: $downloadOut2" + echo "Trying 2nd set of extra headers to download." + if ! downloadOut3="$( \ + curl -fL \ + -H "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-site: same-site" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + "$downloadURL" --remote-header-name --remote-name \ + -w "%{filename_effective}\n%{url_effective}\n")" + then + echo "error downloading $downloadURL with 2nd set of headers." + echo "result: $downloadOut3" + echo "Trying 3rd set of extra headers to download." + if ! downloadOut4="$( \ + curl -fL \ + -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + -H "Referrer Policy: strict-origin-when-cross-origin" \ + "$downloadURL" --remote-header-name --remote-name \ + -w "%{filename_effective}\n%{url_effective}\n")" + then + # we are only here if the download failed + echo "error downloading $downloadURL with 3rd set of headers." + echo "result: $downloadOut4" + echo "no more header sets to try" + # Sometimes a server will give some results to the downloaded output + echo "If any information came out of the previous download attempts, we can try those…" + downloadOuts=( "$downloadOut1" "$downloadOut3" "$downloadOut3" "$downloadOut4" ) + downloadOutCount=${#downloadOuts} + for downloadOut in $downloadOuts ; do + if [[ -n $downloadOut ]]; then + echo "Trying output of this…" + downloadURL="$(echo $downloadOut | tail -1)" + # Last chance for succes on this download + if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then + echo "error downloading $downloadURL using previous output." + echo "result: $downloadOut" + ((downloadOutCount--)) + else + echo "Success on this download." + succesDownloadOut=$downloadOut + break 2 + fi + fi + done + if [[ $downloadOutCount -eq 0 ]]; then echo "No more tries. Cannot continue." exit 1 fi + else + succesDownloadOut=$downloadOut4 fi + else + succesDownloadOut=$downloadOut3 fi + else + succesDownloadOut=$downloadOut2 fi +else + succesDownloadOut=$downloadOut1 fi +downloadOut=$succesDownloadOut # Now we have downloaded the archive, and we need to analyze this # The download have returned both {filename_effective} and {url_effective} @@ -251,7 +324,7 @@ fi identifier=${name:l} # making lower case identifier=${identifier//\%[0-9a-fA-F][0-9a-fA-F]} # removing certain characters -identifier=${identifier//[,._*@$\(\)\-]} # removing more characters from label name +identifier=${identifier//[ ,._*@$\(\)\-]} # removing more characters from label name echo "identifier: $identifier" # github-part to figure out if we can find author and repo, to use our github functions for the label diff --git a/utils/checkLabels.sh b/utils/checkLabels.sh index f9fdef1..fa6144b 100755 --- a/utils/checkLabels.sh +++ b/utils/checkLabels.sh @@ -3,7 +3,7 @@ export PATH=/usr/bin:/bin:/usr/sbin:/sbin # Check Installomator labels from fragments -# 2021 Søren Theilgaard (@theilgaard) +# 2021-2022 Søren Theilgaard (@theilgaard) # This script will test labels and check if download link is active, and if version is defined. # If labels are written to the script only those will be tested. @@ -97,132 +97,172 @@ YELLOW='\033[1;33m' BLUE='\033[1;34m' NC='\033[0m' # No Color -# Labels with the $(arch) call for different versions for Intel and Apple Silicon should be listed here: -archLabels=( $(grep "\$(arch)" ${labels_dir}/* | awk '{print $1}' | sed -E 's/.*\/([a-z0-9\_-]*)\..*/\1/g'| uniq ) ) -echo "${BLUE}Labels with \"\$(arch)\" call:${NC}\n${archLabels}\n" - +# Has label(s) been given as arguments or not, and list those +# Figure out which ones of these include "$(arch)" so those will be testet for both i386 and arm64 architectures if [[ $# -eq 0 ]]; then allLabels=( $(grep -h -E '^([a-z0-9\_-]*)(\)|\|\\)$' ${labels_dir}/*.sh | tr -d ')|\\' | sort) ) + archLabels=( $(grep "\$(arch)" ${labels_dir}/* | awk '{print $1}' | sed -E 's/.*\/([a-z0-9\_-]*)\..*/\1/g'| uniq ) ) else allLabels=( ${=@} ) + # Check if labels exist + for checkLabel in $allLabels; do + if [ ! $(ls ${labels_dir}/${checkLabel}.sh 2>/dev/null) ] ; then + # Remove label from array + allLabels=("${(@)allLabels:#$checkLabel}") + fi + done + # Figure out if labels has "$(arch)" in them + archLabels=( $allLabels ) + for checkLabel in $archLabels; do + if [ ! -n "$(grep "\$(arch)" ${labels_dir}/${checkLabel}.sh 2>/dev/null)" ] ; then + # Remove label from array + archLabels=("${(@)archLabels:#$checkLabel}") + fi + done fi + echo "${BLUE}Total labels:${NC}\n${allLabels}\n" +echo "${BLUE}Labels with \"\$(arch)\" call:${NC}\n${archLabels}\n" secondRoundLabels="" # variable for labels with $(arch) call in them +warningLabels="" # variable for labels with warnings +errorLabels="" # variable for labels with errors countWarning=0 countError=0 + +# Loop through the 2 architectures for fixedArch in i386 arm64; do -echo "${BLUE}Architecture: $fixedArch${NC}" -echo + echo "${BLUE}Architecture: $fixedArch${NC}" + echo -# Go through all labels -for label in $allLabels; do - echo "########## $label" - labelWarning=0; labelError=0; expectedExtension=""; URLextension="" - name=""; type=""; downloadURL=""; appNewVersion=""; expectedTeamID=""; blockingProcesses=""; updateTool=""; updateToolArguments=""; archiveName="" - - #caseLabel - if cat "${labels_dir}/${label}.sh" | grep -v -E '^[a-z0-9\_-]*(\)|\|\\)$' | grep -v ";;" > checkLabelCurrent.sh; then - source checkLabelCurrent.sh + # Loop through all labels + for label in $allLabels; do + echo "########## $label" + labelWarning=0; labelError=0; expectedExtension=""; URLextension="" + name=""; type=""; downloadURL=""; curlOptions=""; appNewVersion=""; expectedTeamID=""; blockingProcesses=""; updateTool=""; updateToolArguments=""; archiveName="" + + #caseLabel + if cat "${labels_dir}/${label}.sh" | grep -v -E '^[a-z0-9\_-]*(\)|\|\\)$' | grep -v ";;" > checkLabelCurrent.sh; then + source checkLabelCurrent.sh - echo "Name: $name" - echo "Download URL: $downloadURL" - echo "Type: $type" - case $type in - dmg|pkg|zip|tbz) - expectedExtension="$type" - ;; - pkgInDmg) - expectedExtension="dmg" - ;; - *InZip) - expectedExtension="zip" - ;; - *) - echo "Cannot handle type $type" - ;; - esac - if [[ "$appNewVersion" == "" ]] ; then - echo "No appNewVersion!" - else - if [[ $( echo "$appNewVersion" | grep -i "[0-9.]" ) == "" || $appNewVersion == "" ]]; then - echo "${RED}-> !! ERROR in appNewVersion${NC}" - labelError=1 + echo "Name: $name" + echo "Download URL: $downloadURL" + echo "Type: $type" + case $type in + dmg|pkg|zip|tbz) + expectedExtension="$type" + ;; + pkgInDmg) + expectedExtension="dmg" + ;; + *InZip) + expectedExtension="zip" + ;; + *) + echo "Cannot handle type $type" + ;; + esac + if [[ "$appNewVersion" == "" ]] ; then + echo "No appNewVersion!" else - if [[ $appNewVersion != $( echo "$appNewVersion" | sed -E 's/[^0-9]*([0-9.]*)[^0-9]*/\1/g' ) ]]; then - echo "${YELLOW}Warning: Version contain not only numbers and dots.${NC}" - labelWarning=1 + if [[ $( echo "$appNewVersion" | grep -i "[0-9.]" ) == "" || $appNewVersion == "" ]]; then + echo "${RED}-> !! ERROR in appNewVersion${NC}" + labelError=1 + else + if [[ $appNewVersion != $( echo "$appNewVersion" | sed -E 's/[^0-9]*([0-9.]*)[^0-9]*/\1/g' ) ]]; then + echo "${YELLOW}Warning: Version contain not only numbers and dots.${NC}" + labelWarning=1 + fi + echo "Version: $appNewVersion" ; fi - echo "Version: $appNewVersion" ; fi - fi - if curl -sfL --output /dev/null -r 0-0 "$downloadURL" ; then - echo "${GREEN}OK: downloadURL works OK${NC}" - if [[ $(echo "$downloadURL" | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' ) == "${expectedExtension}" ]]; then - echo "${GREEN}OK: download extension MATCH on ${expectedExtension}${NC}" - else - if [[ $(echo "$downloadURL" | grep -io "github.com") != "github.com" ]]; then - URLheader=$( curl -fsIL "$downloadURL" ) - if [[ "${URLheader}" != "" ]]; then - URLlocation=$( echo "${URLheader}" | grep -i "^location" ) - URLfilename=$( echo "${URLheader}" | grep -i "filename=" ) - if [[ "${URLlocation}" != "" ]]; then - URLextension=$( echo "${URLlocation}" | tail -1 | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' | tr -d '\r\n' ) + if curl -sfL ${curlOptions} --output /dev/null -r 0-0 "$downloadURL" ; then + echo "${GREEN}OK: downloadURL works OK${NC}" + if [[ $(echo "$downloadURL" | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' ) == "${expectedExtension}" ]]; then + echo "${GREEN}OK: download extension MATCH on ${expectedExtension}${NC}" + else + if [[ $(echo "$downloadURL" | grep -io "github.com") != "github.com" ]]; then + URLheader=$( curl -fsIL "$downloadURL" ) + if [[ "${URLheader}" != "" ]]; then + URLlocation=$( echo "${URLheader}" | grep -i "^location" ) + URLfilename=$( echo "${URLheader}" | grep -i "filename=" ) + if [[ "${URLlocation}" != "" ]]; then + URLextension=$( echo "${URLlocation}" | tail -1 | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' | tr -d '\r\n' ) + else + URLextension=$( echo "${URLfilename}" | tail -1 | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' | tr -d '\r\n' ) + fi + URLextension=${${URLextension:l}%%\?*} + if [[ "${URLextension}" == "${expectedExtension}" ]]; then + echo "${GREEN}OK: download extension MATCH on ${URLextension}${NC}" + else + echo "${RED}-> !! ERROR in download extension, expected ${expectedExtension}, but got ${URLextension}.${NC}" + labelError=1 + fi else - URLextension=$( echo "${URLfilename}" | tail -1 | sed -E 's/.*\.([a-zA-Z]*)\s*/\1/g' | tr -d '\r\n' ) + echo "no header provided from server." fi - URLextension=${${URLextension:l}%%\?*} - if [[ "${URLextension}" == "${expectedExtension}" ]]; then - echo "${GREEN}OK: download extension MATCH on ${URLextension}${NC}" + else + githubPart="$(echo "$downloadURL" | cut -d "/" -f4-6)" + if [[ "$(curl -fsL "$downloadURL" | grep -io "${githubPart}.*\.${expectedExtension}")" != "" ]]; then + echo "${GREEN}OK: download extension MATCH on ${expectedExtension}${NC}" else - echo "${RED}-> !! ERROR in download extension, expected ${expectedExtension}, but got ${URLextension}.${NC}" + echo "${RED}-> !! ERROR in download extension, expected ${expectedExtension}, but it was wrong${NC}" labelError=1 fi - else - echo "no header provided from server." - fi - else - githubPart="$(echo "$downloadURL" | cut -d "/" -f4-6)" - if [[ "$(curl -fsL "$downloadURL" | grep -io "${githubPart}.*\.${expectedExtension}")" != "" ]]; then - echo "${GREEN}OK: download extension MATCH on ${expectedExtension}${NC}" - else - echo "${RED}-> !! ERROR in download extension, expected ${expectedExtension}, but it was wrong${NC}" - labelError=1 fi fi + else + echo "${RED}-> !! ERROR in downloadURL${NC}" + labelError=1 + fi + if [[ $labelWarning != 0 ]]; then; echo "${YELLOW}########## Warning in label: $label${NC}"; ((countWarning++)); warningLabels+=( "$label" ); fi + if [[ $labelError != 0 ]]; then + echo "${RED}########## ERROR in label: $label${NC}" + echo "Testing using Installomator" + exit_status=$( . $repo_dir/assemble.sh $label DEBUG=2 INSTALL=force IGNORE_APP_STORE_APPS=yes BLOCKING_PROCESS_ACTION=ignore | grep exit | tail -1 | sed -E 's/.*exit code ([0-9]).*/\1/g' ) + if [[ ${exit_status} -eq 0 ]] ; then + echo "${GREEN}$label works fine!${NC}" + #errorLabels=("${(@)errorLabels:#$errorLabel}") + else + echo "${RED}$label NOT WORKING!${NC}" + ((countError++)) + errorLabels+=( "$label" ) + fi + + fi + + if (($archLabels[(Ie)$label])); then + secondRoundLabels+=( "$label" ) fi else - echo "${RED}-> !! ERROR in downloadURL${NC}" - labelError=1 - fi - if [[ $labelWarning != 0 ]]; then; echo "${YELLOW}########## Warning in label: $label${NC}"; ((countWarning++)); fi - if [[ $labelError != 0 ]]; then; echo "${RED}########## ERROR in label: $label${NC}"; ((countError++)); fi - - if (($archLabels[(Ie)$label])); then - secondRoundLabels+=( "$label" ) + echo "Label: ${label} is not it's own file in Labels-folder. Skipping" fi + echo + done + if [[ $fixedArch == i386 ]] ; then + errorLabelsi386=( ${=errorLabels} ) else - echo "Label: ${label} is not it's own file in Labels-folder. Skipping" + errorLabelsarm64=( ${=errorLabels} ) fi + + errorLabels="" + allLabels=( ${=secondRoundLabels} ) + archLabels=() echo done -allLabels=( ${=secondRoundLabels} ) -archLabels=() -echo -done rm checkLabelCurrent.sh -#${SELFLOCATION}/Installomator.sh version -#echo - -if [[ countWarning > 0 ]]; then +if [[ countWarning -gt 0 ]]; then echo "${YELLOW}Warnings counted: $countWarning${NC}" + echo "${YELLOW}${warningLabels}${NC}" else echo "${GREEN}No warnings detected!${NC}" fi -if [[ countError > 0 ]]; then +if [[ countError -gt 0 ]]; then echo "${RED}ERRORS counted: $countError${NC}" + echo "${RED}i386 : ${errorLabelsi386}${NC}" + echo "${RED}arm64: ${errorLabelsarm64}${NC}" else echo "${GREEN}No errors detected!${NC}" fi