From a8c9fdbab7a58171ec80104dda3e272c94f39b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Theilgaard?= Date: Mon, 10 Jan 2022 20:26:53 +0100 Subject: [PATCH] Download with extra sets of headers --- CHANGELOG.md | 1 + .../{disabled => labels}/mochakeyboard.sh | 0 fragments/{disabled => labels}/mochatelnet.sh | 0 fragments/{disabled => labels}/mochatn3270.sh | 0 fragments/{disabled => labels}/mochatn3812.sh | 0 fragments/{disabled => labels}/mochatn5250.sh | 0 fragments/main.sh | 73 +++++++++--- utils/buildLabel.sh | 111 +++++++++++++++--- 8 files changed, 149 insertions(+), 36 deletions(-) rename fragments/{disabled => labels}/mochakeyboard.sh (100%) rename fragments/{disabled => labels}/mochatelnet.sh (100%) rename fragments/{disabled => labels}/mochatn3270.sh (100%) rename fragments/{disabled => labels}/mochatn3812.sh (100%) rename fragments/{disabled => labels}/mochatn5250.sh (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dd2a79..11f3056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - New variable `RETURN_LABEL_NAME`. If given the value `1`, like `RETURN_LABEL_NAME=1` then Installomator only returns the name of the label. It makes for a better user friendly message for displaying in DEPNotify if that is integrated. - Changed logic if `IGNORE_APP_STORE_APPS=yes`. Before this version a label like `microsoftonedrive` that was installed from App Store, and that we want to replace with the “ordinary” version, Installomator would still use `updateTool`, even though `IGNORE_APP_STORE_APPS=yes`. So we would have to have `INSTALL=force` in order to have the app replaced, as `updateTool` would be used. But now if `IGNORE_APP_STORE_APPS=yes` then `updateTool` will be not set, and the App Store app will be replaced. BUT if the installed software was not from App Store, then `updateTool` will not be used, and it would be a kind of a forced install (in the example of `microsoftonedrive`), except if the version is the same (where installation is skipped). - Added variable `SYSTEMOWNER` that is used when copying files when installing. Default `0` is to change owner of the app to the current user on the Mac, like this user was installing this app themselves. When using `1` we will put “root:wheel” on the app, which can be useful for shared machines. +- Installomator will normally use `curl`without any headers to download files. But in the `buildLabel.sh` script it was necessary to add extra headers to get the correct download URLs from some servers, and as we get more experienced, various servers require different headers. In this version 3 set of extra headers has been added, so a download will be tried 4 times, but with different headers. Same sets of headers are also in `buildLabel.sh` in the same order. That has made it possible for the "mocha"-software-labels to work. ## v8.0 diff --git a/fragments/disabled/mochakeyboard.sh b/fragments/labels/mochakeyboard.sh similarity index 100% rename from fragments/disabled/mochakeyboard.sh rename to fragments/labels/mochakeyboard.sh diff --git a/fragments/disabled/mochatelnet.sh b/fragments/labels/mochatelnet.sh similarity index 100% rename from fragments/disabled/mochatelnet.sh rename to fragments/labels/mochatelnet.sh diff --git a/fragments/disabled/mochatn3270.sh b/fragments/labels/mochatn3270.sh similarity index 100% rename from fragments/disabled/mochatn3270.sh rename to fragments/labels/mochatn3270.sh diff --git a/fragments/disabled/mochatn3812.sh b/fragments/labels/mochatn3812.sh similarity index 100% rename from fragments/disabled/mochatn3812.sh rename to fragments/labels/mochatn3812.sh diff --git a/fragments/disabled/mochatn5250.sh b/fragments/labels/mochatn5250.sh similarity index 100% rename from fragments/disabled/mochatn5250.sh rename to fragments/labels/mochatn5250.sh diff --git a/fragments/main.sh b/fragments/main.sh index 2eb8781..ac036ad 100644 --- a/fragments/main.sh +++ b/fragments/main.sh @@ -168,7 +168,7 @@ fi if [ -f "$archiveName" ] && [ "$DEBUG" -eq 1 ]; then printlog "$archiveName exists and DEBUG mode 1 enabled, skipping download" else - # download the dmg + # download printlog "Downloading $downloadURL to $archiveName" if [[ $currentUser != "loginwindow" && $NOTIFY == "all" ]]; then printlog "notifying" @@ -178,25 +178,64 @@ else displaynotification "Downloading new $name" "Download in progress …" fi fi - if ! curl --location --fail --silent "$downloadURL" -o "$archiveName"; then - if ! curl -fsL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" -o "$archiveName"; then - if ! curl -fsL -H "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "accept-encoding: gzip, deflate, br" -H "accept-language: en-US,en;q=0.9" -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" -H "sec-fetch-site: same-site" -H "sec-fetch-user: ?1" -H "sec-gpc: 1" -H "upgrade-insecure-requests: 1" "$downloadURL" -o "$archiveName"; then - printlog "error downloading $downloadURL" - message="$name update/installation failed. This will be logged, so IT can follow up." - if [[ $currentUser != "loginwindow" && $NOTIFY == "all" ]]; then - printlog "notifying" - if [[ $updateDetected == "YES" ]]; then - displaynotification "$message" "Error updating $name" - else - displaynotification "$message" "Error installing $name" + if ! curl -fsL "$downloadURL" -o "$archiveName"; then + printlog "Trying 1st set of extra headers to download." + if ! \ + curl -fsL \ + -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + "$downloadURL" -o "$archiveName" + then + printlog "Trying 2nd set of extra headers to download." + if ! \ + curl -fsL \ + -H "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-site: same-site" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + "$downloadURL" -o "$archiveName" + then + printlog "Trying 3rd set of extra headers to download." + if ! \ + curl -fsL \ + -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + -H "Referrer Policy: strict-origin-when-cross-origin" \ + "$downloadURL" -o "$archiveName" + then + printlog "error downloading $downloadURL" + message="$name update/installation failed. This will be logged, so IT can follow up." + if [[ $currentUser != "loginwindow" && $NOTIFY == "all" ]]; then + printlog "notifying" + if [[ $updateDetected == "YES" ]]; then + displaynotification "$message" "Error updating $name" + else + displaynotification "$message" "Error installing $name" + fi fi + cleanupAndExit 2 fi - cleanupAndExit 2 - else - printlog "Downloaded with all headers." fi - else - printlog "Downloaded with exta headers." fi fi fi diff --git a/utils/buildLabel.sh b/utils/buildLabel.sh index 8f96e3c..3dd8a5d 100755 --- a/utils/buildLabel.sh +++ b/utils/buildLabel.sh @@ -153,32 +153,105 @@ echo "Downloading $downloadURL" echo $(basename $downloadURL) # First trying to find redirection headers on the download, as those can contain version numbers echo "Redirecting to (maybe this can help us with version):\n$(curl -fsIL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "Referrer Policy: strict-origin-when-cross-origin" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" | grep -i "^[location|x\-amz\-meta\-version]*")" -# Now downloading without extra headers -if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then - echo "error downloading $downloadURL using standard headers." - echo "result: $downloadOut" - echo "Trying all headers…" # that I know of - if ! downloadOut="$(curl -fL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "Referrer Policy: strict-origin-when-cross-origin" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then - echo "Trying almost all headers…" # that I know of - if ! downloadOut="$(curl -fL -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" -H "accept-encoding: gzip, deflate, br" -H "upgrade-insecure-requests: 1" -H "sec-fetch-dest: document" -H "sec-gpc: 1" -H "sec-fetch-user: ?1" -H "accept-language: en-US,en;q=0.9" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-mode: navigate" "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then - # we are only here if the download failed - echo "error downloading $downloadURL using two different sets of headers." - echo "result: $downloadOut" - # Sometimes a server will give some results to the downloaded output - if [[ -n $downloadOut ]]; then - echo "Trying output of this…" - downloadURL="$(echo $downloadOut | tail -1)" - # Last chance for succes on this download - if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then - echo "error downloading $downloadURL using previous output." - echo "result: $downloadOut" +# Now downloading without various sets of extra headers +if ! downloadOut1="$( \ +curl -fL "$downloadURL" --remote-header-name --remote-name \ +-w "%{filename_effective}\n%{url_effective}\n")" +then + echo "error downloading $downloadURL with no headers." + echo "result: $downloadOut1" + echo "Trying 1st set of extra headers to download." + if ! downloadOut2="$( \ + curl -fL \ + -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + "$downloadURL" --remote-header-name --remote-name \ + -w "%{filename_effective}\n%{url_effective}\n")" + then + echo "error downloading $downloadURL with 1st set of headers." + echo "result: $downloadOut2" + echo "Trying 2nd set of extra headers to download." + if ! downloadOut3="$( \ + curl -fL \ + -H "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-site: same-site" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + "$downloadURL" --remote-header-name --remote-name \ + -w "%{filename_effective}\n%{url_effective}\n")" + then + echo "error downloading $downloadURL with 2nd set of headers." + echo "result: $downloadOut3" + echo "Trying 3rd set of extra headers to download." + if ! downloadOut4="$( \ + curl -fL \ + -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" \ + -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" \ + -H "accept-encoding: gzip, deflate, br" \ + -H "accept-language: en-US,en;q=0.9" \ + -H "sec-fetch-dest: document" \ + -H "sec-fetch-mode: navigate" \ + -H "sec-fetch-user: ?1" \ + -H "sec-gpc: 1" \ + -H "upgrade-insecure-requests: 1" \ + -H "Referrer Policy: strict-origin-when-cross-origin" \ + "$downloadURL" --remote-header-name --remote-name \ + -w "%{filename_effective}\n%{url_effective}\n")" + then + # we are only here if the download failed + echo "error downloading $downloadURL with 3rd set of headers." + echo "result: $downloadOut4" + echo "no more header sets to try" + # Sometimes a server will give some results to the downloaded output + echo "If any information came out of the previous download attempts, we can try those…" + downloadOuts=( "$downloadOut1" "$downloadOut3" "$downloadOut3" "$downloadOut4" ) + downloadOutCount=${#downloadOuts} + for downloadOut in $downloadOuts ; do + if [[ -n $downloadOut ]]; then + echo "Trying output of this…" + downloadURL="$(echo $downloadOut | tail -1)" + # Last chance for succes on this download + if ! downloadOut="$(curl -fL "$downloadURL" --remote-header-name --remote-name -w "%{filename_effective}\n%{url_effective}\n")"; then + echo "error downloading $downloadURL using previous output." + echo "result: $downloadOut" + ((downloadOutCount--)) + else + echo "Success on this download." + succesDownloadOut=$downloadOut + return + fi + fi + done + if [[ $downloadOutCount -eq 0 ]]; then echo "No more tries. Cannot continue." exit 1 fi + else + succesDownloadOut=$downloadOut4 fi + else + succesDownloadOut=$downloadOut3 fi + else + succesDownloadOut=$downloadOut2 fi +else + succesDownloadOut=$downloadOut1 fi +downloadOut=$succesDownloadOut # Now we have downloaded the archive, and we need to analyze this # The download have returned both {filename_effective} and {url_effective}