From 4e43c5e98903618823ce3c56c573b3e880dd8576 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:04:22 +0000 Subject: [PATCH 01/51] Create nzbget-mp4.yml NzbGet, with build in sickbeard_mp4_automator and ffmpeg build script Built for PGBlitz version 8.5.6 Should be fine for versions above this too --- apps/nzbget-mp4.yml | 402 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100644 apps/nzbget-mp4.yml diff --git a/apps/nzbget-mp4.yml b/apps/nzbget-mp4.yml new file mode 100644 index 0000000..87a8703 --- /dev/null +++ b/apps/nzbget-mp4.yml @@ -0,0 +1,402 @@ +#!/bin/bash +# +# NzbGet, with build in sickbeard_mp4_automator and ffmpeg build script +# Built for PGBlitz version 8.5.6 +# Should be fine for versions above this too +# +# Title: PGBlitz (Reference Title File) +# Author(s): Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################# +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'nzbget' + intport: '6789' + extport: '6789' + image: 'linuxserver/nzbget' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + - name: 'Including folders' + include_tasks: '/opt/coreapps/apps/_downloaders.yml' + + - name: Create nzb folder + file: 'path={{item}} state=directory mode=0775 owner=1000 group=1000' + with_items: + - '{{path.stdout}}/nzb' +# force: yes + + - name: 'Including plugins' + include_tasks: '/opt/coreapps/apps/_plugins.yml' + + - name: 'Checking for existing app data' + stat: + path: /opt/appdata/{{pgrole}}/nzbget.conf + register: confcheck + + # EXTRAS FOR MP4 MODS ########################################################## + - name: 'Create {{pgrole}} mp4 directories' + file: 'path={{item}} state=directory mode=0775 owner=1000 group=1000 recurse=yes' + with_items: + - '/opt/appdata/{{pgrole}}' + - '/opt/appdata/{{pgrole}}/cont-init.d' + - '/opt/appdata/{{pgrole}}/installer' + - '/opt/appdata/{{pgrole}}/scripts' + - '/opt/appdata/{{pgrole}}/scripts/MP4_Automator' + - '/opt/appdata/{{pgrole}}/ffmpeg-build' + - '/opt/appdata/{{pgrole}}/services.d' + + - name: 'Copy custom init scripts into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/cont-init.d/30-config + dest: /opt/appdata/{{pgrole}}/cont-init.d + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy custom install scripts into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/installer/installer.sh + dest: /opt/appdata/{{pgrole}}/installer + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy custom mp4 config into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini + dest: /opt/appdata/{{pgrole}}/scripts/MP4_Automator + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy custom NZBGetPostProcess script config into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript + dest: /opt/appdata/{{pgrole}}/ + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy ffmpeg build script into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh + dest: /opt/appdata/{{pgrole}}/ffmpeg-build + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy ffmpeg build script into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg + dest: /opt/appdata/{{pgrole}}/ffmpeg-build + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy custom init scripts into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/services.d/run + dest: /opt/appdata/{{pgrole}}/services.d + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/tmp:/tmp' + - '/opt/appdata/{{pgrole}}/cont-init.d:/etc/cont-init.d' + - '/opt/appdata/{{pgrole}}/services.d:/etc/services.d/nzbget' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + LC_ALL: 'C' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' + + # CONFIGURATION ############################################################# + - name: 'Waiting for {{pgrole}} to initialize' + wait_for: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + state: present + delay: 5 + + - name: 'Stopping {{pgrole}}' + docker_container: + name: '{{pgrole}}' + state: stopped + + - name: Set Main Location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^MainDir\s*=.*' + line: 'MainDir=/config' + state: present + + - name: Set download location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^DestDir\s*=.*' + line: 'DestDir={{path.stdout}}/downloads/{{pgrole}}' + state: present + + - name: Set incomplete location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^InterDir\s*=.*' + line: 'InterDir={{path.stdout}}/incomplete/{{pgrole}}' + state: present + + - name: Set TempDir + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^TempDir\s*=.*' + line: 'TempDir=/tmp' + state: present + + - name: Set NzbDir Location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^NzbDir\s*=.*' + line: 'NzbDir={{path.stdout}}/nzb' + state: present + + - name: ScriptDir + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^ScriptDir\s*=.*' + line: 'ScriptDir=${MainDir}/scripts' + state: present + + # FIRST TIME CONFIGURATION ############################################################# + - name: 'Configuring {{pgrole}} for first time use' + block: + - name: Lowercase & Set Movie Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category1.Name\s*=.*' + line: 'Category1.Name=movies' + state: present + + - name: Set Location of Movies + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category1.DestDir\s*=.*' + line: 'Category1.DestDir=' + state: present + + - name: Set postprocess of Movies + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category1.Extensions\s*=.*' + line: 'Category1.Extensions=MP4_Automator/NZBGetPostProcess.py' + state: present + + - name: Lowercase & Set TV Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category2.Name\s*=.*' + line: 'Category2.Name=tv' + state: present + + - name: Set Location of TV + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category2.DestDir\s*=.*' + line: 'Category2.DestDir=' + state: present + + - name: Set postprocess of TV + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category2.Extensions\s*=.*' + line: 'Category2.Extensions=MP4_Automator/NZBGetPostProcess.py' + state: present + + - name: Lowercase & Set Music Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category3.Name\s*=.*' + line: 'Category3.Name=music' + state: present + + - name: Set Location of Music + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category3.DestDir\s*=.*' + line: 'Category3.DestDir=' + state: present + + - name: Lowercase & Set EBook Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category4.Name\s*=.*' + line: 'Category4.Name=ebooks' + state: present + + - name: Set Location of EBooks + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category4.DestDir\s*=.*' + line: 'Category4.DestDir=' + state: present + + - name: Lowercase & Set abook Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: 'Category5.Name\s*=.*' + line: 'Category5.Name=abooks' + state: present + + - name: Set Location of aBooks + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: 'Category5.DestDir\s*=.*' + line: 'Category5.DestDir=' + state: present + + - name: Set Global Extensions + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Extensions\s*=.*' + line: 'Extensions=unzip.py' + state: present + + - name: Set ScriptOrder + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^ScriptOrder\s*=.*' + line: 'ScriptOrder=unzip.py, MP4_Automator/NZBGetPostProcess.py' + state: present + + - name: Set mp4 script location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^MP4_Automator/NZBGetPostProcess.py:MP4_FOLDER\s*=.*' + line: 'MP4_Automator/NZBGetPostProcess.py:MP4_FOLDER=/config/scripts/MP4_Automator/' + state: present + + - name: Set mp4 conversion true + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^MP4_Automator/NZBGetPostProcess.py:SHOULDCONVERT\s*=.*' + line: 'MP4_Automator/NZBGetPostProcess.py:SHOULDCONVERT=True' + state: present + + - name: Default User + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^ControlUsername\s*=.*' + line: 'ControlUsername=' + state: present + + - name: Default Password + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: 'ControlPassword\s*=.*' + line: 'ControlPassword=' + state: present + + - name: DirectUnpack Set to On + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^DirectUnpack\s*=.*' + line: 'DirectUnpack=yes' + state: present + + - name: HealthCheck + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^HealthCheck\s*=.*' + line: 'HealthCheck=Delete' + state: present + + - name: Set DiskSpace + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^DiskSpace\s*=.*' + line: 'DiskSpace=25000' + state: present + + - name: Remove Generic Task + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '{{ item.regexp }}' + state: absent + with_items: + - { regexp: '.Task1\.Time\=' } + - { regexp: '.Task1\.WeekDays\=' } + - { regexp: '.Task1\.Command\=' } + - { regexp: '.Task1\.Param\=' } + + - name: Unpause + blockinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + block: | + Task1.Time=*,*:00,*:30 + Task1.WeekDays=1-7 + Task1.Command=UnpauseDownload + Task1.Param= + insertafter: '^### SCHEDULER' + when: not confcheck.stat.exists + + - name: Restart {{pgrole}} + docker_container: + name: '{{pgrole}}' + state: started From 01f7ad55c91bd4f3bbf70acfbb7035e427938a16 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:05:31 +0000 Subject: [PATCH 02/51] Create 30-config --- .../nzbget-mp4/cont-init.d/30-config | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 apps/templates/nzbget-mp4/cont-init.d/30-config diff --git a/apps/templates/nzbget-mp4/cont-init.d/30-config b/apps/templates/nzbget-mp4/cont-init.d/30-config new file mode 100644 index 0000000..deaaacd --- /dev/null +++ b/apps/templates/nzbget-mp4/cont-init.d/30-config @@ -0,0 +1,27 @@ +#!/usr/bin/with-contenv bash + +# delete lock file if found +[[ -f /downloads/nzbget.lock ]] && \ + rm /downloads/nzbget.lock + +# check if config file exists in /config +[[ ! -f /config/nzbget.conf ]] && \ + cp /defaults/nzbget.conf /config/nzbget.conf + +# permissions +chown 1000:1000 \ + /downloads +chown 1000:1000 -R \ + /app/nzbget \ + /config +chmod u+rw \ + /config/nzbget.conf + +chmod 777 -R \ + /config +chmod 777 -R \ + /app/nzbget +chmod 777 -R \ + /downloads + +exec /config/installer/installer.sh From 5ac3779584da722357f48511b8e942c7fdcc4a8a Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:06:08 +0000 Subject: [PATCH 03/51] Create build-ffmpeg --- .../nzbget-mp4/ffmpeg-build/build-ffmpeg | 400 ++++++++++++++++++ 1 file changed, 400 insertions(+) create mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg b/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg new file mode 100644 index 0000000..9aad5c4 --- /dev/null +++ b/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg @@ -0,0 +1,400 @@ +#!/bin/bash + +# https://github.com/markus-perl/ffmpeg-build-script + +VERSION=1.1 +CWD=$(pwd) +PACKAGES="$CWD/packages" +WORKSPACE="$CWD/workspace" +CC=clang +LDFLAGS="-L${WORKSPACE}/lib -lm" +CFLAGS="-I${WORKSPACE}/include" +PKG_CONFIG_PATH="${WORKSPACE}/lib/pkgconfig" +ADDITIONAL_CONFIGURE_OPTIONS="" + +# Speed up the process +# Env Var NUMJOBS overrides automatic detection +if [[ -n $NUMJOBS ]]; then + MJOBS=$NUMJOBS +elif [[ -f /proc/cpuinfo ]]; then + MJOBS=$(grep -c processor /proc/cpuinfo) +elif [[ "$OSTYPE" == "darwin"* ]]; then + MJOBS=$(sysctl -n machdep.cpu.thread_count) + ADDITIONAL_CONFIGURE_OPTIONS="--enable-videotoolbox" +else + MJOBS=4 +fi + +make_dir () { + if [ ! -d $1 ]; then + if ! mkdir $1; then + printf "\n Failed to create dir %s" "$1"; + exit 1 + fi + fi +} + +remove_dir () { + if [ -d $1 ]; then + rm -r "$1" + fi +} + +download () { + + DOWNLOAD_PATH=$PACKAGES; + + if [ ! -z "$3" ]; then + mkdir -p $PACKAGES/$3 + DOWNLOAD_PATH=$PACKAGES/$3 + fi; + + if [ ! -f "$DOWNLOAD_PATH/$2" ]; then + + echo "Downloading $1" + curl -L --silent -o "$DOWNLOAD_PATH/$2" "$1" + + EXITCODE=$? + if [ $EXITCODE -ne 0 ]; then + echo "" + echo "Failed to download $1. Exitcode $EXITCODE. Retrying in 10 seconds"; + sleep 10 + curl -L --silent -o "$DOWNLOAD_PATH/$2" "$1" + fi + + EXITCODE=$? + if [ $EXITCODE -ne 0 ]; then + echo "" + echo "Failed to download $1. Exitcode $EXITCODE"; + exit 1 + fi + + echo "... Done" + + if ! tar -xvf "$DOWNLOAD_PATH/$2" -C "$DOWNLOAD_PATH" 2>/dev/null >/dev/null; then + echo "Failed to extract $2"; + exit 1 + fi + + fi +} + +execute () { + echo "$ $*" + + OUTPUT=$($@ 2>&1) + + if [ $? -ne 0 ]; then + echo "$OUTPUT" + echo "" + echo "Failed to Execute $*" >&2 + exit 1 + fi +} + +build () { + echo "" + echo "building $1" + echo "=======================" + + if [ -f "$PACKAGES/$1.done" ]; then + echo "$1 already built. Remove $PACKAGES/$1.done lockfile to rebuild it." + return 1 + fi + + return 0 +} + +command_exists() { + if ! [[ -x $(command -v "$1") ]]; then + return 1 + fi + + return 0 +} + + +build_done () { + touch "$PACKAGES/$1.done" +} + +echo "ffmpeg-build-script v$VERSION" +echo "=========================" +echo "" + +case "$1" in +"--cleanup") + remove_dir $PACKAGES + remove_dir $WORKSPACE + echo "Cleanup done." + echo "" + exit 0 + ;; +"--build") + + ;; +*) + echo "Usage: $0" + echo " --build: start building process" + echo " --cleanup: remove all working dirs" + echo " --help: show this help" + echo "" + exit 0 + ;; +esac + +echo "Using $MJOBS make jobs simultaneously." + +make_dir $PACKAGES +make_dir $WORKSPACE + +export PATH=${WORKSPACE}/bin:$PATH + +if ! command_exists "make"; then + echo "make not installed."; + exit 1 +fi + +if ! command_exists "g++"; then + echo "g++ not installed."; + exit 1 +fi + +if ! command_exists "curl"; then + echo "curl not installed."; + exit 1 +fi + +if build "yasm"; then + download "http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz" "yasm-1.3.0.tar.gz" + cd $PACKAGES/yasm-1.3.0 || exit + execute ./configure --prefix=${WORKSPACE} + execute make -j $MJOBS + execute make install + build_done "yasm" +fi + +if build "nasm"; then + download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.13.03.tar.gz" "nasm.tar.gz" + cd $PACKAGES/nasm-2.13.03 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "nasm" +fi + +if build "opencore"; then + download "http://downloads.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-0.1.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Fopencore-amr%2F&ts=1442256558&use_mirror=netassist" "opencore-amr-0.1.5.tar.gz" + cd $PACKAGES/opencore-amr-0.1.5 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "opencore" +fi + +if build "libvpx"; then + download "https://github.com/webmproject/libvpx/archive/v1.7.0.tar.gz" "libvpx-1.7.0.tar.gz" + cd $PACKAGES/libvpx-*0 || exit + + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Applying Darwin patch" + sed "s/,--version-script//g" build/make/Makefile > build/make/Makefile.patched + sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched > build/make/Makefile + fi + + execute ./configure --prefix=${WORKSPACE} --disable-unit-tests --disable-shared + execute make -j $MJOBS + execute make install + build_done "libvpx" +fi + +if build "lame"; then + download "http://kent.dl.sourceforge.net/project/lame/lame/3.100/lame-3.100.tar.gz" "lame-3.100.tar.gz" + cd $PACKAGES/lame-3.100 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "lame" +fi + +if build "xvidcore"; then + download "http://downloads.xvid.org/downloads/xvidcore-1.3.4.tar.gz" "xvidcore-1.3.4.tar.gz" + cd $PACKAGES/xvidcore || exit + cd build/generic || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + + if [[ -f ${WORKSPACE}/lib/libxvidcore.4.dylib ]]; then + execute rm "${WORKSPACE}/lib/libxvidcore.4.dylib" + fi + + build_done "xvidcore" +fi + +if build "x264"; then + download "http://ftp.videolan.org/pub/x264/snapshots/x264-snapshot-20190204-2245-stable.tar.bz2" "last_x264.tar.bz2" + cd $PACKAGES/x264-snapshot-* || exit + + if [[ "$OSTYPE" == "linux-gnu" ]]; then + execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic CXXFLAGS="-fPIC" + else + execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic + fi + + execute make -j $MJOBS + execute make install + execute make install-lib-static + build_done "x264" +fi + +if build "libogg"; then + download "http://downloads.xiph.org/releases/ogg/libogg-1.3.3.tar.gz" "libogg-1.3.3.tar.gz" + cd $PACKAGES/libogg-1.3.3 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "libogg" +fi + +if build "libvorbis"; then + download "http://downloads.xiph.org/releases/vorbis/libvorbis-1.3.6.tar.gz" "libvorbis-1.3.6.tar.gz" + cd $PACKAGES/libvorbis-1.3.6 || exit + execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest + execute make -j $MJOBS + execute make install + build_done "libvorbis" +fi + +if build "libtheora"; then + download "http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.gz" "libtheora-1.1.1.tar.bz" + cd $PACKAGES/libtheora-1.1.1 || exit + sed "s/-fforce-addr//g" configure > configure.patched + chmod +x configure.patched + mv configure.patched configure + execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --with-vorbis-libraries=${WORKSPACE}/lib --with-vorbis-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm + execute make -j $MJOBS + execute make install + build_done "libtheora" +fi + +if build "pkg-config"; then + download "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" "pkg-config-0.29.2.tar.gz" + cd $PACKAGES/pkg-config-0.29.2 || exit + execute ./configure --silent --prefix=${WORKSPACE} --with-pc-path=${WORKSPACE}/lib/pkgconfig --with-internal-glib + execute make -j $MJOBS + execute make install + build_done "pkg-config" +fi + +if build "cmake"; then + download "https://cmake.org/files/v3.11/cmake-3.11.3.tar.gz" "cmake-3.11.3.tar.gz" + cd $PACKAGES/cmake-3.11.3 || exit + rm Modules/FindJava.cmake + perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt + perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt + execute ./configure --prefix=${WORKSPACE} + execute make -j $MJOBS + execute make install + build_done "cmake" +fi + +if build "vid_stab"; then + download "https://codeload.github.com/georgmartius/vid.stab/legacy.tar.gz/release-0.98b" "vid.stab-0.98b-transcode-1.1-binary-x86_64.tgz" + cd $PACKAGES/georgmartius-vid* || exit + perl -p -i -e "s/vidstab SHARED/vidstab STATIC/" CMakeLists.txt + execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} . + execute make install + build_done "vid_stab" +fi + +if build "x265"; then + download "https://bitbucket.org/multicoreware/x265/downloads/x265_3.0.tar.gz" "x265-3.0.tar.gz" + cd $PACKAGES/x265_* || exit + cd source || exit + execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} -DENABLE_SHARED:bool=off . + execute make -j $MJOBS + execute make install + sed "s/-lx265/-lx265 -lstdc++/g" "$WORKSPACE/lib/pkgconfig/x265.pc" > "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" + mv "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" "$WORKSPACE/lib/pkgconfig/x265.pc" + build_done "x265" +fi + +if build "fdk_aac"; then + download "http://downloads.sourceforge.net/project/opencore-amr/fdk-aac/fdk-aac-0.1.6.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Ffdk-aac%2F&ts=1457561564&use_mirror=kent" "fdk-aac-0.1.6.tar.gz" + cd $PACKAGES/fdk-aac-0.1.6 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "fdk_aac" +fi + + +build "ffmpeg" +download "http://ffmpeg.org/releases/ffmpeg-4.1.tar.bz2" "ffmpeg-snapshot.tar.bz2" +cd $PACKAGES/ffmpeg-4.1 || exit +./configure $ADDITIONAL_CONFIGURE_OPTIONS \ + --pkgconfigdir="$WORKSPACE/lib/pkgconfig" \ + --prefix=${WORKSPACE} \ + --pkg-config-flags="--static" \ + --extra-cflags="-I$WORKSPACE/include" \ + --extra-ldflags="-L$WORKSPACE/lib" \ + --extra-libs="-lpthread -lm" \ + --enable-static \ + --disable-debug \ + --disable-shared \ + --disable-ffplay \ + --disable-doc \ + --enable-gpl \ + --enable-version3 \ + --enable-nonfree \ + --enable-pthreads \ + --enable-libvpx \ + --enable-libmp3lame \ + --enable-libtheora \ + --enable-libvorbis \ + --enable-libx264 \ + --enable-libx265 \ + --enable-runtime-cpudetect \ + --enable-libfdk-aac \ + --enable-avfilter \ + --enable-libopencore_amrwb \ + --enable-libopencore_amrnb \ + --enable-filters \ + --enable-libvidstab + +execute make -j $MJOBS +execute make install + +INSTALL_FOLDER="/usr/bin" +if [[ "$OSTYPE" == "darwin"* ]]; then +INSTALL_FOLDER="/usr/local/bin" +fi + +echo "" +echo "Building done. The binary can be found here: $WORKSPACE/bin/ffmpeg" +echo "" + + +if [[ $AUTOINSTALL == "yes" ]]; then + if command_exists "sudo"; then + sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" + sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" + echo "Done. ffmpeg is now installed to your system" + fi +elif [[ ! $SKIPINSTALL == "yes" ]]; then + if command_exists "sudo"; then + + read -r -p "Install the binary to your $INSTALL_FOLDER folder? [Y/n] " response + + case $response in + [yY][eE][sS]|[yY]) + sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" + sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" + echo "Done. ffmpeg is now installed to your system" + ;; + esac + fi +fi + +exit 0 From df88651991858f9c3beea93a52ef0e21f8f654c1 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:06:28 +0000 Subject: [PATCH 04/51] Create web-install.sh --- .../nzbget-mp4/ffmpeg-build/web-install.sh | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh b/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh new file mode 100644 index 0000000..ffe4158 --- /dev/null +++ b/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Helper script to download and run the build-ffmpeg script. + +make_dir () { + if [ ! -d $1 ]; then + if ! mkdir $1; then + printf "\n Failed to create dir %s" "$1"; + exit 1 + fi + fi +} + +command_exists() { + if ! [[ -x $(command -v "$1") ]]; then + return 1 + fi + + return 0 +} + +TARGET='ffmpeg-build' + +if ! command_exists "curl"; then + echo "curl not installed."; + exit 1 +fi + +echo "ffmpeg-build-script-downloader v0.1" +echo "=========================================" +echo "" + +echo "First we create the ffmpeg build directory $TARGET" +make_dir $TARGET +cd $TARGET + +echo "Now we download and execute the build script" +echo "" + +bash build-ffmpeg --build + From f2a87711f0a184d218ce9184e8588f0c36dfa22b Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:07:02 +0000 Subject: [PATCH 05/51] Create installer.sh --- .../ffmpeg-build/installer/installer.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh b/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh new file mode 100644 index 0000000..2fe24f5 --- /dev/null +++ b/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh @@ -0,0 +1,30 @@ +#!/bin/bash +apk update +apk upgrade +apk add --no-cache git +apk add build-base gcc wget diffutils perl +apk add curl +git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp +mv /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator/ +rm -rf /config/scripts/MP4_Automator/tmp +git unstage +apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev +pip install --upgrade PIP +pip install requests +pip install requests[security] +pip install requests-cache +pip install babelfish +pip install "guessit<2" +pip install "subliminal<2" +pip install qtfaststart +# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 +pip uninstall -y stevedore +pip install stevedore==1.19.1 +#Remove default NZBGetPostProcess script settings, and replace with our own +rm /config/scripts/MP4_Automator/NZBGetPostProcess.py +cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py +#Build ffmpeg +cd /config +. /config/ffmpeg-build/web-install.sh +#Set script file permissions +chmod 777 -R /config/scripts From 7e7e531d5e0b8eb90a2f446d7045709ea2dae677 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:11:27 +0000 Subject: [PATCH 06/51] remove file in wrong location --- .../ffmpeg-build/installer/installer.sh | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh b/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh deleted file mode 100644 index 2fe24f5..0000000 --- a/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -apk update -apk upgrade -apk add --no-cache git -apk add build-base gcc wget diffutils perl -apk add curl -git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp -mv /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator/ -rm -rf /config/scripts/MP4_Automator/tmp -git unstage -apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev -pip install --upgrade PIP -pip install requests -pip install requests[security] -pip install requests-cache -pip install babelfish -pip install "guessit<2" -pip install "subliminal<2" -pip install qtfaststart -# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 -pip uninstall -y stevedore -pip install stevedore==1.19.1 -#Remove default NZBGetPostProcess script settings, and replace with our own -rm /config/scripts/MP4_Automator/NZBGetPostProcess.py -cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py -#Build ffmpeg -cd /config -. /config/ffmpeg-build/web-install.sh -#Set script file permissions -chmod 777 -R /config/scripts From a4c0e34d2d34f75842ad4276e9eee7a85ec2c674 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:12:14 +0000 Subject: [PATCH 07/51] Create installer.sh --- .../nzbget-mp4/installer/installer.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 apps/templates/nzbget-mp4/installer/installer.sh diff --git a/apps/templates/nzbget-mp4/installer/installer.sh b/apps/templates/nzbget-mp4/installer/installer.sh new file mode 100644 index 0000000..2fe24f5 --- /dev/null +++ b/apps/templates/nzbget-mp4/installer/installer.sh @@ -0,0 +1,30 @@ +#!/bin/bash +apk update +apk upgrade +apk add --no-cache git +apk add build-base gcc wget diffutils perl +apk add curl +git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp +mv /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator/ +rm -rf /config/scripts/MP4_Automator/tmp +git unstage +apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev +pip install --upgrade PIP +pip install requests +pip install requests[security] +pip install requests-cache +pip install babelfish +pip install "guessit<2" +pip install "subliminal<2" +pip install qtfaststart +# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 +pip uninstall -y stevedore +pip install stevedore==1.19.1 +#Remove default NZBGetPostProcess script settings, and replace with our own +rm /config/scripts/MP4_Automator/NZBGetPostProcess.py +cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py +#Build ffmpeg +cd /config +. /config/ffmpeg-build/web-install.sh +#Set script file permissions +chmod 777 -R /config/scripts From 6a57cbc8a013f11f93534f48dc73c2eaf7e212a3 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:12:56 +0000 Subject: [PATCH 08/51] Create autoProcess.ini --- .../nzbget-mp4/MP4_Automator/autoProcess.ini | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini diff --git a/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini b/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini new file mode 100644 index 0000000..83b18aa --- /dev/null +++ b/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini @@ -0,0 +1,143 @@ +[SickBeard] +host = sickbeard +port = 8081 +username = +password = +web_root = +ssl = False +api_key = + +[Sonarr] +host = sonarr +port = 8989 +web_root = +ssl = False +apikey = + +[Radarr] +host = radarr +port = 7878 +web_root = +ssl = False +apikey = + +[MP4] +ffmpeg = /config/ffmpeg-build/workspace/bin/ffmpeg +ffprobe = /config/ffmpeg-build/workspace/bin/ffprobe +threads = 1 +output_directory = +copy_to = +move_to = +output_extension = mp4 +output_format = mp4 +delete_original = True +relocate_moov = True +video-codec = h264,x264 +video-bitrate = +video-crf = 18 +video-max-width = +h264-max-level = 4.1 +use-qsv-decoder-with-encoder = True +ios-audio = libfdk_aac +ios-first-track-only = False +ios-audio-filter = dynaudnorm +max-audio-channels = +audio-codec = ac3,mp3,dts,dca,aac,libfdk_aac +audio-language = eng +audio-default-language = eng +audio-channel-bitrate = 256 +audio-filter = +subtitle-codec = srt +subtitle-language = eng +subtitle-default-language = +subtitle-encoding = +fullpathguess = True +convert-mp4 = True +tagfile = True +tag-language = en +download-artwork = Poster +download-subs = True +embed-subs = False +sub-providers = addic7ed,podnapisi,thesubdb,opensubtitles +permissions = 0777 +post-process = False +pix-fmt = +aac_adtstoasc = False +postopts = -preset,slower +preopts = +audio-copy-original = False +enable_dxva2_gpu_decode = False +ios-move-last = False +use-hevc-qsv-decoder = False +embed-only-internal-subs = False +audio-first-track-of-language = False +video-profile = + +[CouchPotato] +host = couchpotato +port = 5050 +username = +password = +web_root = +ssl = False +apikey = +delay = 65 +method = renamer +delete_failed = False + +[uTorrent] +convert = +couchpotato-label = couchpotato +sickbeard-label = sickbeard +sonarr-label = sonarr +bypass-label = bypass +sickrage-label = sickrage +webui = False +action_before = stop +action_after = removedata +host = http://utorrent:8080/ +username = +password = +output_directory = +radarr-label = radarr + +[Deluge] +host = deluge +username = +convert = True +password = +sonarr-label = sonarr +radarr-label = radarr +bypass-label = bypass +sickbeard-label = sickbeard +port = 12569 +sickrage-label = sickrage +couchpotato-label = couchpotato +output_directory = +remove = true + +[SABNZBD] +convert = True +sickrage-category = sickrage +sonarr-category = sonarr +radarr-category = radarr +bypass-category = bypass +couchpotato-category = couchpotato +sickbeard-category = sickbeard +output_directory = + +[Sickrage] +host = sickrage +port = 8081 +username = +password = +web_root = +ssl = False +api_key = + +[Plex] +host = plex +port = 32400 +refresh = False +token = + From a67d15da03bad5a98c285db1526661e1c3b4303a Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:13:32 +0000 Subject: [PATCH 09/51] Create TEMPLATEPPScript --- .../nzbget-mp4/MP4_Automator/TEMPLATEPPScript | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript diff --git a/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript b/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript new file mode 100644 index 0000000..6eb7055 --- /dev/null +++ b/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# +############################################################################## +### NZBGET POST-PROCESSING SCRIPT ### + +# Modified to enable multiple bypass categories, +# as per: https://github.com/mdhiggins/sickbeard_mp4_automator/issues/509 +# +# Converts files and passes them to Sonarr for further processing. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### + +# Change to full path to MP4 Automator folder. No quotes and a trailing / +#MP4_FOLDER=~/sickbeard_mp4_automator/ + +# Convert file before passing to destination (True, False) +#SHOULDCONVERT=False + +# Category for Couchpotato +#CP_CAT=Couchpotato + +# Category for Sonarr +#SONARR_CAT=Sonarr + +# Category for Radarr +#RADARR_CAT=Radarr + +# Category for Sickbeard +#SICKBEARD_CAT=Sickbeard + +# Category for Sickrage +#SICKRAGE_CAT=Sickrage + +# Category list (comma seperated) for bypassing any further processing but still converting +#BYPASS_CAT=tv,movies + +# Custom output_directory setting +#OUTPUT_DIR= + +### NZBGET POST-PROCESSING SCRIPT ### +############################################################################## + +import os +import sys +import re +import json +import traceback + +# Sanity checks for path string +MP4folder = os.environ['NZBPO_MP4_FOLDER'].strip() +MP4folder = MP4folder.replace('"', '') +MP4folder = MP4folder.replace("'", "") +MP4folder = MP4folder.replace("\\", "/") +if not(MP4folder.endswith("/")): + MP4folder += "/" +#DEBUG#print MP4folder+" the original is "+os.environ['NZBPO_MP4_FOLDER'] + +output_dir = None +if 'NZBPO_OUTPUT_DIR' in os.environ: + output_dir = os.environ['NZBPO_OUTPUT_DIR'].strip() + if len(output_dir) > 0: + output_dir = output_dir.replace('"', '') + output_dir = output_dir.replace("'", "") + output_dir = output_dir.replace("\\", "/") + if not(output_dir.endswith("/")): + output_dir += "/" + #DEBUG#print Overriding output directory + +sys.path.append(MP4folder) +try: + from readSettings import ReadSettings + from mkvtomp4 import MkvtoMp4 + from autoprocess import autoProcessMovie, autoProcessTV, autoProcessTVSR, sonarr, radarr + import logging + from logging.config import fileConfig +except ImportError: + print("[ERROR] Wrong path to sickbeard_mp4_automator: " + os.environ['NZBPO_MP4_FOLDER']) + print("[ERROR] %s" % traceback.print_exc()) + sys.exit(0) + +# Setup Logging +logpath = '/var/log/sickbeard_mp4_automator' +if os.name == 'nt': + logpath = MP4folder +elif not os.path.isdir(logpath): + try: + os.mkdir(logpath) + except: + logpath = MP4folder +configPath = os.path.abspath(os.path.join(MP4folder, 'logging.ini')).replace("\\", "\\\\") +logPath = os.path.abspath(os.path.join(logpath, 'index.log')).replace("\\", "\\\\") +fileConfig(configPath, defaults={'logfilename': logPath}) +log = logging.getLogger("NZBGetPostProcess") + +# Determine if conversion will take place +shouldConvert = (os.environ['NZBPO_SHOULDCONVERT'].lower() in ("yes", "true", "t", "1")) + +if 'NZBOP_SCRIPTDIR' in os.environ and not os.environ['NZBOP_VERSION'][0:5] < '11.0': + log.info("Script triggered from NZBGet (11.0 or later).") + + path = os.environ['NZBPP_DIRECTORY'] # Path to NZB directory + nzb = os.environ['NZBPP_NZBFILENAME'] # Original NZB name + category = os.environ['NZBPP_CATEGORY'] # NZB Category to determine destination + #DEBUG#print "Category is %s." % category + + couchcat = os.environ['NZBPO_CP_CAT'].lower() + sonarrcat = os.environ['NZBPO_SONARR_CAT'].lower() + radarrcat = os.environ['NZBPO_RADARR_CAT'].lower() + sickbeardcat = os.environ['NZBPO_SICKBEARD_CAT'].lower() + sickragecat = os.environ['NZBPO_SICKRAGE_CAT'].lower() + bypass = os.environ['NZBPO_BYPASS_CAT'].lower().replace(' ','').split(',') + + categories = [sickbeardcat, couchcat, sonarrcat, radarrcat, sickragecat] + + log.debug("Path: %s" % path) + log.debug("NZB: %s" % nzb) + log.debug("Category: %s" % category) + log.debug("Categories: %s" % categories) + + # NZBGet argv: all passed as environment variables. + clientAgent = "nzbget" + # Exit codes used by NZBGet + POSTPROCESS_PARCHECK = 92 + POSTPROCESS_SUCCESS = 93 + POSTPROCESS_ERROR = 94 + POSTPROCESS_NONE = 95 + + # Check nzbget.conf options + status = 0 + + if os.environ['NZBOP_UNPACK'] != 'yes': + log.error("Please enable option \"Unpack\" in nzbget configuration file, exiting.") + sys.exit(POSTPROCESS_NONE) + + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '3': + log.error("Par-check successful, but Par-repair disabled, exiting") + sys.exit(POSTPROCESS_NONE) + + if os.environ['NZBPP_PARSTATUS'] == '1': + log.error("Par-check failed, setting status \"failed\".") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + log.error("Unpack failed, setting status \"failed\".") + status = 1 + sys.exit(POSTPROCESS_NONE) + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] != '2': + # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check + + for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): + for file in filenames: + fileExtension = os.path.splitext(file)[1] + + if fileExtension in ['.par2']: + log.error("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\".") + status = 1 + break + + if os.path.isfile(os.path.join(os.environ['NZBPP_DIRECTORY'], "_brokenlog.txt")) and not status == 1: + log.error("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting.") + status = 1 + + if not status == 1: + log.error("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful.") + + # Check if destination directory exists (important for reprocessing of history items) + if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + log.error("Post-Process: Nothing to post-process: destination directory ", os.environ['NZBPP_DIRECTORY'], "doesn't exist.") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Make sure one of the appropriate categories is set + if category.lower() not in categories and category.lower() not in bypass: + log.error("Post-Process: No valid category detected. Category was %s." % (category)) + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Make sure there are no duplicate categories + if len(categories) != len(set(categories)): + log.error("Duplicate category detected. Category names must be unique.") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # All checks done, now launching the script. + settings = ReadSettings(MP4folder, "autoProcess.ini") + + if shouldConvert: + if output_dir: + settings.output_dir = output_dir + converter = MkvtoMp4(settings, logger=log) + for r, d, f in os.walk(path): + for files in f: + inputfile = os.path.join(r, files) + #DEBUG#print inputfile + #Ignores files under 50MB + if os.path.getsize(inputfile) > 50000000: + if MkvtoMp4(settings, logger=log).validSource(inputfile): + try: + output = converter.process(inputfile) + log.info("Successfully processed %s." % inputfile) + except: + log.exception("File processing failed.") + if converter.output_dir: + path = converter.output_dir + if (category.lower() == categories[0]): + #DEBUG#print "Sickbeard Processing Activated" + autoProcessTV.processEpisode(path, settings, nzb) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() == categories[1]): + #DEBUG#print "CouchPotato Processing Activated" + autoProcessMovie.process(path, settings, nzb, status) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() == categories[2]): + #DEBUG#print "Sonarr Processing Activated" + success = sonarr.processEpisode(path, settings, True) + if success: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_ERROR) + elif (category.lower() == categories[3]): + #DEBUG#print "Radarr Processing Activated" + success = radarr.processMovie(path, settings, True) + if success: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_ERROR) + elif (category.lower() == categories[4]): + #DEBUG#print "Sickrage Processing Activated" + autoProcessTVSR.processEpisode(path, settings, nzb) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() in bypass): + #DEBUG#print "Bypass Further Processing" + sys.exit(POSTPROCESS_NONE) + +else: + log.error("This script can only be called from NZBGet (11.0 or later).") + sys.exit(0) From 068c47edd10e88f369568de78777bae27e1ce895 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:14:14 +0000 Subject: [PATCH 10/51] Create run --- apps/templates/nzbget-mp4/services.d/run | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 apps/templates/nzbget-mp4/services.d/run diff --git a/apps/templates/nzbget-mp4/services.d/run b/apps/templates/nzbget-mp4/services.d/run new file mode 100644 index 0000000..ad735b4 --- /dev/null +++ b/apps/templates/nzbget-mp4/services.d/run @@ -0,0 +1,7 @@ +#!/usr/bin/with-contenv bash + +umask 000 + +exec \ + s6-envuidgid -nB 1000:1000 /app/nzbget/nzbget -s -c /config/nzbget.conf \ + -o OutputMode=log From 53b8e77fb7d5a23ff61a87108ea9c3971a058e2e Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:49:14 +0000 Subject: [PATCH 11/51] changed role name and ports --- apps/nzbget-mp4.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/nzbget-mp4.yml b/apps/nzbget-mp4.yml index 87a8703..b9650cd 100644 --- a/apps/nzbget-mp4.yml +++ b/apps/nzbget-mp4.yml @@ -17,17 +17,17 @@ - name: 'Set Known Facts' set_fact: - pgrole: 'nzbget' - intport: '6789' - extport: '6789' + pgrole: 'nzbget-mp4' + intport: '6790' + extport: '6790' image: 'linuxserver/nzbget' # CORE (MANDATORY) ############################################################ - name: 'Including cron job' - include_tasks: '/opt/coreapps/apps/_core.yml' + include_tasks: '/opt/communityapps/apps/_core.yml' - name: 'Including folders' - include_tasks: '/opt/coreapps/apps/_downloaders.yml' + include_tasks: '/opt/communityapps/apps/_downloaders.yml' - name: Create nzb folder file: 'path={{item}} state=directory mode=0775 owner=1000 group=1000' @@ -36,7 +36,7 @@ # force: yes - name: 'Including plugins' - include_tasks: '/opt/coreapps/apps/_plugins.yml' + include_tasks: '/opt/communityapps/apps/_plugins.yml' - name: 'Checking for existing app data' stat: From bc8627c4a01f79fd05cb72443e24f3b579095b46 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:51:44 +0000 Subject: [PATCH 12/51] Create nzbget-mp4 --- apps/image/nzbget-mp4 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 apps/image/nzbget-mp4 diff --git a/apps/image/nzbget-mp4 b/apps/image/nzbget-mp4 new file mode 100644 index 0000000..d8f3f35 --- /dev/null +++ b/apps/image/nzbget-mp4 @@ -0,0 +1,2 @@ +linuxserver/nzbget +linuxserver/nzbget:testing From 080ce649962fd0e95e771cdec4d3536539cc527d Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:54:20 +0000 Subject: [PATCH 13/51] Create __init__.py --- apps/templates/nzbget-mp4/scripts/rarfile/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 apps/templates/nzbget-mp4/scripts/rarfile/__init__.py diff --git a/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py b/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py @@ -0,0 +1 @@ + From 01be43661f2d50ff15fa262f93f0d75326e8edcf Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:54:59 +0000 Subject: [PATCH 14/51] Create rarfile.py --- .../nzbget-mp4/scripts/rarfile/rarfile.py | 2931 +++++++++++++++++ 1 file changed, 2931 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py diff --git a/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py b/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py new file mode 100644 index 0000000..6a3a647 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py @@ -0,0 +1,2931 @@ +# rarfile.py +# +# Copyright (c) 2005-2016 Marko Kreen +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +r"""RAR archive reader. + +This is Python module for Rar archive reading. The interface +is made as :mod:`zipfile`-like as possible. + +Basic logic: + - Parse archive structure with Python. + - Extract non-compressed files with Python + - Extract compressed files with unrar. + - Optionally write compressed data to temp file to speed up unrar, + otherwise it needs to scan whole archive on each execution. + +Example:: + + import rarfile + + rf = rarfile.RarFile('myarchive.rar') + for f in rf.infolist(): + print f.filename, f.file_size + if f.filename == 'README': + print(rf.read(f)) + +Archive files can also be accessed via file-like object returned +by :meth:`RarFile.open`:: + + import rarfile + + with rarfile.RarFile('archive.rar') as rf: + with rf.open('README') as f: + for ln in f: + print(ln.strip()) + +There are few module-level parameters to tune behaviour, +here they are with defaults, and reason to change it:: + + import rarfile + + # Set to full path of unrar.exe if it is not in PATH + rarfile.UNRAR_TOOL = "unrar" + + # Set to '\\' to be more compatible with old rarfile + rarfile.PATH_SEP = '/' + +For more details, refer to source. + +""" + +from __future__ import division, print_function + +## +## Imports and compat - support both Python 2.x and 3.x +## + +import sys +import os +import errno +import struct + +from struct import pack, unpack, Struct +from binascii import crc32, hexlify +from tempfile import mkstemp +from subprocess import Popen, PIPE, STDOUT +from io import RawIOBase +from hashlib import sha1, sha256 +from hmac import HMAC +from datetime import datetime, timedelta, tzinfo + +# fixed offset timezone, for UTC +try: + from datetime import timezone +except ImportError: + class timezone(tzinfo): + """Compat timezone.""" + __slots__ = ('_ofs', '_name') + _DST = timedelta(0) + + def __init__(self, offset, name): + super(timezone, self).__init__() + self._ofs, self._name = offset, name + + def utcoffset(self, dt): + return self._ofs + + def tzname(self, dt): + return self._name + + def dst(self, dt): + return self._DST + +# only needed for encryped headers +try: + try: + from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf import pbkdf2 + + class AES_CBC_Decrypt(object): + """Decrypt API""" + def __init__(self, key, iv): + ciph = Cipher(algorithms.AES(key), modes.CBC(iv), default_backend()) + self.decrypt = ciph.decryptor().update + + def pbkdf2_sha256(password, salt, iters): + """PBKDF2 with HMAC-SHA256""" + ctx = pbkdf2.PBKDF2HMAC(hashes.SHA256(), 32, salt, iters, default_backend()) + return ctx.derive(password) + + except ImportError: + from Crypto.Cipher import AES + from Crypto.Protocol import KDF + + class AES_CBC_Decrypt(object): + """Decrypt API""" + def __init__(self, key, iv): + self.decrypt = AES.new(key, AES.MODE_CBC, iv).decrypt + + def pbkdf2_sha256(password, salt, iters): + """PBKDF2 with HMAC-SHA256""" + return KDF.PBKDF2(password, salt, 32, iters, hmac_sha256) + + _have_crypto = 1 +except ImportError: + _have_crypto = 0 + +try: + from pyblake2 import blake2s + _have_blake2 = True +except ImportError: + _have_blake2 = False + +# compat with 2.x +if sys.hexversion < 0x3000000: + def rar_crc32(data, prev=0): + """CRC32 with unsigned values. + """ + if (prev > 0) and (prev & 0x80000000): + prev -= (1 << 32) + res = crc32(data, prev) + if res < 0: + res += (1 << 32) + return res + tohex = hexlify + _byte_code = ord +else: # pragma: no cover + def tohex(data): + """Return hex string.""" + return hexlify(data).decode('ascii') + rar_crc32 = crc32 + unicode = str + _byte_code = int # noqa + + +__version__ = '3.0' + +# export only interesting items +__all__ = ['is_rarfile', 'RarInfo', 'RarFile', 'RarExtFile'] + +## +## Module configuration. Can be tuned after importing. +## + +#: default fallback charset +DEFAULT_CHARSET = "windows-1252" + +#: list of encodings to try, with fallback to DEFAULT_CHARSET if none succeed +TRY_ENCODINGS = ('utf8', 'utf-16le') + +#: 'unrar', 'rar' or full path to either one +UNRAR_TOOL = "unrar" + +#: Command line args to use for opening file for reading. +OPEN_ARGS = ('p', '-inul') + +#: Command line args to use for extracting file to disk. +EXTRACT_ARGS = ('x', '-y', '-idq') + +#: args for testrar() +TEST_ARGS = ('t', '-idq') + +# +# Allow use of tool that is not compatible with unrar. +# +# By default use 'bsdtar' which is 'tar' program that +# sits on top of libarchive. +# +# Problems with libarchive RAR backend: +# - Does not support solid archives. +# - Does not support password-protected archives. +# + +ALT_TOOL = 'bsdtar' +ALT_OPEN_ARGS = ('-x', '--to-stdout', '-f') +ALT_EXTRACT_ARGS = ('-x', '-f') +ALT_TEST_ARGS = ('-t', '-f') +ALT_CHECK_ARGS = ('--help',) + +#: whether to speed up decompression by using tmp archive +USE_EXTRACT_HACK = 1 + +#: limit the filesize for tmp archive usage +HACK_SIZE_LIMIT = 20 * 1024 * 1024 + +#: Separator for path name components. RAR internally uses '\\'. +#: Use '/' to be similar with zipfile. +PATH_SEP = '/' + +## +## rar constants +## + +# block types +RAR_BLOCK_MARK = 0x72 # r +RAR_BLOCK_MAIN = 0x73 # s +RAR_BLOCK_FILE = 0x74 # t +RAR_BLOCK_OLD_COMMENT = 0x75 # u +RAR_BLOCK_OLD_EXTRA = 0x76 # v +RAR_BLOCK_OLD_SUB = 0x77 # w +RAR_BLOCK_OLD_RECOVERY = 0x78 # x +RAR_BLOCK_OLD_AUTH = 0x79 # y +RAR_BLOCK_SUB = 0x7a # z +RAR_BLOCK_ENDARC = 0x7b # { + +# flags for RAR_BLOCK_MAIN +RAR_MAIN_VOLUME = 0x0001 +RAR_MAIN_COMMENT = 0x0002 +RAR_MAIN_LOCK = 0x0004 +RAR_MAIN_SOLID = 0x0008 +RAR_MAIN_NEWNUMBERING = 0x0010 +RAR_MAIN_AUTH = 0x0020 +RAR_MAIN_RECOVERY = 0x0040 +RAR_MAIN_PASSWORD = 0x0080 +RAR_MAIN_FIRSTVOLUME = 0x0100 +RAR_MAIN_ENCRYPTVER = 0x0200 + +# flags for RAR_BLOCK_FILE +RAR_FILE_SPLIT_BEFORE = 0x0001 +RAR_FILE_SPLIT_AFTER = 0x0002 +RAR_FILE_PASSWORD = 0x0004 +RAR_FILE_COMMENT = 0x0008 +RAR_FILE_SOLID = 0x0010 +RAR_FILE_DICTMASK = 0x00e0 +RAR_FILE_DICT64 = 0x0000 +RAR_FILE_DICT128 = 0x0020 +RAR_FILE_DICT256 = 0x0040 +RAR_FILE_DICT512 = 0x0060 +RAR_FILE_DICT1024 = 0x0080 +RAR_FILE_DICT2048 = 0x00a0 +RAR_FILE_DICT4096 = 0x00c0 +RAR_FILE_DIRECTORY = 0x00e0 +RAR_FILE_LARGE = 0x0100 +RAR_FILE_UNICODE = 0x0200 +RAR_FILE_SALT = 0x0400 +RAR_FILE_VERSION = 0x0800 +RAR_FILE_EXTTIME = 0x1000 +RAR_FILE_EXTFLAGS = 0x2000 + +# flags for RAR_BLOCK_ENDARC +RAR_ENDARC_NEXT_VOLUME = 0x0001 +RAR_ENDARC_DATACRC = 0x0002 +RAR_ENDARC_REVSPACE = 0x0004 +RAR_ENDARC_VOLNR = 0x0008 + +# flags common to all blocks +RAR_SKIP_IF_UNKNOWN = 0x4000 +RAR_LONG_BLOCK = 0x8000 + +# Host OS types +RAR_OS_MSDOS = 0 +RAR_OS_OS2 = 1 +RAR_OS_WIN32 = 2 +RAR_OS_UNIX = 3 +RAR_OS_MACOS = 4 +RAR_OS_BEOS = 5 + +# Compression methods - '0'..'5' +RAR_M0 = 0x30 +RAR_M1 = 0x31 +RAR_M2 = 0x32 +RAR_M3 = 0x33 +RAR_M4 = 0x34 +RAR_M5 = 0x35 + +# +# RAR5 constants +# + +RAR5_BLOCK_MAIN = 1 +RAR5_BLOCK_FILE = 2 +RAR5_BLOCK_SERVICE = 3 +RAR5_BLOCK_ENCRYPTION = 4 +RAR5_BLOCK_ENDARC = 5 + +RAR5_BLOCK_FLAG_EXTRA_DATA = 0x01 +RAR5_BLOCK_FLAG_DATA_AREA = 0x02 +RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN = 0x04 +RAR5_BLOCK_FLAG_SPLIT_BEFORE = 0x08 +RAR5_BLOCK_FLAG_SPLIT_AFTER = 0x10 +RAR5_BLOCK_FLAG_DEPENDS_PREV = 0x20 +RAR5_BLOCK_FLAG_KEEP_WITH_PARENT = 0x40 + +RAR5_MAIN_FLAG_ISVOL = 0x01 +RAR5_MAIN_FLAG_HAS_VOLNR = 0x02 +RAR5_MAIN_FLAG_SOLID = 0x04 +RAR5_MAIN_FLAG_RECOVERY = 0x08 +RAR5_MAIN_FLAG_LOCKED = 0x10 + +RAR5_FILE_FLAG_ISDIR = 0x01 +RAR5_FILE_FLAG_HAS_MTIME = 0x02 +RAR5_FILE_FLAG_HAS_CRC32 = 0x04 +RAR5_FILE_FLAG_UNKNOWN_SIZE = 0x08 + +RAR5_COMPR_SOLID = 0x40 + +RAR5_ENC_FLAG_HAS_CHECKVAL = 0x01 + +RAR5_ENDARC_FLAG_NEXT_VOL = 0x01 + +RAR5_XFILE_ENCRYPTION = 1 +RAR5_XFILE_HASH = 2 +RAR5_XFILE_TIME = 3 +RAR5_XFILE_VERSION = 4 +RAR5_XFILE_REDIR = 5 +RAR5_XFILE_OWNER = 6 +RAR5_XFILE_SERVICE = 7 + +RAR5_XTIME_UNIXTIME = 0x01 +RAR5_XTIME_HAS_MTIME = 0x02 +RAR5_XTIME_HAS_CTIME = 0x04 +RAR5_XTIME_HAS_ATIME = 0x08 + +RAR5_XENC_CIPHER_AES256 = 0 + +RAR5_XENC_CHECKVAL = 0x01 +RAR5_XENC_TWEAKED = 0x02 + +RAR5_XHASH_BLAKE2SP = 0 + +RAR5_XREDIR_UNIX_SYMLINK = 1 +RAR5_XREDIR_WINDOWS_SYMLINK = 2 +RAR5_XREDIR_WINDOWS_JUNCTION = 3 +RAR5_XREDIR_HARD_LINK = 4 +RAR5_XREDIR_FILE_COPY = 5 + +RAR5_XREDIR_ISDIR = 0x01 + +RAR5_XOWNER_UNAME = 0x01 +RAR5_XOWNER_GNAME = 0x02 +RAR5_XOWNER_UID = 0x04 +RAR5_XOWNER_GID = 0x08 + +RAR5_OS_WINDOWS = 0 +RAR5_OS_UNIX = 1 + +## +## internal constants +## + +RAR_ID = b"Rar!\x1a\x07\x00" +RAR5_ID = b"Rar!\x1a\x07\x01\x00" +ZERO = b'\0' +EMPTY = b'' +UTC = timezone(timedelta(0), 'UTC') +BSIZE = 32 * 1024 + +def _get_rar_version(xfile): + '''Check quickly whether file is rar archive. + ''' + with XFile(xfile) as fd: + buf = fd.read(len(RAR5_ID)) + if buf.startswith(RAR_ID): + return 3 + elif buf.startswith(RAR5_ID): + return 5 + return 0 + +## +## Public interface +## + +def is_rarfile(xfile): + '''Check quickly whether file is rar archive. + ''' + return _get_rar_version(xfile) > 0 + +class Error(Exception): + """Base class for rarfile errors.""" + +class BadRarFile(Error): + """Incorrect data in archive.""" + +class NotRarFile(Error): + """The file is not RAR archive.""" + +class BadRarName(Error): + """Cannot guess multipart name components.""" + +class NoRarEntry(Error): + """File not found in RAR""" + +class PasswordRequired(Error): + """File requires password""" + +class NeedFirstVolume(Error): + """Need to start from first volume.""" + +class NoCrypto(Error): + """Cannot parse encrypted headers - no crypto available.""" + +class RarExecError(Error): + """Problem reported by unrar/rar.""" + +class RarWarning(RarExecError): + """Non-fatal error""" + +class RarFatalError(RarExecError): + """Fatal error""" + +class RarCRCError(RarExecError): + """CRC error during unpacking""" + +class RarLockedArchiveError(RarExecError): + """Must not modify locked archive""" + +class RarWriteError(RarExecError): + """Write error""" + +class RarOpenError(RarExecError): + """Open error""" + +class RarUserError(RarExecError): + """User error""" + +class RarMemoryError(RarExecError): + """Memory error""" + +class RarCreateError(RarExecError): + """Create error""" + +class RarNoFilesError(RarExecError): + """No files that match pattern were found""" + +class RarUserBreak(RarExecError): + """User stop""" + +class RarWrongPassword(RarExecError): + """Incorrect password""" + +class RarUnknownError(RarExecError): + """Unknown exit code""" + +class RarSignalExit(RarExecError): + """Unrar exited with signal""" + +class RarCannotExec(RarExecError): + """Executable not found.""" + + +class RarInfo(object): + r'''An entry in rar archive. + + RAR3 extended timestamps are :class:`datetime.datetime` objects without timezone. + RAR5 extended timestamps are :class:`datetime.datetime` objects with UTC timezone. + + Attributes: + + filename + File name with relative path. + Path separator is '/'. Always unicode string. + + date_time + File modification timestamp. As tuple of (year, month, day, hour, minute, second). + RAR5 allows archives where it is missing, it's None then. + + file_size + Uncompressed size. + + compress_size + Compressed size. + + compress_type + Compression method: one of :data:`RAR_M0` .. :data:`RAR_M5` constants. + + extract_version + Minimal Rar version needed for decompressing. As (major*10 + minor), + so 2.9 is 29. + + RAR3: 10, 20, 29 + + RAR5 does not have such field in archive, it's simply set to 50. + + host_os + Host OS type, one of RAR_OS_* constants. + + RAR3: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`, :data:`RAR_OS_MSDOS`, + :data:`RAR_OS_OS2`, :data:`RAR_OS_BEOS`. + + RAR5: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`. + + mode + File attributes. May be either dos-style or unix-style, depending on host_os. + + mtime + File modification time. Same value as :attr:`date_time` + but as :class:`datetime.datetime` object with extended precision. + + ctime + Optional time field: creation time. As :class:`datetime.datetime` object. + + atime + Optional time field: last access time. As :class:`datetime.datetime` object. + + arctime + Optional time field: archival time. As :class:`datetime.datetime` object. + (RAR3-only) + + CRC + CRC-32 of uncompressed file, unsigned int. + + RAR5: may be None. + + blake2sp_hash + Blake2SP hash over decompressed data. (RAR5-only) + + comment + Optional file comment field. Unicode string. (RAR3-only) + + file_redir + If not None, file is link of some sort. Contains tuple of (type, flags, target). + (RAR5-only) + + Type is one of constants: + + :data:`RAR5_XREDIR_UNIX_SYMLINK` + unix symlink to target. + :data:`RAR5_XREDIR_WINDOWS_SYMLINK` + windows symlink to target. + :data:`RAR5_XREDIR_WINDOWS_JUNCTION` + windows junction. + :data:`RAR5_XREDIR_HARD_LINK` + hard link to target. + :data:`RAR5_XREDIR_FILE_COPY` + current file is copy of another archive entry. + + Flags may contain :data:`RAR5_XREDIR_ISDIR` bit. + + volume + Volume nr, starting from 0. + + volume_file + Volume file name, where file starts. + + ''' + + # zipfile-compatible fields + filename = None + file_size = None + compress_size = None + date_time = None + comment = None + CRC = None + volume = None + orig_filename = None + + # optional extended time fields, datetime() objects. + mtime = None + ctime = None + atime = None + + extract_version = None + mode = None + host_os = None + compress_type = None + + # rar3-only fields + comment = None + arctime = None + + # rar5-only fields + blake2sp_hash = None + file_redir = None + + # internal fields + flags = 0 + type = None + + def isdir(self): + """Returns True if entry is a directory. + """ + if self.type == RAR_BLOCK_FILE: + return (self.flags & RAR_FILE_DIRECTORY) == RAR_FILE_DIRECTORY + return False + + def needs_password(self): + """Returns True if data is stored password-protected. + """ + if self.type == RAR_BLOCK_FILE: + return (self.flags & RAR_FILE_PASSWORD) > 0 + return False + + +class RarFile(object): + '''Parse RAR structure, provide access to files in archive. + ''' + + #: Archive comment. Unicode string or None. + comment = None + + def __init__(self, rarfile, mode="r", charset=None, info_callback=None, + crc_check=True, errors="stop"): + """Open and parse a RAR archive. + + Parameters: + + rarfile + archive file name + mode + only 'r' is supported. + charset + fallback charset to use, if filenames are not already Unicode-enabled. + info_callback + debug callback, gets to see all archive entries. + crc_check + set to False to disable CRC checks + errors + Either "stop" to quietly stop parsing on errors, + or "strict" to raise errors. Default is "stop". + """ + self._rarfile = rarfile + self._charset = charset or DEFAULT_CHARSET + self._info_callback = info_callback + self._crc_check = crc_check + self._password = None + self._file_parser = None + + if errors == "stop": + self._strict = False + elif errors == "strict": + self._strict = True + else: + raise ValueError("Invalid value for 'errors' parameter.") + + if mode != "r": + raise NotImplementedError("RarFile supports only mode=r") + + self._parse() + + def __enter__(self): + return self + + def __exit__(self, typ, value, traceback): + self.close() + + def setpassword(self, password): + '''Sets the password to use when extracting.''' + self._password = password + if self._file_parser: + if self._file_parser.has_header_encryption(): + self._file_parser = None + if not self._file_parser: + self._parse() + else: + self._file_parser.setpassword(self._password) + + def needs_password(self): + '''Returns True if any archive entries require password for extraction.''' + return self._file_parser.needs_password() + + def namelist(self): + '''Return list of filenames in archive.''' + return [f.filename for f in self.infolist()] + + def infolist(self): + '''Return RarInfo objects for all files/directories in archive.''' + return self._file_parser.infolist() + + def volumelist(self): + '''Returns filenames of archive volumes. + + In case of single-volume archive, the list contains + just the name of main archive file. + ''' + return self._file_parser.volumelist() + + def getinfo(self, fname): + '''Return RarInfo for file. + ''' + return self._file_parser.getinfo(fname) + + def open(self, fname, mode='r', psw=None): + '''Returns file-like object (:class:`RarExtFile`), + from where the data can be read. + + The object implements :class:`io.RawIOBase` interface, so it can + be further wrapped with :class:`io.BufferedReader` + and :class:`io.TextIOWrapper`. + + On older Python where io module is not available, it implements + only .read(), .seek(), .tell() and .close() methods. + + The object is seekable, although the seeking is fast only on + uncompressed files, on compressed files the seeking is implemented + by reading ahead and/or restarting the decompression. + + Parameters: + + fname + file name or RarInfo instance. + mode + must be 'r' + psw + password to use for extracting. + ''' + + if mode != 'r': + raise NotImplementedError("RarFile.open() supports only mode=r") + + # entry lookup + inf = self.getinfo(fname) + if inf.isdir(): + raise TypeError("Directory does not have any data: " + inf.filename) + + # check password + if inf.needs_password(): + psw = psw or self._password + if psw is None: + raise PasswordRequired("File %s requires password" % inf.filename) + else: + psw = None + + return self._file_parser.open(inf, psw) + + def read(self, fname, psw=None): + """Return uncompressed data for archive entry. + + For longer files using :meth:`RarFile.open` may be better idea. + + Parameters: + + fname + filename or RarInfo instance + psw + password to use for extracting. + """ + + with self.open(fname, 'r', psw) as f: + return f.read() + + def close(self): + """Release open resources.""" + pass + + def printdir(self): + """Print archive file list to stdout.""" + for f in self.infolist(): + print(f.filename) + + def extract(self, member, path=None, pwd=None): + """Extract single file into current directory. + + Parameters: + + member + filename or :class:`RarInfo` instance + path + optional destination path + pwd + optional password to use + """ + if isinstance(member, RarInfo): + fname = member.filename + else: + fname = member + self._extract([fname], path, pwd) + + def extractall(self, path=None, members=None, pwd=None): + """Extract all files into current directory. + + Parameters: + + path + optional destination path + members + optional filename or :class:`RarInfo` instance list to extract + pwd + optional password to use + """ + fnlist = [] + if members is not None: + for m in members: + if isinstance(m, RarInfo): + fnlist.append(m.filename) + else: + fnlist.append(m) + self._extract(fnlist, path, pwd) + + def testrar(self): + """Let 'unrar' test the archive. + """ + cmd = [UNRAR_TOOL] + list(TEST_ARGS) + add_password_arg(cmd, self._password) + cmd.append('--') + with XTempFile(self._rarfile) as rarfile: + cmd.append(rarfile) + p = custom_popen(cmd) + output = p.communicate()[0] + check_returncode(p, output) + + def strerror(self): + """Return error string if parsing failed, + or None if no problems. + """ + if not self._file_parser: + return "Not a RAR file" + return self._file_parser.strerror() + + ## + ## private methods + ## + + def _parse(self): + ver = _get_rar_version(self._rarfile) + if ver == 3: + p3 = RAR3Parser(self._rarfile, self._password, self._crc_check, + self._charset, self._strict, self._info_callback) + self._file_parser = p3 # noqa + elif ver == 5: + p5 = RAR5Parser(self._rarfile, self._password, self._crc_check, + self._charset, self._strict, self._info_callback) + self._file_parser = p5 # noqa + else: + raise BadRarFile("Not a RAR file") + + self._file_parser.parse() + self.comment = self._file_parser.comment + + # call unrar to extract a file + def _extract(self, fnlist, path=None, psw=None): + cmd = [UNRAR_TOOL] + list(EXTRACT_ARGS) + + # pasoword + psw = psw or self._password + add_password_arg(cmd, psw) + cmd.append('--') + + # rar file + with XTempFile(self._rarfile) as rarfn: + cmd.append(rarfn) + + # file list + for fn in fnlist: + if os.sep != PATH_SEP: + fn = fn.replace(PATH_SEP, os.sep) + cmd.append(fn) + + # destination path + if path is not None: + cmd.append(path + os.sep) + + # call + p = custom_popen(cmd) + output = p.communicate()[0] + check_returncode(p, output) + +# +# File format parsing +# + +class CommonParser(object): + """Shared parser parts.""" + _main = None + _hdrenc_main = None + _needs_password = False + _fd = None + _expect_sig = None + _parse_error = None + _password = None + comment = None + + def __init__(self, rarfile, password, crc_check, charset, strict, info_cb): + self._rarfile = rarfile + self._password = password + self._crc_check = crc_check + self._charset = charset + self._strict = strict + self._info_callback = info_cb + self._info_list = [] + self._info_map = {} + self._vol_list = [] + + def has_header_encryption(self): + """Returns True if headers are encrypted + """ + if self._hdrenc_main: + return True + if self._main: + if self._main.flags & RAR_MAIN_PASSWORD: + return True + return False + + def setpassword(self, psw): + """Set cached password.""" + self._password = psw + + def volumelist(self): + """Volume files""" + return self._vol_list + + def needs_password(self): + """Is password required""" + return self._needs_password + + def strerror(self): + """Last error""" + return self._parse_error + + def infolist(self): + """List of RarInfo records. + """ + return self._info_list + + def getinfo(self, fname): + """Return RarInfo for filename + """ + # accept both ways here + if PATH_SEP == '/': + fname2 = fname.replace("\\", "/") + else: + fname2 = fname.replace("/", "\\") + + try: + return self._info_map[fname] + except KeyError: + try: + return self._info_map[fname2] + except KeyError: + raise NoRarEntry("No such file: %s" % fname) + + # read rar + def parse(self): + """Process file.""" + self._fd = None + try: + self._parse_real() + finally: + if self._fd: + self._fd.close() + self._fd = None + + def _parse_real(self): + fd = XFile(self._rarfile) + self._fd = fd + sig = fd.read(len(self._expect_sig)) + if sig != self._expect_sig: + if isinstance(self._rarfile, (str, unicode)): + raise NotRarFile("Not a Rar archive: {}".format(self._rarfile)) + raise NotRarFile("Not a Rar archive") + + volume = 0 # first vol (.rar) is 0 + more_vols = False + endarc = False + volfile = self._rarfile + self._vol_list = [self._rarfile] + while 1: + if endarc: + h = None # don't read past ENDARC + else: + h = self._parse_header(fd) + if not h: + if more_vols: + volume += 1 + fd.close() + try: + volfile = self._next_volname(volfile) + fd = XFile(volfile) + except IOError: + self._set_error("Cannot open next volume: %s", volfile) + break + self._fd = fd + sig = fd.read(len(self._expect_sig)) + if sig != self._expect_sig: + self._set_error("Invalid volume sig: %s", volfile) + break + more_vols = False + endarc = False + self._vol_list.append(volfile) + continue + break + h.volume = volume + h.volume_file = volfile + + if h.type == RAR_BLOCK_MAIN and not self._main: + self._main = h + if h.flags & RAR_MAIN_NEWNUMBERING: + # RAR 2.x does not set FIRSTVOLUME, + # so check it only if NEWNUMBERING is used + if (h.flags & RAR_MAIN_FIRSTVOLUME) == 0: + raise NeedFirstVolume("Need to start from first volume") + if h.flags & RAR_MAIN_PASSWORD: + self._needs_password = True + if not self._password: + break + elif h.type == RAR_BLOCK_ENDARC: + more_vols = (h.flags & RAR_ENDARC_NEXT_VOLUME) > 0 + endarc = True + elif h.type == RAR_BLOCK_FILE: + # RAR 2.x does not write RAR_BLOCK_ENDARC + if h.flags & RAR_FILE_SPLIT_AFTER: + more_vols = True + # RAR 2.x does not set RAR_MAIN_FIRSTVOLUME + if volume == 0 and h.flags & RAR_FILE_SPLIT_BEFORE: + raise NeedFirstVolume("Need to start from first volume") + + if h.needs_password(): + self._needs_password = True + + # store it + self.process_entry(fd, h) + + if self._info_callback: + self._info_callback(h) + + # go to next header + if h.add_size > 0: + fd.seek(h.data_offset + h.add_size, 0) + + def process_entry(self, fd, item): + """Examine item, add into lookup cache.""" + raise NotImplementedError() + + def _decrypt_header(self, fd): + raise NotImplementedError('_decrypt_header') + + def _parse_block_header(self, fd): + raise NotImplementedError('_parse_block_header') + + def _open_hack(self, inf, psw): + raise NotImplementedError('_open_hack') + + # read single header + def _parse_header(self, fd): + try: + # handle encrypted headers + if (self._main and self._main.flags & RAR_MAIN_PASSWORD) or self._hdrenc_main: + if not self._password: + return + fd = self._decrypt_header(fd) + + # now read actual header + return self._parse_block_header(fd) + except struct.error: + self._set_error('Broken header in RAR file') + return None + + # given current vol name, construct next one + def _next_volname(self, volfile): + if is_filelike(volfile): + raise IOError("Working on single FD") + if self._main.flags & RAR_MAIN_NEWNUMBERING: + return _next_newvol(volfile) + return _next_oldvol(volfile) + + def _set_error(self, msg, *args): + if args: + msg = msg % args + self._parse_error = msg + if self._strict: + raise BadRarFile(msg) + + def open(self, inf, psw): + """Return stream object for file data.""" + + if inf.file_redir: + # cannot leave to unrar as it expects copied file to exist + if inf.file_redir[0] in (RAR5_XREDIR_FILE_COPY, RAR5_XREDIR_HARD_LINK): + inf = self.getinfo(inf.file_redir[2]) + if not inf: + raise BadRarFile('cannot find copied file') + + if inf.flags & RAR_FILE_SPLIT_BEFORE: + raise NeedFirstVolume("Partial file, please start from first volume: " + inf.filename) + + # is temp write usable? + use_hack = 1 + if not self._main: + use_hack = 0 + elif self._main._must_disable_hack(): + use_hack = 0 + elif inf._must_disable_hack(): + use_hack = 0 + elif is_filelike(self._rarfile): + pass + elif inf.file_size > HACK_SIZE_LIMIT: + use_hack = 0 + elif not USE_EXTRACT_HACK: + use_hack = 0 + + # now extract + if inf.compress_type == RAR_M0 and (inf.flags & RAR_FILE_PASSWORD) == 0 and inf.file_redir is None: + return self._open_clear(inf) + elif use_hack: + return self._open_hack(inf, psw) + elif is_filelike(self._rarfile): + return self._open_unrar_membuf(self._rarfile, inf, psw) + else: + return self._open_unrar(self._rarfile, inf, psw) + + def _open_clear(self, inf): + return DirectReader(self, inf) + + def _open_hack_core(self, inf, psw, prefix, suffix): + + size = inf.compress_size + inf.header_size + rf = XFile(inf.volume_file, 0) + rf.seek(inf.header_offset) + + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + + try: + tmpf.write(prefix) + while size > 0: + if size > BSIZE: + buf = rf.read(BSIZE) + else: + buf = rf.read(size) + if not buf: + raise BadRarFile('read failed: ' + inf.filename) + tmpf.write(buf) + size -= len(buf) + tmpf.write(suffix) + tmpf.close() + rf.close() + except: + rf.close() + tmpf.close() + os.unlink(tmpname) + raise + + return self._open_unrar(tmpname, inf, psw, tmpname) + + # write in-memory archive to temp file - needed for solid archives + def _open_unrar_membuf(self, memfile, inf, psw): + tmpname = membuf_tempfile(memfile) + return self._open_unrar(tmpname, inf, psw, tmpname, force_file=True) + + # extract using unrar + def _open_unrar(self, rarfile, inf, psw=None, tmpfile=None, force_file=False): + cmd = [UNRAR_TOOL] + list(OPEN_ARGS) + add_password_arg(cmd, psw) + cmd.append("--") + cmd.append(rarfile) + + # not giving filename avoids encoding related problems + if not tmpfile or force_file: + fn = inf.filename + if PATH_SEP != os.sep: + fn = fn.replace(PATH_SEP, os.sep) + cmd.append(fn) + + # read from unrar pipe + return PipeReader(self, inf, cmd, tmpfile) + +# +# RAR3 format +# + +class Rar3Info(RarInfo): + """RAR3 specific fields.""" + extract_version = 15 + salt = None + add_size = 0 + header_crc = None + header_size = None + header_offset = None + data_offset = None + _md_class = None + _md_expect = None + + # make sure some rar5 fields are always present + file_redir = None + blake2sp_hash = None + + def _must_disable_hack(self): + if self.type == RAR_BLOCK_FILE: + if self.flags & RAR_FILE_PASSWORD: + return True + elif self.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): + return True + elif self.type == RAR_BLOCK_MAIN: + if self.flags & (RAR_MAIN_SOLID | RAR_MAIN_PASSWORD): + return True + return False + + +class RAR3Parser(CommonParser): + """Parse RAR3 file format. + """ + _expect_sig = RAR_ID + _last_aes_key = (None, None, None) # (salt, key, iv) + + def _decrypt_header(self, fd): + if not _have_crypto: + raise NoCrypto('Cannot parse encrypted headers - no crypto') + salt = fd.read(8) + if self._last_aes_key[0] == salt: + key, iv = self._last_aes_key[1:] + else: + key, iv = rar3_s2k(self._password, salt) + self._last_aes_key = (salt, key, iv) + return HeaderDecrypt(fd, key, iv) + + # common header + def _parse_block_header(self, fd): + h = Rar3Info() + h.header_offset = fd.tell() + + # read and parse base header + buf = fd.read(S_BLK_HDR.size) + if not buf: + return None + t = S_BLK_HDR.unpack_from(buf) + h.header_crc, h.type, h.flags, h.header_size = t + + # read full header + if h.header_size > S_BLK_HDR.size: + hdata = buf + fd.read(h.header_size - S_BLK_HDR.size) + else: + hdata = buf + h.data_offset = fd.tell() + + # unexpected EOF? + if len(hdata) != h.header_size: + self._set_error('Unexpected EOF when reading header') + return None + + pos = S_BLK_HDR.size + + # block has data assiciated with it? + if h.flags & RAR_LONG_BLOCK: + h.add_size, pos = load_le32(hdata, pos) + else: + h.add_size = 0 + + # parse interesting ones, decide header boundaries for crc + if h.type == RAR_BLOCK_MARK: + return h + elif h.type == RAR_BLOCK_MAIN: + pos += 6 + if h.flags & RAR_MAIN_ENCRYPTVER: + pos += 1 + crc_pos = pos + if h.flags & RAR_MAIN_COMMENT: + self._parse_subblocks(h, hdata, pos) + elif h.type == RAR_BLOCK_FILE: + pos = self._parse_file_header(h, hdata, pos - 4) + crc_pos = pos + if h.flags & RAR_FILE_COMMENT: + pos = self._parse_subblocks(h, hdata, pos) + elif h.type == RAR_BLOCK_SUB: + pos = self._parse_file_header(h, hdata, pos - 4) + crc_pos = h.header_size + elif h.type == RAR_BLOCK_OLD_AUTH: + pos += 8 + crc_pos = pos + elif h.type == RAR_BLOCK_OLD_EXTRA: + pos += 7 + crc_pos = pos + else: + crc_pos = h.header_size + + # check crc + if h.type == RAR_BLOCK_OLD_SUB: + crcdat = hdata[2:] + fd.read(h.add_size) + else: + crcdat = hdata[2:crc_pos] + + calc_crc = rar_crc32(crcdat) & 0xFFFF + + # return good header + if h.header_crc == calc_crc: + return h + + # header parsing failed. + self._set_error('Header CRC error (%02x): exp=%x got=%x (xlen = %d)', + h.type, h.header_crc, calc_crc, len(crcdat)) + + # instead panicing, send eof + return None + + # read file-specific header + def _parse_file_header(self, h, hdata, pos): + fld = S_FILE_HDR.unpack_from(hdata, pos) + pos += S_FILE_HDR.size + + h.compress_size = fld[0] + h.file_size = fld[1] + h.host_os = fld[2] + h.CRC = fld[3] + h.date_time = parse_dos_time(fld[4]) + h.mtime = to_datetime(h.date_time) + h.extract_version = fld[5] + h.compress_type = fld[6] + name_size = fld[7] + h.mode = fld[8] + + h._md_class = CRC32Context + h._md_expect = h.CRC + + if h.flags & RAR_FILE_LARGE: + h1, pos = load_le32(hdata, pos) + h2, pos = load_le32(hdata, pos) + h.compress_size |= h1 << 32 + h.file_size |= h2 << 32 + h.add_size = h.compress_size + + name, pos = load_bytes(hdata, name_size, pos) + if h.flags & RAR_FILE_UNICODE: + nul = name.find(ZERO) + h.orig_filename = name[:nul] + u = UnicodeFilename(h.orig_filename, name[nul + 1:]) + h.filename = u.decode() + + # if parsing failed fall back to simple name + if u.failed: + h.filename = self._decode(h.orig_filename) + else: + h.orig_filename = name + h.filename = self._decode(name) + + # change separator, if requested + if PATH_SEP != '\\': + h.filename = h.filename.replace('\\', PATH_SEP) + + if h.flags & RAR_FILE_SALT: + h.salt, pos = load_bytes(hdata, 8, pos) + else: + h.salt = None + + # optional extended time stamps + if h.flags & RAR_FILE_EXTTIME: + pos = _parse_ext_time(h, hdata, pos) + else: + h.mtime = h.atime = h.ctime = h.arctime = None + + return pos + + # find old-style comment subblock + def _parse_subblocks(self, h, hdata, pos): + while pos < len(hdata): + # ordinary block header + t = S_BLK_HDR.unpack_from(hdata, pos) + ___scrc, stype, sflags, slen = t + pos_next = pos + slen + pos += S_BLK_HDR.size + + # corrupt header + if pos_next < pos: + break + + # followed by block-specific header + if stype == RAR_BLOCK_OLD_COMMENT and pos + S_COMMENT_HDR.size <= pos_next: + declen, ver, meth, crc = S_COMMENT_HDR.unpack_from(hdata, pos) + pos += S_COMMENT_HDR.size + data = hdata[pos : pos_next] + cmt = rar3_decompress(ver, meth, data, declen, sflags, + crc, self._password) + if not self._crc_check: + h.comment = self._decode_comment(cmt) + elif rar_crc32(cmt) & 0xFFFF == crc: + h.comment = self._decode_comment(cmt) + + pos = pos_next + return pos + + def _read_comment_v3(self, inf, psw=None): + + # read data + with XFile(inf.volume_file) as rf: + rf.seek(inf.data_offset) + data = rf.read(inf.compress_size) + + # decompress + cmt = rar3_decompress(inf.extract_version, inf.compress_type, data, + inf.file_size, inf.flags, inf.CRC, psw, inf.salt) + + # check crc + if self._crc_check: + crc = rar_crc32(cmt) + if crc != inf.CRC: + return None + + return self._decode_comment(cmt) + + def _decode(self, val): + for c in TRY_ENCODINGS: + try: + return val.decode(c) + except UnicodeError: + pass + return val.decode(self._charset, 'replace') + + def _decode_comment(self, val): + return self._decode(val) + + def process_entry(self, fd, item): + if item.type == RAR_BLOCK_FILE: + # use only first part + if (item.flags & RAR_FILE_SPLIT_BEFORE) == 0: + self._info_map[item.filename] = item + self._info_list.append(item) + elif len(self._info_list) > 0: + # final crc is in last block + old = self._info_list[-1] + old.CRC = item.CRC + old._md_expect = item._md_expect + old.compress_size += item.compress_size + + # parse new-style comment + if item.type == RAR_BLOCK_SUB and item.filename == 'CMT': + if item.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): + pass + elif item.flags & RAR_FILE_SOLID: + # file comment + cmt = self._read_comment_v3(item, self._password) + if len(self._info_list) > 0: + old = self._info_list[-1] + old.comment = cmt + else: + # archive comment + cmt = self._read_comment_v3(item, self._password) + self.comment = cmt + + if item.type == RAR_BLOCK_MAIN: + if item.flags & RAR_MAIN_COMMENT: + self.comment = item.comment + if item.flags & RAR_MAIN_PASSWORD: + self._needs_password = True + + # put file compressed data into temporary .rar archive, and run + # unrar on that, thus avoiding unrar going over whole archive + def _open_hack(self, inf, psw): + # create main header: crc, type, flags, size, res1, res2 + prefix = RAR_ID + S_BLK_HDR.pack(0x90CF, 0x73, 0, 13) + ZERO * (2 + 4) + return self._open_hack_core(inf, psw, prefix, EMPTY) + +# +# RAR5 format +# + +class Rar5Info(RarInfo): + """Shared fields for RAR5 records. + """ + extract_version = 50 + header_crc = None + header_size = None + header_offset = None + data_offset = None + + # type=all + block_type = None + block_flags = None + add_size = 0 + block_extra_size = 0 + + # type=MAIN + volume_number = None + _md_class = None + _md_expect = None + + def _must_disable_hack(self): + return False + + +class Rar5BaseFile(Rar5Info): + """Shared sturct for file & service record. + """ + type = -1 + file_flags = None + file_encryption = (0, 0, 0, EMPTY, EMPTY, EMPTY) + file_compress_flags = None + file_redir = None + file_owner = None + file_version = None + blake2sp_hash = None + + def _must_disable_hack(self): + if self.flags & RAR_FILE_PASSWORD: + return True + if self.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): + return True + if self.file_compress_flags & RAR5_COMPR_SOLID: + return True + if self.file_redir: + return True + return False + + +class Rar5FileInfo(Rar5BaseFile): + """RAR5 file record. + """ + type = RAR_BLOCK_FILE + + +class Rar5ServiceInfo(Rar5BaseFile): + """RAR5 service record. + """ + type = RAR_BLOCK_SUB + + +class Rar5MainInfo(Rar5Info): + """RAR5 archive main record. + """ + type = RAR_BLOCK_MAIN + main_flags = None + main_volume_number = None + + def _must_disable_hack(self): + if self.main_flags & RAR5_MAIN_FLAG_SOLID: + return True + return False + + +class Rar5EncryptionInfo(Rar5Info): + """RAR5 archive header encryption record. + """ + type = RAR5_BLOCK_ENCRYPTION + encryption_algo = None + encryption_flags = None + encryption_kdf_count = None + encryption_salt = None + encryption_check_value = None + + def needs_password(self): + return True + + +class Rar5EndArcInfo(Rar5Info): + """RAR5 end of archive record. + """ + type = RAR_BLOCK_ENDARC + endarc_flags = None + + +class RAR5Parser(CommonParser): + """Parse RAR5 format. + """ + _expect_sig = RAR5_ID + _hdrenc_main = None + + # AES encrypted headers + _last_aes256_key = (-1, None, None) # (kdf_count, salt, key) + + def _gen_key(self, kdf_count, salt): + if self._last_aes256_key[:2] == (kdf_count, salt): + return self._last_aes256_key[2] + if kdf_count > 24: + raise BadRarFile('Too large kdf_count') + psw = self._password + if isinstance(psw, unicode): + psw = psw.encode('utf8') + key = pbkdf2_sha256(psw, salt, 1 << kdf_count) + self._last_aes256_key = (kdf_count, salt, key) + return key + + def _decrypt_header(self, fd): + if not _have_crypto: + raise NoCrypto('Cannot parse encrypted headers - no crypto') + h = self._hdrenc_main + key = self._gen_key(h.encryption_kdf_count, h.encryption_salt) + iv = fd.read(16) + return HeaderDecrypt(fd, key, iv) + + # common header + def _parse_block_header(self, fd): + header_offset = fd.tell() + + preload = 4 + 3 + start_bytes = fd.read(preload) + header_crc, pos = load_le32(start_bytes, 0) + hdrlen, pos = load_vint(start_bytes, pos) + if hdrlen > 2 * 1024 * 1024: + return None + header_size = pos + hdrlen + + # read full header, check for EOF + hdata = start_bytes + fd.read(header_size - len(start_bytes)) + if len(hdata) != header_size: + self._set_error('Unexpected EOF when reading header') + return None + data_offset = fd.tell() + + calc_crc = rar_crc32(memoryview(hdata)[4:]) + if header_crc != calc_crc: + # header parsing failed. + self._set_error('Header CRC error: exp=%x got=%x (xlen = %d)', + header_crc, calc_crc, len(hdata)) + return None + + block_type, pos = load_vint(hdata, pos) + + if block_type == RAR5_BLOCK_MAIN: + h, pos = self._parse_block_common(Rar5MainInfo(), hdata) + h = self._parse_main_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_FILE: + h, pos = self._parse_block_common(Rar5FileInfo(), hdata) + h = self._parse_file_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_SERVICE: + h, pos = self._parse_block_common(Rar5ServiceInfo(), hdata) + h = self._parse_file_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_ENCRYPTION: + h, pos = self._parse_block_common(Rar5EncryptionInfo(), hdata) + h = self._parse_encryption_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_ENDARC: + h, pos = self._parse_block_common(Rar5EndArcInfo(), hdata) + h = self._parse_endarc_block(h, hdata, pos) + else: + h = None + if h: + h.header_offset = header_offset + h.data_offset = data_offset + return h + + def _parse_block_common(self, h, hdata): + h.header_crc, pos = load_le32(hdata, 0) + hdrlen, pos = load_vint(hdata, pos) + h.header_size = hdrlen + pos + h.block_type, pos = load_vint(hdata, pos) + h.block_flags, pos = load_vint(hdata, pos) + + if h.block_flags & RAR5_BLOCK_FLAG_EXTRA_DATA: + h.block_extra_size, pos = load_vint(hdata, pos) + if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: + h.add_size, pos = load_vint(hdata, pos) + + h.compress_size = h.add_size + + if h.block_flags & RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN: + h.flags |= RAR_SKIP_IF_UNKNOWN + if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: + h.flags |= RAR_LONG_BLOCK + return h, pos + + def _parse_main_block(self, h, hdata, pos): + h.main_flags, pos = load_vint(hdata, pos) + if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR: + h.main_volume_number = load_vint(hdata, pos) + + h.flags |= RAR_MAIN_NEWNUMBERING + if h.main_flags & RAR5_MAIN_FLAG_SOLID: + h.flags |= RAR_MAIN_SOLID + if h.main_flags & RAR5_MAIN_FLAG_ISVOL: + h.flags |= RAR_MAIN_VOLUME + if h.main_flags & RAR5_MAIN_FLAG_RECOVERY: + h.flags |= RAR_MAIN_RECOVERY + if self._hdrenc_main: + h.flags |= RAR_MAIN_PASSWORD + if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR == 0: + h.flags |= RAR_MAIN_FIRSTVOLUME + + return h + + def _parse_file_block(self, h, hdata, pos): + h.file_flags, pos = load_vint(hdata, pos) + h.file_size, pos = load_vint(hdata, pos) + h.mode, pos = load_vint(hdata, pos) + + if h.file_flags & RAR5_FILE_FLAG_HAS_MTIME: + h.mtime, pos = load_unixtime(hdata, pos) + h.date_time = h.mtime.timetuple()[:6] + if h.file_flags & RAR5_FILE_FLAG_HAS_CRC32: + h.CRC, pos = load_le32(hdata, pos) + h._md_class = CRC32Context + h._md_expect = h.CRC + + h.file_compress_flags, pos = load_vint(hdata, pos) + h.file_host_os, pos = load_vint(hdata, pos) + h.orig_filename, pos = load_vstr(hdata, pos) + h.filename = h.orig_filename.decode('utf8', 'replace') + + # use compatible values + if h.file_host_os == RAR5_OS_WINDOWS: + h.host_os = RAR_OS_WIN32 + else: + h.host_os = RAR_OS_UNIX + h.compress_type = RAR_M0 + ((h.file_compress_flags >> 7) & 7) + + if h.block_extra_size: + # allow 1 byte of garbage + while pos < len(hdata) - 1: + xsize, pos = load_vint(hdata, pos) + xdata, pos = load_bytes(hdata, xsize, pos) + self._process_file_extra(h, xdata) + + if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE: + h.flags |= RAR_FILE_SPLIT_BEFORE + if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_AFTER: + h.flags |= RAR_FILE_SPLIT_AFTER + if h.file_flags & RAR5_FILE_FLAG_ISDIR: + h.flags |= RAR_FILE_DIRECTORY + if h.file_compress_flags & RAR5_COMPR_SOLID: + h.flags |= RAR_FILE_SOLID + + return h + + def _parse_endarc_block(self, h, hdata, pos): + h.endarc_flags, pos = load_vint(hdata, pos) + if h.endarc_flags & RAR5_ENDARC_FLAG_NEXT_VOL: + h.flags |= RAR_ENDARC_NEXT_VOLUME + return h + + def _parse_encryption_block(self, h, hdata, pos): + h.encryption_algo, pos = load_vint(hdata, pos) + h.encryption_flags, pos = load_vint(hdata, pos) + h.encryption_kdf_count, pos = load_byte(hdata, pos) + h.encryption_salt, pos = load_bytes(hdata, 16, pos) + if h.encryption_flags & RAR5_ENC_FLAG_HAS_CHECKVAL: + h.encryption_check_value = load_bytes(hdata, 12, pos) + if h.encryption_algo != RAR5_XENC_CIPHER_AES256: + raise BadRarFile('Unsupported header encryption cipher') + self._hdrenc_main = h + return h + + # file extra record + def _process_file_extra(self, h, xdata): + xtype, pos = load_vint(xdata, 0) + if xtype == RAR5_XFILE_TIME: + self._parse_file_xtime(h, xdata, pos) + elif xtype == RAR5_XFILE_ENCRYPTION: + self._parse_file_encryption(h, xdata, pos) + elif xtype == RAR5_XFILE_HASH: + self._parse_file_hash(h, xdata, pos) + elif xtype == RAR5_XFILE_VERSION: + self._parse_file_version(h, xdata, pos) + elif xtype == RAR5_XFILE_REDIR: + self._parse_file_redir(h, xdata, pos) + elif xtype == RAR5_XFILE_OWNER: + self._parse_file_owner(h, xdata, pos) + elif xtype == RAR5_XFILE_SERVICE: + pass + else: + pass + + # extra block for file time record + def _parse_file_xtime(self, h, xdata, pos): + tflags, pos = load_vint(xdata, pos) + ldr = load_windowstime + if tflags & RAR5_XTIME_UNIXTIME: + ldr = load_unixtime + if tflags & RAR5_XTIME_HAS_MTIME: + h.mtime, pos = ldr(xdata, pos) + h.date_time = h.mtime.timetuple()[:6] + if tflags & RAR5_XTIME_HAS_CTIME: + h.ctime, pos = ldr(xdata, pos) + if tflags & RAR5_XTIME_HAS_ATIME: + h.atime, pos = ldr(xdata, pos) + + # just remember encryption info + def _parse_file_encryption(self, h, xdata, pos): + algo, pos = load_vint(xdata, pos) + flags, pos = load_vint(xdata, pos) + kdf_count, pos = load_byte(xdata, pos) + salt, pos = load_bytes(xdata, 16, pos) + iv, pos = load_bytes(xdata, 16, pos) + checkval = None + if flags & RAR5_XENC_CHECKVAL: + checkval, pos = load_bytes(xdata, 12, pos) + if flags & RAR5_XENC_TWEAKED: + h._md_expect = None + h._md_class = NoHashContext + + h.file_encryption = (algo, flags, kdf_count, salt, iv, checkval) + h.flags |= RAR_FILE_PASSWORD + + def _parse_file_hash(self, h, xdata, pos): + hash_type, pos = load_vint(xdata, pos) + if hash_type == RAR5_XHASH_BLAKE2SP: + h.blake2sp_hash, pos = load_bytes(xdata, 32, pos) + if _have_blake2 and (h.file_encryption[1] & RAR5_XENC_TWEAKED) == 0: + h._md_class = Blake2SP + h._md_expect = h.blake2sp_hash + + def _parse_file_version(self, h, xdata, pos): + flags, pos = load_vint(xdata, pos) + version, pos = load_vint(xdata, pos) + h.file_version = (flags, version) + + def _parse_file_redir(self, h, xdata, pos): + redir_type, pos = load_vint(xdata, pos) + redir_flags, pos = load_vint(xdata, pos) + redir_name, pos = load_vstr(xdata, pos) + redir_name = redir_name.decode('utf8', 'replace') + h.file_redir = (redir_type, redir_flags, redir_name) + + def _parse_file_owner(self, h, xdata, pos): + user_name = group_name = user_id = group_id = None + + flags, pos = load_vint(xdata, pos) + if flags & RAR5_XOWNER_UNAME: + user_name, pos = load_vstr(xdata, pos) + if flags & RAR5_XOWNER_GNAME: + group_name, pos = load_vstr(xdata, pos) + if flags & RAR5_XOWNER_UID: + user_id, pos = load_vint(xdata, pos) + if flags & RAR5_XOWNER_GID: + group_id, pos = load_vint(xdata, pos) + + h.file_owner = (user_name, group_name, user_id, group_id) + + def process_entry(self, fd, item): + if item.block_type == RAR5_BLOCK_FILE: + # use only first part + if (item.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE) == 0: + self._info_map[item.filename] = item + self._info_list.append(item) + elif len(self._info_list) > 0: + # final crc is in last block + old = self._info_list[-1] + old.CRC = item.CRC + old._md_expect = item._md_expect + old.blake2sp_hash = item.blake2sp_hash + old.compress_size += item.compress_size + elif item.block_type == RAR5_BLOCK_SERVICE: + if item.filename == 'CMT': + self._load_comment(fd, item) + + def _load_comment(self, fd, item): + if item.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): + return None + if item.compress_type != RAR_M0: + return None + + if item.flags & RAR_FILE_PASSWORD: + algo, ___flags, kdf_count, salt, iv, ___checkval = item.file_encryption + if algo != RAR5_XENC_CIPHER_AES256: + return None + key = self._gen_key(kdf_count, salt) + f = HeaderDecrypt(fd, key, iv) + cmt = f.read(item.file_size) + else: + # archive comment + with self._open_clear(item) as cmtstream: + cmt = cmtstream.read() + + # rar bug? - appends zero to comment + cmt = cmt.split(ZERO, 1)[0] + self.comment = cmt.decode('utf8') + + def _open_hack(self, inf, psw): + # len, type, blk_flags, flags + main_hdr = b'\x03\x01\x00\x00' + endarc_hdr = b'\x03\x05\x00\x00' + main_hdr = S_LONG.pack(rar_crc32(main_hdr)) + main_hdr + endarc_hdr = S_LONG.pack(rar_crc32(endarc_hdr)) + endarc_hdr + return self._open_hack_core(inf, psw, RAR5_ID + main_hdr, endarc_hdr) + +## +## Utility classes +## + +class UnicodeFilename(object): + """Handle RAR3 unicode filename decompression. + """ + def __init__(self, name, encdata): + self.std_name = bytearray(name) + self.encdata = bytearray(encdata) + self.pos = self.encpos = 0 + self.buf = bytearray() + self.failed = 0 + + def enc_byte(self): + """Copy encoded byte.""" + try: + c = self.encdata[self.encpos] + self.encpos += 1 + return c + except IndexError: + self.failed = 1 + return 0 + + def std_byte(self): + """Copy byte from 8-bit representation.""" + try: + return self.std_name[self.pos] + except IndexError: + self.failed = 1 + return ord('?') + + def put(self, lo, hi): + """Copy 16-bit value to result.""" + self.buf.append(lo) + self.buf.append(hi) + self.pos += 1 + + def decode(self): + """Decompress compressed UTF16 value.""" + hi = self.enc_byte() + flagbits = 0 + while self.encpos < len(self.encdata): + if flagbits == 0: + flags = self.enc_byte() + flagbits = 8 + flagbits -= 2 + t = (flags >> flagbits) & 3 + if t == 0: + self.put(self.enc_byte(), 0) + elif t == 1: + self.put(self.enc_byte(), hi) + elif t == 2: + self.put(self.enc_byte(), self.enc_byte()) + else: + n = self.enc_byte() + if n & 0x80: + c = self.enc_byte() + for _ in range((n & 0x7f) + 2): + lo = (self.std_byte() + c) & 0xFF + self.put(lo, hi) + else: + for _ in range(n + 2): + self.put(self.std_byte(), 0) + return self.buf.decode("utf-16le", "replace") + + +class RarExtFile(RawIOBase): + """Base class for file-like object that :meth:`RarFile.open` returns. + + Provides public methods and common crc checking. + + Behaviour: + - no short reads - .read() and .readinfo() read as much as requested. + - no internal buffer, use io.BufferedReader for that. + """ + + #: Filename of the archive entry + name = None + + def __init__(self, parser, inf): + super(RarExtFile, self).__init__() + + # standard io.* properties + self.name = inf.filename + self.mode = 'rb' + + self._parser = parser + self._inf = inf + self._fd = None + self._remain = 0 + self._returncode = 0 + + self._md_context = None + + self._open() + + def _open(self): + if self._fd: + self._fd.close() + md_class = self._inf._md_class or NoHashContext + self._md_context = md_class() + self._fd = None + self._remain = self._inf.file_size + + def read(self, cnt=None): + """Read all or specified amount of data from archive entry.""" + + # sanitize cnt + if cnt is None or cnt < 0: + cnt = self._remain + elif cnt > self._remain: + cnt = self._remain + if cnt == 0: + return EMPTY + + # actual read + data = self._read(cnt) + if data: + self._md_context.update(data) + self._remain -= len(data) + if len(data) != cnt: + raise BadRarFile("Failed the read enough data") + + # done? + if not data or self._remain == 0: + # self.close() + self._check() + return data + + def _check(self): + """Check final CRC.""" + final = self._md_context.digest() + exp = self._inf._md_expect + if exp is None: + return + if final is None: + return + if self._returncode: + check_returncode(self, '') + if self._remain != 0: + raise BadRarFile("Failed the read enough data") + if final != exp: + raise BadRarFile("Corrupt file - CRC check failed: %s - exp=%r got=%r" % ( + self._inf.filename, exp, final)) + + def _read(self, cnt): + """Actual read that gets sanitized cnt.""" + + def close(self): + """Close open resources.""" + + super(RarExtFile, self).close() + + if self._fd: + self._fd.close() + self._fd = None + + def __del__(self): + """Hook delete to make sure tempfile is removed.""" + self.close() + + def readinto(self, buf): + """Zero-copy read directly into buffer. + + Returns bytes read. + """ + raise NotImplementedError('readinto') + + def tell(self): + """Return current reading position in uncompressed data.""" + return self._inf.file_size - self._remain + + def seek(self, ofs, whence=0): + """Seek in data. + + On uncompressed files, the seeking works by actual + seeks so it's fast. On compresses files its slow + - forward seeking happends by reading ahead, + backwards by re-opening and decompressing from the start. + """ + + # disable crc check when seeking + self._md_context = NoHashContext() + + fsize = self._inf.file_size + cur_ofs = self.tell() + + if whence == 0: # seek from beginning of file + new_ofs = ofs + elif whence == 1: # seek from current position + new_ofs = cur_ofs + ofs + elif whence == 2: # seek from end of file + new_ofs = fsize + ofs + else: + raise ValueError('Invalid value for whence') + + # sanity check + if new_ofs < 0: + new_ofs = 0 + elif new_ofs > fsize: + new_ofs = fsize + + # do the actual seek + if new_ofs >= cur_ofs: + self._skip(new_ofs - cur_ofs) + else: + # reopen and seek + self._open() + self._skip(new_ofs) + return self.tell() + + def _skip(self, cnt): + """Read and discard data""" + while cnt > 0: + if cnt > 8192: + buf = self.read(8192) + else: + buf = self.read(cnt) + if not buf: + break + cnt -= len(buf) + + def readable(self): + """Returns True""" + return True + + def writable(self): + """Returns False. + + Writing is not supported.""" + return False + + def seekable(self): + """Returns True. + + Seeking is supported, although it's slow on compressed files. + """ + return True + + def readall(self): + """Read all remaining data""" + # avoid RawIOBase default impl + return self.read() + + +class PipeReader(RarExtFile): + """Read data from pipe, handle tempfile cleanup.""" + + def __init__(self, rf, inf, cmd, tempfile=None): + self._cmd = cmd + self._proc = None + self._tempfile = tempfile + super(PipeReader, self).__init__(rf, inf) + + def _close_proc(self): + if not self._proc: + return + if self._proc.stdout: + self._proc.stdout.close() + if self._proc.stdin: + self._proc.stdin.close() + if self._proc.stderr: + self._proc.stderr.close() + self._proc.wait() + self._returncode = self._proc.returncode + self._proc = None + + def _open(self): + super(PipeReader, self)._open() + + # stop old process + self._close_proc() + + # launch new process + self._returncode = 0 + self._proc = custom_popen(self._cmd) + self._fd = self._proc.stdout + + # avoid situation where unrar waits on stdin + if self._proc.stdin: + self._proc.stdin.close() + + def _read(self, cnt): + """Read from pipe.""" + + # normal read is usually enough + data = self._fd.read(cnt) + if len(data) == cnt or not data: + return data + + # short read, try looping + buf = [data] + cnt -= len(data) + while cnt > 0: + data = self._fd.read(cnt) + if not data: + break + cnt -= len(data) + buf.append(data) + return EMPTY.join(buf) + + def close(self): + """Close open resources.""" + + self._close_proc() + super(PipeReader, self).close() + + if self._tempfile: + try: + os.unlink(self._tempfile) + except OSError: + pass + self._tempfile = None + + def readinto(self, buf): + """Zero-copy read directly into buffer.""" + cnt = len(buf) + if cnt > self._remain: + cnt = self._remain + vbuf = memoryview(buf) + res = got = 0 + while got < cnt: + res = self._fd.readinto(vbuf[got : cnt]) + if not res: + break + self._md_context.update(vbuf[got : got + res]) + self._remain -= res + got += res + return got + + +class DirectReader(RarExtFile): + """Read uncompressed data directly from archive. + """ + _cur = None + _cur_avail = None + _volfile = None + + def _open(self): + super(DirectReader, self)._open() + + self._volfile = self._inf.volume_file + self._fd = XFile(self._volfile, 0) + self._fd.seek(self._inf.header_offset, 0) + self._cur = self._parser._parse_header(self._fd) + self._cur_avail = self._cur.add_size + + def _skip(self, cnt): + """RAR Seek, skipping through rar files to get to correct position + """ + + while cnt > 0: + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # fd is in read pos, do the read + if cnt > self._cur_avail: + cnt -= self._cur_avail + self._remain -= self._cur_avail + self._cur_avail = 0 + else: + self._fd.seek(cnt, 1) + self._cur_avail -= cnt + self._remain -= cnt + cnt = 0 + + def _read(self, cnt): + """Read from potentially multi-volume archive.""" + + buf = [] + while cnt > 0: + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # fd is in read pos, do the read + if cnt > self._cur_avail: + data = self._fd.read(self._cur_avail) + else: + data = self._fd.read(cnt) + if not data: + break + + # got some data + cnt -= len(data) + self._cur_avail -= len(data) + buf.append(data) + + if len(buf) == 1: + return buf[0] + return EMPTY.join(buf) + + def _open_next(self): + """Proceed to next volume.""" + + # is the file split over archives? + if (self._cur.flags & RAR_FILE_SPLIT_AFTER) == 0: + return False + + if self._fd: + self._fd.close() + self._fd = None + + # open next part + self._volfile = self._parser._next_volname(self._volfile) + fd = open(self._volfile, "rb", 0) + self._fd = fd + sig = fd.read(len(self._parser._expect_sig)) + if sig != self._parser._expect_sig: + raise BadRarFile("Invalid signature") + + # loop until first file header + while 1: + cur = self._parser._parse_header(fd) + if not cur: + raise BadRarFile("Unexpected EOF") + if cur.type in (RAR_BLOCK_MARK, RAR_BLOCK_MAIN): + if cur.add_size: + fd.seek(cur.add_size, 1) + continue + if cur.orig_filename != self._inf.orig_filename: + raise BadRarFile("Did not found file entry") + self._cur = cur + self._cur_avail = cur.add_size + return True + + def readinto(self, buf): + """Zero-copy read directly into buffer.""" + got = 0 + vbuf = memoryview(buf) + while got < len(buf): + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # length for next read + cnt = len(buf) - got + if cnt > self._cur_avail: + cnt = self._cur_avail + + # read into temp view + res = self._fd.readinto(vbuf[got : got + cnt]) + if not res: + break + self._md_context.update(vbuf[got : got + res]) + self._cur_avail -= res + self._remain -= res + got += res + return got + + +class HeaderDecrypt(object): + """File-like object that decrypts from another file""" + def __init__(self, f, key, iv): + self.f = f + self.ciph = AES_CBC_Decrypt(key, iv) + self.buf = EMPTY + + def tell(self): + """Current file pos - works only on block boundaries.""" + return self.f.tell() + + def read(self, cnt=None): + """Read and decrypt.""" + if cnt > 8 * 1024: + raise BadRarFile('Bad count to header decrypt - wrong password?') + + # consume old data + if cnt <= len(self.buf): + res = self.buf[:cnt] + self.buf = self.buf[cnt:] + return res + res = self.buf + self.buf = EMPTY + cnt -= len(res) + + # decrypt new data + blklen = 16 + while cnt > 0: + enc = self.f.read(blklen) + if len(enc) < blklen: + break + dec = self.ciph.decrypt(enc) + if cnt >= len(dec): + res += dec + cnt -= len(dec) + else: + res += dec[:cnt] + self.buf = dec[cnt:] + cnt = 0 + + return res + + +# handle (filename|filelike) object +class XFile(object): + """Input may be filename or file object. + """ + __slots__ = ('_fd', '_need_close') + + def __init__(self, xfile, bufsize=1024): + if is_filelike(xfile): + self._need_close = False + self._fd = xfile + self._fd.seek(0) + else: + self._need_close = True + self._fd = open(xfile, 'rb', bufsize) + + def read(self, n=None): + """Read from file.""" + return self._fd.read(n) + + def tell(self): + """Return file pos.""" + return self._fd.tell() + + def seek(self, ofs, whence=0): + """Move file pos.""" + return self._fd.seek(ofs, whence) + + def readinto(self, dst): + """Read into buffer.""" + return self._fd.readinto(dst) + + def close(self): + """Close file object.""" + if self._need_close: + self._fd.close() + + def __enter__(self): + return self + + def __exit__(self, typ, val, tb): + self.close() + + +class NoHashContext(object): + """No-op hash function.""" + def __init__(self, data=None): + """Initialize""" + def update(self, data): + """Update data""" + def digest(self): + """Final hash""" + def hexdigest(self): + """Hexadecimal digest.""" + + +class CRC32Context(object): + """Hash context that uses CRC32.""" + __slots__ = ['_crc'] + + def __init__(self, data=None): + self._crc = 0 + if data: + self.update(data) + + def update(self, data): + """Process data.""" + self._crc = rar_crc32(data, self._crc) + + def digest(self): + """Final hash.""" + return self._crc + + def hexdigest(self): + """Hexadecimal digest.""" + return '%08x' % self.digest() + + +class Blake2SP(object): + """Blake2sp hash context. + """ + __slots__ = ['_thread', '_buf', '_cur', '_digest'] + digest_size = 32 + block_size = 64 + parallelism = 8 + + def __init__(self, data=None): + self._buf = b'' + self._cur = 0 + self._digest = None + self._thread = [] + + for i in range(self.parallelism): + ctx = self._blake2s(i, 0, i == (self.parallelism - 1)) + self._thread.append(ctx) + + if data: + self.update(data) + + def _blake2s(self, ofs, depth, is_last): + return blake2s(node_offset=ofs, node_depth=depth, last_node=is_last, + depth=2, inner_size=32, fanout=self.parallelism) + + def _add_block(self, blk): + self._thread[self._cur].update(blk) + self._cur = (self._cur + 1) % self.parallelism + + def update(self, data): + """Hash data. + """ + view = memoryview(data) + bs = self.block_size + if self._buf: + need = bs - len(self._buf) + if len(view) < need: + self._buf += view.tobytes() + return + self._add_block(self._buf + view[:need].tobytes()) + view = view[need:] + while len(view) >= bs: + self._add_block(view[:bs]) + view = view[bs:] + self._buf = view.tobytes() + + def digest(self): + """Return final digest value. + """ + if self._digest is None: + if self._buf: + self._add_block(self._buf) + self._buf = EMPTY + ctx = self._blake2s(0, 1, True) + for t in self._thread: + ctx.update(t.digest()) + self._digest = ctx.digest() + return self._digest + + def hexdigest(self): + """Hexadecimal digest.""" + return tohex(self.digest()) + +## +## Utility functions +## + +S_LONG = Struct(' len(buf): + raise BadRarFile('cannot load byte') + return S_BYTE.unpack_from(buf, pos)[0], end + +def load_le32(buf, pos): + """Load little-endian 32-bit integer""" + end = pos + 4 + if end > len(buf): + raise BadRarFile('cannot load le32') + return S_LONG.unpack_from(buf, pos)[0], pos + 4 + +def load_bytes(buf, num, pos): + """Load sequence of bytes""" + end = pos + num + if end > len(buf): + raise BadRarFile('cannot load bytes') + return buf[pos : end], end + +def load_vstr(buf, pos): + """Load bytes prefixed by vint length""" + slen, pos = load_vint(buf, pos) + return load_bytes(buf, slen, pos) + +def load_dostime(buf, pos): + """Load LE32 dos timestamp""" + stamp, pos = load_le32(buf, pos) + tup = parse_dos_time(stamp) + return to_datetime(tup), pos + +def load_unixtime(buf, pos): + """Load LE32 unix timestamp""" + secs, pos = load_le32(buf, pos) + dt = datetime.fromtimestamp(secs, UTC) + return dt, pos + +def load_windowstime(buf, pos): + """Load LE64 windows timestamp""" + # unix epoch (1970) in seconds from windows epoch (1601) + unix_epoch = 11644473600 + val1, pos = load_le32(buf, pos) + val2, pos = load_le32(buf, pos) + secs, n1secs = divmod((val2 << 32) | val1, 10000000) + dt = datetime.fromtimestamp(secs - unix_epoch, UTC) + dt = dt.replace(microsecond=n1secs // 10) + return dt, pos + +# new-style next volume +def _next_newvol(volfile): + i = len(volfile) - 1 + while i >= 0: + if volfile[i] >= '0' and volfile[i] <= '9': + return _inc_volname(volfile, i) + i -= 1 + raise BadRarName("Cannot construct volume name: " + volfile) + +# old-style next volume +def _next_oldvol(volfile): + # rar -> r00 + if volfile[-4:].lower() == '.rar': + return volfile[:-2] + '00' + return _inc_volname(volfile, len(volfile) - 1) + +# increase digits with carry, otherwise just increment char +def _inc_volname(volfile, i): + fn = list(volfile) + while i >= 0: + if fn[i] != '9': + fn[i] = chr(ord(fn[i]) + 1) + break + fn[i] = '0' + i -= 1 + return ''.join(fn) + +# rar3 extended time fields +def _parse_ext_time(h, data, pos): + # flags and rest of data can be missing + flags = 0 + if pos + 2 <= len(data): + flags = S_SHORT.unpack_from(data, pos)[0] + pos += 2 + + mtime, pos = _parse_xtime(flags >> 3 * 4, data, pos, h.mtime) + h.ctime, pos = _parse_xtime(flags >> 2 * 4, data, pos) + h.atime, pos = _parse_xtime(flags >> 1 * 4, data, pos) + h.arctime, pos = _parse_xtime(flags >> 0 * 4, data, pos) + if mtime: + h.mtime = mtime + h.date_time = mtime.timetuple()[:6] + return pos + +# rar3 one extended time field +def _parse_xtime(flag, data, pos, basetime=None): + res = None + if flag & 8: + if not basetime: + basetime, pos = load_dostime(data, pos) + + # load second fractions + rem = 0 + cnt = flag & 3 + for _ in range(cnt): + b, pos = load_byte(data, pos) + rem = (b << 16) | (rem >> 8) + + # convert 100ns units to microseconds + usec = rem // 10 + if usec > 1000000: + usec = 999999 + + # dostime has room for 30 seconds only, correct if needed + if flag & 4 and basetime.second < 59: + res = basetime.replace(microsecond=usec, second=basetime.second + 1) + else: + res = basetime.replace(microsecond=usec) + return res, pos + +def is_filelike(obj): + """Filename or file object? + """ + if isinstance(obj, str) or isinstance(obj, unicode): + return False + res = True + for a in ('read', 'tell', 'seek'): + res = res and hasattr(obj, a) + if not res: + raise ValueError("Invalid object passed as file") + return True + +def rar3_s2k(psw, salt): + """String-to-key hash for RAR3. + """ + if not isinstance(psw, unicode): + psw = psw.decode('utf8') + seed = psw.encode('utf-16le') + salt + iv = EMPTY + h = sha1() + for i in range(16): + for j in range(0x4000): + cnt = S_LONG.pack(i * 0x4000 + j) + h.update(seed + cnt[:3]) + if j == 0: + iv += h.digest()[19:20] + key_be = h.digest()[:16] + key_le = pack("LLLL", key_be)) + return key_le, iv + +def rar3_decompress(vers, meth, data, declen=0, flags=0, crc=0, psw=None, salt=None): + """Decompress blob of compressed data. + + Used for data with non-standard header - eg. comments. + """ + # already uncompressed? + if meth == RAR_M0 and (flags & RAR_FILE_PASSWORD) == 0: + return data + + # take only necessary flags + flags = flags & (RAR_FILE_PASSWORD | RAR_FILE_SALT | RAR_FILE_DICTMASK) + flags |= RAR_LONG_BLOCK + + # file header + fname = b'data' + date = 0 + mode = 0x20 + fhdr = S_FILE_HDR.pack(len(data), declen, RAR_OS_MSDOS, crc, + date, vers, meth, len(fname), mode) + fhdr += fname + if flags & RAR_FILE_SALT: + if not salt: + return EMPTY + fhdr += salt + + # full header + hlen = S_BLK_HDR.size + len(fhdr) + hdr = S_BLK_HDR.pack(0, RAR_BLOCK_FILE, flags, hlen) + fhdr + hcrc = rar_crc32(hdr[2:]) & 0xFFFF + hdr = S_BLK_HDR.pack(hcrc, RAR_BLOCK_FILE, flags, hlen) + fhdr + + # archive main header + mh = S_BLK_HDR.pack(0x90CF, RAR_BLOCK_MAIN, 0, 13) + ZERO * (2 + 4) + + # decompress via temp rar + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + try: + tmpf.write(RAR_ID + mh + hdr + data) + tmpf.close() + + cmd = [UNRAR_TOOL] + list(OPEN_ARGS) + add_password_arg(cmd, psw, (flags & RAR_FILE_PASSWORD)) + cmd.append(tmpname) + + p = custom_popen(cmd) + return p.communicate()[0] + finally: + tmpf.close() + os.unlink(tmpname) + +def to_datetime(t): + """Convert 6-part time tuple into datetime object. + """ + if t is None: + return None + + # extract values + year, mon, day, h, m, s = t + + # assume the values are valid + try: + return datetime(year, mon, day, h, m, s) + except ValueError: + pass + + # sanitize invalid values + mday = (0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) + if mon < 1: + mon = 1 + if mon > 12: + mon = 12 + if day < 1: + day = 1 + if day > mday[mon]: + day = mday[mon] + if h > 23: + h = 23 + if m > 59: + m = 59 + if s > 59: + s = 59 + if mon == 2 and day == 29: + try: + return datetime(year, mon, day, h, m, s) + except ValueError: + day = 28 + return datetime(year, mon, day, h, m, s) + +def parse_dos_time(stamp): + """Parse standard 32-bit DOS timestamp. + """ + sec, stamp = stamp & 0x1F, stamp >> 5 + mn, stamp = stamp & 0x3F, stamp >> 6 + hr, stamp = stamp & 0x1F, stamp >> 5 + day, stamp = stamp & 0x1F, stamp >> 5 + mon, stamp = stamp & 0x0F, stamp >> 4 + yr = (stamp & 0x7F) + 1980 + return (yr, mon, day, hr, mn, sec * 2) + +def custom_popen(cmd): + """Disconnect cmd from parent fds, read only from stdout. + """ + # needed for py2exe + creationflags = 0 + if sys.platform == 'win32': + creationflags = 0x08000000 # CREATE_NO_WINDOW + + # run command + try: + p = Popen(cmd, bufsize=0, stdout=PIPE, stdin=PIPE, stderr=STDOUT, + creationflags=creationflags) + except OSError as ex: + if ex.errno == errno.ENOENT: + raise RarCannotExec("Unrar not installed? (rarfile.UNRAR_TOOL=%r)" % UNRAR_TOOL) + raise + return p + +def custom_check(cmd, ignore_retcode=False): + """Run command, collect output, raise error if needed. + """ + p = custom_popen(cmd) + out, _ = p.communicate() + if p.returncode and not ignore_retcode: + raise RarExecError("Check-run failed") + return out + +def add_password_arg(cmd, psw, ___required=False): + """Append password switch to commandline. + """ + if UNRAR_TOOL == ALT_TOOL: + return + if psw is not None: + cmd.append('-p' + psw) + else: + cmd.append('-p-') + +def check_returncode(p, out): + """Raise exception according to unrar exit code. + """ + code = p.returncode + if code == 0: + return + + # map return code to exception class, codes from rar.txt + errmap = [None, + RarWarning, RarFatalError, RarCRCError, RarLockedArchiveError, # 1..4 + RarWriteError, RarOpenError, RarUserError, RarMemoryError, # 5..8 + RarCreateError, RarNoFilesError, RarWrongPassword] # 9..11 + if UNRAR_TOOL == ALT_TOOL: + errmap = [None] + if code > 0 and code < len(errmap): + exc = errmap[code] + elif code == 255: + exc = RarUserBreak + elif code < 0: + exc = RarSignalExit + else: + exc = RarUnknownError + + # format message + if out: + msg = "%s [%d]: %s" % (exc.__doc__, p.returncode, out) + else: + msg = "%s [%d]" % (exc.__doc__, p.returncode) + + raise exc(msg) + +def hmac_sha256(key, data): + """HMAC-SHA256""" + return HMAC(key, data, sha256).digest() + +def membuf_tempfile(memfile): + memfile.seek(0, 0) + + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + + try: + while True: + buf = memfile.read(BSIZE) + if not buf: + break + tmpf.write(buf) + tmpf.close() + except: + tmpf.close() + os.unlink(tmpname) + raise + return tmpname + +class XTempFile(object): + __slots__ = ('_tmpfile', '_filename') + + def __init__(self, rarfile): + if is_filelike(rarfile): + self._tmpfile = membuf_tempfile(rarfile) + self._filename = self._tmpfile + else: + self._tmpfile = None + self._filename = rarfile + + def __enter__(self): + return self._filename + + def __exit__(self, exc_type, exc_value, tb): + if self._tmpfile: + try: + os.unlink(self._tmpfile) + except OSError: + pass + self._tmpfile = None + +# +# Check if unrar works +# + +ORIG_UNRAR_TOOL = UNRAR_TOOL +ORIG_OPEN_ARGS = OPEN_ARGS +ORIG_EXTRACT_ARGS = EXTRACT_ARGS +ORIG_TEST_ARGS = TEST_ARGS + +def _check_unrar_tool(): + global UNRAR_TOOL, OPEN_ARGS, EXTRACT_ARGS, TEST_ARGS + try: + # does UNRAR_TOOL work? + custom_check([ORIG_UNRAR_TOOL], True) + + UNRAR_TOOL = ORIG_UNRAR_TOOL + OPEN_ARGS = ORIG_OPEN_ARGS + EXTRACT_ARGS = ORIG_EXTRACT_ARGS + TEST_ARGS = ORIG_TEST_ARGS + except RarCannotExec: + try: + # does ALT_TOOL work? + custom_check([ALT_TOOL] + list(ALT_CHECK_ARGS), True) + # replace config + UNRAR_TOOL = ALT_TOOL + OPEN_ARGS = ALT_OPEN_ARGS + EXTRACT_ARGS = ALT_EXTRACT_ARGS + TEST_ARGS = ALT_TEST_ARGS + except RarCannotExec: + # no usable tool, only uncompressed archives work + pass + +_check_unrar_tool() From 470e8c621ba703a40e5c5f0c01b8d03aead5e97d Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:55:39 +0000 Subject: [PATCH 15/51] Create DeleteSamples.py --- .../nzbget-mp4/scripts/DeleteSamples.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/DeleteSamples.py diff --git a/apps/templates/nzbget-mp4/scripts/DeleteSamples.py b/apps/templates/nzbget-mp4/scripts/DeleteSamples.py new file mode 100644 index 0000000..395628f --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/DeleteSamples.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +def is_sample(filePath, inputName, maxSampleSize, SampleIDs): + # 200 MB in bytes + SIZE_CUTOFF = int(maxSampleSize) * 1024 * 1024 + if os.path.getsize(filePath) < SIZE_CUTOFF: + if 'SizeOnly' in SampleIDs: + return True + # Ignore 'sample' in files unless 'sample' in Torrent Name + for ident in SampleIDs: + if ident.lower() in filePath.lower() and not ident.lower() in inputName.lower(): + return True + # Return False if none of these were met. + return False + +if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + +if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + +print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) +status = 0 +if os.environ.has_key('NZBPP_TOTALSTATUS'): + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + +else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." + print "Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." + print "Please check your Par-check/repair settings for future downloads." + +# Check if destination directory exists (important for reprocessing of history items) +if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + +# All checks done, now launching the script. +if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + +mediaContainer = os.environ['NZBPO_MEDIAEXTENSIONS'].split(',') +SampleIDs = os.environ['NZBPO_SAMPLEIDS'].split(',') +for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): + for file in filenames: + filePath = os.path.join(dirpath, file) + fileName, fileExtension = os.path.splitext(file) + if fileExtension in mediaContainer or ".*" in mediaContainer : # If the file is a video file + if is_sample(filePath, os.environ['NZBPP_NZBNAME'], os.environ['NZBPO_MAXSAMPLESIZE'], SampleIDs): # Ignore samples + print "Deleting sample file: ", filePath + try: + os.unlink(filePath) + except: + print "Error: unable to delete file", filePath + sys.exit(NZBGET_POSTPROCESS_ERROR) +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From bfeef308c55b3903eec671025493e600ad21e624 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:56:01 +0000 Subject: [PATCH 16/51] Create flatten.py --- apps/templates/nzbget-mp4/scripts/flatten.py | 106 +++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/flatten.py diff --git a/apps/templates/nzbget-mp4/scripts/flatten.py b/apps/templates/nzbget-mp4/scripts/flatten.py new file mode 100644 index 0000000..73dd0d5 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/flatten.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys +import shutil + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + +if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + +print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) +status = 0 +if os.environ.has_key('NZBPP_TOTALSTATUS'): + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + +else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "[ERROR] Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "[ERROR] Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." + print "[ERROR] Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." + print "[WARNING] Please check your Par-check/repair settings for future downloads." + +# Check if destination directory exists (important for reprocessing of history items) +if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "[ERROR] Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + +# All checks done, now launching the script. +if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + +def removeEmptyFolders(path, removeRoot=True): + #Function to remove empty folders + if not os.path.isdir(path): + return + + # remove empty subfolders + print "[INFO] Checking for empty folders in:%s" % path + files = os.listdir(path) + if len(files): + for f in files: + fullpath = os.path.join(path, f) + if os.path.isdir(fullpath): + removeEmptyFolders(fullpath) + + # if folder empty, delete it + files = os.listdir(path) + if len(files) == 0 and removeRoot: + print "[INFO] Removing empty folder:%s" % path + os.rmdir(path) + +directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) +if os.environ['NZBPO_DESTINATIONDIRECTORY'] and os.path.isdir(os.environ['NZBPO_DESTINATIONDIRECTORY']): + destination = os.environ['NZBPO_DESTINATIONDIRECTORY'] + if os.environ['NZBPO_APPENDCATEGORIES'] == 'yes': + destination = os.path.join(destination, os.environ['NZBPP_CATEGORY']) +else: + destination = directory +print "Flattening directory: %s" % (directory) +for dirpath, dirnames, filenames in os.walk(directory): + for fileName in filenames: + outputFile = os.path.join(dirpath, fileName) + if dirpath == directory: + continue + target = os.path.join(destination, fileName) + try: + shutil.move(outputFile, target) + except: + print "[ERROR] Could not flatten %s" % outputFile +removeEmptyFolders(directory) # Cleanup empty directories +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From d64d8fccdc910ffc02a9ae5cb90e76a234bf5258 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:56:18 +0000 Subject: [PATCH 17/51] Create hash.py --- apps/templates/nzbget-mp4/scripts/hash.py | 167 ++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/hash.py diff --git a/apps/templates/nzbget-mp4/scripts/hash.py b/apps/templates/nzbget-mp4/scripts/hash.py new file mode 100644 index 0000000..f2fbec1 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/hash.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Original Author: clinton-hall +# https://github.com/clinton-hall/GetScripts/blob/master/SafeRename.py +# +# Modified By: desimaniac (No Acknowledgement of Source Above) +################################################################################ +import os +import re +import shutil +import sys + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +############################################################ +# EXTENSION STUFF +############################################################ + +def do_check(): + if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + + if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + + print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) + + status = 0 + if 'NZBPP_TOTALSTATUS' in os.environ: + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "[ERROR] Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "[ERROR] Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. " \ + "Setting status \"failed\"." + print "[ERROR] Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is " \ + "ok so handle as though download successful." + print "[WARNING] Please check your Par-check/repair settings for future downloads." + + # Check if destination directory exists (important for reprocessing of history items) + if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "[ERROR] Nothing to post-process: destination directory", os.environ[ + 'NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + + # All checks done, now launching the script. + if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + + +def get_file_name(path): + try: + file_name = os.path.basename(path) + extensions = re.findall(r'\.([^.]+)', file_name) + ext = '.'.join(extensions) + name = file_name.replace(".%s" % ext, '') + return name, ext + except Exception: + pass + return None + + +def is_file_hash(file_name): + hash_regexp = [ + r'^[a-fA-F0-9]{40}$', + r'^[a-fA-F0-9]{32}$', + r'^[a-f0-9]{128}$', + r'^[a-zA-Z0-9]{42}$' + ] + for hash in hash_regexp: + if re.match(hash, file_name): + return True + return False + + +def find_files(folder, extension=None, depth=None): + file_list = [] + start_count = folder.count(os.sep) + for path, subdirs, files in os.walk(folder, topdown=True): + for name in files: + if depth and path.count(os.sep) - start_count >= depth: + del subdirs[:] + continue + file = os.path.join(path, name) + if not extension: + file_list.append(file) + else: + if file.lower().endswith(extension.lower()): + file_list.append(file) + + return sorted(file_list, key=lambda x: x.count(os.path.sep), reverse=True) + + +############################################################ +# MAIN +############################################################ + +# do checks +do_check() + +# retrieve required variables +directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) +nzb_name = os.environ['NZBPP_NZBFILENAME'] +if nzb_name is None: + print("[ERROR] Unable to retrieve NZBPP_NZBFILENAME") + sys.exit(NZBGET_POSTPROCESS_ERROR) +nzb_name = nzb_name.replace('.nzb', '') + +print("[INFO] Using \"%s\" for hashed filenames" % nzb_name) +print("[INFO] Scanning \"%s\" for hashed filenames" % directory) + +# scan for files +found_files = find_files(directory) +if not found_files: + print("[INFO] No files were found in \"%s\"" % directory) + sys.exit(NZBGET_POSTPROCESS_NONE) +else: + print("[INFO] Found %d files to check for hashed filenames" % len(found_files)) + # loop files checking for file hash + moved_files = 0 + for found_file_path in found_files: + # set variable + dir_name = os.path.dirname(found_file_path) + file_name, file_ext = get_file_name(found_file_path) + + # is this a file hash + if is_file_hash(file_name): + new_file_path = os.path.join(dir_name, "%s.%s" % (nzb_name, file_ext)) + print("[INFO] Moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) + try: + shutil.move(found_file_path, new_file_path) + moved_files += 1 + except Exception: + print("[ERROR] Failed moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) + + print("[INFO] Finished processing \"%s\", moved %d files" % (directory, moved_files)) + +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From 4ca7a8e0d67c1133c2c8e1456ff4383b8658dab7 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:56:40 +0000 Subject: [PATCH 18/51] Create reverse_name.py --- .../nzbget-mp4/scripts/reverse_name.py | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/reverse_name.py diff --git a/apps/templates/nzbget-mp4/scripts/reverse_name.py b/apps/templates/nzbget-mp4/scripts/reverse_name.py new file mode 100644 index 0000000..98819d0 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/reverse_name.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys +import re +import locale + +reverse_list = [r"\.\d{2}e\d{2}s\.", r"\.p0612\.", r"\.[pi]0801\.", r"\.p027\.", r"\.[pi]675\.", r"\.[pi]084\.", r"\.p063\.", r"\b[45]62[xh]\.", r"\.yarulb\.", r"\.vtd[hp]\.", + r'\.(?:ld[.-]?)?bew\.', r"\.pir.?(shv|dov|bew|dvd|db|rb)\.", r"\brdvd\.", r"\.vts\.", r"\.reneercs\.", r"\.dcv\.", r"\b(pir|mac)dh\b", r"\.reporp\.", r"\.kcaper\.", + r"\.lanretni\.", r"\b3ca\b", r"\bcaa\b", r"\b3pm\b", r"\.cstn\.", r"\.5r\.", r"\brcs\b"] +reverse_pattern = re.compile('|'.join(reverse_list), flags=re.IGNORECASE) +season_pattern = re.compile(r"(.*\.\d{2}e\d{2}s\.)(.*)", flags=re.IGNORECASE) +word_pattern = re.compile(r"([^A-Z0-9]*[A-Z0-9]+)") +char_replace = [[r"(\w)1\.(\w)",r"\1i\2"] +] +garbage_name = re.compile(r"^[a-zA-Z0-9]{2,}$") +media_list = [r"\.s\d{2}e\d{2}\.", r"\.2160p\.", r"\.1080[pi]\.", r"\.720p\.", r"\.576[pi]\.", r"\.480[pi]\.", r"\.360p\.", r"\.[xh]26[45]\b", r"\.bluray\.", r"\.[hp]dtv\.", + r'\.web(?:[.-]?dl)?\.', r"\.(vhs|vod|dvd|web|bd|br).?rip\.", r"\.dvdr\b", r"\.stv\.", r"\.screener\.", r"\.vcd\.", r"\bhd(cam|rip)\b", r"\.proper\.", r"\.repack\.", + r"\.internal\.", r"\bac3\b", r"\baac\b", r"\bmp3\b", r"\.ntsc\.", r"\.pal\.", r"\.secam\.", r"\bdivx\b", r"\bxvid\b", r"\.r5\.", r"\.scr\."] +media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) +media_extentions = [".mkv", ".mp4", ".avi", ".wmv", ".divx", ".xvid"] + +if 'nt' == os.name: + import ctypes + + class WinEnv: + def __init__(self): + pass + + @staticmethod + def get_environment_variable(name): + name = unicode(name) # ensures string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + env_value = None + if n: + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + env_value = buf.value + return env_value + + def __getitem__(self, key): + return self.get_environment_variable(key) + + def get(self, key, default=None): + r = self.get_environment_variable(key) + return r if r is not None else default + + evn = WinEnv() +else: + class LinuxEnv(object): + def __init__(self, environ): + self.environ = environ + + def __getitem__(self, key): + v = self.environ.get(key) + try: + return v.decode(SYS_ENCODING) if isinstance(v, str) else v + except (UnicodeDecodeError, UnicodeEncodeError): + return v + + def get(self, key, default=None): + v = self[key] + return v if v is not None else default + + evn = LinuxEnv(os.environ) + +SYS_ENCODING = None + +try: + locale.setlocale(locale.LC_ALL, '') +except (locale.Error, IOError): + pass +try: + SYS_ENCODING = locale.getpreferredencoding() +except (locale.Error, IOError): + pass + +if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + SYS_ENCODING = 'UTF-8' + + +class ek: + def __init__(self): + pass + + @staticmethod + def fix_string_encoding(x): + if str == type(x): + try: + return x.decode(SYS_ENCODING) + except UnicodeDecodeError: + return None + elif unicode == type(x): + return x + return None + + @staticmethod + def fix_out_encoding(x): + if isinstance(x, basestring): + return ek.fix_string_encoding(x) + return x + + @staticmethod + def fix_list_encoding(x): + if type(x) not in (list, tuple): + return x + return filter(lambda i: None is not i, map(ek.fix_out_encoding, x)) + + @staticmethod + def encode_item(x): + try: + return x.encode(SYS_ENCODING) + except UnicodeEncodeError: + return x.encode(SYS_ENCODING, 'ignore') + + @staticmethod + def win_encode_unicode(x): + if isinstance(x, str): + try: + return x.decode('UTF-8') + except UnicodeDecodeError: + return x + return x + + @staticmethod + def ek(func, *args, **kwargs): + if 'nt' == os.name: + # convert all str parameter values to unicode + args = tuple([x if not isinstance(x, str) else ek.win_encode_unicode(x) for x in args]) + kwargs = {k: x if not isinstance(x, str) else ek.win_encode_unicode(x) for k, x in + kwargs.iteritems()} + func_result = func(*args, **kwargs) + else: + func_result = func(*[ek.encode_item(x) if type(x) == str else x for x in args], **kwargs) + + if type(func_result) in (list, tuple): + return ek.fix_list_encoding(func_result) + elif str == type(func_result): + return ek.fix_string_encoding(func_result) + return func_result + + +class logger: + INFO = 'INFO' + DETAIL = 'DETAIL' + ERROR = 'ERROR' + WARNING = 'WARNING' + + @staticmethod + def log(message, msg_type=INFO): + print('[%s] %s' % (msg_type, message)) + + +def tryInt(s, s_default=0): + try: + return int(s) + except: + return s_default + +# NZBGet V11+ +# Check if the script is called from nzbget 11.0 or later +nzbget_version = evn.get('NZBOP_VERSION', '0.1') +nzbget_version = tryInt(nzbget_version[:nzbget_version.find(".")]) +if nzbget_version >= 11: + logger.log("Script triggered from NZBGet (11.0 or later).") + + # NZBGet argv: all passed as environment variables. + clientAgent = "nzbget" + # Exit codes used by NZBGet + POSTPROCESS_PARCHECK=92 + POSTPROCESS_SUCCESS=93 + POSTPROCESS_ERROR=94 + POSTPROCESS_NONE=95 + + # Check nzbget.conf options + status = 0 + + if evn['NZBOP_UNPACK'] != 'yes': + logger.log("Please enable option \"Unpack\" in nzbget configuration file, exiting") + sys.exit(POSTPROCESS_NONE) + + parstatus = evn['NZBPP_PARSTATUS'] + + # Check par status + if parstatus == '3': + logger.log("Par-check successful, but Par-repair disabled, exiting") + sys.exit(POSTPROCESS_NONE) + + if parstatus == '1': + logger.log("Par-check failed, setting status \"failed\"") + status = 1 + sys.exit(POSTPROCESS_NONE) + + unpackstatus = evn['NZBPP_UNPACKSTATUS'] + + # Check unpack status + if unpackstatus == '1': + logger.log("Unpack failed, setting status \"failed\"") + status = 1 + sys.exit(POSTPROCESS_NONE) + + directory = evn['NZBPP_DIRECTORY'] + + if unpackstatus == '0' and parstatus != '2': + # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check + + for dirpath, dirnames, filenames in ek.ek(os.walk, directory): + for file in filenames: + fileExtension = ek.ek(os.path.splitext, file)[1] + + if fileExtension in ['.par2']: + logger.log("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\"g") + status = 1 + break + + if ek.ek(os.path.isfile, ek.ek(os.path.join, directory, "_brokenlog.txt")) and not status == 1: + logger.log("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting") + status = 1 + + if not status == 1: + logger.log("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful") + + # Check if destination directory exists (important for reprocessing of history items) + if not ek.ek(os.path.isdir, directory): + logger.log("Post-Process: Nothing to post-process: destination directory %s doesn't exist" % directory) + status = 1 + + # All checks done, now launching the script. + + rd = False + videos = 0 + old_name = None + new_name = None + base_name = ek.ek(os.path.basename, directory) + for dirpath, dirnames, filenames in ek.ek(os.walk, directory): + for file in filenames: + + filePath = ek.ek(os.path.join, dirpath, file) + fileName, fileExtension = ek.ek(os.path.splitext, file) + dirname = ek.ek(os.path.dirname, filePath) + + if reverse_pattern.search(fileName) is not None: + na_parts = season_pattern.search(fileName) + if na_parts is not None: + word_p = word_pattern.findall(na_parts.group(2)) + new_words = "" + for wp in word_p: + if wp[0] == ".": + new_words += "." + new_words += re.sub(r"\W","",wp) + for cr in char_replace: + new_words = re.sub(cr[0],cr[1],new_words) + new_filename = new_words[::-1] + na_parts.group(1)[::-1] + else: + new_filename = fileName[::-1] + logger.log("reversing filename from: %s to %s" % (fileName, new_filename)) + try: + ek.ek(os.rename, filePath, ek.ek(os.path.join, dirpath, new_filename + fileExtension)) + rd = True + except Exception,e: + logger.log(e, logger.ERROR) + logger.log("Error: unable to rename file %s" % file, logger.ERROR) + pass + elif (fileExtension.lower() in media_extentions) and (garbage_name.search(fileName) is not None) and (media_pattern.search(base_name) is not None): + videos += 1 + old_name = filePath + new_name = ek.ek(os.path.join, dirname, '%s%s' % (base_name, fileExtension)) + + if not rd and videos == 1 and old_name is not None and new_name is not None: + logger.log("renaming the File %s to the Dirname %s" % (ek.ek(os.path.basename, old_name), base_name)) + try: + ek.ek(os.rename, old_name, new_name) + rd = True + except Exception,e: + logger.log(e, logger.ERROR) + logger.log("Error unable to rename file %s" % old_name, logger.ERROR) + pass + + if rd: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_NONE) + +else: + logger.log("This script can only be called from NZBGet (11.0 or later).", logger.ERROR) + sys.exit(0) From 007070dcbb4f36df14861f00160a164d8cf93777 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:57:00 +0000 Subject: [PATCH 19/51] Create unzip.py --- apps/templates/nzbget-mp4/scripts/unzip.py | 294 +++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/unzip.py diff --git a/apps/templates/nzbget-mp4/scripts/unzip.py b/apps/templates/nzbget-mp4/scripts/unzip.py new file mode 100644 index 0000000..521a50f --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/unzip.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# +############################################################################## +### NZBGET SCAN SCRIPT ### + +# Unzips zipped nzbs. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### +### NZBGET SCAN SCRIPT ### +############################################################################## + +import os, zipfile, tarfile, gzip, pickle, datetime, re, struct, locale +import rarfile.rarfile as rarfile + +from gzip import FEXTRA, FNAME + +if 'nt' == os.name: + import ctypes + + class WinEnv: + def __init__(self): + pass + + @staticmethod + def get_environment_variable(name): + name = unicode(name) # ensures string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + result = None + if n: + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + result = buf.value + return result + + def __getitem__(self, key): + return self.get_environment_variable(key) + + def get(self, key, default=None): + r = self.get_environment_variable(key) + return r if r is not None else default + + env_var = WinEnv() +else: + class LinuxEnv(object): + def __init__(self, environ): + self.environ = environ + + def __getitem__(self, key): + v = self.environ.get(key) + try: + return v.decode(SYS_ENCODING) if isinstance(v, str) else v + except (UnicodeDecodeError, UnicodeEncodeError): + return v + + def get(self, key, default=None): + v = self[key] + return v if v is not None else default + + env_var = LinuxEnv(os.environ) + + +SYS_ENCODING = None +try: + locale.setlocale(locale.LC_ALL, '') +except (locale.Error, IOError): + pass +try: + SYS_ENCODING = locale.getpreferredencoding() +except (locale.Error, IOError): + pass +if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + SYS_ENCODING = 'UTF-8' + + +filename = env_var.get('NZBNP_FILENAME') +if re.search(r"\.tar\.gz$", filename, flags=re.I) is None: + ext = os.path.splitext(filename)[1].lower() +else: + ext = '.tar.gz' +cat = env_var.get('NZBNP_CATEGORY') +dir = env_var.get('NZBNP_DIRECTORY') +prio = env_var.get('NZBNP_PRIORITY') +top = env_var.get('NZBNP_TOP') +pause = env_var.get('NZBNP_PAUSED') +if 'NZBNP_DUPEKEY' in os.environ: + dupekey = env_var.get('NZBNP_DUPEKEY') + dupescore = env_var.get('NZBNP_DUPESCORE') + dupemode = env_var.get('NZBNP_DUPEMODE') +else: + dupekey = None + dupescore = None + dupemode = None + +tmp_zipinfo = os.path.join(os.environ.get('NZBOP_TEMPDIR'), r'nzbget\unzip_scan\info') +nzb_list = [] + +def read_gzip_info(gzipfile): + gf = gzipfile.fileobj + pos = gf.tell() + + # Read archive size + gf.seek(-4, 2) + size = struct.unpack('/dev/null >/dev/null; then + echo "Failed to extract $2"; + exit 1 + fi + + fi +} + +execute () { + echo "$ $*" + + OUTPUT=$($@ 2>&1) + + if [ $? -ne 0 ]; then + echo "$OUTPUT" + echo "" + echo "Failed to Execute $*" >&2 + exit 1 + fi +} + +build () { + echo "" + echo "building $1" + echo "=======================" + + if [ -f "$PACKAGES/$1.done" ]; then + echo "$1 already built. Remove $PACKAGES/$1.done lockfile to rebuild it." + return 1 + fi + + return 0 +} + +command_exists() { + if ! [[ -x $(command -v "$1") ]]; then + return 1 + fi + + return 0 +} + + +build_done () { + touch "$PACKAGES/$1.done" +} + +echo "ffmpeg-build-script v$VERSION" +echo "=========================" +echo "" + +case "$1" in +"--cleanup") + remove_dir $PACKAGES + remove_dir $WORKSPACE + echo "Cleanup done." + echo "" + exit 0 + ;; +"--build") + + ;; +*) + echo "Usage: $0" + echo " --build: start building process" + echo " --cleanup: remove all working dirs" + echo " --help: show this help" + echo "" + exit 0 + ;; +esac + +echo "Using $MJOBS make jobs simultaneously." + +make_dir $PACKAGES +make_dir $WORKSPACE + +export PATH=${WORKSPACE}/bin:$PATH + +if ! command_exists "make"; then + echo "make not installed."; + exit 1 +fi + +if ! command_exists "g++"; then + echo "g++ not installed."; + exit 1 +fi + +if ! command_exists "curl"; then + echo "curl not installed."; + exit 1 +fi + +if build "yasm"; then + download "http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz" "yasm-1.3.0.tar.gz" + cd $PACKAGES/yasm-1.3.0 || exit + execute ./configure --prefix=${WORKSPACE} + execute make -j $MJOBS + execute make install + build_done "yasm" +fi + +if build "nasm"; then + download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.13.03.tar.gz" "nasm.tar.gz" + cd $PACKAGES/nasm-2.13.03 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "nasm" +fi + +if build "opencore"; then + download "http://downloads.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-0.1.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Fopencore-amr%2F&ts=1442256558&use_mirror=netassist" "opencore-amr-0.1.5.tar.gz" + cd $PACKAGES/opencore-amr-0.1.5 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "opencore" +fi + +if build "libvpx"; then + download "https://github.com/webmproject/libvpx/archive/v1.7.0.tar.gz" "libvpx-1.7.0.tar.gz" + cd $PACKAGES/libvpx-*0 || exit + + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Applying Darwin patch" + sed "s/,--version-script//g" build/make/Makefile > build/make/Makefile.patched + sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched > build/make/Makefile + fi + + execute ./configure --prefix=${WORKSPACE} --disable-unit-tests --disable-shared + execute make -j $MJOBS + execute make install + build_done "libvpx" +fi + +if build "lame"; then + download "http://kent.dl.sourceforge.net/project/lame/lame/3.100/lame-3.100.tar.gz" "lame-3.100.tar.gz" + cd $PACKAGES/lame-3.100 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "lame" +fi + +if build "xvidcore"; then + download "http://downloads.xvid.org/downloads/xvidcore-1.3.4.tar.gz" "xvidcore-1.3.4.tar.gz" + cd $PACKAGES/xvidcore || exit + cd build/generic || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + + if [[ -f ${WORKSPACE}/lib/libxvidcore.4.dylib ]]; then + execute rm "${WORKSPACE}/lib/libxvidcore.4.dylib" + fi + + build_done "xvidcore" +fi + +if build "x264"; then + download "http://ftp.videolan.org/pub/x264/snapshots/x264-snapshot-20190204-2245-stable.tar.bz2" "last_x264.tar.bz2" + cd $PACKAGES/x264-snapshot-* || exit + + if [[ "$OSTYPE" == "linux-gnu" ]]; then + execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic CXXFLAGS="-fPIC" + else + execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic + fi + + execute make -j $MJOBS + execute make install + execute make install-lib-static + build_done "x264" +fi + +if build "libogg"; then + download "http://downloads.xiph.org/releases/ogg/libogg-1.3.3.tar.gz" "libogg-1.3.3.tar.gz" + cd $PACKAGES/libogg-1.3.3 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "libogg" +fi + +if build "libvorbis"; then + download "http://downloads.xiph.org/releases/vorbis/libvorbis-1.3.6.tar.gz" "libvorbis-1.3.6.tar.gz" + cd $PACKAGES/libvorbis-1.3.6 || exit + execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest + execute make -j $MJOBS + execute make install + build_done "libvorbis" +fi + +if build "libtheora"; then + download "http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.gz" "libtheora-1.1.1.tar.bz" + cd $PACKAGES/libtheora-1.1.1 || exit + sed "s/-fforce-addr//g" configure > configure.patched + chmod +x configure.patched + mv configure.patched configure + execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --with-vorbis-libraries=${WORKSPACE}/lib --with-vorbis-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm + execute make -j $MJOBS + execute make install + build_done "libtheora" +fi + +if build "pkg-config"; then + download "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" "pkg-config-0.29.2.tar.gz" + cd $PACKAGES/pkg-config-0.29.2 || exit + execute ./configure --silent --prefix=${WORKSPACE} --with-pc-path=${WORKSPACE}/lib/pkgconfig --with-internal-glib + execute make -j $MJOBS + execute make install + build_done "pkg-config" +fi + +if build "cmake"; then + download "https://cmake.org/files/v3.11/cmake-3.11.3.tar.gz" "cmake-3.11.3.tar.gz" + cd $PACKAGES/cmake-3.11.3 || exit + rm Modules/FindJava.cmake + perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt + perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt + execute ./configure --prefix=${WORKSPACE} + execute make -j $MJOBS + execute make install + build_done "cmake" +fi + +if build "vid_stab"; then + download "https://codeload.github.com/georgmartius/vid.stab/legacy.tar.gz/release-0.98b" "vid.stab-0.98b-transcode-1.1-binary-x86_64.tgz" + cd $PACKAGES/georgmartius-vid* || exit + perl -p -i -e "s/vidstab SHARED/vidstab STATIC/" CMakeLists.txt + execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} . + execute make install + build_done "vid_stab" +fi + +if build "x265"; then + download "https://bitbucket.org/multicoreware/x265/downloads/x265_3.0.tar.gz" "x265-3.0.tar.gz" + cd $PACKAGES/x265_* || exit + cd source || exit + execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} -DENABLE_SHARED:bool=off . + execute make -j $MJOBS + execute make install + sed "s/-lx265/-lx265 -lstdc++/g" "$WORKSPACE/lib/pkgconfig/x265.pc" > "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" + mv "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" "$WORKSPACE/lib/pkgconfig/x265.pc" + build_done "x265" +fi + +if build "fdk_aac"; then + download "http://downloads.sourceforge.net/project/opencore-amr/fdk-aac/fdk-aac-0.1.6.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Ffdk-aac%2F&ts=1457561564&use_mirror=kent" "fdk-aac-0.1.6.tar.gz" + cd $PACKAGES/fdk-aac-0.1.6 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "fdk_aac" +fi + + +build "ffmpeg" +download "http://ffmpeg.org/releases/ffmpeg-4.1.tar.bz2" "ffmpeg-snapshot.tar.bz2" +cd $PACKAGES/ffmpeg-4.1 || exit +./configure $ADDITIONAL_CONFIGURE_OPTIONS \ + --pkgconfigdir="$WORKSPACE/lib/pkgconfig" \ + --prefix=${WORKSPACE} \ + --pkg-config-flags="--static" \ + --extra-cflags="-I$WORKSPACE/include" \ + --extra-ldflags="-L$WORKSPACE/lib" \ + --extra-libs="-lpthread -lm" \ + --enable-static \ + --disable-debug \ + --disable-shared \ + --disable-ffplay \ + --disable-doc \ + --enable-gpl \ + --enable-version3 \ + --enable-nonfree \ + --enable-pthreads \ + --enable-libvpx \ + --enable-libmp3lame \ + --enable-libtheora \ + --enable-libvorbis \ + --enable-libx264 \ + --enable-libx265 \ + --enable-runtime-cpudetect \ + --enable-libfdk-aac \ + --enable-avfilter \ + --enable-libopencore_amrwb \ + --enable-libopencore_amrnb \ + --enable-filters \ + --enable-libvidstab + +execute make -j $MJOBS +execute make install + +INSTALL_FOLDER="/usr/bin" +if [[ "$OSTYPE" == "darwin"* ]]; then +INSTALL_FOLDER="/usr/local/bin" +fi + +echo "" +echo "Building done. The binary can be found here: $WORKSPACE/bin/ffmpeg" +echo "" + + +if [[ $AUTOINSTALL == "yes" ]]; then + if command_exists "sudo"; then + sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" + sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" + echo "Done. ffmpeg is now installed to your system" + fi +elif [[ ! $SKIPINSTALL == "yes" ]]; then + if command_exists "sudo"; then + + read -r -p "Install the binary to your $INSTALL_FOLDER folder? [Y/n] " response + + case $response in + [yY][eE][sS]|[yY]) + sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" + sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" + echo "Done. ffmpeg is now installed to your system" + ;; + esac + fi +fi + +exit 0 From c15975c0af634b5953b234adbbb3416d5dcf483f Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:15:12 +0000 Subject: [PATCH 26/51] Create web-install.sh --- .../nzgbet/ffmpeg-build/web-install.sh | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 apps/templates/nzgbet/ffmpeg-build/web-install.sh diff --git a/apps/templates/nzgbet/ffmpeg-build/web-install.sh b/apps/templates/nzgbet/ffmpeg-build/web-install.sh new file mode 100644 index 0000000..ffe4158 --- /dev/null +++ b/apps/templates/nzgbet/ffmpeg-build/web-install.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Helper script to download and run the build-ffmpeg script. + +make_dir () { + if [ ! -d $1 ]; then + if ! mkdir $1; then + printf "\n Failed to create dir %s" "$1"; + exit 1 + fi + fi +} + +command_exists() { + if ! [[ -x $(command -v "$1") ]]; then + return 1 + fi + + return 0 +} + +TARGET='ffmpeg-build' + +if ! command_exists "curl"; then + echo "curl not installed."; + exit 1 +fi + +echo "ffmpeg-build-script-downloader v0.1" +echo "=========================================" +echo "" + +echo "First we create the ffmpeg build directory $TARGET" +make_dir $TARGET +cd $TARGET + +echo "Now we download and execute the build script" +echo "" + +bash build-ffmpeg --build + From e4f0b4aec7162c3a963c9117377d2279dd235cc6 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:16:32 +0000 Subject: [PATCH 27/51] Create installer.sh --- apps/templates/nzgbet/installer/installer.sh | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 apps/templates/nzgbet/installer/installer.sh diff --git a/apps/templates/nzgbet/installer/installer.sh b/apps/templates/nzgbet/installer/installer.sh new file mode 100644 index 0000000..8c26cf9 --- /dev/null +++ b/apps/templates/nzgbet/installer/installer.sh @@ -0,0 +1,30 @@ +#!/bin/bash +apk update +apk upgrade +apk add --no-cache git +apk add build-base gcc wget diffutils perl +apk add curl +git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp +cp -r /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator +rm -rf /config/scripts/MP4_Automator/tmp +git unstage +apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev +pip install --upgrade PIP +pip install requests +pip install requests[security] +pip install requests-cache +pip install babelfish +pip install "guessit<2" +pip install "subliminal<2" +pip install qtfaststart +# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 +pip uninstall -y stevedore +pip install stevedore==1.19.1 +#Remove default NZBGetPostProcess script settings, and replace with our own +rm /config/scripts/MP4_Automator/NZBGetPostProcess.py +cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py +#Build ffmpeg +cd /config +. /config/ffmpeg-build/web-install.sh +#Set script file permissions +chmod 777 -R /config/scripts From c3834b613886a341720b2be0bbc95c84d35413ab Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:17:23 +0000 Subject: [PATCH 28/51] Create autoProcess.ini --- .../nzgbet/mp4_automator/autoProcess.ini | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 apps/templates/nzgbet/mp4_automator/autoProcess.ini diff --git a/apps/templates/nzgbet/mp4_automator/autoProcess.ini b/apps/templates/nzgbet/mp4_automator/autoProcess.ini new file mode 100644 index 0000000..83b18aa --- /dev/null +++ b/apps/templates/nzgbet/mp4_automator/autoProcess.ini @@ -0,0 +1,143 @@ +[SickBeard] +host = sickbeard +port = 8081 +username = +password = +web_root = +ssl = False +api_key = + +[Sonarr] +host = sonarr +port = 8989 +web_root = +ssl = False +apikey = + +[Radarr] +host = radarr +port = 7878 +web_root = +ssl = False +apikey = + +[MP4] +ffmpeg = /config/ffmpeg-build/workspace/bin/ffmpeg +ffprobe = /config/ffmpeg-build/workspace/bin/ffprobe +threads = 1 +output_directory = +copy_to = +move_to = +output_extension = mp4 +output_format = mp4 +delete_original = True +relocate_moov = True +video-codec = h264,x264 +video-bitrate = +video-crf = 18 +video-max-width = +h264-max-level = 4.1 +use-qsv-decoder-with-encoder = True +ios-audio = libfdk_aac +ios-first-track-only = False +ios-audio-filter = dynaudnorm +max-audio-channels = +audio-codec = ac3,mp3,dts,dca,aac,libfdk_aac +audio-language = eng +audio-default-language = eng +audio-channel-bitrate = 256 +audio-filter = +subtitle-codec = srt +subtitle-language = eng +subtitle-default-language = +subtitle-encoding = +fullpathguess = True +convert-mp4 = True +tagfile = True +tag-language = en +download-artwork = Poster +download-subs = True +embed-subs = False +sub-providers = addic7ed,podnapisi,thesubdb,opensubtitles +permissions = 0777 +post-process = False +pix-fmt = +aac_adtstoasc = False +postopts = -preset,slower +preopts = +audio-copy-original = False +enable_dxva2_gpu_decode = False +ios-move-last = False +use-hevc-qsv-decoder = False +embed-only-internal-subs = False +audio-first-track-of-language = False +video-profile = + +[CouchPotato] +host = couchpotato +port = 5050 +username = +password = +web_root = +ssl = False +apikey = +delay = 65 +method = renamer +delete_failed = False + +[uTorrent] +convert = +couchpotato-label = couchpotato +sickbeard-label = sickbeard +sonarr-label = sonarr +bypass-label = bypass +sickrage-label = sickrage +webui = False +action_before = stop +action_after = removedata +host = http://utorrent:8080/ +username = +password = +output_directory = +radarr-label = radarr + +[Deluge] +host = deluge +username = +convert = True +password = +sonarr-label = sonarr +radarr-label = radarr +bypass-label = bypass +sickbeard-label = sickbeard +port = 12569 +sickrage-label = sickrage +couchpotato-label = couchpotato +output_directory = +remove = true + +[SABNZBD] +convert = True +sickrage-category = sickrage +sonarr-category = sonarr +radarr-category = radarr +bypass-category = bypass +couchpotato-category = couchpotato +sickbeard-category = sickbeard +output_directory = + +[Sickrage] +host = sickrage +port = 8081 +username = +password = +web_root = +ssl = False +api_key = + +[Plex] +host = plex +port = 32400 +refresh = False +token = + From 1f2f5abb1dc82d2fc80a18b02c6dd30b3538f607 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:17:44 +0000 Subject: [PATCH 29/51] Create TEMPLATEPPScript --- .../nzgbet/mp4_automator/TEMPLATEPPScript | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript diff --git a/apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript b/apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript new file mode 100644 index 0000000..6eb7055 --- /dev/null +++ b/apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# +############################################################################## +### NZBGET POST-PROCESSING SCRIPT ### + +# Modified to enable multiple bypass categories, +# as per: https://github.com/mdhiggins/sickbeard_mp4_automator/issues/509 +# +# Converts files and passes them to Sonarr for further processing. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### + +# Change to full path to MP4 Automator folder. No quotes and a trailing / +#MP4_FOLDER=~/sickbeard_mp4_automator/ + +# Convert file before passing to destination (True, False) +#SHOULDCONVERT=False + +# Category for Couchpotato +#CP_CAT=Couchpotato + +# Category for Sonarr +#SONARR_CAT=Sonarr + +# Category for Radarr +#RADARR_CAT=Radarr + +# Category for Sickbeard +#SICKBEARD_CAT=Sickbeard + +# Category for Sickrage +#SICKRAGE_CAT=Sickrage + +# Category list (comma seperated) for bypassing any further processing but still converting +#BYPASS_CAT=tv,movies + +# Custom output_directory setting +#OUTPUT_DIR= + +### NZBGET POST-PROCESSING SCRIPT ### +############################################################################## + +import os +import sys +import re +import json +import traceback + +# Sanity checks for path string +MP4folder = os.environ['NZBPO_MP4_FOLDER'].strip() +MP4folder = MP4folder.replace('"', '') +MP4folder = MP4folder.replace("'", "") +MP4folder = MP4folder.replace("\\", "/") +if not(MP4folder.endswith("/")): + MP4folder += "/" +#DEBUG#print MP4folder+" the original is "+os.environ['NZBPO_MP4_FOLDER'] + +output_dir = None +if 'NZBPO_OUTPUT_DIR' in os.environ: + output_dir = os.environ['NZBPO_OUTPUT_DIR'].strip() + if len(output_dir) > 0: + output_dir = output_dir.replace('"', '') + output_dir = output_dir.replace("'", "") + output_dir = output_dir.replace("\\", "/") + if not(output_dir.endswith("/")): + output_dir += "/" + #DEBUG#print Overriding output directory + +sys.path.append(MP4folder) +try: + from readSettings import ReadSettings + from mkvtomp4 import MkvtoMp4 + from autoprocess import autoProcessMovie, autoProcessTV, autoProcessTVSR, sonarr, radarr + import logging + from logging.config import fileConfig +except ImportError: + print("[ERROR] Wrong path to sickbeard_mp4_automator: " + os.environ['NZBPO_MP4_FOLDER']) + print("[ERROR] %s" % traceback.print_exc()) + sys.exit(0) + +# Setup Logging +logpath = '/var/log/sickbeard_mp4_automator' +if os.name == 'nt': + logpath = MP4folder +elif not os.path.isdir(logpath): + try: + os.mkdir(logpath) + except: + logpath = MP4folder +configPath = os.path.abspath(os.path.join(MP4folder, 'logging.ini')).replace("\\", "\\\\") +logPath = os.path.abspath(os.path.join(logpath, 'index.log')).replace("\\", "\\\\") +fileConfig(configPath, defaults={'logfilename': logPath}) +log = logging.getLogger("NZBGetPostProcess") + +# Determine if conversion will take place +shouldConvert = (os.environ['NZBPO_SHOULDCONVERT'].lower() in ("yes", "true", "t", "1")) + +if 'NZBOP_SCRIPTDIR' in os.environ and not os.environ['NZBOP_VERSION'][0:5] < '11.0': + log.info("Script triggered from NZBGet (11.0 or later).") + + path = os.environ['NZBPP_DIRECTORY'] # Path to NZB directory + nzb = os.environ['NZBPP_NZBFILENAME'] # Original NZB name + category = os.environ['NZBPP_CATEGORY'] # NZB Category to determine destination + #DEBUG#print "Category is %s." % category + + couchcat = os.environ['NZBPO_CP_CAT'].lower() + sonarrcat = os.environ['NZBPO_SONARR_CAT'].lower() + radarrcat = os.environ['NZBPO_RADARR_CAT'].lower() + sickbeardcat = os.environ['NZBPO_SICKBEARD_CAT'].lower() + sickragecat = os.environ['NZBPO_SICKRAGE_CAT'].lower() + bypass = os.environ['NZBPO_BYPASS_CAT'].lower().replace(' ','').split(',') + + categories = [sickbeardcat, couchcat, sonarrcat, radarrcat, sickragecat] + + log.debug("Path: %s" % path) + log.debug("NZB: %s" % nzb) + log.debug("Category: %s" % category) + log.debug("Categories: %s" % categories) + + # NZBGet argv: all passed as environment variables. + clientAgent = "nzbget" + # Exit codes used by NZBGet + POSTPROCESS_PARCHECK = 92 + POSTPROCESS_SUCCESS = 93 + POSTPROCESS_ERROR = 94 + POSTPROCESS_NONE = 95 + + # Check nzbget.conf options + status = 0 + + if os.environ['NZBOP_UNPACK'] != 'yes': + log.error("Please enable option \"Unpack\" in nzbget configuration file, exiting.") + sys.exit(POSTPROCESS_NONE) + + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '3': + log.error("Par-check successful, but Par-repair disabled, exiting") + sys.exit(POSTPROCESS_NONE) + + if os.environ['NZBPP_PARSTATUS'] == '1': + log.error("Par-check failed, setting status \"failed\".") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + log.error("Unpack failed, setting status \"failed\".") + status = 1 + sys.exit(POSTPROCESS_NONE) + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] != '2': + # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check + + for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): + for file in filenames: + fileExtension = os.path.splitext(file)[1] + + if fileExtension in ['.par2']: + log.error("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\".") + status = 1 + break + + if os.path.isfile(os.path.join(os.environ['NZBPP_DIRECTORY'], "_brokenlog.txt")) and not status == 1: + log.error("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting.") + status = 1 + + if not status == 1: + log.error("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful.") + + # Check if destination directory exists (important for reprocessing of history items) + if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + log.error("Post-Process: Nothing to post-process: destination directory ", os.environ['NZBPP_DIRECTORY'], "doesn't exist.") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Make sure one of the appropriate categories is set + if category.lower() not in categories and category.lower() not in bypass: + log.error("Post-Process: No valid category detected. Category was %s." % (category)) + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Make sure there are no duplicate categories + if len(categories) != len(set(categories)): + log.error("Duplicate category detected. Category names must be unique.") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # All checks done, now launching the script. + settings = ReadSettings(MP4folder, "autoProcess.ini") + + if shouldConvert: + if output_dir: + settings.output_dir = output_dir + converter = MkvtoMp4(settings, logger=log) + for r, d, f in os.walk(path): + for files in f: + inputfile = os.path.join(r, files) + #DEBUG#print inputfile + #Ignores files under 50MB + if os.path.getsize(inputfile) > 50000000: + if MkvtoMp4(settings, logger=log).validSource(inputfile): + try: + output = converter.process(inputfile) + log.info("Successfully processed %s." % inputfile) + except: + log.exception("File processing failed.") + if converter.output_dir: + path = converter.output_dir + if (category.lower() == categories[0]): + #DEBUG#print "Sickbeard Processing Activated" + autoProcessTV.processEpisode(path, settings, nzb) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() == categories[1]): + #DEBUG#print "CouchPotato Processing Activated" + autoProcessMovie.process(path, settings, nzb, status) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() == categories[2]): + #DEBUG#print "Sonarr Processing Activated" + success = sonarr.processEpisode(path, settings, True) + if success: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_ERROR) + elif (category.lower() == categories[3]): + #DEBUG#print "Radarr Processing Activated" + success = radarr.processMovie(path, settings, True) + if success: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_ERROR) + elif (category.lower() == categories[4]): + #DEBUG#print "Sickrage Processing Activated" + autoProcessTVSR.processEpisode(path, settings, nzb) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() in bypass): + #DEBUG#print "Bypass Further Processing" + sys.exit(POSTPROCESS_NONE) + +else: + log.error("This script can only be called from NZBGet (11.0 or later).") + sys.exit(0) From b31e90f6b162a857d6b440c6590e7bb966460649 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:18:23 +0000 Subject: [PATCH 30/51] Create __init__.py --- apps/templates/nzgbet/scripts/rarfile/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 apps/templates/nzgbet/scripts/rarfile/__init__.py diff --git a/apps/templates/nzgbet/scripts/rarfile/__init__.py b/apps/templates/nzgbet/scripts/rarfile/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/apps/templates/nzgbet/scripts/rarfile/__init__.py @@ -0,0 +1 @@ + From b596ffa4a8f7f9246473149a1f419887e48c9af6 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:18:52 +0000 Subject: [PATCH 31/51] Create rarfile.py --- .../nzgbet/scripts/rarfile/rarfile.py | 2932 +++++++++++++++++ 1 file changed, 2932 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/rarfile/rarfile.py diff --git a/apps/templates/nzgbet/scripts/rarfile/rarfile.py b/apps/templates/nzgbet/scripts/rarfile/rarfile.py new file mode 100644 index 0000000..c6e0d44 --- /dev/null +++ b/apps/templates/nzgbet/scripts/rarfile/rarfile.py @@ -0,0 +1,2932 @@ +# rarfile.py +# +# Copyright (c) 2005-2016 Marko Kreen +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +r"""RAR archive reader. + +This is Python module for Rar archive reading. The interface +is made as :mod:`zipfile`-like as possible. + +Basic logic: + - Parse archive structure with Python. + - Extract non-compressed files with Python + - Extract compressed files with unrar. + - Optionally write compressed data to temp file to speed up unrar, + otherwise it needs to scan whole archive on each execution. + +Example:: + + import rarfile + + rf = rarfile.RarFile('myarchive.rar') + for f in rf.infolist(): + print f.filename, f.file_size + if f.filename == 'README': + print(rf.read(f)) + +Archive files can also be accessed via file-like object returned +by :meth:`RarFile.open`:: + + import rarfile + + with rarfile.RarFile('archive.rar') as rf: + with rf.open('README') as f: + for ln in f: + print(ln.strip()) + +There are few module-level parameters to tune behaviour, +here they are with defaults, and reason to change it:: + + import rarfile + + # Set to full path of unrar.exe if it is not in PATH + rarfile.UNRAR_TOOL = "unrar" + + # Set to '\\' to be more compatible with old rarfile + rarfile.PATH_SEP = '/' + +For more details, refer to source. + +""" + +from __future__ import division, print_function + +## +## Imports and compat - support both Python 2.x and 3.x +## + +import sys +import os +import errno +import struct + +from struct import pack, unpack, Struct +from binascii import crc32, hexlify +from tempfile import mkstemp +from subprocess import Popen, PIPE, STDOUT +from io import RawIOBase +from hashlib import sha1, sha256 +from hmac import HMAC +from datetime import datetime, timedelta, tzinfo + +# fixed offset timezone, for UTC +try: + from datetime import timezone +except ImportError: + class timezone(tzinfo): + """Compat timezone.""" + __slots__ = ('_ofs', '_name') + _DST = timedelta(0) + + def __init__(self, offset, name): + super(timezone, self).__init__() + self._ofs, self._name = offset, name + + def utcoffset(self, dt): + return self._ofs + + def tzname(self, dt): + return self._name + + def dst(self, dt): + return self._DST + +# only needed for encryped headers +try: + try: + from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf import pbkdf2 + + class AES_CBC_Decrypt(object): + """Decrypt API""" + def __init__(self, key, iv): + ciph = Cipher(algorithms.AES(key), modes.CBC(iv), default_backend()) + self.decrypt = ciph.decryptor().update + + def pbkdf2_sha256(password, salt, iters): + """PBKDF2 with HMAC-SHA256""" + ctx = pbkdf2.PBKDF2HMAC(hashes.SHA256(), 32, salt, iters, default_backend()) + return ctx.derive(password) + + except ImportError: + from Crypto.Cipher import AES + from Crypto.Protocol import KDF + + class AES_CBC_Decrypt(object): + """Decrypt API""" + def __init__(self, key, iv): + self.decrypt = AES.new(key, AES.MODE_CBC, iv).decrypt + + def pbkdf2_sha256(password, salt, iters): + """PBKDF2 with HMAC-SHA256""" + return KDF.PBKDF2(password, salt, 32, iters, hmac_sha256) + + _have_crypto = 1 +except ImportError: + _have_crypto = 0 + +try: + from pyblake2 import blake2s + _have_blake2 = True +except ImportError: + _have_blake2 = False + +# compat with 2.x +if sys.hexversion < 0x3000000: + def rar_crc32(data, prev=0): + """CRC32 with unsigned values. + """ + if (prev > 0) and (prev & 0x80000000): + prev -= (1 << 32) + res = crc32(data, prev) + if res < 0: + res += (1 << 32) + return res + tohex = hexlify + _byte_code = ord +else: # pragma: no cover + def tohex(data): + """Return hex string.""" + return hexlify(data).decode('ascii') + rar_crc32 = crc32 + unicode = str + _byte_code = int # noqa + + +__version__ = '3.0' + +# export only interesting items +__all__ = ['is_rarfile', 'RarInfo', 'RarFile', 'RarExtFile'] + +## +## Module configuration. Can be tuned after importing. +## + +#: default fallback charset +DEFAULT_CHARSET = "windows-1252" + +#: list of encodings to try, with fallback to DEFAULT_CHARSET if none succeed +TRY_ENCODINGS = ('utf8', 'utf-16le') + +#: 'unrar', 'rar' or full path to either one +UNRAR_TOOL = "unrar" + +#: Command line args to use for opening file for reading. +OPEN_ARGS = ('p', '-inul') + +#: Command line args to use for extracting file to disk. +EXTRACT_ARGS = ('x', '-y', '-idq') + +#: args for testrar() +TEST_ARGS = ('t', '-idq') + +# +# Allow use of tool that is not compatible with unrar. +# +# By default use 'bsdtar' which is 'tar' program that +# sits on top of libarchive. +# +# Problems with libarchive RAR backend: +# - Does not support solid archives. +# - Does not support password-protected archives. +# + +ALT_TOOL = 'bsdtar' +ALT_OPEN_ARGS = ('-x', '--to-stdout', '-f') +ALT_EXTRACT_ARGS = ('-x', '-f') +ALT_TEST_ARGS = ('-t', '-f') +ALT_CHECK_ARGS = ('--help',) + +#: whether to speed up decompression by using tmp archive +USE_EXTRACT_HACK = 1 + +#: limit the filesize for tmp archive usage +HACK_SIZE_LIMIT = 20 * 1024 * 1024 + +#: Separator for path name components. RAR internally uses '\\'. +#: Use '/' to be similar with zipfile. +PATH_SEP = '/' + +## +## rar constants +## + +# block types +RAR_BLOCK_MARK = 0x72 # r +RAR_BLOCK_MAIN = 0x73 # s +RAR_BLOCK_FILE = 0x74 # t +RAR_BLOCK_OLD_COMMENT = 0x75 # u +RAR_BLOCK_OLD_EXTRA = 0x76 # v +RAR_BLOCK_OLD_SUB = 0x77 # w +RAR_BLOCK_OLD_RECOVERY = 0x78 # x +RAR_BLOCK_OLD_AUTH = 0x79 # y +RAR_BLOCK_SUB = 0x7a # z +RAR_BLOCK_ENDARC = 0x7b # { + +# flags for RAR_BLOCK_MAIN +RAR_MAIN_VOLUME = 0x0001 +RAR_MAIN_COMMENT = 0x0002 +RAR_MAIN_LOCK = 0x0004 +RAR_MAIN_SOLID = 0x0008 +RAR_MAIN_NEWNUMBERING = 0x0010 +RAR_MAIN_AUTH = 0x0020 +RAR_MAIN_RECOVERY = 0x0040 +RAR_MAIN_PASSWORD = 0x0080 +RAR_MAIN_FIRSTVOLUME = 0x0100 +RAR_MAIN_ENCRYPTVER = 0x0200 + +# flags for RAR_BLOCK_FILE +RAR_FILE_SPLIT_BEFORE = 0x0001 +RAR_FILE_SPLIT_AFTER = 0x0002 +RAR_FILE_PASSWORD = 0x0004 +RAR_FILE_COMMENT = 0x0008 +RAR_FILE_SOLID = 0x0010 +RAR_FILE_DICTMASK = 0x00e0 +RAR_FILE_DICT64 = 0x0000 +RAR_FILE_DICT128 = 0x0020 +RAR_FILE_DICT256 = 0x0040 +RAR_FILE_DICT512 = 0x0060 +RAR_FILE_DICT1024 = 0x0080 +RAR_FILE_DICT2048 = 0x00a0 +RAR_FILE_DICT4096 = 0x00c0 +RAR_FILE_DIRECTORY = 0x00e0 +RAR_FILE_LARGE = 0x0100 +RAR_FILE_UNICODE = 0x0200 +RAR_FILE_SALT = 0x0400 +RAR_FILE_VERSION = 0x0800 +RAR_FILE_EXTTIME = 0x1000 +RAR_FILE_EXTFLAGS = 0x2000 + +# flags for RAR_BLOCK_ENDARC +RAR_ENDARC_NEXT_VOLUME = 0x0001 +RAR_ENDARC_DATACRC = 0x0002 +RAR_ENDARC_REVSPACE = 0x0004 +RAR_ENDARC_VOLNR = 0x0008 + +# flags common to all blocks +RAR_SKIP_IF_UNKNOWN = 0x4000 +RAR_LONG_BLOCK = 0x8000 + +# Host OS types +RAR_OS_MSDOS = 0 +RAR_OS_OS2 = 1 +RAR_OS_WIN32 = 2 +RAR_OS_UNIX = 3 +RAR_OS_MACOS = 4 +RAR_OS_BEOS = 5 + +# Compression methods - '0'..'5' +RAR_M0 = 0x30 +RAR_M1 = 0x31 +RAR_M2 = 0x32 +RAR_M3 = 0x33 +RAR_M4 = 0x34 +RAR_M5 = 0x35 + +# +# RAR5 constants +# + +RAR5_BLOCK_MAIN = 1 +RAR5_BLOCK_FILE = 2 +RAR5_BLOCK_SERVICE = 3 +RAR5_BLOCK_ENCRYPTION = 4 +RAR5_BLOCK_ENDARC = 5 + +RAR5_BLOCK_FLAG_EXTRA_DATA = 0x01 +RAR5_BLOCK_FLAG_DATA_AREA = 0x02 +RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN = 0x04 +RAR5_BLOCK_FLAG_SPLIT_BEFORE = 0x08 +RAR5_BLOCK_FLAG_SPLIT_AFTER = 0x10 +RAR5_BLOCK_FLAG_DEPENDS_PREV = 0x20 +RAR5_BLOCK_FLAG_KEEP_WITH_PARENT = 0x40 + +RAR5_MAIN_FLAG_ISVOL = 0x01 +RAR5_MAIN_FLAG_HAS_VOLNR = 0x02 +RAR5_MAIN_FLAG_SOLID = 0x04 +RAR5_MAIN_FLAG_RECOVERY = 0x08 +RAR5_MAIN_FLAG_LOCKED = 0x10 + +RAR5_FILE_FLAG_ISDIR = 0x01 +RAR5_FILE_FLAG_HAS_MTIME = 0x02 +RAR5_FILE_FLAG_HAS_CRC32 = 0x04 +RAR5_FILE_FLAG_UNKNOWN_SIZE = 0x08 + +RAR5_COMPR_SOLID = 0x40 + +RAR5_ENC_FLAG_HAS_CHECKVAL = 0x01 + +RAR5_ENDARC_FLAG_NEXT_VOL = 0x01 + +RAR5_XFILE_ENCRYPTION = 1 +RAR5_XFILE_HASH = 2 +RAR5_XFILE_TIME = 3 +RAR5_XFILE_VERSION = 4 +RAR5_XFILE_REDIR = 5 +RAR5_XFILE_OWNER = 6 +RAR5_XFILE_SERVICE = 7 + +RAR5_XTIME_UNIXTIME = 0x01 +RAR5_XTIME_HAS_MTIME = 0x02 +RAR5_XTIME_HAS_CTIME = 0x04 +RAR5_XTIME_HAS_ATIME = 0x08 + +RAR5_XENC_CIPHER_AES256 = 0 + +RAR5_XENC_CHECKVAL = 0x01 +RAR5_XENC_TWEAKED = 0x02 + +RAR5_XHASH_BLAKE2SP = 0 + +RAR5_XREDIR_UNIX_SYMLINK = 1 +RAR5_XREDIR_WINDOWS_SYMLINK = 2 +RAR5_XREDIR_WINDOWS_JUNCTION = 3 +RAR5_XREDIR_HARD_LINK = 4 +RAR5_XREDIR_FILE_COPY = 5 + +RAR5_XREDIR_ISDIR = 0x01 + +RAR5_XOWNER_UNAME = 0x01 +RAR5_XOWNER_GNAME = 0x02 +RAR5_XOWNER_UID = 0x04 +RAR5_XOWNER_GID = 0x08 + +RAR5_OS_WINDOWS = 0 +RAR5_OS_UNIX = 1 + +## +## internal constants +## + +RAR_ID = b"Rar!\x1a\x07\x00" +RAR5_ID = b"Rar!\x1a\x07\x01\x00" +ZERO = b'\0' +EMPTY = b'' +UTC = timezone(timedelta(0), 'UTC') +BSIZE = 32 * 1024 + +def _get_rar_version(xfile): + '''Check quickly whether file is rar archive. + ''' + with XFile(xfile) as fd: + buf = fd.read(len(RAR5_ID)) + if buf.startswith(RAR_ID): + return 3 + elif buf.startswith(RAR5_ID): + return 5 + return 0 + +## +## Public interface +## + +def is_rarfile(xfile): + '''Check quickly whether file is rar archive. + ''' + return _get_rar_version(xfile) > 0 + +class Error(Exception): + """Base class for rarfile errors.""" + +class BadRarFile(Error): + """Incorrect data in archive.""" + +class NotRarFile(Error): + """The file is not RAR archive.""" + +class BadRarName(Error): + """Cannot guess multipart name components.""" + +class NoRarEntry(Error): + """File not found in RAR""" + +class PasswordRequired(Error): + """File requires password""" + +class NeedFirstVolume(Error): + """Need to start from first volume.""" + +class NoCrypto(Error): + """Cannot parse encrypted headers - no crypto available.""" + +class RarExecError(Error): + """Problem reported by unrar/rar.""" + +class RarWarning(RarExecError): + """Non-fatal error""" + +class RarFatalError(RarExecError): + """Fatal error""" + +class RarCRCError(RarExecError): + """CRC error during unpacking""" + +class RarLockedArchiveError(RarExecError): + """Must not modify locked archive""" + +class RarWriteError(RarExecError): + """Write error""" + +class RarOpenError(RarExecError): + """Open error""" + +class RarUserError(RarExecError): + """User error""" + +class RarMemoryError(RarExecError): + """Memory error""" + +class RarCreateError(RarExecError): + """Create error""" + +class RarNoFilesError(RarExecError): + """No files that match pattern were found""" + +class RarUserBreak(RarExecError): + """User stop""" + +class RarWrongPassword(RarExecError): + """Incorrect password""" + +class RarUnknownError(RarExecError): + """Unknown exit code""" + +class RarSignalExit(RarExecError): + """Unrar exited with signal""" + +class RarCannotExec(RarExecError): + """Executable not found.""" + + +class RarInfo(object): + r'''An entry in rar archive. + + RAR3 extended timestamps are :class:`datetime.datetime` objects without timezone. + RAR5 extended timestamps are :class:`datetime.datetime` objects with UTC timezone. + + Attributes: + + filename + File name with relative path. + Path separator is '/'. Always unicode string. + + date_time + File modification timestamp. As tuple of (year, month, day, hour, minute, second). + RAR5 allows archives where it is missing, it's None then. + + file_size + Uncompressed size. + + compress_size + Compressed size. + + compress_type + Compression method: one of :data:`RAR_M0` .. :data:`RAR_M5` constants. + + extract_version + Minimal Rar version needed for decompressing. As (major*10 + minor), + so 2.9 is 29. + + RAR3: 10, 20, 29 + + RAR5 does not have such field in archive, it's simply set to 50. + + host_os + Host OS type, one of RAR_OS_* constants. + + RAR3: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`, :data:`RAR_OS_MSDOS`, + :data:`RAR_OS_OS2`, :data:`RAR_OS_BEOS`. + + RAR5: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`. + + mode + File attributes. May be either dos-style or unix-style, depending on host_os. + + mtime + File modification time. Same value as :attr:`date_time` + but as :class:`datetime.datetime` object with extended precision. + + ctime + Optional time field: creation time. As :class:`datetime.datetime` object. + + atime + Optional time field: last access time. As :class:`datetime.datetime` object. + + arctime + Optional time field: archival time. As :class:`datetime.datetime` object. + (RAR3-only) + + CRC + CRC-32 of uncompressed file, unsigned int. + + RAR5: may be None. + + blake2sp_hash + Blake2SP hash over decompressed data. (RAR5-only) + + comment + Optional file comment field. Unicode string. (RAR3-only) + + file_redir + If not None, file is link of some sort. Contains tuple of (type, flags, target). + (RAR5-only) + + Type is one of constants: + + :data:`RAR5_XREDIR_UNIX_SYMLINK` + unix symlink to target. + :data:`RAR5_XREDIR_WINDOWS_SYMLINK` + windows symlink to target. + :data:`RAR5_XREDIR_WINDOWS_JUNCTION` + windows junction. + :data:`RAR5_XREDIR_HARD_LINK` + hard link to target. + :data:`RAR5_XREDIR_FILE_COPY` + current file is copy of another archive entry. + + Flags may contain :data:`RAR5_XREDIR_ISDIR` bit. + + volume + Volume nr, starting from 0. + + volume_file + Volume file name, where file starts. + + ''' + + # zipfile-compatible fields + filename = None + file_size = None + compress_size = None + date_time = None + comment = None + CRC = None + volume = None + orig_filename = None + + # optional extended time fields, datetime() objects. + mtime = None + ctime = None + atime = None + + extract_version = None + mode = None + host_os = None + compress_type = None + + # rar3-only fields + comment = None + arctime = None + + # rar5-only fields + blake2sp_hash = None + file_redir = None + + # internal fields + flags = 0 + type = None + + def isdir(self): + """Returns True if entry is a directory. + """ + if self.type == RAR_BLOCK_FILE: + return (self.flags & RAR_FILE_DIRECTORY) == RAR_FILE_DIRECTORY + return False + + def needs_password(self): + """Returns True if data is stored password-protected. + """ + if self.type == RAR_BLOCK_FILE: + return (self.flags & RAR_FILE_PASSWORD) > 0 + return False + + +class RarFile(object): + '''Parse RAR structure, provide access to files in archive. + ''' + + #: Archive comment. Unicode string or None. + comment = None + + def __init__(self, rarfile, mode="r", charset=None, info_callback=None, + crc_check=True, errors="stop"): + """Open and parse a RAR archive. + + Parameters: + + rarfile + archive file name + mode + only 'r' is supported. + charset + fallback charset to use, if filenames are not already Unicode-enabled. + info_callback + debug callback, gets to see all archive entries. + crc_check + set to False to disable CRC checks + errors + Either "stop" to quietly stop parsing on errors, + or "strict" to raise errors. Default is "stop". + """ + self._rarfile = rarfile + self._charset = charset or DEFAULT_CHARSET + self._info_callback = info_callback + self._crc_check = crc_check + self._password = None + self._file_parser = None + + if errors == "stop": + self._strict = False + elif errors == "strict": + self._strict = True + else: + raise ValueError("Invalid value for 'errors' parameter.") + + if mode != "r": + raise NotImplementedError("RarFile supports only mode=r") + + self._parse() + + def __enter__(self): + return self + + def __exit__(self, typ, value, traceback): + self.close() + + def setpassword(self, password): + '''Sets the password to use when extracting.''' + self._password = password + if self._file_parser: + if self._file_parser.has_header_encryption(): + self._file_parser = None + if not self._file_parser: + self._parse() + else: + self._file_parser.setpassword(self._password) + + def needs_password(self): + '''Returns True if any archive entries require password for extraction.''' + return self._file_parser.needs_password() + + def namelist(self): + '''Return list of filenames in archive.''' + return [f.filename for f in self.infolist()] + + def infolist(self): + '''Return RarInfo objects for all files/directories in archive.''' + return self._file_parser.infolist() + + def volumelist(self): + '''Returns filenames of archive volumes. + + In case of single-volume archive, the list contains + just the name of main archive file. + ''' + return self._file_parser.volumelist() + + def getinfo(self, fname): + '''Return RarInfo for file. + ''' + return self._file_parser.getinfo(fname) + + def open(self, fname, mode='r', psw=None): + '''Returns file-like object (:class:`RarExtFile`), + from where the data can be read. + + The object implements :class:`io.RawIOBase` interface, so it can + be further wrapped with :class:`io.BufferedReader` + and :class:`io.TextIOWrapper`. + + On older Python where io module is not available, it implements + only .read(), .seek(), .tell() and .close() methods. + + The object is seekable, although the seeking is fast only on + uncompressed files, on compressed files the seeking is implemented + by reading ahead and/or restarting the decompression. + + Parameters: + + fname + file name or RarInfo instance. + mode + must be 'r' + psw + password to use for extracting. + ''' + + if mode != 'r': + raise NotImplementedError("RarFile.open() supports only mode=r") + + # entry lookup + inf = self.getinfo(fname) + if inf.isdir(): + raise TypeError("Directory does not have any data: " + inf.filename) + + # check password + if inf.needs_password(): + psw = psw or self._password + if psw is None: + raise PasswordRequired("File %s requires password" % inf.filename) + else: + psw = None + + return self._file_parser.open(inf, psw) + + def read(self, fname, psw=None): + """Return uncompressed data for archive entry. + + For longer files using :meth:`RarFile.open` may be better idea. + + Parameters: + + fname + filename or RarInfo instance + psw + password to use for extracting. + """ + + with self.open(fname, 'r', psw) as f: + return f.read() + + def close(self): + """Release open resources.""" + pass + + def printdir(self): + """Print archive file list to stdout.""" + for f in self.infolist(): + print(f.filename) + + def extract(self, member, path=None, pwd=None): + """Extract single file into current directory. + + Parameters: + + member + filename or :class:`RarInfo` instance + path + optional destination path + pwd + optional password to use + """ + if isinstance(member, RarInfo): + fname = member.filename + else: + fname = member + self._extract([fname], path, pwd) + + def extractall(self, path=None, members=None, pwd=None): + """Extract all files into current directory. + + Parameters: + + path + optional destination path + members + optional filename or :class:`RarInfo` instance list to extract + pwd + optional password to use + """ + fnlist = [] + if members is not None: + for m in members: + if isinstance(m, RarInfo): + fnlist.append(m.filename) + else: + fnlist.append(m) + self._extract(fnlist, path, pwd) + + def testrar(self): + """Let 'unrar' test the archive. + """ + cmd = [UNRAR_TOOL] + list(TEST_ARGS) + add_password_arg(cmd, self._password) + cmd.append('--') + with XTempFile(self._rarfile) as rarfile: + cmd.append(rarfile) + p = custom_popen(cmd) + output = p.communicate()[0] + check_returncode(p, output) + + def strerror(self): + """Return error string if parsing failed, + or None if no problems. + """ + if not self._file_parser: + return "Not a RAR file" + return self._file_parser.strerror() + + ## + ## private methods + ## + + def _parse(self): + ver = _get_rar_version(self._rarfile) + if ver == 3: + p3 = RAR3Parser(self._rarfile, self._password, self._crc_check, + self._charset, self._strict, self._info_callback) + self._file_parser = p3 # noqa + elif ver == 5: + p5 = RAR5Parser(self._rarfile, self._password, self._crc_check, + self._charset, self._strict, self._info_callback) + self._file_parser = p5 # noqa + else: + raise BadRarFile("Not a RAR file") + + self._file_parser.parse() + self.comment = self._file_parser.comment + + # call unrar to extract a file + def _extract(self, fnlist, path=None, psw=None): + cmd = [UNRAR_TOOL] + list(EXTRACT_ARGS) + + # pasoword + psw = psw or self._password + add_password_arg(cmd, psw) + cmd.append('--') + + # rar file + with XTempFile(self._rarfile) as rarfn: + cmd.append(rarfn) + + # file list + for fn in fnlist: + if os.sep != PATH_SEP: + fn = fn.replace(PATH_SEP, os.sep) + cmd.append(fn) + + # destination path + if path is not None: + cmd.append(path + os.sep) + + # call + p = custom_popen(cmd) + output = p.communicate()[0] + check_returncode(p, output) + +# +# File format parsing +# + +class CommonParser(object): + """Shared parser parts.""" + _main = None + _hdrenc_main = None + _needs_password = False + _fd = None + _expect_sig = None + _parse_error = None + _password = None + comment = None + + def __init__(self, rarfile, password, crc_check, charset, strict, info_cb): + self._rarfile = rarfile + self._password = password + self._crc_check = crc_check + self._charset = charset + self._strict = strict + self._info_callback = info_cb + self._info_list = [] + self._info_map = {} + self._vol_list = [] + + def has_header_encryption(self): + """Returns True if headers are encrypted + """ + if self._hdrenc_main: + return True + if self._main: + if self._main.flags & RAR_MAIN_PASSWORD: + return True + return False + + def setpassword(self, psw): + """Set cached password.""" + self._password = psw + + def volumelist(self): + """Volume files""" + return self._vol_list + + def needs_password(self): + """Is password required""" + return self._needs_password + + def strerror(self): + """Last error""" + return self._parse_error + + def infolist(self): + """List of RarInfo records. + """ + return self._info_list + + def getinfo(self, fname): + """Return RarInfo for filename + """ + # accept both ways here + if PATH_SEP == '/': + fname2 = fname.replace("\\", "/") + else: + fname2 = fname.replace("/", "\\") + + try: + return self._info_map[fname] + except KeyError: + try: + return self._info_map[fname2] + except KeyError: + raise NoRarEntry("No such file: %s" % fname) + + # read rar + def parse(self): + """Process file.""" + self._fd = None + try: + self._parse_real() + finally: + if self._fd: + self._fd.close() + self._fd = None + + def _parse_real(self): + fd = XFile(self._rarfile) + self._fd = fd + sig = fd.read(len(self._expect_sig)) + if sig != self._expect_sig: + if isinstance(self._rarfile, (str, unicode)): + raise NotRarFile("Not a Rar archive: {}".format(self._rarfile)) + raise NotRarFile("Not a Rar archive") + + volume = 0 # first vol (.rar) is 0 + more_vols = False + endarc = False + volfile = self._rarfile + self._vol_list = [self._rarfile] + while 1: + if endarc: + h = None # don't read past ENDARC + else: + h = self._parse_header(fd) + if not h: + if more_vols: + volume += 1 + fd.close() + try: + volfile = self._next_volname(volfile) + fd = XFile(volfile) + except IOError: + self._set_error("Cannot open next volume: %s", volfile) + break + self._fd = fd + sig = fd.read(len(self._expect_sig)) + if sig != self._expect_sig: + self._set_error("Invalid volume sig: %s", volfile) + break + more_vols = False + endarc = False + self._vol_list.append(volfile) + continue + break + h.volume = volume + h.volume_file = volfile + + if h.type == RAR_BLOCK_MAIN and not self._main: + self._main = h + if h.flags & RAR_MAIN_NEWNUMBERING: + # RAR 2.x does not set FIRSTVOLUME, + # so check it only if NEWNUMBERING is used + if (h.flags & RAR_MAIN_FIRSTVOLUME) == 0: + raise NeedFirstVolume("Need to start from first volume") + if h.flags & RAR_MAIN_PASSWORD: + self._needs_password = True + if not self._password: + break + elif h.type == RAR_BLOCK_ENDARC: + more_vols = (h.flags & RAR_ENDARC_NEXT_VOLUME) > 0 + endarc = True + elif h.type == RAR_BLOCK_FILE: + # RAR 2.x does not write RAR_BLOCK_ENDARC + if h.flags & RAR_FILE_SPLIT_AFTER: + more_vols = True + # RAR 2.x does not set RAR_MAIN_FIRSTVOLUME + if volume == 0 and h.flags & RAR_FILE_SPLIT_BEFORE: + raise NeedFirstVolume("Need to start from first volume") + + if h.needs_password(): + self._needs_password = True + + # store it + self.process_entry(fd, h) + + if self._info_callback: + self._info_callback(h) + + # go to next header + if h.add_size > 0: + fd.seek(h.data_offset + h.add_size, 0) + + def process_entry(self, fd, item): + """Examine item, add into lookup cache.""" + raise NotImplementedError() + + def _decrypt_header(self, fd): + raise NotImplementedError('_decrypt_header') + + def _parse_block_header(self, fd): + raise NotImplementedError('_parse_block_header') + + def _open_hack(self, inf, psw): + raise NotImplementedError('_open_hack') + + # read single header + def _parse_header(self, fd): + try: + # handle encrypted headers + if (self._main and self._main.flags & RAR_MAIN_PASSWORD) or self._hdrenc_main: + if not self._password: + return + fd = self._decrypt_header(fd) + + # now read actual header + return self._parse_block_header(fd) + except struct.error: + self._set_error('Broken header in RAR file') + return None + + # given current vol name, construct next one + def _next_volname(self, volfile): + if is_filelike(volfile): + raise IOError("Working on single FD") + if self._main.flags & RAR_MAIN_NEWNUMBERING: + return _next_newvol(volfile) + return _next_oldvol(volfile) + + def _set_error(self, msg, *args): + if args: + msg = msg % args + self._parse_error = msg + if self._strict: + raise BadRarFile(msg) + + def open(self, inf, psw): + """Return stream object for file data.""" + + if inf.file_redir: + # cannot leave to unrar as it expects copied file to exist + if inf.file_redir[0] in (RAR5_XREDIR_FILE_COPY, RAR5_XREDIR_HARD_LINK): + inf = self.getinfo(inf.file_redir[2]) + if not inf: + raise BadRarFile('cannot find copied file') + + if inf.flags & RAR_FILE_SPLIT_BEFORE: + raise NeedFirstVolume("Partial file, please start from first volume: " + inf.filename) + + # is temp write usable? + use_hack = 1 + if not self._main: + use_hack = 0 + elif self._main._must_disable_hack(): + use_hack = 0 + elif inf._must_disable_hack(): + use_hack = 0 + elif is_filelike(self._rarfile): + pass + elif inf.file_size > HACK_SIZE_LIMIT: + use_hack = 0 + elif not USE_EXTRACT_HACK: + use_hack = 0 + + # now extract + if inf.compress_type == RAR_M0 and (inf.flags & RAR_FILE_PASSWORD) == 0 and inf.file_redir is None: + return self._open_clear(inf) + elif use_hack: + return self._open_hack(inf, psw) + elif is_filelike(self._rarfile): + return self._open_unrar_membuf(self._rarfile, inf, psw) + else: + return self._open_unrar(self._rarfile, inf, psw) + + def _open_clear(self, inf): + return DirectReader(self, inf) + + def _open_hack_core(self, inf, psw, prefix, suffix): + + size = inf.compress_size + inf.header_size + rf = XFile(inf.volume_file, 0) + rf.seek(inf.header_offset) + + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + + try: + tmpf.write(prefix) + while size > 0: + if size > BSIZE: + buf = rf.read(BSIZE) + else: + buf = rf.read(size) + if not buf: + raise BadRarFile('read failed: ' + inf.filename) + tmpf.write(buf) + size -= len(buf) + tmpf.write(suffix) + tmpf.close() + rf.close() + except: + rf.close() + tmpf.close() + os.unlink(tmpname) + raise + + return self._open_unrar(tmpname, inf, psw, tmpname) + + # write in-memory archive to temp file - needed for solid archives + def _open_unrar_membuf(self, memfile, inf, psw): + tmpname = membuf_tempfile(memfile) + return self._open_unrar(tmpname, inf, psw, tmpname, force_file=True) + + # extract using unrar + def _open_unrar(self, rarfile, inf, psw=None, tmpfile=None, force_file=False): + cmd = [UNRAR_TOOL] + list(OPEN_ARGS) + add_password_arg(cmd, psw) + cmd.append("--") + cmd.append(rarfile) + + # not giving filename avoids encoding related problems + if not tmpfile or force_file: + fn = inf.filename + if PATH_SEP != os.sep: + fn = fn.replace(PATH_SEP, os.sep) + cmd.append(fn) + + # read from unrar pipe + return PipeReader(self, inf, cmd, tmpfile) + +# +# RAR3 format +# + +class Rar3Info(RarInfo): + """RAR3 specific fields.""" + extract_version = 15 + salt = None + add_size = 0 + header_crc = None + header_size = None + header_offset = None + data_offset = None + _md_class = None + _md_expect = None + + # make sure some rar5 fields are always present + file_redir = None + blake2sp_hash = None + + def _must_disable_hack(self): + if self.type == RAR_BLOCK_FILE: + if self.flags & RAR_FILE_PASSWORD: + return True + elif self.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): + return True + elif self.type == RAR_BLOCK_MAIN: + if self.flags & (RAR_MAIN_SOLID | RAR_MAIN_PASSWORD): + return True + return False + + +class RAR3Parser(CommonParser): + """Parse RAR3 file format. + """ + _expect_sig = RAR_ID + _last_aes_key = (None, None, None) # (salt, key, iv) + + def _decrypt_header(self, fd): + if not _have_crypto: + raise NoCrypto('Cannot parse encrypted headers - no crypto') + salt = fd.read(8) + if self._last_aes_key[0] == salt: + key, iv = self._last_aes_key[1:] + else: + key, iv = rar3_s2k(self._password, salt) + self._last_aes_key = (salt, key, iv) + return HeaderDecrypt(fd, key, iv) + + # common header + def _parse_block_header(self, fd): + h = Rar3Info() + h.header_offset = fd.tell() + + # read and parse base header + buf = fd.read(S_BLK_HDR.size) + if not buf: + return None + t = S_BLK_HDR.unpack_from(buf) + h.header_crc, h.type, h.flags, h.header_size = t + + # read full header + if h.header_size > S_BLK_HDR.size: + hdata = buf + fd.read(h.header_size - S_BLK_HDR.size) + else: + hdata = buf + h.data_offset = fd.tell() + + # unexpected EOF? + if len(hdata) != h.header_size: + self._set_error('Unexpected EOF when reading header') + return None + + pos = S_BLK_HDR.size + + # block has data assiciated with it? + if h.flags & RAR_LONG_BLOCK: + h.add_size, pos = load_le32(hdata, pos) + else: + h.add_size = 0 + + # parse interesting ones, decide header boundaries for crc + if h.type == RAR_BLOCK_MARK: + return h + elif h.type == RAR_BLOCK_MAIN: + pos += 6 + if h.flags & RAR_MAIN_ENCRYPTVER: + pos += 1 + crc_pos = pos + if h.flags & RAR_MAIN_COMMENT: + self._parse_subblocks(h, hdata, pos) + elif h.type == RAR_BLOCK_FILE: + pos = self._parse_file_header(h, hdata, pos - 4) + crc_pos = pos + if h.flags & RAR_FILE_COMMENT: + pos = self._parse_subblocks(h, hdata, pos) + elif h.type == RAR_BLOCK_SUB: + pos = self._parse_file_header(h, hdata, pos - 4) + crc_pos = h.header_size + elif h.type == RAR_BLOCK_OLD_AUTH: + pos += 8 + crc_pos = pos + elif h.type == RAR_BLOCK_OLD_EXTRA: + pos += 7 + crc_pos = pos + else: + crc_pos = h.header_size + + # check crc + if h.type == RAR_BLOCK_OLD_SUB: + crcdat = hdata[2:] + fd.read(h.add_size) + else: + crcdat = hdata[2:crc_pos] + + calc_crc = rar_crc32(crcdat) & 0xFFFF + + # return good header + if h.header_crc == calc_crc: + return h + + # header parsing failed. + self._set_error('Header CRC error (%02x): exp=%x got=%x (xlen = %d)', + h.type, h.header_crc, calc_crc, len(crcdat)) + + # instead panicing, send eof + return None + + # read file-specific header + def _parse_file_header(self, h, hdata, pos): + fld = S_FILE_HDR.unpack_from(hdata, pos) + pos += S_FILE_HDR.size + + h.compress_size = fld[0] + h.file_size = fld[1] + h.host_os = fld[2] + h.CRC = fld[3] + h.date_time = parse_dos_time(fld[4]) + h.mtime = to_datetime(h.date_time) + h.extract_version = fld[5] + h.compress_type = fld[6] + name_size = fld[7] + h.mode = fld[8] + + h._md_class = CRC32Context + h._md_expect = h.CRC + + if h.flags & RAR_FILE_LARGE: + h1, pos = load_le32(hdata, pos) + h2, pos = load_le32(hdata, pos) + h.compress_size |= h1 << 32 + h.file_size |= h2 << 32 + h.add_size = h.compress_size + + name, pos = load_bytes(hdata, name_size, pos) + if h.flags & RAR_FILE_UNICODE: + nul = name.find(ZERO) + h.orig_filename = name[:nul] + u = UnicodeFilename(h.orig_filename, name[nul + 1:]) + h.filename = u.decode() + + # if parsing failed fall back to simple name + if u.failed: + h.filename = self._decode(h.orig_filename) + else: + h.orig_filename = name + h.filename = self._decode(name) + + # change separator, if requested + if PATH_SEP != '\\': + h.filename = h.filename.replace('\\', PATH_SEP) + + if h.flags & RAR_FILE_SALT: + h.salt, pos = load_bytes(hdata, 8, pos) + else: + h.salt = None + + # optional extended time stamps + if h.flags & RAR_FILE_EXTTIME: + pos = _parse_ext_time(h, hdata, pos) + else: + h.mtime = h.atime = h.ctime = h.arctime = None + + return pos + + # find old-style comment subblock + def _parse_subblocks(self, h, hdata, pos): + while pos < len(hdata): + # ordinary block header + t = S_BLK_HDR.unpack_from(hdata, pos) + ___scrc, stype, sflags, slen = t + pos_next = pos + slen + pos += S_BLK_HDR.size + + # corrupt header + if pos_next < pos: + break + + # followed by block-specific header + if stype == RAR_BLOCK_OLD_COMMENT and pos + S_COMMENT_HDR.size <= pos_next: + declen, ver, meth, crc = S_COMMENT_HDR.unpack_from(hdata, pos) + pos += S_COMMENT_HDR.size + data = hdata[pos : pos_next] + cmt = rar3_decompress(ver, meth, data, declen, sflags, + crc, self._password) + if not self._crc_check: + h.comment = self._decode_comment(cmt) + elif rar_crc32(cmt) & 0xFFFF == crc: + h.comment = self._decode_comment(cmt) + + pos = pos_next + return pos + + def _read_comment_v3(self, inf, psw=None): + + # read data + with XFile(inf.volume_file) as rf: + rf.seek(inf.data_offset) + data = rf.read(inf.compress_size) + + # decompress + cmt = rar3_decompress(inf.extract_version, inf.compress_type, data, + inf.file_size, inf.flags, inf.CRC, psw, inf.salt) + + # check crc + if self._crc_check: + crc = rar_crc32(cmt) + if crc != inf.CRC: + return None + + return self._decode_comment(cmt) + + def _decode(self, val): + for c in TRY_ENCODINGS: + try: + return val.decode(c) + except UnicodeError: + pass + return val.decode(self._charset, 'replace') + + def _decode_comment(self, val): + return self._decode(val) + + def process_entry(self, fd, item): + if item.type == RAR_BLOCK_FILE: + # use only first part + if (item.flags & RAR_FILE_SPLIT_BEFORE) == 0: + self._info_map[item.filename] = item + self._info_list.append(item) + elif len(self._info_list) > 0: + # final crc is in last block + old = self._info_list[-1] + old.CRC = item.CRC + old._md_expect = item._md_expect + old.compress_size += item.compress_size + + # parse new-style comment + if item.type == RAR_BLOCK_SUB and item.filename == 'CMT': + if item.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): + pass + elif item.flags & RAR_FILE_SOLID: + # file comment + cmt = self._read_comment_v3(item, self._password) + if len(self._info_list) > 0: + old = self._info_list[-1] + old.comment = cmt + else: + # archive comment + cmt = self._read_comment_v3(item, self._password) + self.comment = cmt + + if item.type == RAR_BLOCK_MAIN: + if item.flags & RAR_MAIN_COMMENT: + self.comment = item.comment + if item.flags & RAR_MAIN_PASSWORD: + self._needs_password = True + + # put file compressed data into temporary .rar archive, and run + # unrar on that, thus avoiding unrar going over whole archive + def _open_hack(self, inf, psw): + # create main header: crc, type, flags, size, res1, res2 + prefix = RAR_ID + S_BLK_HDR.pack(0x90CF, 0x73, 0, 13) + ZERO * (2 + 4) + return self._open_hack_core(inf, psw, prefix, EMPTY) + +# +# RAR5 format +# + +class Rar5Info(RarInfo): + """Shared fields for RAR5 records. + """ + extract_version = 50 + header_crc = None + header_size = None + header_offset = None + data_offset = None + + # type=all + block_type = None + block_flags = None + add_size = 0 + block_extra_size = 0 + + # type=MAIN + volume_number = None + _md_class = None + _md_expect = None + + def _must_disable_hack(self): + return False + + +class Rar5BaseFile(Rar5Info): + """Shared sturct for file & service record. + """ + type = -1 + file_flags = None + file_encryption = (0, 0, 0, EMPTY, EMPTY, EMPTY) + file_compress_flags = None + file_redir = None + file_owner = None + file_version = None + blake2sp_hash = None + + def _must_disable_hack(self): + if self.flags & RAR_FILE_PASSWORD: + return True + if self.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): + return True + if self.file_compress_flags & RAR5_COMPR_SOLID: + return True + if self.file_redir: + return True + return False + + +class Rar5FileInfo(Rar5BaseFile): + """RAR5 file record. + """ + type = RAR_BLOCK_FILE + + +class Rar5ServiceInfo(Rar5BaseFile): + """RAR5 service record. + """ + type = RAR_BLOCK_SUB + + +class Rar5MainInfo(Rar5Info): + """RAR5 archive main record. + """ + type = RAR_BLOCK_MAIN + main_flags = None + main_volume_number = None + + def _must_disable_hack(self): + if self.main_flags & RAR5_MAIN_FLAG_SOLID: + return True + return False + + +class Rar5EncryptionInfo(Rar5Info): + """RAR5 archive header encryption record. + """ + type = RAR5_BLOCK_ENCRYPTION + encryption_algo = None + encryption_flags = None + encryption_kdf_count = None + encryption_salt = None + encryption_check_value = None + + def needs_password(self): + return True + + +class Rar5EndArcInfo(Rar5Info): + """RAR5 end of archive record. + """ + type = RAR_BLOCK_ENDARC + endarc_flags = None + + +class RAR5Parser(CommonParser): + """Parse RAR5 format. + """ + _expect_sig = RAR5_ID + _hdrenc_main = None + + # AES encrypted headers + _last_aes256_key = (-1, None, None) # (kdf_count, salt, key) + + def _gen_key(self, kdf_count, salt): + if self._last_aes256_key[:2] == (kdf_count, salt): + return self._last_aes256_key[2] + if kdf_count > 24: + raise BadRarFile('Too large kdf_count') + psw = self._password + if isinstance(psw, unicode): + psw = psw.encode('utf8') + key = pbkdf2_sha256(psw, salt, 1 << kdf_count) + self._last_aes256_key = (kdf_count, salt, key) + return key + + def _decrypt_header(self, fd): + if not _have_crypto: + raise NoCrypto('Cannot parse encrypted headers - no crypto') + h = self._hdrenc_main + key = self._gen_key(h.encryption_kdf_count, h.encryption_salt) + iv = fd.read(16) + return HeaderDecrypt(fd, key, iv) + + # common header + def _parse_block_header(self, fd): + header_offset = fd.tell() + + preload = 4 + 3 + start_bytes = fd.read(preload) + header_crc, pos = load_le32(start_bytes, 0) + hdrlen, pos = load_vint(start_bytes, pos) + if hdrlen > 2 * 1024 * 1024: + return None + header_size = pos + hdrlen + + # read full header, check for EOF + hdata = start_bytes + fd.read(header_size - len(start_bytes)) + if len(hdata) != header_size: + self._set_error('Unexpected EOF when reading header') + return None + data_offset = fd.tell() + + calc_crc = rar_crc32(memoryview(hdata)[4:]) + if header_crc != calc_crc: + # header parsing failed. + self._set_error('Header CRC error: exp=%x got=%x (xlen = %d)', + header_crc, calc_crc, len(hdata)) + return None + + block_type, pos = load_vint(hdata, pos) + + if block_type == RAR5_BLOCK_MAIN: + h, pos = self._parse_block_common(Rar5MainInfo(), hdata) + h = self._parse_main_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_FILE: + h, pos = self._parse_block_common(Rar5FileInfo(), hdata) + h = self._parse_file_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_SERVICE: + h, pos = self._parse_block_common(Rar5ServiceInfo(), hdata) + h = self._parse_file_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_ENCRYPTION: + h, pos = self._parse_block_common(Rar5EncryptionInfo(), hdata) + h = self._parse_encryption_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_ENDARC: + h, pos = self._parse_block_common(Rar5EndArcInfo(), hdata) + h = self._parse_endarc_block(h, hdata, pos) + else: + h = None + if h: + h.header_offset = header_offset + h.data_offset = data_offset + return h + + def _parse_block_common(self, h, hdata): + h.header_crc, pos = load_le32(hdata, 0) + hdrlen, pos = load_vint(hdata, pos) + h.header_size = hdrlen + pos + h.block_type, pos = load_vint(hdata, pos) + h.block_flags, pos = load_vint(hdata, pos) + + if h.block_flags & RAR5_BLOCK_FLAG_EXTRA_DATA: + h.block_extra_size, pos = load_vint(hdata, pos) + if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: + h.add_size, pos = load_vint(hdata, pos) + + h.compress_size = h.add_size + + if h.block_flags & RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN: + h.flags |= RAR_SKIP_IF_UNKNOWN + if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: + h.flags |= RAR_LONG_BLOCK + return h, pos + + def _parse_main_block(self, h, hdata, pos): + h.main_flags, pos = load_vint(hdata, pos) + if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR: + h.main_volume_number = load_vint(hdata, pos) + + h.flags |= RAR_MAIN_NEWNUMBERING + if h.main_flags & RAR5_MAIN_FLAG_SOLID: + h.flags |= RAR_MAIN_SOLID + if h.main_flags & RAR5_MAIN_FLAG_ISVOL: + h.flags |= RAR_MAIN_VOLUME + if h.main_flags & RAR5_MAIN_FLAG_RECOVERY: + h.flags |= RAR_MAIN_RECOVERY + if self._hdrenc_main: + h.flags |= RAR_MAIN_PASSWORD + if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR == 0: + h.flags |= RAR_MAIN_FIRSTVOLUME + + return h + + def _parse_file_block(self, h, hdata, pos): + h.file_flags, pos = load_vint(hdata, pos) + h.file_size, pos = load_vint(hdata, pos) + h.mode, pos = load_vint(hdata, pos) + + if h.file_flags & RAR5_FILE_FLAG_HAS_MTIME: + h.mtime, pos = load_unixtime(hdata, pos) + h.date_time = h.mtime.timetuple()[:6] + if h.file_flags & RAR5_FILE_FLAG_HAS_CRC32: + h.CRC, pos = load_le32(hdata, pos) + h._md_class = CRC32Context + h._md_expect = h.CRC + + h.file_compress_flags, pos = load_vint(hdata, pos) + h.file_host_os, pos = load_vint(hdata, pos) + h.orig_filename, pos = load_vstr(hdata, pos) + h.filename = h.orig_filename.decode('utf8', 'replace') + + # use compatible values + if h.file_host_os == RAR5_OS_WINDOWS: + h.host_os = RAR_OS_WIN32 + else: + h.host_os = RAR_OS_UNIX + h.compress_type = RAR_M0 + ((h.file_compress_flags >> 7) & 7) + + if h.block_extra_size: + # allow 1 byte of garbage + while pos < len(hdata) - 1: + xsize, pos = load_vint(hdata, pos) + xdata, pos = load_bytes(hdata, xsize, pos) + self._process_file_extra(h, xdata) + + if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE: + h.flags |= RAR_FILE_SPLIT_BEFORE + if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_AFTER: + h.flags |= RAR_FILE_SPLIT_AFTER + if h.file_flags & RAR5_FILE_FLAG_ISDIR: + h.flags |= RAR_FILE_DIRECTORY + if h.file_compress_flags & RAR5_COMPR_SOLID: + h.flags |= RAR_FILE_SOLID + + return h + + def _parse_endarc_block(self, h, hdata, pos): + h.endarc_flags, pos = load_vint(hdata, pos) + if h.endarc_flags & RAR5_ENDARC_FLAG_NEXT_VOL: + h.flags |= RAR_ENDARC_NEXT_VOLUME + return h + + def _parse_encryption_block(self, h, hdata, pos): + h.encryption_algo, pos = load_vint(hdata, pos) + h.encryption_flags, pos = load_vint(hdata, pos) + h.encryption_kdf_count, pos = load_byte(hdata, pos) + h.encryption_salt, pos = load_bytes(hdata, 16, pos) + if h.encryption_flags & RAR5_ENC_FLAG_HAS_CHECKVAL: + h.encryption_check_value = load_bytes(hdata, 12, pos) + if h.encryption_algo != RAR5_XENC_CIPHER_AES256: + raise BadRarFile('Unsupported header encryption cipher') + self._hdrenc_main = h + return h + + # file extra record + def _process_file_extra(self, h, xdata): + xtype, pos = load_vint(xdata, 0) + if xtype == RAR5_XFILE_TIME: + self._parse_file_xtime(h, xdata, pos) + elif xtype == RAR5_XFILE_ENCRYPTION: + self._parse_file_encryption(h, xdata, pos) + elif xtype == RAR5_XFILE_HASH: + self._parse_file_hash(h, xdata, pos) + elif xtype == RAR5_XFILE_VERSION: + self._parse_file_version(h, xdata, pos) + elif xtype == RAR5_XFILE_REDIR: + self._parse_file_redir(h, xdata, pos) + elif xtype == RAR5_XFILE_OWNER: + self._parse_file_owner(h, xdata, pos) + elif xtype == RAR5_XFILE_SERVICE: + pass + else: + pass + + # extra block for file time record + def _parse_file_xtime(self, h, xdata, pos): + tflags, pos = load_vint(xdata, pos) + ldr = load_windowstime + if tflags & RAR5_XTIME_UNIXTIME: + ldr = load_unixtime + if tflags & RAR5_XTIME_HAS_MTIME: + h.mtime, pos = ldr(xdata, pos) + h.date_time = h.mtime.timetuple()[:6] + if tflags & RAR5_XTIME_HAS_CTIME: + h.ctime, pos = ldr(xdata, pos) + if tflags & RAR5_XTIME_HAS_ATIME: + h.atime, pos = ldr(xdata, pos) + + # just remember encryption info + def _parse_file_encryption(self, h, xdata, pos): + algo, pos = load_vint(xdata, pos) + flags, pos = load_vint(xdata, pos) + kdf_count, pos = load_byte(xdata, pos) + salt, pos = load_bytes(xdata, 16, pos) + iv, pos = load_bytes(xdata, 16, pos) + checkval = None + if flags & RAR5_XENC_CHECKVAL: + checkval, pos = load_bytes(xdata, 12, pos) + if flags & RAR5_XENC_TWEAKED: + h._md_expect = None + h._md_class = NoHashContext + + h.file_encryption = (algo, flags, kdf_count, salt, iv, checkval) + h.flags |= RAR_FILE_PASSWORD + + def _parse_file_hash(self, h, xdata, pos): + hash_type, pos = load_vint(xdata, pos) + if hash_type == RAR5_XHASH_BLAKE2SP: + h.blake2sp_hash, pos = load_bytes(xdata, 32, pos) + if _have_blake2 and (h.file_encryption[1] & RAR5_XENC_TWEAKED) == 0: + h._md_class = Blake2SP + h._md_expect = h.blake2sp_hash + + def _parse_file_version(self, h, xdata, pos): + flags, pos = load_vint(xdata, pos) + version, pos = load_vint(xdata, pos) + h.file_version = (flags, version) + + def _parse_file_redir(self, h, xdata, pos): + redir_type, pos = load_vint(xdata, pos) + redir_flags, pos = load_vint(xdata, pos) + redir_name, pos = load_vstr(xdata, pos) + redir_name = redir_name.decode('utf8', 'replace') + h.file_redir = (redir_type, redir_flags, redir_name) + + def _parse_file_owner(self, h, xdata, pos): + user_name = group_name = user_id = group_id = None + + flags, pos = load_vint(xdata, pos) + if flags & RAR5_XOWNER_UNAME: + user_name, pos = load_vstr(xdata, pos) + if flags & RAR5_XOWNER_GNAME: + group_name, pos = load_vstr(xdata, pos) + if flags & RAR5_XOWNER_UID: + user_id, pos = load_vint(xdata, pos) + if flags & RAR5_XOWNER_GID: + group_id, pos = load_vint(xdata, pos) + + h.file_owner = (user_name, group_name, user_id, group_id) + + def process_entry(self, fd, item): + if item.block_type == RAR5_BLOCK_FILE: + # use only first part + if (item.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE) == 0: + self._info_map[item.filename] = item + self._info_list.append(item) + elif len(self._info_list) > 0: + # final crc is in last block + old = self._info_list[-1] + old.CRC = item.CRC + old._md_expect = item._md_expect + old.blake2sp_hash = item.blake2sp_hash + old.compress_size += item.compress_size + elif item.block_type == RAR5_BLOCK_SERVICE: + if item.filename == 'CMT': + self._load_comment(fd, item) + + def _load_comment(self, fd, item): + if item.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): + return None + if item.compress_type != RAR_M0: + return None + + if item.flags & RAR_FILE_PASSWORD: + algo, ___flags, kdf_count, salt, iv, ___checkval = item.file_encryption + if algo != RAR5_XENC_CIPHER_AES256: + return None + key = self._gen_key(kdf_count, salt) + f = HeaderDecrypt(fd, key, iv) + cmt = f.read(item.file_size) + else: + # archive comment + with self._open_clear(item) as cmtstream: + cmt = cmtstream.read() + + # rar bug? - appends zero to comment + cmt = cmt.split(ZERO, 1)[0] + self.comment = cmt.decode('utf8') + + def _open_hack(self, inf, psw): + # len, type, blk_flags, flags + main_hdr = b'\x03\x01\x00\x00' + endarc_hdr = b'\x03\x05\x00\x00' + main_hdr = S_LONG.pack(rar_crc32(main_hdr)) + main_hdr + endarc_hdr = S_LONG.pack(rar_crc32(endarc_hdr)) + endarc_hdr + return self._open_hack_core(inf, psw, RAR5_ID + main_hdr, endarc_hdr) + +## +## Utility classes +## + +class UnicodeFilename(object): + """Handle RAR3 unicode filename decompression. + """ + def __init__(self, name, encdata): + self.std_name = bytearray(name) + self.encdata = bytearray(encdata) + self.pos = self.encpos = 0 + self.buf = bytearray() + self.failed = 0 + + def enc_byte(self): + """Copy encoded byte.""" + try: + c = self.encdata[self.encpos] + self.encpos += 1 + return c + except IndexError: + self.failed = 1 + return 0 + + def std_byte(self): + """Copy byte from 8-bit representation.""" + try: + return self.std_name[self.pos] + except IndexError: + self.failed = 1 + return ord('?') + + def put(self, lo, hi): + """Copy 16-bit value to result.""" + self.buf.append(lo) + self.buf.append(hi) + self.pos += 1 + + def decode(self): + """Decompress compressed UTF16 value.""" + hi = self.enc_byte() + flagbits = 0 + while self.encpos < len(self.encdata): + if flagbits == 0: + flags = self.enc_byte() + flagbits = 8 + flagbits -= 2 + t = (flags >> flagbits) & 3 + if t == 0: + self.put(self.enc_byte(), 0) + elif t == 1: + self.put(self.enc_byte(), hi) + elif t == 2: + self.put(self.enc_byte(), self.enc_byte()) + else: + n = self.enc_byte() + if n & 0x80: + c = self.enc_byte() + for _ in range((n & 0x7f) + 2): + lo = (self.std_byte() + c) & 0xFF + self.put(lo, hi) + else: + for _ in range(n + 2): + self.put(self.std_byte(), 0) + return self.buf.decode("utf-16le", "replace") + + +class RarExtFile(RawIOBase): + """Base class for file-like object that :meth:`RarFile.open` returns. + + Provides public methods and common crc checking. + + Behaviour: + - no short reads - .read() and .readinfo() read as much as requested. + - no internal buffer, use io.BufferedReader for that. + """ + + #: Filename of the archive entry + name = None + + def __init__(self, parser, inf): + super(RarExtFile, self).__init__() + + # standard io.* properties + self.name = inf.filename + self.mode = 'rb' + + self._parser = parser + self._inf = inf + self._fd = None + self._remain = 0 + self._returncode = 0 + + self._md_context = None + + self._open() + + def _open(self): + if self._fd: + self._fd.close() + md_class = self._inf._md_class or NoHashContext + self._md_context = md_class() + self._fd = None + self._remain = self._inf.file_size + + def read(self, cnt=None): + """Read all or specified amount of data from archive entry.""" + + # sanitize cnt + if cnt is None or cnt < 0: + cnt = self._remain + elif cnt > self._remain: + cnt = self._remain + if cnt == 0: + return EMPTY + + # actual read + data = self._read(cnt) + if data: + self._md_context.update(data) + self._remain -= len(data) + if len(data) != cnt: + raise BadRarFile("Failed the read enough data") + + # done? + if not data or self._remain == 0: + # self.close() + self._check() + return data + + def _check(self): + """Check final CRC.""" + final = self._md_context.digest() + exp = self._inf._md_expect + if exp is None: + return + if final is None: + return + if self._returncode: + check_returncode(self, '') + if self._remain != 0: + raise BadRarFile("Failed the read enough data") + if final != exp: + raise BadRarFile("Corrupt file - CRC check failed: %s - exp=%r got=%r" % ( + self._inf.filename, exp, final)) + + def _read(self, cnt): + """Actual read that gets sanitized cnt.""" + + def close(self): + """Close open resources.""" + + super(RarExtFile, self).close() + + if self._fd: + self._fd.close() + self._fd = None + + def __del__(self): + """Hook delete to make sure tempfile is removed.""" + self.close() + + def readinto(self, buf): + """Zero-copy read directly into buffer. + + Returns bytes read. + """ + raise NotImplementedError('readinto') + + def tell(self): + """Return current reading position in uncompressed data.""" + return self._inf.file_size - self._remain + + def seek(self, ofs, whence=0): + """Seek in data. + + On uncompressed files, the seeking works by actual + seeks so it's fast. On compresses files its slow + - forward seeking happends by reading ahead, + backwards by re-opening and decompressing from the start. + """ + + # disable crc check when seeking + self._md_context = NoHashContext() + + fsize = self._inf.file_size + cur_ofs = self.tell() + + if whence == 0: # seek from beginning of file + new_ofs = ofs + elif whence == 1: # seek from current position + new_ofs = cur_ofs + ofs + elif whence == 2: # seek from end of file + new_ofs = fsize + ofs + else: + raise ValueError('Invalid value for whence') + + # sanity check + if new_ofs < 0: + new_ofs = 0 + elif new_ofs > fsize: + new_ofs = fsize + + # do the actual seek + if new_ofs >= cur_ofs: + self._skip(new_ofs - cur_ofs) + else: + # reopen and seek + self._open() + self._skip(new_ofs) + return self.tell() + + def _skip(self, cnt): + """Read and discard data""" + while cnt > 0: + if cnt > 8192: + buf = self.read(8192) + else: + buf = self.read(cnt) + if not buf: + break + cnt -= len(buf) + + def readable(self): + """Returns True""" + return True + + def writable(self): + """Returns False. + + Writing is not supported.""" + return False + + def seekable(self): + """Returns True. + + Seeking is supported, although it's slow on compressed files. + """ + return True + + def readall(self): + """Read all remaining data""" + # avoid RawIOBase default impl + return self.read() + + +class PipeReader(RarExtFile): + """Read data from pipe, handle tempfile cleanup.""" + + def __init__(self, rf, inf, cmd, tempfile=None): + self._cmd = cmd + self._proc = None + self._tempfile = tempfile + super(PipeReader, self).__init__(rf, inf) + + def _close_proc(self): + if not self._proc: + return + if self._proc.stdout: + self._proc.stdout.close() + if self._proc.stdin: + self._proc.stdin.close() + if self._proc.stderr: + self._proc.stderr.close() + self._proc.wait() + self._returncode = self._proc.returncode + self._proc = None + + def _open(self): + super(PipeReader, self)._open() + + # stop old process + self._close_proc() + + # launch new process + self._returncode = 0 + self._proc = custom_popen(self._cmd) + self._fd = self._proc.stdout + + # avoid situation where unrar waits on stdin + if self._proc.stdin: + self._proc.stdin.close() + + def _read(self, cnt): + """Read from pipe.""" + + # normal read is usually enough + data = self._fd.read(cnt) + if len(data) == cnt or not data: + return data + + # short read, try looping + buf = [data] + cnt -= len(data) + while cnt > 0: + data = self._fd.read(cnt) + if not data: + break + cnt -= len(data) + buf.append(data) + return EMPTY.join(buf) + + def close(self): + """Close open resources.""" + + self._close_proc() + super(PipeReader, self).close() + + if self._tempfile: + try: + os.unlink(self._tempfile) + except OSError: + pass + self._tempfile = None + + def readinto(self, buf): + """Zero-copy read directly into buffer.""" + cnt = len(buf) + if cnt > self._remain: + cnt = self._remain + vbuf = memoryview(buf) + res = got = 0 + while got < cnt: + res = self._fd.readinto(vbuf[got : cnt]) + if not res: + break + self._md_context.update(vbuf[got : got + res]) + self._remain -= res + got += res + return got + + +class DirectReader(RarExtFile): + """Read uncompressed data directly from archive. + """ + _cur = None + _cur_avail = None + _volfile = None + + def _open(self): + super(DirectReader, self)._open() + + self._volfile = self._inf.volume_file + self._fd = XFile(self._volfile, 0) + self._fd.seek(self._inf.header_offset, 0) + self._cur = self._parser._parse_header(self._fd) + self._cur_avail = self._cur.add_size + + def _skip(self, cnt): + """RAR Seek, skipping through rar files to get to correct position + """ + + while cnt > 0: + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # fd is in read pos, do the read + if cnt > self._cur_avail: + cnt -= self._cur_avail + self._remain -= self._cur_avail + self._cur_avail = 0 + else: + self._fd.seek(cnt, 1) + self._cur_avail -= cnt + self._remain -= cnt + cnt = 0 + + def _read(self, cnt): + """Read from potentially multi-volume archive.""" + + buf = [] + while cnt > 0: + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # fd is in read pos, do the read + if cnt > self._cur_avail: + data = self._fd.read(self._cur_avail) + else: + data = self._fd.read(cnt) + if not data: + break + + # got some data + cnt -= len(data) + self._cur_avail -= len(data) + buf.append(data) + + if len(buf) == 1: + return buf[0] + return EMPTY.join(buf) + + def _open_next(self): + """Proceed to next volume.""" + + # is the file split over archives? + if (self._cur.flags & RAR_FILE_SPLIT_AFTER) == 0: + return False + + if self._fd: + self._fd.close() + self._fd = None + + # open next part + self._volfile = self._parser._next_volname(self._volfile) + fd = open(self._volfile, "rb", 0) + self._fd = fd + sig = fd.read(len(self._parser._expect_sig)) + if sig != self._parser._expect_sig: + raise BadRarFile("Invalid signature") + + # loop until first file header + while 1: + cur = self._parser._parse_header(fd) + if not cur: + raise BadRarFile("Unexpected EOF") + if cur.type in (RAR_BLOCK_MARK, RAR_BLOCK_MAIN): + if cur.add_size: + fd.seek(cur.add_size, 1) + continue + if cur.orig_filename != self._inf.orig_filename: + raise BadRarFile("Did not found file entry") + self._cur = cur + self._cur_avail = cur.add_size + return True + + def readinto(self, buf): + """Zero-copy read directly into buffer.""" + got = 0 + vbuf = memoryview(buf) + while got < len(buf): + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # length for next read + cnt = len(buf) - got + if cnt > self._cur_avail: + cnt = self._cur_avail + + # read into temp view + res = self._fd.readinto(vbuf[got : got + cnt]) + if not res: + break + self._md_context.update(vbuf[got : got + res]) + self._cur_avail -= res + self._remain -= res + got += res + return got + + +class HeaderDecrypt(object): + """File-like object that decrypts from another file""" + def __init__(self, f, key, iv): + self.f = f + self.ciph = AES_CBC_Decrypt(key, iv) + self.buf = EMPTY + + def tell(self): + """Current file pos - works only on block boundaries.""" + return self.f.tell() + + def read(self, cnt=None): + """Read and decrypt.""" + if cnt > 8 * 1024: + raise BadRarFile('Bad count to header decrypt - wrong password?') + + # consume old data + if cnt <= len(self.buf): + res = self.buf[:cnt] + self.buf = self.buf[cnt:] + return res + res = self.buf + self.buf = EMPTY + cnt -= len(res) + + # decrypt new data + blklen = 16 + while cnt > 0: + enc = self.f.read(blklen) + if len(enc) < blklen: + break + dec = self.ciph.decrypt(enc) + if cnt >= len(dec): + res += dec + cnt -= len(dec) + else: + res += dec[:cnt] + self.buf = dec[cnt:] + cnt = 0 + + return res + + +# handle (filename|filelike) object +class XFile(object): + """Input may be filename or file object. + """ + __slots__ = ('_fd', '_need_close') + + def __init__(self, xfile, bufsize=1024): + if is_filelike(xfile): + self._need_close = False + self._fd = xfile + self._fd.seek(0) + else: + self._need_close = True + self._fd = open(xfile, 'rb', bufsize) + + def read(self, n=None): + """Read from file.""" + return self._fd.read(n) + + def tell(self): + """Return file pos.""" + return self._fd.tell() + + def seek(self, ofs, whence=0): + """Move file pos.""" + return self._fd.seek(ofs, whence) + + def readinto(self, dst): + """Read into buffer.""" + return self._fd.readinto(dst) + + def close(self): + """Close file object.""" + if self._need_close: + self._fd.close() + + def __enter__(self): + return self + + def __exit__(self, typ, val, tb): + self.close() + + +class NoHashContext(object): + """No-op hash function.""" + def __init__(self, data=None): + """Initialize""" + def update(self, data): + """Update data""" + def digest(self): + """Final hash""" + def hexdigest(self): + """Hexadecimal digest.""" + + +class CRC32Context(object): + """Hash context that uses CRC32.""" + __slots__ = ['_crc'] + + def __init__(self, data=None): + self._crc = 0 + if data: + self.update(data) + + def update(self, data): + """Process data.""" + self._crc = rar_crc32(data, self._crc) + + def digest(self): + """Final hash.""" + return self._crc + + def hexdigest(self): + """Hexadecimal digest.""" + return '%08x' % self.digest() + + +class Blake2SP(object): + """Blake2sp hash context. + """ + __slots__ = ['_thread', '_buf', '_cur', '_digest'] + digest_size = 32 + block_size = 64 + parallelism = 8 + + def __init__(self, data=None): + self._buf = b'' + self._cur = 0 + self._digest = None + self._thread = [] + + for i in range(self.parallelism): + ctx = self._blake2s(i, 0, i == (self.parallelism - 1)) + self._thread.append(ctx) + + if data: + self.update(data) + + def _blake2s(self, ofs, depth, is_last): + return blake2s(node_offset=ofs, node_depth=depth, last_node=is_last, + depth=2, inner_size=32, fanout=self.parallelism) + + def _add_block(self, blk): + self._thread[self._cur].update(blk) + self._cur = (self._cur + 1) % self.parallelism + + def update(self, data): + """Hash data. + """ + view = memoryview(data) + bs = self.block_size + if self._buf: + need = bs - len(self._buf) + if len(view) < need: + self._buf += view.tobytes() + return + self._add_block(self._buf + view[:need].tobytes()) + view = view[need:] + while len(view) >= bs: + self._add_block(view[:bs]) + view = view[bs:] + self._buf = view.tobytes() + + def digest(self): + """Return final digest value. + """ + if self._digest is None: + if self._buf: + self._add_block(self._buf) + self._buf = EMPTY + ctx = self._blake2s(0, 1, True) + for t in self._thread: + ctx.update(t.digest()) + self._digest = ctx.digest() + return self._digest + + def hexdigest(self): + """Hexadecimal digest.""" + return tohex(self.digest()) + +## +## Utility functions +## + +S_LONG = Struct(' len(buf): + raise BadRarFile('cannot load byte') + return S_BYTE.unpack_from(buf, pos)[0], end + +def load_le32(buf, pos): + """Load little-endian 32-bit integer""" + end = pos + 4 + if end > len(buf): + raise BadRarFile('cannot load le32') + return S_LONG.unpack_from(buf, pos)[0], pos + 4 + +def load_bytes(buf, num, pos): + """Load sequence of bytes""" + end = pos + num + if end > len(buf): + raise BadRarFile('cannot load bytes') + return buf[pos : end], end + +def load_vstr(buf, pos): + """Load bytes prefixed by vint length""" + slen, pos = load_vint(buf, pos) + return load_bytes(buf, slen, pos) + +def load_dostime(buf, pos): + """Load LE32 dos timestamp""" + stamp, pos = load_le32(buf, pos) + tup = parse_dos_time(stamp) + return to_datetime(tup), pos + +def load_unixtime(buf, pos): + """Load LE32 unix timestamp""" + secs, pos = load_le32(buf, pos) + dt = datetime.fromtimestamp(secs, UTC) + return dt, pos + +def load_windowstime(buf, pos): + """Load LE64 windows timestamp""" + # unix epoch (1970) in seconds from windows epoch (1601) + unix_epoch = 11644473600 + val1, pos = load_le32(buf, pos) + val2, pos = load_le32(buf, pos) + secs, n1secs = divmod((val2 << 32) | val1, 10000000) + dt = datetime.fromtimestamp(secs - unix_epoch, UTC) + dt = dt.replace(microsecond=n1secs // 10) + return dt, pos + +# new-style next volume +def _next_newvol(volfile): + i = len(volfile) - 1 + while i >= 0: + if volfile[i] >= '0' and volfile[i] <= '9': + return _inc_volname(volfile, i) + i -= 1 + raise BadRarName("Cannot construct volume name: " + volfile) + +# old-style next volume +def _next_oldvol(volfile): + # rar -> r00 + if volfile[-4:].lower() == '.rar': + return volfile[:-2] + '00' + return _inc_volname(volfile, len(volfile) - 1) + +# increase digits with carry, otherwise just increment char +def _inc_volname(volfile, i): + fn = list(volfile) + while i >= 0: + if fn[i] != '9': + fn[i] = chr(ord(fn[i]) + 1) + break + fn[i] = '0' + i -= 1 + return ''.join(fn) + +# rar3 extended time fields +def _parse_ext_time(h, data, pos): + # flags and rest of data can be missing + flags = 0 + if pos + 2 <= len(data): + flags = S_SHORT.unpack_from(data, pos)[0] + pos += 2 + + mtime, pos = _parse_xtime(flags >> 3 * 4, data, pos, h.mtime) + h.ctime, pos = _parse_xtime(flags >> 2 * 4, data, pos) + h.atime, pos = _parse_xtime(flags >> 1 * 4, data, pos) + h.arctime, pos = _parse_xtime(flags >> 0 * 4, data, pos) + if mtime: + h.mtime = mtime + h.date_time = mtime.timetuple()[:6] + return pos + +# rar3 one extended time field +def _parse_xtime(flag, data, pos, basetime=None): + res = None + if flag & 8: + if not basetime: + basetime, pos = load_dostime(data, pos) + + # load second fractions + rem = 0 + cnt = flag & 3 + for _ in range(cnt): + b, pos = load_byte(data, pos) + rem = (b << 16) | (rem >> 8) + + # convert 100ns units to microseconds + usec = rem // 10 + if usec > 1000000: + usec = 999999 + + # dostime has room for 30 seconds only, correct if needed + if flag & 4 and basetime.second < 59: + res = basetime.replace(microsecond=usec, second=basetime.second + 1) + else: + res = basetime.replace(microsecond=usec) + return res, pos + +def is_filelike(obj): + """Filename or file object? + """ + if isinstance(obj, str) or isinstance(obj, unicode): + return False + res = True + for a in ('read', 'tell', 'seek'): + res = res and hasattr(obj, a) + if not res: + raise ValueError("Invalid object passed as file") + return True + +def rar3_s2k(psw, salt): + """String-to-key hash for RAR3. + """ + if not isinstance(psw, unicode): + psw = psw.decode('utf8') + seed = psw.encode('utf-16le') + salt + iv = EMPTY + h = sha1() + for i in range(16): + for j in range(0x4000): + cnt = S_LONG.pack(i * 0x4000 + j) + h.update(seed + cnt[:3]) + if j == 0: + iv += h.digest()[19:20] + key_be = h.digest()[:16] + key_le = pack("LLLL", key_be)) + return key_le, iv + +def rar3_decompress(vers, meth, data, declen=0, flags=0, crc=0, psw=None, salt=None): + """Decompress blob of compressed data. + + Used for data with non-standard header - eg. comments. + """ + # already uncompressed? + if meth == RAR_M0 and (flags & RAR_FILE_PASSWORD) == 0: + return data + + # take only necessary flags + flags = flags & (RAR_FILE_PASSWORD | RAR_FILE_SALT | RAR_FILE_DICTMASK) + flags |= RAR_LONG_BLOCK + + # file header + fname = b'data' + date = 0 + mode = 0x20 + fhdr = S_FILE_HDR.pack(len(data), declen, RAR_OS_MSDOS, crc, + date, vers, meth, len(fname), mode) + fhdr += fname + if flags & RAR_FILE_SALT: + if not salt: + return EMPTY + fhdr += salt + + # full header + hlen = S_BLK_HDR.size + len(fhdr) + hdr = S_BLK_HDR.pack(0, RAR_BLOCK_FILE, flags, hlen) + fhdr + hcrc = rar_crc32(hdr[2:]) & 0xFFFF + hdr = S_BLK_HDR.pack(hcrc, RAR_BLOCK_FILE, flags, hlen) + fhdr + + # archive main header + mh = S_BLK_HDR.pack(0x90CF, RAR_BLOCK_MAIN, 0, 13) + ZERO * (2 + 4) + + # decompress via temp rar + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + try: + tmpf.write(RAR_ID + mh + hdr + data) + tmpf.close() + + cmd = [UNRAR_TOOL] + list(OPEN_ARGS) + add_password_arg(cmd, psw, (flags & RAR_FILE_PASSWORD)) + cmd.append(tmpname) + + p = custom_popen(cmd) + return p.communicate()[0] + finally: + tmpf.close() + os.unlink(tmpname) + +def to_datetime(t): + """Convert 6-part time tuple into datetime object. + """ + if t is None: + return None + + # extract values + year, mon, day, h, m, s = t + + # assume the values are valid + try: + return datetime(year, mon, day, h, m, s) + except ValueError: + pass + + # sanitize invalid values + mday = (0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) + if mon < 1: + mon = 1 + if mon > 12: + mon = 12 + if day < 1: + day = 1 + if day > mday[mon]: + day = mday[mon] + if h > 23: + h = 23 + if m > 59: + m = 59 + if s > 59: + s = 59 + if mon == 2 and day == 29: + try: + return datetime(year, mon, day, h, m, s) + except ValueError: + day = 28 + return datetime(year, mon, day, h, m, s) + +def parse_dos_time(stamp): + """Parse standard 32-bit DOS timestamp. + """ + sec, stamp = stamp & 0x1F, stamp >> 5 + mn, stamp = stamp & 0x3F, stamp >> 6 + hr, stamp = stamp & 0x1F, stamp >> 5 + day, stamp = stamp & 0x1F, stamp >> 5 + mon, stamp = stamp & 0x0F, stamp >> 4 + yr = (stamp & 0x7F) + 1980 + return (yr, mon, day, hr, mn, sec * 2) + +def custom_popen(cmd): + """Disconnect cmd from parent fds, read only from stdout. + """ + # needed for py2exe + creationflags = 0 + if sys.platform == 'win32': + creationflags = 0x08000000 # CREATE_NO_WINDOW + + # run command + try: + p = Popen(cmd, bufsize=0, stdout=PIPE, stdin=PIPE, stderr=STDOUT, + creationflags=creationflags) + except OSError as ex: + if ex.errno == errno.ENOENT: + raise RarCannotExec("Unrar not installed? (rarfile.UNRAR_TOOL=%r)" % UNRAR_TOOL) + raise + return p + +def custom_check(cmd, ignore_retcode=False): + """Run command, collect output, raise error if needed. + """ + p = custom_popen(cmd) + out, _ = p.communicate() + if p.returncode and not ignore_retcode: + raise RarExecError("Check-run failed") + return out + +def add_password_arg(cmd, psw, ___required=False): + """Append password switch to commandline. + """ + if UNRAR_TOOL == ALT_TOOL: + return + if psw is not None: + cmd.append('-p' + psw) + else: + cmd.append('-p-') + +def check_returncode(p, out): + """Raise exception according to unrar exit code. + """ + code = p.returncode + if code == 0: + return + + # map return code to exception class, codes from rar.txt + errmap = [None, + RarWarning, RarFatalError, RarCRCError, RarLockedArchiveError, # 1..4 + RarWriteError, RarOpenError, RarUserError, RarMemoryError, # 5..8 + RarCreateError, RarNoFilesError, RarWrongPassword] # 9..11 + if UNRAR_TOOL == ALT_TOOL: + errmap = [None] + if code > 0 and code < len(errmap): + exc = errmap[code] + elif code == 255: + exc = RarUserBreak + elif code < 0: + exc = RarSignalExit + else: + exc = RarUnknownError + + # format message + if out: + msg = "%s [%d]: %s" % (exc.__doc__, p.returncode, out) + else: + msg = "%s [%d]" % (exc.__doc__, p.returncode) + + raise exc(msg) + +def hmac_sha256(key, data): + """HMAC-SHA256""" + return HMAC(key, data, sha256).digest() + +def membuf_tempfile(memfile): + memfile.seek(0, 0) + + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + + try: + while True: + buf = memfile.read(BSIZE) + if not buf: + break + tmpf.write(buf) + tmpf.close() + except: + tmpf.close() + os.unlink(tmpname) + raise + return tmpname + +class XTempFile(object): + __slots__ = ('_tmpfile', '_filename') + + def __init__(self, rarfile): + if is_filelike(rarfile): + self._tmpfile = membuf_tempfile(rarfile) + self._filename = self._tmpfile + else: + self._tmpfile = None + self._filename = rarfile + + def __enter__(self): + return self._filename + + def __exit__(self, exc_type, exc_value, tb): + if self._tmpfile: + try: + os.unlink(self._tmpfile) + except OSError: + pass + self._tmpfile = None + +# +# Check if unrar works +# + +ORIG_UNRAR_TOOL = UNRAR_TOOL +ORIG_OPEN_ARGS = OPEN_ARGS +ORIG_EXTRACT_ARGS = EXTRACT_ARGS +ORIG_TEST_ARGS = TEST_ARGS + +def _check_unrar_tool(): + global UNRAR_TOOL, OPEN_ARGS, EXTRACT_ARGS, TEST_ARGS + try: + # does UNRAR_TOOL work? + custom_check([ORIG_UNRAR_TOOL], True) + + UNRAR_TOOL = ORIG_UNRAR_TOOL + OPEN_ARGS = ORIG_OPEN_ARGS + EXTRACT_ARGS = ORIG_EXTRACT_ARGS + TEST_ARGS = ORIG_TEST_ARGS + except RarCannotExec: + try: + # does ALT_TOOL work? + custom_check([ALT_TOOL] + list(ALT_CHECK_ARGS), True) + # replace config + UNRAR_TOOL = ALT_TOOL + OPEN_ARGS = ALT_OPEN_ARGS + EXTRACT_ARGS = ALT_EXTRACT_ARGS + TEST_ARGS = ALT_TEST_ARGS + except RarCannotExec: + # no usable tool, only uncompressed archives work + pass + +_check_unrar_tool() + From d8e8cff0733afb26f82fcdf4b27c45c900b9a6a4 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:19:16 +0000 Subject: [PATCH 32/51] Create DeleteSamples.py --- .../templates/nzgbet/scripts/DeleteSamples.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/DeleteSamples.py diff --git a/apps/templates/nzgbet/scripts/DeleteSamples.py b/apps/templates/nzgbet/scripts/DeleteSamples.py new file mode 100644 index 0000000..395628f --- /dev/null +++ b/apps/templates/nzgbet/scripts/DeleteSamples.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +def is_sample(filePath, inputName, maxSampleSize, SampleIDs): + # 200 MB in bytes + SIZE_CUTOFF = int(maxSampleSize) * 1024 * 1024 + if os.path.getsize(filePath) < SIZE_CUTOFF: + if 'SizeOnly' in SampleIDs: + return True + # Ignore 'sample' in files unless 'sample' in Torrent Name + for ident in SampleIDs: + if ident.lower() in filePath.lower() and not ident.lower() in inputName.lower(): + return True + # Return False if none of these were met. + return False + +if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + +if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + +print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) +status = 0 +if os.environ.has_key('NZBPP_TOTALSTATUS'): + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + +else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." + print "Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." + print "Please check your Par-check/repair settings for future downloads." + +# Check if destination directory exists (important for reprocessing of history items) +if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + +# All checks done, now launching the script. +if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + +mediaContainer = os.environ['NZBPO_MEDIAEXTENSIONS'].split(',') +SampleIDs = os.environ['NZBPO_SAMPLEIDS'].split(',') +for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): + for file in filenames: + filePath = os.path.join(dirpath, file) + fileName, fileExtension = os.path.splitext(file) + if fileExtension in mediaContainer or ".*" in mediaContainer : # If the file is a video file + if is_sample(filePath, os.environ['NZBPP_NZBNAME'], os.environ['NZBPO_MAXSAMPLESIZE'], SampleIDs): # Ignore samples + print "Deleting sample file: ", filePath + try: + os.unlink(filePath) + except: + print "Error: unable to delete file", filePath + sys.exit(NZBGET_POSTPROCESS_ERROR) +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From 17ef17859feb676dd3da81ecb258f91d8d5bdf28 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:19:32 +0000 Subject: [PATCH 33/51] Create flatten.py --- apps/templates/nzgbet/scripts/flatten.py | 106 +++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/flatten.py diff --git a/apps/templates/nzgbet/scripts/flatten.py b/apps/templates/nzgbet/scripts/flatten.py new file mode 100644 index 0000000..73dd0d5 --- /dev/null +++ b/apps/templates/nzgbet/scripts/flatten.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys +import shutil + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + +if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + +print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) +status = 0 +if os.environ.has_key('NZBPP_TOTALSTATUS'): + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + +else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "[ERROR] Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "[ERROR] Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." + print "[ERROR] Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." + print "[WARNING] Please check your Par-check/repair settings for future downloads." + +# Check if destination directory exists (important for reprocessing of history items) +if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "[ERROR] Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + +# All checks done, now launching the script. +if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + +def removeEmptyFolders(path, removeRoot=True): + #Function to remove empty folders + if not os.path.isdir(path): + return + + # remove empty subfolders + print "[INFO] Checking for empty folders in:%s" % path + files = os.listdir(path) + if len(files): + for f in files: + fullpath = os.path.join(path, f) + if os.path.isdir(fullpath): + removeEmptyFolders(fullpath) + + # if folder empty, delete it + files = os.listdir(path) + if len(files) == 0 and removeRoot: + print "[INFO] Removing empty folder:%s" % path + os.rmdir(path) + +directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) +if os.environ['NZBPO_DESTINATIONDIRECTORY'] and os.path.isdir(os.environ['NZBPO_DESTINATIONDIRECTORY']): + destination = os.environ['NZBPO_DESTINATIONDIRECTORY'] + if os.environ['NZBPO_APPENDCATEGORIES'] == 'yes': + destination = os.path.join(destination, os.environ['NZBPP_CATEGORY']) +else: + destination = directory +print "Flattening directory: %s" % (directory) +for dirpath, dirnames, filenames in os.walk(directory): + for fileName in filenames: + outputFile = os.path.join(dirpath, fileName) + if dirpath == directory: + continue + target = os.path.join(destination, fileName) + try: + shutil.move(outputFile, target) + except: + print "[ERROR] Could not flatten %s" % outputFile +removeEmptyFolders(directory) # Cleanup empty directories +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From e4496b31fcc9b8999205acf14dbabb61c32ffd44 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:19:51 +0000 Subject: [PATCH 34/51] Create hash.py --- apps/templates/nzgbet/scripts/hash.py | 167 ++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/hash.py diff --git a/apps/templates/nzgbet/scripts/hash.py b/apps/templates/nzgbet/scripts/hash.py new file mode 100644 index 0000000..f2fbec1 --- /dev/null +++ b/apps/templates/nzgbet/scripts/hash.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Original Author: clinton-hall +# https://github.com/clinton-hall/GetScripts/blob/master/SafeRename.py +# +# Modified By: desimaniac (No Acknowledgement of Source Above) +################################################################################ +import os +import re +import shutil +import sys + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +############################################################ +# EXTENSION STUFF +############################################################ + +def do_check(): + if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + + if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + + print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) + + status = 0 + if 'NZBPP_TOTALSTATUS' in os.environ: + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "[ERROR] Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "[ERROR] Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. " \ + "Setting status \"failed\"." + print "[ERROR] Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is " \ + "ok so handle as though download successful." + print "[WARNING] Please check your Par-check/repair settings for future downloads." + + # Check if destination directory exists (important for reprocessing of history items) + if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "[ERROR] Nothing to post-process: destination directory", os.environ[ + 'NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + + # All checks done, now launching the script. + if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + + +def get_file_name(path): + try: + file_name = os.path.basename(path) + extensions = re.findall(r'\.([^.]+)', file_name) + ext = '.'.join(extensions) + name = file_name.replace(".%s" % ext, '') + return name, ext + except Exception: + pass + return None + + +def is_file_hash(file_name): + hash_regexp = [ + r'^[a-fA-F0-9]{40}$', + r'^[a-fA-F0-9]{32}$', + r'^[a-f0-9]{128}$', + r'^[a-zA-Z0-9]{42}$' + ] + for hash in hash_regexp: + if re.match(hash, file_name): + return True + return False + + +def find_files(folder, extension=None, depth=None): + file_list = [] + start_count = folder.count(os.sep) + for path, subdirs, files in os.walk(folder, topdown=True): + for name in files: + if depth and path.count(os.sep) - start_count >= depth: + del subdirs[:] + continue + file = os.path.join(path, name) + if not extension: + file_list.append(file) + else: + if file.lower().endswith(extension.lower()): + file_list.append(file) + + return sorted(file_list, key=lambda x: x.count(os.path.sep), reverse=True) + + +############################################################ +# MAIN +############################################################ + +# do checks +do_check() + +# retrieve required variables +directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) +nzb_name = os.environ['NZBPP_NZBFILENAME'] +if nzb_name is None: + print("[ERROR] Unable to retrieve NZBPP_NZBFILENAME") + sys.exit(NZBGET_POSTPROCESS_ERROR) +nzb_name = nzb_name.replace('.nzb', '') + +print("[INFO] Using \"%s\" for hashed filenames" % nzb_name) +print("[INFO] Scanning \"%s\" for hashed filenames" % directory) + +# scan for files +found_files = find_files(directory) +if not found_files: + print("[INFO] No files were found in \"%s\"" % directory) + sys.exit(NZBGET_POSTPROCESS_NONE) +else: + print("[INFO] Found %d files to check for hashed filenames" % len(found_files)) + # loop files checking for file hash + moved_files = 0 + for found_file_path in found_files: + # set variable + dir_name = os.path.dirname(found_file_path) + file_name, file_ext = get_file_name(found_file_path) + + # is this a file hash + if is_file_hash(file_name): + new_file_path = os.path.join(dir_name, "%s.%s" % (nzb_name, file_ext)) + print("[INFO] Moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) + try: + shutil.move(found_file_path, new_file_path) + moved_files += 1 + except Exception: + print("[ERROR] Failed moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) + + print("[INFO] Finished processing \"%s\", moved %d files" % (directory, moved_files)) + +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From b4834608ba66469e090e4bb1eb69939d79e166f3 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:20:07 +0000 Subject: [PATCH 35/51] Create reverse_name.py --- apps/templates/nzgbet/scripts/reverse_name.py | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/reverse_name.py diff --git a/apps/templates/nzgbet/scripts/reverse_name.py b/apps/templates/nzgbet/scripts/reverse_name.py new file mode 100644 index 0000000..98819d0 --- /dev/null +++ b/apps/templates/nzgbet/scripts/reverse_name.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys +import re +import locale + +reverse_list = [r"\.\d{2}e\d{2}s\.", r"\.p0612\.", r"\.[pi]0801\.", r"\.p027\.", r"\.[pi]675\.", r"\.[pi]084\.", r"\.p063\.", r"\b[45]62[xh]\.", r"\.yarulb\.", r"\.vtd[hp]\.", + r'\.(?:ld[.-]?)?bew\.', r"\.pir.?(shv|dov|bew|dvd|db|rb)\.", r"\brdvd\.", r"\.vts\.", r"\.reneercs\.", r"\.dcv\.", r"\b(pir|mac)dh\b", r"\.reporp\.", r"\.kcaper\.", + r"\.lanretni\.", r"\b3ca\b", r"\bcaa\b", r"\b3pm\b", r"\.cstn\.", r"\.5r\.", r"\brcs\b"] +reverse_pattern = re.compile('|'.join(reverse_list), flags=re.IGNORECASE) +season_pattern = re.compile(r"(.*\.\d{2}e\d{2}s\.)(.*)", flags=re.IGNORECASE) +word_pattern = re.compile(r"([^A-Z0-9]*[A-Z0-9]+)") +char_replace = [[r"(\w)1\.(\w)",r"\1i\2"] +] +garbage_name = re.compile(r"^[a-zA-Z0-9]{2,}$") +media_list = [r"\.s\d{2}e\d{2}\.", r"\.2160p\.", r"\.1080[pi]\.", r"\.720p\.", r"\.576[pi]\.", r"\.480[pi]\.", r"\.360p\.", r"\.[xh]26[45]\b", r"\.bluray\.", r"\.[hp]dtv\.", + r'\.web(?:[.-]?dl)?\.', r"\.(vhs|vod|dvd|web|bd|br).?rip\.", r"\.dvdr\b", r"\.stv\.", r"\.screener\.", r"\.vcd\.", r"\bhd(cam|rip)\b", r"\.proper\.", r"\.repack\.", + r"\.internal\.", r"\bac3\b", r"\baac\b", r"\bmp3\b", r"\.ntsc\.", r"\.pal\.", r"\.secam\.", r"\bdivx\b", r"\bxvid\b", r"\.r5\.", r"\.scr\."] +media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) +media_extentions = [".mkv", ".mp4", ".avi", ".wmv", ".divx", ".xvid"] + +if 'nt' == os.name: + import ctypes + + class WinEnv: + def __init__(self): + pass + + @staticmethod + def get_environment_variable(name): + name = unicode(name) # ensures string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + env_value = None + if n: + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + env_value = buf.value + return env_value + + def __getitem__(self, key): + return self.get_environment_variable(key) + + def get(self, key, default=None): + r = self.get_environment_variable(key) + return r if r is not None else default + + evn = WinEnv() +else: + class LinuxEnv(object): + def __init__(self, environ): + self.environ = environ + + def __getitem__(self, key): + v = self.environ.get(key) + try: + return v.decode(SYS_ENCODING) if isinstance(v, str) else v + except (UnicodeDecodeError, UnicodeEncodeError): + return v + + def get(self, key, default=None): + v = self[key] + return v if v is not None else default + + evn = LinuxEnv(os.environ) + +SYS_ENCODING = None + +try: + locale.setlocale(locale.LC_ALL, '') +except (locale.Error, IOError): + pass +try: + SYS_ENCODING = locale.getpreferredencoding() +except (locale.Error, IOError): + pass + +if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + SYS_ENCODING = 'UTF-8' + + +class ek: + def __init__(self): + pass + + @staticmethod + def fix_string_encoding(x): + if str == type(x): + try: + return x.decode(SYS_ENCODING) + except UnicodeDecodeError: + return None + elif unicode == type(x): + return x + return None + + @staticmethod + def fix_out_encoding(x): + if isinstance(x, basestring): + return ek.fix_string_encoding(x) + return x + + @staticmethod + def fix_list_encoding(x): + if type(x) not in (list, tuple): + return x + return filter(lambda i: None is not i, map(ek.fix_out_encoding, x)) + + @staticmethod + def encode_item(x): + try: + return x.encode(SYS_ENCODING) + except UnicodeEncodeError: + return x.encode(SYS_ENCODING, 'ignore') + + @staticmethod + def win_encode_unicode(x): + if isinstance(x, str): + try: + return x.decode('UTF-8') + except UnicodeDecodeError: + return x + return x + + @staticmethod + def ek(func, *args, **kwargs): + if 'nt' == os.name: + # convert all str parameter values to unicode + args = tuple([x if not isinstance(x, str) else ek.win_encode_unicode(x) for x in args]) + kwargs = {k: x if not isinstance(x, str) else ek.win_encode_unicode(x) for k, x in + kwargs.iteritems()} + func_result = func(*args, **kwargs) + else: + func_result = func(*[ek.encode_item(x) if type(x) == str else x for x in args], **kwargs) + + if type(func_result) in (list, tuple): + return ek.fix_list_encoding(func_result) + elif str == type(func_result): + return ek.fix_string_encoding(func_result) + return func_result + + +class logger: + INFO = 'INFO' + DETAIL = 'DETAIL' + ERROR = 'ERROR' + WARNING = 'WARNING' + + @staticmethod + def log(message, msg_type=INFO): + print('[%s] %s' % (msg_type, message)) + + +def tryInt(s, s_default=0): + try: + return int(s) + except: + return s_default + +# NZBGet V11+ +# Check if the script is called from nzbget 11.0 or later +nzbget_version = evn.get('NZBOP_VERSION', '0.1') +nzbget_version = tryInt(nzbget_version[:nzbget_version.find(".")]) +if nzbget_version >= 11: + logger.log("Script triggered from NZBGet (11.0 or later).") + + # NZBGet argv: all passed as environment variables. + clientAgent = "nzbget" + # Exit codes used by NZBGet + POSTPROCESS_PARCHECK=92 + POSTPROCESS_SUCCESS=93 + POSTPROCESS_ERROR=94 + POSTPROCESS_NONE=95 + + # Check nzbget.conf options + status = 0 + + if evn['NZBOP_UNPACK'] != 'yes': + logger.log("Please enable option \"Unpack\" in nzbget configuration file, exiting") + sys.exit(POSTPROCESS_NONE) + + parstatus = evn['NZBPP_PARSTATUS'] + + # Check par status + if parstatus == '3': + logger.log("Par-check successful, but Par-repair disabled, exiting") + sys.exit(POSTPROCESS_NONE) + + if parstatus == '1': + logger.log("Par-check failed, setting status \"failed\"") + status = 1 + sys.exit(POSTPROCESS_NONE) + + unpackstatus = evn['NZBPP_UNPACKSTATUS'] + + # Check unpack status + if unpackstatus == '1': + logger.log("Unpack failed, setting status \"failed\"") + status = 1 + sys.exit(POSTPROCESS_NONE) + + directory = evn['NZBPP_DIRECTORY'] + + if unpackstatus == '0' and parstatus != '2': + # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check + + for dirpath, dirnames, filenames in ek.ek(os.walk, directory): + for file in filenames: + fileExtension = ek.ek(os.path.splitext, file)[1] + + if fileExtension in ['.par2']: + logger.log("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\"g") + status = 1 + break + + if ek.ek(os.path.isfile, ek.ek(os.path.join, directory, "_brokenlog.txt")) and not status == 1: + logger.log("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting") + status = 1 + + if not status == 1: + logger.log("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful") + + # Check if destination directory exists (important for reprocessing of history items) + if not ek.ek(os.path.isdir, directory): + logger.log("Post-Process: Nothing to post-process: destination directory %s doesn't exist" % directory) + status = 1 + + # All checks done, now launching the script. + + rd = False + videos = 0 + old_name = None + new_name = None + base_name = ek.ek(os.path.basename, directory) + for dirpath, dirnames, filenames in ek.ek(os.walk, directory): + for file in filenames: + + filePath = ek.ek(os.path.join, dirpath, file) + fileName, fileExtension = ek.ek(os.path.splitext, file) + dirname = ek.ek(os.path.dirname, filePath) + + if reverse_pattern.search(fileName) is not None: + na_parts = season_pattern.search(fileName) + if na_parts is not None: + word_p = word_pattern.findall(na_parts.group(2)) + new_words = "" + for wp in word_p: + if wp[0] == ".": + new_words += "." + new_words += re.sub(r"\W","",wp) + for cr in char_replace: + new_words = re.sub(cr[0],cr[1],new_words) + new_filename = new_words[::-1] + na_parts.group(1)[::-1] + else: + new_filename = fileName[::-1] + logger.log("reversing filename from: %s to %s" % (fileName, new_filename)) + try: + ek.ek(os.rename, filePath, ek.ek(os.path.join, dirpath, new_filename + fileExtension)) + rd = True + except Exception,e: + logger.log(e, logger.ERROR) + logger.log("Error: unable to rename file %s" % file, logger.ERROR) + pass + elif (fileExtension.lower() in media_extentions) and (garbage_name.search(fileName) is not None) and (media_pattern.search(base_name) is not None): + videos += 1 + old_name = filePath + new_name = ek.ek(os.path.join, dirname, '%s%s' % (base_name, fileExtension)) + + if not rd and videos == 1 and old_name is not None and new_name is not None: + logger.log("renaming the File %s to the Dirname %s" % (ek.ek(os.path.basename, old_name), base_name)) + try: + ek.ek(os.rename, old_name, new_name) + rd = True + except Exception,e: + logger.log(e, logger.ERROR) + logger.log("Error unable to rename file %s" % old_name, logger.ERROR) + pass + + if rd: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_NONE) + +else: + logger.log("This script can only be called from NZBGet (11.0 or later).", logger.ERROR) + sys.exit(0) From 85837ddeade65829d65b6228e651ef050cdb5ba8 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:20:22 +0000 Subject: [PATCH 36/51] Create unzip.py --- apps/templates/nzgbet/scripts/unzip.py | 294 +++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/unzip.py diff --git a/apps/templates/nzgbet/scripts/unzip.py b/apps/templates/nzgbet/scripts/unzip.py new file mode 100644 index 0000000..521a50f --- /dev/null +++ b/apps/templates/nzgbet/scripts/unzip.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# +############################################################################## +### NZBGET SCAN SCRIPT ### + +# Unzips zipped nzbs. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### +### NZBGET SCAN SCRIPT ### +############################################################################## + +import os, zipfile, tarfile, gzip, pickle, datetime, re, struct, locale +import rarfile.rarfile as rarfile + +from gzip import FEXTRA, FNAME + +if 'nt' == os.name: + import ctypes + + class WinEnv: + def __init__(self): + pass + + @staticmethod + def get_environment_variable(name): + name = unicode(name) # ensures string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + result = None + if n: + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + result = buf.value + return result + + def __getitem__(self, key): + return self.get_environment_variable(key) + + def get(self, key, default=None): + r = self.get_environment_variable(key) + return r if r is not None else default + + env_var = WinEnv() +else: + class LinuxEnv(object): + def __init__(self, environ): + self.environ = environ + + def __getitem__(self, key): + v = self.environ.get(key) + try: + return v.decode(SYS_ENCODING) if isinstance(v, str) else v + except (UnicodeDecodeError, UnicodeEncodeError): + return v + + def get(self, key, default=None): + v = self[key] + return v if v is not None else default + + env_var = LinuxEnv(os.environ) + + +SYS_ENCODING = None +try: + locale.setlocale(locale.LC_ALL, '') +except (locale.Error, IOError): + pass +try: + SYS_ENCODING = locale.getpreferredencoding() +except (locale.Error, IOError): + pass +if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + SYS_ENCODING = 'UTF-8' + + +filename = env_var.get('NZBNP_FILENAME') +if re.search(r"\.tar\.gz$", filename, flags=re.I) is None: + ext = os.path.splitext(filename)[1].lower() +else: + ext = '.tar.gz' +cat = env_var.get('NZBNP_CATEGORY') +dir = env_var.get('NZBNP_DIRECTORY') +prio = env_var.get('NZBNP_PRIORITY') +top = env_var.get('NZBNP_TOP') +pause = env_var.get('NZBNP_PAUSED') +if 'NZBNP_DUPEKEY' in os.environ: + dupekey = env_var.get('NZBNP_DUPEKEY') + dupescore = env_var.get('NZBNP_DUPESCORE') + dupemode = env_var.get('NZBNP_DUPEMODE') +else: + dupekey = None + dupescore = None + dupemode = None + +tmp_zipinfo = os.path.join(os.environ.get('NZBOP_TEMPDIR'), r'nzbget\unzip_scan\info') +nzb_list = [] + +def read_gzip_info(gzipfile): + gf = gzipfile.fileobj + pos = gf.tell() + + # Read archive size + gf.seek(-4, 2) + size = struct.unpack('= 11: - logger.log("Script triggered from NZBGet (11.0 or later).") - - # NZBGet argv: all passed as environment variables. - clientAgent = "nzbget" - # Exit codes used by NZBGet - POSTPROCESS_PARCHECK=92 - POSTPROCESS_SUCCESS=93 - POSTPROCESS_ERROR=94 - POSTPROCESS_NONE=95 - - # Check nzbget.conf options - status = 0 - - if evn['NZBOP_UNPACK'] != 'yes': - logger.log("Please enable option \"Unpack\" in nzbget configuration file, exiting") - sys.exit(POSTPROCESS_NONE) - - parstatus = evn['NZBPP_PARSTATUS'] - - # Check par status - if parstatus == '3': - logger.log("Par-check successful, but Par-repair disabled, exiting") - sys.exit(POSTPROCESS_NONE) - - if parstatus == '1': - logger.log("Par-check failed, setting status \"failed\"") - status = 1 - sys.exit(POSTPROCESS_NONE) - - unpackstatus = evn['NZBPP_UNPACKSTATUS'] - - # Check unpack status - if unpackstatus == '1': - logger.log("Unpack failed, setting status \"failed\"") - status = 1 - sys.exit(POSTPROCESS_NONE) - - directory = evn['NZBPP_DIRECTORY'] - - if unpackstatus == '0' and parstatus != '2': - # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check - - for dirpath, dirnames, filenames in ek.ek(os.walk, directory): - for file in filenames: - fileExtension = ek.ek(os.path.splitext, file)[1] - - if fileExtension in ['.par2']: - logger.log("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\"g") - status = 1 - break - - if ek.ek(os.path.isfile, ek.ek(os.path.join, directory, "_brokenlog.txt")) and not status == 1: - logger.log("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting") - status = 1 - - if not status == 1: - logger.log("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful") - - # Check if destination directory exists (important for reprocessing of history items) - if not ek.ek(os.path.isdir, directory): - logger.log("Post-Process: Nothing to post-process: destination directory %s doesn't exist" % directory) - status = 1 - - # All checks done, now launching the script. - - rd = False - videos = 0 - old_name = None - new_name = None - base_name = ek.ek(os.path.basename, directory) - for dirpath, dirnames, filenames in ek.ek(os.walk, directory): - for file in filenames: - - filePath = ek.ek(os.path.join, dirpath, file) - fileName, fileExtension = ek.ek(os.path.splitext, file) - dirname = ek.ek(os.path.dirname, filePath) - - if reverse_pattern.search(fileName) is not None: - na_parts = season_pattern.search(fileName) - if na_parts is not None: - word_p = word_pattern.findall(na_parts.group(2)) - new_words = "" - for wp in word_p: - if wp[0] == ".": - new_words += "." - new_words += re.sub(r"\W","",wp) - for cr in char_replace: - new_words = re.sub(cr[0],cr[1],new_words) - new_filename = new_words[::-1] + na_parts.group(1)[::-1] - else: - new_filename = fileName[::-1] - logger.log("reversing filename from: %s to %s" % (fileName, new_filename)) - try: - ek.ek(os.rename, filePath, ek.ek(os.path.join, dirpath, new_filename + fileExtension)) - rd = True - except Exception,e: - logger.log(e, logger.ERROR) - logger.log("Error: unable to rename file %s" % file, logger.ERROR) - pass - elif (fileExtension.lower() in media_extentions) and (garbage_name.search(fileName) is not None) and (media_pattern.search(base_name) is not None): - videos += 1 - old_name = filePath - new_name = ek.ek(os.path.join, dirname, '%s%s' % (base_name, fileExtension)) - - if not rd and videos == 1 and old_name is not None and new_name is not None: - logger.log("renaming the File %s to the Dirname %s" % (ek.ek(os.path.basename, old_name), base_name)) - try: - ek.ek(os.rename, old_name, new_name) - rd = True - except Exception,e: - logger.log(e, logger.ERROR) - logger.log("Error unable to rename file %s" % old_name, logger.ERROR) - pass - - if rd: - sys.exit(POSTPROCESS_SUCCESS) - else: - sys.exit(POSTPROCESS_NONE) - -else: - logger.log("This script can only be called from NZBGet (11.0 or later).", logger.ERROR) - sys.exit(0) From 5aa46bf941837c2e065df21472cc0986962da901 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:35:41 +0000 Subject: [PATCH 41/51] Delete hash.py --- apps/templates/nzbget-mp4/scripts/hash.py | 167 ---------------------- 1 file changed, 167 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/hash.py diff --git a/apps/templates/nzbget-mp4/scripts/hash.py b/apps/templates/nzbget-mp4/scripts/hash.py deleted file mode 100644 index f2fbec1..0000000 --- a/apps/templates/nzbget-mp4/scripts/hash.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python -# -# Title: PGBlitz (Reference Title File) -# Maintainer: Admin9705 -# URL: https://pgblitz.com - http://github.pgblitz.com -# GNU: General Public License v3.0 -# -# Original Author: clinton-hall -# https://github.com/clinton-hall/GetScripts/blob/master/SafeRename.py -# -# Modified By: desimaniac (No Acknowledgement of Source Above) -################################################################################ -import os -import re -import shutil -import sys - -# NZBGet Exit Codes -NZBGET_POSTPROCESS_PARCHECK = 92 -NZBGET_POSTPROCESS_SUCCESS = 93 -NZBGET_POSTPROCESS_ERROR = 94 -NZBGET_POSTPROCESS_NONE = 95 - -############################################################ -# EXTENSION STUFF -############################################################ - -def do_check(): - if not os.environ.has_key('NZBOP_SCRIPTDIR'): - print "This script can only be called from NZBGet (11.0 or later)." - sys.exit(0) - - if os.environ['NZBOP_VERSION'][0:5] < '11.0': - print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) - sys.exit(0) - - print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) - - status = 0 - if 'NZBPP_TOTALSTATUS' in os.environ: - if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': - print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) - status = 1 - else: - # Check par status - if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': - print "[ERROR] Par-repair failed, setting status \"failed\"." - status = 1 - - # Check unpack status - if os.environ['NZBPP_UNPACKSTATUS'] == '1': - print "[ERROR] Unpack failed, setting status \"failed\"." - status = 1 - - if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': - # Unpack was skipped due to nzb-file properties or due to errors during par-check - - if os.environ['NZBPP_HEALTH'] < 1000: - print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. " \ - "Setting status \"failed\"." - print "[ERROR] Please check your Par-check/repair settings for future downloads." - status = 1 - - else: - print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is " \ - "ok so handle as though download successful." - print "[WARNING] Please check your Par-check/repair settings for future downloads." - - # Check if destination directory exists (important for reprocessing of history items) - if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): - print "[ERROR] Nothing to post-process: destination directory", os.environ[ - 'NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." - status = 1 - - # All checks done, now launching the script. - if status == 1: - sys.exit(NZBGET_POSTPROCESS_NONE) - - -def get_file_name(path): - try: - file_name = os.path.basename(path) - extensions = re.findall(r'\.([^.]+)', file_name) - ext = '.'.join(extensions) - name = file_name.replace(".%s" % ext, '') - return name, ext - except Exception: - pass - return None - - -def is_file_hash(file_name): - hash_regexp = [ - r'^[a-fA-F0-9]{40}$', - r'^[a-fA-F0-9]{32}$', - r'^[a-f0-9]{128}$', - r'^[a-zA-Z0-9]{42}$' - ] - for hash in hash_regexp: - if re.match(hash, file_name): - return True - return False - - -def find_files(folder, extension=None, depth=None): - file_list = [] - start_count = folder.count(os.sep) - for path, subdirs, files in os.walk(folder, topdown=True): - for name in files: - if depth and path.count(os.sep) - start_count >= depth: - del subdirs[:] - continue - file = os.path.join(path, name) - if not extension: - file_list.append(file) - else: - if file.lower().endswith(extension.lower()): - file_list.append(file) - - return sorted(file_list, key=lambda x: x.count(os.path.sep), reverse=True) - - -############################################################ -# MAIN -############################################################ - -# do checks -do_check() - -# retrieve required variables -directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) -nzb_name = os.environ['NZBPP_NZBFILENAME'] -if nzb_name is None: - print("[ERROR] Unable to retrieve NZBPP_NZBFILENAME") - sys.exit(NZBGET_POSTPROCESS_ERROR) -nzb_name = nzb_name.replace('.nzb', '') - -print("[INFO] Using \"%s\" for hashed filenames" % nzb_name) -print("[INFO] Scanning \"%s\" for hashed filenames" % directory) - -# scan for files -found_files = find_files(directory) -if not found_files: - print("[INFO] No files were found in \"%s\"" % directory) - sys.exit(NZBGET_POSTPROCESS_NONE) -else: - print("[INFO] Found %d files to check for hashed filenames" % len(found_files)) - # loop files checking for file hash - moved_files = 0 - for found_file_path in found_files: - # set variable - dir_name = os.path.dirname(found_file_path) - file_name, file_ext = get_file_name(found_file_path) - - # is this a file hash - if is_file_hash(file_name): - new_file_path = os.path.join(dir_name, "%s.%s" % (nzb_name, file_ext)) - print("[INFO] Moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) - try: - shutil.move(found_file_path, new_file_path) - moved_files += 1 - except Exception: - print("[ERROR] Failed moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) - - print("[INFO] Finished processing \"%s\", moved %d files" % (directory, moved_files)) - -sys.exit(NZBGET_POSTPROCESS_SUCCESS) From b74965ad0469263daa401fa15c37308b6d61bb8c Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:35:51 +0000 Subject: [PATCH 42/51] Delete flatten.py --- apps/templates/nzbget-mp4/scripts/flatten.py | 106 ------------------- 1 file changed, 106 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/flatten.py diff --git a/apps/templates/nzbget-mp4/scripts/flatten.py b/apps/templates/nzbget-mp4/scripts/flatten.py deleted file mode 100644 index 73dd0d5..0000000 --- a/apps/templates/nzbget-mp4/scripts/flatten.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python -# -# Title: PGBlitz (Reference Title File) -# Maintainer: Admin9705 -# URL: https://pgblitz.com - http://github.pgblitz.com -# GNU: General Public License v3.0 -# -# Additions: clinton-hall - https://github.com/Prinz23 -################################################################################ -import os -import sys -import shutil - -# NZBGet Exit Codes -NZBGET_POSTPROCESS_PARCHECK = 92 -NZBGET_POSTPROCESS_SUCCESS = 93 -NZBGET_POSTPROCESS_ERROR = 94 -NZBGET_POSTPROCESS_NONE = 95 - -if not os.environ.has_key('NZBOP_SCRIPTDIR'): - print "This script can only be called from NZBGet (11.0 or later)." - sys.exit(0) - -if os.environ['NZBOP_VERSION'][0:5] < '11.0': - print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) - sys.exit(0) - -print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) -status = 0 -if os.environ.has_key('NZBPP_TOTALSTATUS'): - if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': - print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) - status = 1 - -else: - # Check par status - if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': - print "[ERROR] Par-repair failed, setting status \"failed\"." - status = 1 - - # Check unpack status - if os.environ['NZBPP_UNPACKSTATUS'] == '1': - print "[ERROR] Unpack failed, setting status \"failed\"." - status = 1 - - if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': - # Unpack was skipped due to nzb-file properties or due to errors during par-check - - if os.environ['NZBPP_HEALTH'] < 1000: - print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." - print "[ERROR] Please check your Par-check/repair settings for future downloads." - status = 1 - - else: - print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." - print "[WARNING] Please check your Par-check/repair settings for future downloads." - -# Check if destination directory exists (important for reprocessing of history items) -if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): - print "[ERROR] Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." - status = 1 - -# All checks done, now launching the script. -if status == 1: - sys.exit(NZBGET_POSTPROCESS_NONE) - -def removeEmptyFolders(path, removeRoot=True): - #Function to remove empty folders - if not os.path.isdir(path): - return - - # remove empty subfolders - print "[INFO] Checking for empty folders in:%s" % path - files = os.listdir(path) - if len(files): - for f in files: - fullpath = os.path.join(path, f) - if os.path.isdir(fullpath): - removeEmptyFolders(fullpath) - - # if folder empty, delete it - files = os.listdir(path) - if len(files) == 0 and removeRoot: - print "[INFO] Removing empty folder:%s" % path - os.rmdir(path) - -directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) -if os.environ['NZBPO_DESTINATIONDIRECTORY'] and os.path.isdir(os.environ['NZBPO_DESTINATIONDIRECTORY']): - destination = os.environ['NZBPO_DESTINATIONDIRECTORY'] - if os.environ['NZBPO_APPENDCATEGORIES'] == 'yes': - destination = os.path.join(destination, os.environ['NZBPP_CATEGORY']) -else: - destination = directory -print "Flattening directory: %s" % (directory) -for dirpath, dirnames, filenames in os.walk(directory): - for fileName in filenames: - outputFile = os.path.join(dirpath, fileName) - if dirpath == directory: - continue - target = os.path.join(destination, fileName) - try: - shutil.move(outputFile, target) - except: - print "[ERROR] Could not flatten %s" % outputFile -removeEmptyFolders(directory) # Cleanup empty directories -sys.exit(NZBGET_POSTPROCESS_SUCCESS) From 56e3b16ab377ffa1442eadb2733044db8769ab4f Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:36:03 +0000 Subject: [PATCH 43/51] Delete DeleteSamples.py --- .../nzbget-mp4/scripts/DeleteSamples.py | 93 ------------------- 1 file changed, 93 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/DeleteSamples.py diff --git a/apps/templates/nzbget-mp4/scripts/DeleteSamples.py b/apps/templates/nzbget-mp4/scripts/DeleteSamples.py deleted file mode 100644 index 395628f..0000000 --- a/apps/templates/nzbget-mp4/scripts/DeleteSamples.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -# Title: PGBlitz (Reference Title File) -# Maintainer: Admin9705 -# URL: https://pgblitz.com - http://github.pgblitz.com -# GNU: General Public License v3.0 -# -# Additions: clinton-hall - https://github.com/Prinz23 -################################################################################ -import os -import sys - -# NZBGet Exit Codes -NZBGET_POSTPROCESS_PARCHECK = 92 -NZBGET_POSTPROCESS_SUCCESS = 93 -NZBGET_POSTPROCESS_ERROR = 94 -NZBGET_POSTPROCESS_NONE = 95 - -def is_sample(filePath, inputName, maxSampleSize, SampleIDs): - # 200 MB in bytes - SIZE_CUTOFF = int(maxSampleSize) * 1024 * 1024 - if os.path.getsize(filePath) < SIZE_CUTOFF: - if 'SizeOnly' in SampleIDs: - return True - # Ignore 'sample' in files unless 'sample' in Torrent Name - for ident in SampleIDs: - if ident.lower() in filePath.lower() and not ident.lower() in inputName.lower(): - return True - # Return False if none of these were met. - return False - -if not os.environ.has_key('NZBOP_SCRIPTDIR'): - print "This script can only be called from NZBGet (11.0 or later)." - sys.exit(0) - -if os.environ['NZBOP_VERSION'][0:5] < '11.0': - print "NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) - sys.exit(0) - -print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) -status = 0 -if os.environ.has_key('NZBPP_TOTALSTATUS'): - if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': - print "Download failed with status %s." % (os.environ['NZBPP_STATUS']) - status = 1 - -else: - # Check par status - if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': - print "Par-repair failed, setting status \"failed\"." - status = 1 - - # Check unpack status - if os.environ['NZBPP_UNPACKSTATUS'] == '1': - print "Unpack failed, setting status \"failed\"." - status = 1 - - if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': - # Unpack was skipped due to nzb-file properties or due to errors during par-check - - if os.environ['NZBPP_HEALTH'] < 1000: - print "Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." - print "Please check your Par-check/repair settings for future downloads." - status = 1 - - else: - print "Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." - print "Please check your Par-check/repair settings for future downloads." - -# Check if destination directory exists (important for reprocessing of history items) -if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): - print "Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." - status = 1 - -# All checks done, now launching the script. -if status == 1: - sys.exit(NZBGET_POSTPROCESS_NONE) - -mediaContainer = os.environ['NZBPO_MEDIAEXTENSIONS'].split(',') -SampleIDs = os.environ['NZBPO_SAMPLEIDS'].split(',') -for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): - for file in filenames: - filePath = os.path.join(dirpath, file) - fileName, fileExtension = os.path.splitext(file) - if fileExtension in mediaContainer or ".*" in mediaContainer : # If the file is a video file - if is_sample(filePath, os.environ['NZBPP_NZBNAME'], os.environ['NZBPO_MAXSAMPLESIZE'], SampleIDs): # Ignore samples - print "Deleting sample file: ", filePath - try: - os.unlink(filePath) - except: - print "Error: unable to delete file", filePath - sys.exit(NZBGET_POSTPROCESS_ERROR) -sys.exit(NZBGET_POSTPROCESS_SUCCESS) From f0ba686d0e6a97e27fce220dbd6bfca970b80e18 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:36:20 +0000 Subject: [PATCH 44/51] Delete rarfile.py --- .../nzbget-mp4/scripts/rarfile/rarfile.py | 2931 ----------------- 1 file changed, 2931 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py diff --git a/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py b/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py deleted file mode 100644 index 6a3a647..0000000 --- a/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py +++ /dev/null @@ -1,2931 +0,0 @@ -# rarfile.py -# -# Copyright (c) 2005-2016 Marko Kreen -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -r"""RAR archive reader. - -This is Python module for Rar archive reading. The interface -is made as :mod:`zipfile`-like as possible. - -Basic logic: - - Parse archive structure with Python. - - Extract non-compressed files with Python - - Extract compressed files with unrar. - - Optionally write compressed data to temp file to speed up unrar, - otherwise it needs to scan whole archive on each execution. - -Example:: - - import rarfile - - rf = rarfile.RarFile('myarchive.rar') - for f in rf.infolist(): - print f.filename, f.file_size - if f.filename == 'README': - print(rf.read(f)) - -Archive files can also be accessed via file-like object returned -by :meth:`RarFile.open`:: - - import rarfile - - with rarfile.RarFile('archive.rar') as rf: - with rf.open('README') as f: - for ln in f: - print(ln.strip()) - -There are few module-level parameters to tune behaviour, -here they are with defaults, and reason to change it:: - - import rarfile - - # Set to full path of unrar.exe if it is not in PATH - rarfile.UNRAR_TOOL = "unrar" - - # Set to '\\' to be more compatible with old rarfile - rarfile.PATH_SEP = '/' - -For more details, refer to source. - -""" - -from __future__ import division, print_function - -## -## Imports and compat - support both Python 2.x and 3.x -## - -import sys -import os -import errno -import struct - -from struct import pack, unpack, Struct -from binascii import crc32, hexlify -from tempfile import mkstemp -from subprocess import Popen, PIPE, STDOUT -from io import RawIOBase -from hashlib import sha1, sha256 -from hmac import HMAC -from datetime import datetime, timedelta, tzinfo - -# fixed offset timezone, for UTC -try: - from datetime import timezone -except ImportError: - class timezone(tzinfo): - """Compat timezone.""" - __slots__ = ('_ofs', '_name') - _DST = timedelta(0) - - def __init__(self, offset, name): - super(timezone, self).__init__() - self._ofs, self._name = offset, name - - def utcoffset(self, dt): - return self._ofs - - def tzname(self, dt): - return self._name - - def dst(self, dt): - return self._DST - -# only needed for encryped headers -try: - try: - from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.kdf import pbkdf2 - - class AES_CBC_Decrypt(object): - """Decrypt API""" - def __init__(self, key, iv): - ciph = Cipher(algorithms.AES(key), modes.CBC(iv), default_backend()) - self.decrypt = ciph.decryptor().update - - def pbkdf2_sha256(password, salt, iters): - """PBKDF2 with HMAC-SHA256""" - ctx = pbkdf2.PBKDF2HMAC(hashes.SHA256(), 32, salt, iters, default_backend()) - return ctx.derive(password) - - except ImportError: - from Crypto.Cipher import AES - from Crypto.Protocol import KDF - - class AES_CBC_Decrypt(object): - """Decrypt API""" - def __init__(self, key, iv): - self.decrypt = AES.new(key, AES.MODE_CBC, iv).decrypt - - def pbkdf2_sha256(password, salt, iters): - """PBKDF2 with HMAC-SHA256""" - return KDF.PBKDF2(password, salt, 32, iters, hmac_sha256) - - _have_crypto = 1 -except ImportError: - _have_crypto = 0 - -try: - from pyblake2 import blake2s - _have_blake2 = True -except ImportError: - _have_blake2 = False - -# compat with 2.x -if sys.hexversion < 0x3000000: - def rar_crc32(data, prev=0): - """CRC32 with unsigned values. - """ - if (prev > 0) and (prev & 0x80000000): - prev -= (1 << 32) - res = crc32(data, prev) - if res < 0: - res += (1 << 32) - return res - tohex = hexlify - _byte_code = ord -else: # pragma: no cover - def tohex(data): - """Return hex string.""" - return hexlify(data).decode('ascii') - rar_crc32 = crc32 - unicode = str - _byte_code = int # noqa - - -__version__ = '3.0' - -# export only interesting items -__all__ = ['is_rarfile', 'RarInfo', 'RarFile', 'RarExtFile'] - -## -## Module configuration. Can be tuned after importing. -## - -#: default fallback charset -DEFAULT_CHARSET = "windows-1252" - -#: list of encodings to try, with fallback to DEFAULT_CHARSET if none succeed -TRY_ENCODINGS = ('utf8', 'utf-16le') - -#: 'unrar', 'rar' or full path to either one -UNRAR_TOOL = "unrar" - -#: Command line args to use for opening file for reading. -OPEN_ARGS = ('p', '-inul') - -#: Command line args to use for extracting file to disk. -EXTRACT_ARGS = ('x', '-y', '-idq') - -#: args for testrar() -TEST_ARGS = ('t', '-idq') - -# -# Allow use of tool that is not compatible with unrar. -# -# By default use 'bsdtar' which is 'tar' program that -# sits on top of libarchive. -# -# Problems with libarchive RAR backend: -# - Does not support solid archives. -# - Does not support password-protected archives. -# - -ALT_TOOL = 'bsdtar' -ALT_OPEN_ARGS = ('-x', '--to-stdout', '-f') -ALT_EXTRACT_ARGS = ('-x', '-f') -ALT_TEST_ARGS = ('-t', '-f') -ALT_CHECK_ARGS = ('--help',) - -#: whether to speed up decompression by using tmp archive -USE_EXTRACT_HACK = 1 - -#: limit the filesize for tmp archive usage -HACK_SIZE_LIMIT = 20 * 1024 * 1024 - -#: Separator for path name components. RAR internally uses '\\'. -#: Use '/' to be similar with zipfile. -PATH_SEP = '/' - -## -## rar constants -## - -# block types -RAR_BLOCK_MARK = 0x72 # r -RAR_BLOCK_MAIN = 0x73 # s -RAR_BLOCK_FILE = 0x74 # t -RAR_BLOCK_OLD_COMMENT = 0x75 # u -RAR_BLOCK_OLD_EXTRA = 0x76 # v -RAR_BLOCK_OLD_SUB = 0x77 # w -RAR_BLOCK_OLD_RECOVERY = 0x78 # x -RAR_BLOCK_OLD_AUTH = 0x79 # y -RAR_BLOCK_SUB = 0x7a # z -RAR_BLOCK_ENDARC = 0x7b # { - -# flags for RAR_BLOCK_MAIN -RAR_MAIN_VOLUME = 0x0001 -RAR_MAIN_COMMENT = 0x0002 -RAR_MAIN_LOCK = 0x0004 -RAR_MAIN_SOLID = 0x0008 -RAR_MAIN_NEWNUMBERING = 0x0010 -RAR_MAIN_AUTH = 0x0020 -RAR_MAIN_RECOVERY = 0x0040 -RAR_MAIN_PASSWORD = 0x0080 -RAR_MAIN_FIRSTVOLUME = 0x0100 -RAR_MAIN_ENCRYPTVER = 0x0200 - -# flags for RAR_BLOCK_FILE -RAR_FILE_SPLIT_BEFORE = 0x0001 -RAR_FILE_SPLIT_AFTER = 0x0002 -RAR_FILE_PASSWORD = 0x0004 -RAR_FILE_COMMENT = 0x0008 -RAR_FILE_SOLID = 0x0010 -RAR_FILE_DICTMASK = 0x00e0 -RAR_FILE_DICT64 = 0x0000 -RAR_FILE_DICT128 = 0x0020 -RAR_FILE_DICT256 = 0x0040 -RAR_FILE_DICT512 = 0x0060 -RAR_FILE_DICT1024 = 0x0080 -RAR_FILE_DICT2048 = 0x00a0 -RAR_FILE_DICT4096 = 0x00c0 -RAR_FILE_DIRECTORY = 0x00e0 -RAR_FILE_LARGE = 0x0100 -RAR_FILE_UNICODE = 0x0200 -RAR_FILE_SALT = 0x0400 -RAR_FILE_VERSION = 0x0800 -RAR_FILE_EXTTIME = 0x1000 -RAR_FILE_EXTFLAGS = 0x2000 - -# flags for RAR_BLOCK_ENDARC -RAR_ENDARC_NEXT_VOLUME = 0x0001 -RAR_ENDARC_DATACRC = 0x0002 -RAR_ENDARC_REVSPACE = 0x0004 -RAR_ENDARC_VOLNR = 0x0008 - -# flags common to all blocks -RAR_SKIP_IF_UNKNOWN = 0x4000 -RAR_LONG_BLOCK = 0x8000 - -# Host OS types -RAR_OS_MSDOS = 0 -RAR_OS_OS2 = 1 -RAR_OS_WIN32 = 2 -RAR_OS_UNIX = 3 -RAR_OS_MACOS = 4 -RAR_OS_BEOS = 5 - -# Compression methods - '0'..'5' -RAR_M0 = 0x30 -RAR_M1 = 0x31 -RAR_M2 = 0x32 -RAR_M3 = 0x33 -RAR_M4 = 0x34 -RAR_M5 = 0x35 - -# -# RAR5 constants -# - -RAR5_BLOCK_MAIN = 1 -RAR5_BLOCK_FILE = 2 -RAR5_BLOCK_SERVICE = 3 -RAR5_BLOCK_ENCRYPTION = 4 -RAR5_BLOCK_ENDARC = 5 - -RAR5_BLOCK_FLAG_EXTRA_DATA = 0x01 -RAR5_BLOCK_FLAG_DATA_AREA = 0x02 -RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN = 0x04 -RAR5_BLOCK_FLAG_SPLIT_BEFORE = 0x08 -RAR5_BLOCK_FLAG_SPLIT_AFTER = 0x10 -RAR5_BLOCK_FLAG_DEPENDS_PREV = 0x20 -RAR5_BLOCK_FLAG_KEEP_WITH_PARENT = 0x40 - -RAR5_MAIN_FLAG_ISVOL = 0x01 -RAR5_MAIN_FLAG_HAS_VOLNR = 0x02 -RAR5_MAIN_FLAG_SOLID = 0x04 -RAR5_MAIN_FLAG_RECOVERY = 0x08 -RAR5_MAIN_FLAG_LOCKED = 0x10 - -RAR5_FILE_FLAG_ISDIR = 0x01 -RAR5_FILE_FLAG_HAS_MTIME = 0x02 -RAR5_FILE_FLAG_HAS_CRC32 = 0x04 -RAR5_FILE_FLAG_UNKNOWN_SIZE = 0x08 - -RAR5_COMPR_SOLID = 0x40 - -RAR5_ENC_FLAG_HAS_CHECKVAL = 0x01 - -RAR5_ENDARC_FLAG_NEXT_VOL = 0x01 - -RAR5_XFILE_ENCRYPTION = 1 -RAR5_XFILE_HASH = 2 -RAR5_XFILE_TIME = 3 -RAR5_XFILE_VERSION = 4 -RAR5_XFILE_REDIR = 5 -RAR5_XFILE_OWNER = 6 -RAR5_XFILE_SERVICE = 7 - -RAR5_XTIME_UNIXTIME = 0x01 -RAR5_XTIME_HAS_MTIME = 0x02 -RAR5_XTIME_HAS_CTIME = 0x04 -RAR5_XTIME_HAS_ATIME = 0x08 - -RAR5_XENC_CIPHER_AES256 = 0 - -RAR5_XENC_CHECKVAL = 0x01 -RAR5_XENC_TWEAKED = 0x02 - -RAR5_XHASH_BLAKE2SP = 0 - -RAR5_XREDIR_UNIX_SYMLINK = 1 -RAR5_XREDIR_WINDOWS_SYMLINK = 2 -RAR5_XREDIR_WINDOWS_JUNCTION = 3 -RAR5_XREDIR_HARD_LINK = 4 -RAR5_XREDIR_FILE_COPY = 5 - -RAR5_XREDIR_ISDIR = 0x01 - -RAR5_XOWNER_UNAME = 0x01 -RAR5_XOWNER_GNAME = 0x02 -RAR5_XOWNER_UID = 0x04 -RAR5_XOWNER_GID = 0x08 - -RAR5_OS_WINDOWS = 0 -RAR5_OS_UNIX = 1 - -## -## internal constants -## - -RAR_ID = b"Rar!\x1a\x07\x00" -RAR5_ID = b"Rar!\x1a\x07\x01\x00" -ZERO = b'\0' -EMPTY = b'' -UTC = timezone(timedelta(0), 'UTC') -BSIZE = 32 * 1024 - -def _get_rar_version(xfile): - '''Check quickly whether file is rar archive. - ''' - with XFile(xfile) as fd: - buf = fd.read(len(RAR5_ID)) - if buf.startswith(RAR_ID): - return 3 - elif buf.startswith(RAR5_ID): - return 5 - return 0 - -## -## Public interface -## - -def is_rarfile(xfile): - '''Check quickly whether file is rar archive. - ''' - return _get_rar_version(xfile) > 0 - -class Error(Exception): - """Base class for rarfile errors.""" - -class BadRarFile(Error): - """Incorrect data in archive.""" - -class NotRarFile(Error): - """The file is not RAR archive.""" - -class BadRarName(Error): - """Cannot guess multipart name components.""" - -class NoRarEntry(Error): - """File not found in RAR""" - -class PasswordRequired(Error): - """File requires password""" - -class NeedFirstVolume(Error): - """Need to start from first volume.""" - -class NoCrypto(Error): - """Cannot parse encrypted headers - no crypto available.""" - -class RarExecError(Error): - """Problem reported by unrar/rar.""" - -class RarWarning(RarExecError): - """Non-fatal error""" - -class RarFatalError(RarExecError): - """Fatal error""" - -class RarCRCError(RarExecError): - """CRC error during unpacking""" - -class RarLockedArchiveError(RarExecError): - """Must not modify locked archive""" - -class RarWriteError(RarExecError): - """Write error""" - -class RarOpenError(RarExecError): - """Open error""" - -class RarUserError(RarExecError): - """User error""" - -class RarMemoryError(RarExecError): - """Memory error""" - -class RarCreateError(RarExecError): - """Create error""" - -class RarNoFilesError(RarExecError): - """No files that match pattern were found""" - -class RarUserBreak(RarExecError): - """User stop""" - -class RarWrongPassword(RarExecError): - """Incorrect password""" - -class RarUnknownError(RarExecError): - """Unknown exit code""" - -class RarSignalExit(RarExecError): - """Unrar exited with signal""" - -class RarCannotExec(RarExecError): - """Executable not found.""" - - -class RarInfo(object): - r'''An entry in rar archive. - - RAR3 extended timestamps are :class:`datetime.datetime` objects without timezone. - RAR5 extended timestamps are :class:`datetime.datetime` objects with UTC timezone. - - Attributes: - - filename - File name with relative path. - Path separator is '/'. Always unicode string. - - date_time - File modification timestamp. As tuple of (year, month, day, hour, minute, second). - RAR5 allows archives where it is missing, it's None then. - - file_size - Uncompressed size. - - compress_size - Compressed size. - - compress_type - Compression method: one of :data:`RAR_M0` .. :data:`RAR_M5` constants. - - extract_version - Minimal Rar version needed for decompressing. As (major*10 + minor), - so 2.9 is 29. - - RAR3: 10, 20, 29 - - RAR5 does not have such field in archive, it's simply set to 50. - - host_os - Host OS type, one of RAR_OS_* constants. - - RAR3: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`, :data:`RAR_OS_MSDOS`, - :data:`RAR_OS_OS2`, :data:`RAR_OS_BEOS`. - - RAR5: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`. - - mode - File attributes. May be either dos-style or unix-style, depending on host_os. - - mtime - File modification time. Same value as :attr:`date_time` - but as :class:`datetime.datetime` object with extended precision. - - ctime - Optional time field: creation time. As :class:`datetime.datetime` object. - - atime - Optional time field: last access time. As :class:`datetime.datetime` object. - - arctime - Optional time field: archival time. As :class:`datetime.datetime` object. - (RAR3-only) - - CRC - CRC-32 of uncompressed file, unsigned int. - - RAR5: may be None. - - blake2sp_hash - Blake2SP hash over decompressed data. (RAR5-only) - - comment - Optional file comment field. Unicode string. (RAR3-only) - - file_redir - If not None, file is link of some sort. Contains tuple of (type, flags, target). - (RAR5-only) - - Type is one of constants: - - :data:`RAR5_XREDIR_UNIX_SYMLINK` - unix symlink to target. - :data:`RAR5_XREDIR_WINDOWS_SYMLINK` - windows symlink to target. - :data:`RAR5_XREDIR_WINDOWS_JUNCTION` - windows junction. - :data:`RAR5_XREDIR_HARD_LINK` - hard link to target. - :data:`RAR5_XREDIR_FILE_COPY` - current file is copy of another archive entry. - - Flags may contain :data:`RAR5_XREDIR_ISDIR` bit. - - volume - Volume nr, starting from 0. - - volume_file - Volume file name, where file starts. - - ''' - - # zipfile-compatible fields - filename = None - file_size = None - compress_size = None - date_time = None - comment = None - CRC = None - volume = None - orig_filename = None - - # optional extended time fields, datetime() objects. - mtime = None - ctime = None - atime = None - - extract_version = None - mode = None - host_os = None - compress_type = None - - # rar3-only fields - comment = None - arctime = None - - # rar5-only fields - blake2sp_hash = None - file_redir = None - - # internal fields - flags = 0 - type = None - - def isdir(self): - """Returns True if entry is a directory. - """ - if self.type == RAR_BLOCK_FILE: - return (self.flags & RAR_FILE_DIRECTORY) == RAR_FILE_DIRECTORY - return False - - def needs_password(self): - """Returns True if data is stored password-protected. - """ - if self.type == RAR_BLOCK_FILE: - return (self.flags & RAR_FILE_PASSWORD) > 0 - return False - - -class RarFile(object): - '''Parse RAR structure, provide access to files in archive. - ''' - - #: Archive comment. Unicode string or None. - comment = None - - def __init__(self, rarfile, mode="r", charset=None, info_callback=None, - crc_check=True, errors="stop"): - """Open and parse a RAR archive. - - Parameters: - - rarfile - archive file name - mode - only 'r' is supported. - charset - fallback charset to use, if filenames are not already Unicode-enabled. - info_callback - debug callback, gets to see all archive entries. - crc_check - set to False to disable CRC checks - errors - Either "stop" to quietly stop parsing on errors, - or "strict" to raise errors. Default is "stop". - """ - self._rarfile = rarfile - self._charset = charset or DEFAULT_CHARSET - self._info_callback = info_callback - self._crc_check = crc_check - self._password = None - self._file_parser = None - - if errors == "stop": - self._strict = False - elif errors == "strict": - self._strict = True - else: - raise ValueError("Invalid value for 'errors' parameter.") - - if mode != "r": - raise NotImplementedError("RarFile supports only mode=r") - - self._parse() - - def __enter__(self): - return self - - def __exit__(self, typ, value, traceback): - self.close() - - def setpassword(self, password): - '''Sets the password to use when extracting.''' - self._password = password - if self._file_parser: - if self._file_parser.has_header_encryption(): - self._file_parser = None - if not self._file_parser: - self._parse() - else: - self._file_parser.setpassword(self._password) - - def needs_password(self): - '''Returns True if any archive entries require password for extraction.''' - return self._file_parser.needs_password() - - def namelist(self): - '''Return list of filenames in archive.''' - return [f.filename for f in self.infolist()] - - def infolist(self): - '''Return RarInfo objects for all files/directories in archive.''' - return self._file_parser.infolist() - - def volumelist(self): - '''Returns filenames of archive volumes. - - In case of single-volume archive, the list contains - just the name of main archive file. - ''' - return self._file_parser.volumelist() - - def getinfo(self, fname): - '''Return RarInfo for file. - ''' - return self._file_parser.getinfo(fname) - - def open(self, fname, mode='r', psw=None): - '''Returns file-like object (:class:`RarExtFile`), - from where the data can be read. - - The object implements :class:`io.RawIOBase` interface, so it can - be further wrapped with :class:`io.BufferedReader` - and :class:`io.TextIOWrapper`. - - On older Python where io module is not available, it implements - only .read(), .seek(), .tell() and .close() methods. - - The object is seekable, although the seeking is fast only on - uncompressed files, on compressed files the seeking is implemented - by reading ahead and/or restarting the decompression. - - Parameters: - - fname - file name or RarInfo instance. - mode - must be 'r' - psw - password to use for extracting. - ''' - - if mode != 'r': - raise NotImplementedError("RarFile.open() supports only mode=r") - - # entry lookup - inf = self.getinfo(fname) - if inf.isdir(): - raise TypeError("Directory does not have any data: " + inf.filename) - - # check password - if inf.needs_password(): - psw = psw or self._password - if psw is None: - raise PasswordRequired("File %s requires password" % inf.filename) - else: - psw = None - - return self._file_parser.open(inf, psw) - - def read(self, fname, psw=None): - """Return uncompressed data for archive entry. - - For longer files using :meth:`RarFile.open` may be better idea. - - Parameters: - - fname - filename or RarInfo instance - psw - password to use for extracting. - """ - - with self.open(fname, 'r', psw) as f: - return f.read() - - def close(self): - """Release open resources.""" - pass - - def printdir(self): - """Print archive file list to stdout.""" - for f in self.infolist(): - print(f.filename) - - def extract(self, member, path=None, pwd=None): - """Extract single file into current directory. - - Parameters: - - member - filename or :class:`RarInfo` instance - path - optional destination path - pwd - optional password to use - """ - if isinstance(member, RarInfo): - fname = member.filename - else: - fname = member - self._extract([fname], path, pwd) - - def extractall(self, path=None, members=None, pwd=None): - """Extract all files into current directory. - - Parameters: - - path - optional destination path - members - optional filename or :class:`RarInfo` instance list to extract - pwd - optional password to use - """ - fnlist = [] - if members is not None: - for m in members: - if isinstance(m, RarInfo): - fnlist.append(m.filename) - else: - fnlist.append(m) - self._extract(fnlist, path, pwd) - - def testrar(self): - """Let 'unrar' test the archive. - """ - cmd = [UNRAR_TOOL] + list(TEST_ARGS) - add_password_arg(cmd, self._password) - cmd.append('--') - with XTempFile(self._rarfile) as rarfile: - cmd.append(rarfile) - p = custom_popen(cmd) - output = p.communicate()[0] - check_returncode(p, output) - - def strerror(self): - """Return error string if parsing failed, - or None if no problems. - """ - if not self._file_parser: - return "Not a RAR file" - return self._file_parser.strerror() - - ## - ## private methods - ## - - def _parse(self): - ver = _get_rar_version(self._rarfile) - if ver == 3: - p3 = RAR3Parser(self._rarfile, self._password, self._crc_check, - self._charset, self._strict, self._info_callback) - self._file_parser = p3 # noqa - elif ver == 5: - p5 = RAR5Parser(self._rarfile, self._password, self._crc_check, - self._charset, self._strict, self._info_callback) - self._file_parser = p5 # noqa - else: - raise BadRarFile("Not a RAR file") - - self._file_parser.parse() - self.comment = self._file_parser.comment - - # call unrar to extract a file - def _extract(self, fnlist, path=None, psw=None): - cmd = [UNRAR_TOOL] + list(EXTRACT_ARGS) - - # pasoword - psw = psw or self._password - add_password_arg(cmd, psw) - cmd.append('--') - - # rar file - with XTempFile(self._rarfile) as rarfn: - cmd.append(rarfn) - - # file list - for fn in fnlist: - if os.sep != PATH_SEP: - fn = fn.replace(PATH_SEP, os.sep) - cmd.append(fn) - - # destination path - if path is not None: - cmd.append(path + os.sep) - - # call - p = custom_popen(cmd) - output = p.communicate()[0] - check_returncode(p, output) - -# -# File format parsing -# - -class CommonParser(object): - """Shared parser parts.""" - _main = None - _hdrenc_main = None - _needs_password = False - _fd = None - _expect_sig = None - _parse_error = None - _password = None - comment = None - - def __init__(self, rarfile, password, crc_check, charset, strict, info_cb): - self._rarfile = rarfile - self._password = password - self._crc_check = crc_check - self._charset = charset - self._strict = strict - self._info_callback = info_cb - self._info_list = [] - self._info_map = {} - self._vol_list = [] - - def has_header_encryption(self): - """Returns True if headers are encrypted - """ - if self._hdrenc_main: - return True - if self._main: - if self._main.flags & RAR_MAIN_PASSWORD: - return True - return False - - def setpassword(self, psw): - """Set cached password.""" - self._password = psw - - def volumelist(self): - """Volume files""" - return self._vol_list - - def needs_password(self): - """Is password required""" - return self._needs_password - - def strerror(self): - """Last error""" - return self._parse_error - - def infolist(self): - """List of RarInfo records. - """ - return self._info_list - - def getinfo(self, fname): - """Return RarInfo for filename - """ - # accept both ways here - if PATH_SEP == '/': - fname2 = fname.replace("\\", "/") - else: - fname2 = fname.replace("/", "\\") - - try: - return self._info_map[fname] - except KeyError: - try: - return self._info_map[fname2] - except KeyError: - raise NoRarEntry("No such file: %s" % fname) - - # read rar - def parse(self): - """Process file.""" - self._fd = None - try: - self._parse_real() - finally: - if self._fd: - self._fd.close() - self._fd = None - - def _parse_real(self): - fd = XFile(self._rarfile) - self._fd = fd - sig = fd.read(len(self._expect_sig)) - if sig != self._expect_sig: - if isinstance(self._rarfile, (str, unicode)): - raise NotRarFile("Not a Rar archive: {}".format(self._rarfile)) - raise NotRarFile("Not a Rar archive") - - volume = 0 # first vol (.rar) is 0 - more_vols = False - endarc = False - volfile = self._rarfile - self._vol_list = [self._rarfile] - while 1: - if endarc: - h = None # don't read past ENDARC - else: - h = self._parse_header(fd) - if not h: - if more_vols: - volume += 1 - fd.close() - try: - volfile = self._next_volname(volfile) - fd = XFile(volfile) - except IOError: - self._set_error("Cannot open next volume: %s", volfile) - break - self._fd = fd - sig = fd.read(len(self._expect_sig)) - if sig != self._expect_sig: - self._set_error("Invalid volume sig: %s", volfile) - break - more_vols = False - endarc = False - self._vol_list.append(volfile) - continue - break - h.volume = volume - h.volume_file = volfile - - if h.type == RAR_BLOCK_MAIN and not self._main: - self._main = h - if h.flags & RAR_MAIN_NEWNUMBERING: - # RAR 2.x does not set FIRSTVOLUME, - # so check it only if NEWNUMBERING is used - if (h.flags & RAR_MAIN_FIRSTVOLUME) == 0: - raise NeedFirstVolume("Need to start from first volume") - if h.flags & RAR_MAIN_PASSWORD: - self._needs_password = True - if not self._password: - break - elif h.type == RAR_BLOCK_ENDARC: - more_vols = (h.flags & RAR_ENDARC_NEXT_VOLUME) > 0 - endarc = True - elif h.type == RAR_BLOCK_FILE: - # RAR 2.x does not write RAR_BLOCK_ENDARC - if h.flags & RAR_FILE_SPLIT_AFTER: - more_vols = True - # RAR 2.x does not set RAR_MAIN_FIRSTVOLUME - if volume == 0 and h.flags & RAR_FILE_SPLIT_BEFORE: - raise NeedFirstVolume("Need to start from first volume") - - if h.needs_password(): - self._needs_password = True - - # store it - self.process_entry(fd, h) - - if self._info_callback: - self._info_callback(h) - - # go to next header - if h.add_size > 0: - fd.seek(h.data_offset + h.add_size, 0) - - def process_entry(self, fd, item): - """Examine item, add into lookup cache.""" - raise NotImplementedError() - - def _decrypt_header(self, fd): - raise NotImplementedError('_decrypt_header') - - def _parse_block_header(self, fd): - raise NotImplementedError('_parse_block_header') - - def _open_hack(self, inf, psw): - raise NotImplementedError('_open_hack') - - # read single header - def _parse_header(self, fd): - try: - # handle encrypted headers - if (self._main and self._main.flags & RAR_MAIN_PASSWORD) or self._hdrenc_main: - if not self._password: - return - fd = self._decrypt_header(fd) - - # now read actual header - return self._parse_block_header(fd) - except struct.error: - self._set_error('Broken header in RAR file') - return None - - # given current vol name, construct next one - def _next_volname(self, volfile): - if is_filelike(volfile): - raise IOError("Working on single FD") - if self._main.flags & RAR_MAIN_NEWNUMBERING: - return _next_newvol(volfile) - return _next_oldvol(volfile) - - def _set_error(self, msg, *args): - if args: - msg = msg % args - self._parse_error = msg - if self._strict: - raise BadRarFile(msg) - - def open(self, inf, psw): - """Return stream object for file data.""" - - if inf.file_redir: - # cannot leave to unrar as it expects copied file to exist - if inf.file_redir[0] in (RAR5_XREDIR_FILE_COPY, RAR5_XREDIR_HARD_LINK): - inf = self.getinfo(inf.file_redir[2]) - if not inf: - raise BadRarFile('cannot find copied file') - - if inf.flags & RAR_FILE_SPLIT_BEFORE: - raise NeedFirstVolume("Partial file, please start from first volume: " + inf.filename) - - # is temp write usable? - use_hack = 1 - if not self._main: - use_hack = 0 - elif self._main._must_disable_hack(): - use_hack = 0 - elif inf._must_disable_hack(): - use_hack = 0 - elif is_filelike(self._rarfile): - pass - elif inf.file_size > HACK_SIZE_LIMIT: - use_hack = 0 - elif not USE_EXTRACT_HACK: - use_hack = 0 - - # now extract - if inf.compress_type == RAR_M0 and (inf.flags & RAR_FILE_PASSWORD) == 0 and inf.file_redir is None: - return self._open_clear(inf) - elif use_hack: - return self._open_hack(inf, psw) - elif is_filelike(self._rarfile): - return self._open_unrar_membuf(self._rarfile, inf, psw) - else: - return self._open_unrar(self._rarfile, inf, psw) - - def _open_clear(self, inf): - return DirectReader(self, inf) - - def _open_hack_core(self, inf, psw, prefix, suffix): - - size = inf.compress_size + inf.header_size - rf = XFile(inf.volume_file, 0) - rf.seek(inf.header_offset) - - tmpfd, tmpname = mkstemp(suffix='.rar') - tmpf = os.fdopen(tmpfd, "wb") - - try: - tmpf.write(prefix) - while size > 0: - if size > BSIZE: - buf = rf.read(BSIZE) - else: - buf = rf.read(size) - if not buf: - raise BadRarFile('read failed: ' + inf.filename) - tmpf.write(buf) - size -= len(buf) - tmpf.write(suffix) - tmpf.close() - rf.close() - except: - rf.close() - tmpf.close() - os.unlink(tmpname) - raise - - return self._open_unrar(tmpname, inf, psw, tmpname) - - # write in-memory archive to temp file - needed for solid archives - def _open_unrar_membuf(self, memfile, inf, psw): - tmpname = membuf_tempfile(memfile) - return self._open_unrar(tmpname, inf, psw, tmpname, force_file=True) - - # extract using unrar - def _open_unrar(self, rarfile, inf, psw=None, tmpfile=None, force_file=False): - cmd = [UNRAR_TOOL] + list(OPEN_ARGS) - add_password_arg(cmd, psw) - cmd.append("--") - cmd.append(rarfile) - - # not giving filename avoids encoding related problems - if not tmpfile or force_file: - fn = inf.filename - if PATH_SEP != os.sep: - fn = fn.replace(PATH_SEP, os.sep) - cmd.append(fn) - - # read from unrar pipe - return PipeReader(self, inf, cmd, tmpfile) - -# -# RAR3 format -# - -class Rar3Info(RarInfo): - """RAR3 specific fields.""" - extract_version = 15 - salt = None - add_size = 0 - header_crc = None - header_size = None - header_offset = None - data_offset = None - _md_class = None - _md_expect = None - - # make sure some rar5 fields are always present - file_redir = None - blake2sp_hash = None - - def _must_disable_hack(self): - if self.type == RAR_BLOCK_FILE: - if self.flags & RAR_FILE_PASSWORD: - return True - elif self.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): - return True - elif self.type == RAR_BLOCK_MAIN: - if self.flags & (RAR_MAIN_SOLID | RAR_MAIN_PASSWORD): - return True - return False - - -class RAR3Parser(CommonParser): - """Parse RAR3 file format. - """ - _expect_sig = RAR_ID - _last_aes_key = (None, None, None) # (salt, key, iv) - - def _decrypt_header(self, fd): - if not _have_crypto: - raise NoCrypto('Cannot parse encrypted headers - no crypto') - salt = fd.read(8) - if self._last_aes_key[0] == salt: - key, iv = self._last_aes_key[1:] - else: - key, iv = rar3_s2k(self._password, salt) - self._last_aes_key = (salt, key, iv) - return HeaderDecrypt(fd, key, iv) - - # common header - def _parse_block_header(self, fd): - h = Rar3Info() - h.header_offset = fd.tell() - - # read and parse base header - buf = fd.read(S_BLK_HDR.size) - if not buf: - return None - t = S_BLK_HDR.unpack_from(buf) - h.header_crc, h.type, h.flags, h.header_size = t - - # read full header - if h.header_size > S_BLK_HDR.size: - hdata = buf + fd.read(h.header_size - S_BLK_HDR.size) - else: - hdata = buf - h.data_offset = fd.tell() - - # unexpected EOF? - if len(hdata) != h.header_size: - self._set_error('Unexpected EOF when reading header') - return None - - pos = S_BLK_HDR.size - - # block has data assiciated with it? - if h.flags & RAR_LONG_BLOCK: - h.add_size, pos = load_le32(hdata, pos) - else: - h.add_size = 0 - - # parse interesting ones, decide header boundaries for crc - if h.type == RAR_BLOCK_MARK: - return h - elif h.type == RAR_BLOCK_MAIN: - pos += 6 - if h.flags & RAR_MAIN_ENCRYPTVER: - pos += 1 - crc_pos = pos - if h.flags & RAR_MAIN_COMMENT: - self._parse_subblocks(h, hdata, pos) - elif h.type == RAR_BLOCK_FILE: - pos = self._parse_file_header(h, hdata, pos - 4) - crc_pos = pos - if h.flags & RAR_FILE_COMMENT: - pos = self._parse_subblocks(h, hdata, pos) - elif h.type == RAR_BLOCK_SUB: - pos = self._parse_file_header(h, hdata, pos - 4) - crc_pos = h.header_size - elif h.type == RAR_BLOCK_OLD_AUTH: - pos += 8 - crc_pos = pos - elif h.type == RAR_BLOCK_OLD_EXTRA: - pos += 7 - crc_pos = pos - else: - crc_pos = h.header_size - - # check crc - if h.type == RAR_BLOCK_OLD_SUB: - crcdat = hdata[2:] + fd.read(h.add_size) - else: - crcdat = hdata[2:crc_pos] - - calc_crc = rar_crc32(crcdat) & 0xFFFF - - # return good header - if h.header_crc == calc_crc: - return h - - # header parsing failed. - self._set_error('Header CRC error (%02x): exp=%x got=%x (xlen = %d)', - h.type, h.header_crc, calc_crc, len(crcdat)) - - # instead panicing, send eof - return None - - # read file-specific header - def _parse_file_header(self, h, hdata, pos): - fld = S_FILE_HDR.unpack_from(hdata, pos) - pos += S_FILE_HDR.size - - h.compress_size = fld[0] - h.file_size = fld[1] - h.host_os = fld[2] - h.CRC = fld[3] - h.date_time = parse_dos_time(fld[4]) - h.mtime = to_datetime(h.date_time) - h.extract_version = fld[5] - h.compress_type = fld[6] - name_size = fld[7] - h.mode = fld[8] - - h._md_class = CRC32Context - h._md_expect = h.CRC - - if h.flags & RAR_FILE_LARGE: - h1, pos = load_le32(hdata, pos) - h2, pos = load_le32(hdata, pos) - h.compress_size |= h1 << 32 - h.file_size |= h2 << 32 - h.add_size = h.compress_size - - name, pos = load_bytes(hdata, name_size, pos) - if h.flags & RAR_FILE_UNICODE: - nul = name.find(ZERO) - h.orig_filename = name[:nul] - u = UnicodeFilename(h.orig_filename, name[nul + 1:]) - h.filename = u.decode() - - # if parsing failed fall back to simple name - if u.failed: - h.filename = self._decode(h.orig_filename) - else: - h.orig_filename = name - h.filename = self._decode(name) - - # change separator, if requested - if PATH_SEP != '\\': - h.filename = h.filename.replace('\\', PATH_SEP) - - if h.flags & RAR_FILE_SALT: - h.salt, pos = load_bytes(hdata, 8, pos) - else: - h.salt = None - - # optional extended time stamps - if h.flags & RAR_FILE_EXTTIME: - pos = _parse_ext_time(h, hdata, pos) - else: - h.mtime = h.atime = h.ctime = h.arctime = None - - return pos - - # find old-style comment subblock - def _parse_subblocks(self, h, hdata, pos): - while pos < len(hdata): - # ordinary block header - t = S_BLK_HDR.unpack_from(hdata, pos) - ___scrc, stype, sflags, slen = t - pos_next = pos + slen - pos += S_BLK_HDR.size - - # corrupt header - if pos_next < pos: - break - - # followed by block-specific header - if stype == RAR_BLOCK_OLD_COMMENT and pos + S_COMMENT_HDR.size <= pos_next: - declen, ver, meth, crc = S_COMMENT_HDR.unpack_from(hdata, pos) - pos += S_COMMENT_HDR.size - data = hdata[pos : pos_next] - cmt = rar3_decompress(ver, meth, data, declen, sflags, - crc, self._password) - if not self._crc_check: - h.comment = self._decode_comment(cmt) - elif rar_crc32(cmt) & 0xFFFF == crc: - h.comment = self._decode_comment(cmt) - - pos = pos_next - return pos - - def _read_comment_v3(self, inf, psw=None): - - # read data - with XFile(inf.volume_file) as rf: - rf.seek(inf.data_offset) - data = rf.read(inf.compress_size) - - # decompress - cmt = rar3_decompress(inf.extract_version, inf.compress_type, data, - inf.file_size, inf.flags, inf.CRC, psw, inf.salt) - - # check crc - if self._crc_check: - crc = rar_crc32(cmt) - if crc != inf.CRC: - return None - - return self._decode_comment(cmt) - - def _decode(self, val): - for c in TRY_ENCODINGS: - try: - return val.decode(c) - except UnicodeError: - pass - return val.decode(self._charset, 'replace') - - def _decode_comment(self, val): - return self._decode(val) - - def process_entry(self, fd, item): - if item.type == RAR_BLOCK_FILE: - # use only first part - if (item.flags & RAR_FILE_SPLIT_BEFORE) == 0: - self._info_map[item.filename] = item - self._info_list.append(item) - elif len(self._info_list) > 0: - # final crc is in last block - old = self._info_list[-1] - old.CRC = item.CRC - old._md_expect = item._md_expect - old.compress_size += item.compress_size - - # parse new-style comment - if item.type == RAR_BLOCK_SUB and item.filename == 'CMT': - if item.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): - pass - elif item.flags & RAR_FILE_SOLID: - # file comment - cmt = self._read_comment_v3(item, self._password) - if len(self._info_list) > 0: - old = self._info_list[-1] - old.comment = cmt - else: - # archive comment - cmt = self._read_comment_v3(item, self._password) - self.comment = cmt - - if item.type == RAR_BLOCK_MAIN: - if item.flags & RAR_MAIN_COMMENT: - self.comment = item.comment - if item.flags & RAR_MAIN_PASSWORD: - self._needs_password = True - - # put file compressed data into temporary .rar archive, and run - # unrar on that, thus avoiding unrar going over whole archive - def _open_hack(self, inf, psw): - # create main header: crc, type, flags, size, res1, res2 - prefix = RAR_ID + S_BLK_HDR.pack(0x90CF, 0x73, 0, 13) + ZERO * (2 + 4) - return self._open_hack_core(inf, psw, prefix, EMPTY) - -# -# RAR5 format -# - -class Rar5Info(RarInfo): - """Shared fields for RAR5 records. - """ - extract_version = 50 - header_crc = None - header_size = None - header_offset = None - data_offset = None - - # type=all - block_type = None - block_flags = None - add_size = 0 - block_extra_size = 0 - - # type=MAIN - volume_number = None - _md_class = None - _md_expect = None - - def _must_disable_hack(self): - return False - - -class Rar5BaseFile(Rar5Info): - """Shared sturct for file & service record. - """ - type = -1 - file_flags = None - file_encryption = (0, 0, 0, EMPTY, EMPTY, EMPTY) - file_compress_flags = None - file_redir = None - file_owner = None - file_version = None - blake2sp_hash = None - - def _must_disable_hack(self): - if self.flags & RAR_FILE_PASSWORD: - return True - if self.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): - return True - if self.file_compress_flags & RAR5_COMPR_SOLID: - return True - if self.file_redir: - return True - return False - - -class Rar5FileInfo(Rar5BaseFile): - """RAR5 file record. - """ - type = RAR_BLOCK_FILE - - -class Rar5ServiceInfo(Rar5BaseFile): - """RAR5 service record. - """ - type = RAR_BLOCK_SUB - - -class Rar5MainInfo(Rar5Info): - """RAR5 archive main record. - """ - type = RAR_BLOCK_MAIN - main_flags = None - main_volume_number = None - - def _must_disable_hack(self): - if self.main_flags & RAR5_MAIN_FLAG_SOLID: - return True - return False - - -class Rar5EncryptionInfo(Rar5Info): - """RAR5 archive header encryption record. - """ - type = RAR5_BLOCK_ENCRYPTION - encryption_algo = None - encryption_flags = None - encryption_kdf_count = None - encryption_salt = None - encryption_check_value = None - - def needs_password(self): - return True - - -class Rar5EndArcInfo(Rar5Info): - """RAR5 end of archive record. - """ - type = RAR_BLOCK_ENDARC - endarc_flags = None - - -class RAR5Parser(CommonParser): - """Parse RAR5 format. - """ - _expect_sig = RAR5_ID - _hdrenc_main = None - - # AES encrypted headers - _last_aes256_key = (-1, None, None) # (kdf_count, salt, key) - - def _gen_key(self, kdf_count, salt): - if self._last_aes256_key[:2] == (kdf_count, salt): - return self._last_aes256_key[2] - if kdf_count > 24: - raise BadRarFile('Too large kdf_count') - psw = self._password - if isinstance(psw, unicode): - psw = psw.encode('utf8') - key = pbkdf2_sha256(psw, salt, 1 << kdf_count) - self._last_aes256_key = (kdf_count, salt, key) - return key - - def _decrypt_header(self, fd): - if not _have_crypto: - raise NoCrypto('Cannot parse encrypted headers - no crypto') - h = self._hdrenc_main - key = self._gen_key(h.encryption_kdf_count, h.encryption_salt) - iv = fd.read(16) - return HeaderDecrypt(fd, key, iv) - - # common header - def _parse_block_header(self, fd): - header_offset = fd.tell() - - preload = 4 + 3 - start_bytes = fd.read(preload) - header_crc, pos = load_le32(start_bytes, 0) - hdrlen, pos = load_vint(start_bytes, pos) - if hdrlen > 2 * 1024 * 1024: - return None - header_size = pos + hdrlen - - # read full header, check for EOF - hdata = start_bytes + fd.read(header_size - len(start_bytes)) - if len(hdata) != header_size: - self._set_error('Unexpected EOF when reading header') - return None - data_offset = fd.tell() - - calc_crc = rar_crc32(memoryview(hdata)[4:]) - if header_crc != calc_crc: - # header parsing failed. - self._set_error('Header CRC error: exp=%x got=%x (xlen = %d)', - header_crc, calc_crc, len(hdata)) - return None - - block_type, pos = load_vint(hdata, pos) - - if block_type == RAR5_BLOCK_MAIN: - h, pos = self._parse_block_common(Rar5MainInfo(), hdata) - h = self._parse_main_block(h, hdata, pos) - elif block_type == RAR5_BLOCK_FILE: - h, pos = self._parse_block_common(Rar5FileInfo(), hdata) - h = self._parse_file_block(h, hdata, pos) - elif block_type == RAR5_BLOCK_SERVICE: - h, pos = self._parse_block_common(Rar5ServiceInfo(), hdata) - h = self._parse_file_block(h, hdata, pos) - elif block_type == RAR5_BLOCK_ENCRYPTION: - h, pos = self._parse_block_common(Rar5EncryptionInfo(), hdata) - h = self._parse_encryption_block(h, hdata, pos) - elif block_type == RAR5_BLOCK_ENDARC: - h, pos = self._parse_block_common(Rar5EndArcInfo(), hdata) - h = self._parse_endarc_block(h, hdata, pos) - else: - h = None - if h: - h.header_offset = header_offset - h.data_offset = data_offset - return h - - def _parse_block_common(self, h, hdata): - h.header_crc, pos = load_le32(hdata, 0) - hdrlen, pos = load_vint(hdata, pos) - h.header_size = hdrlen + pos - h.block_type, pos = load_vint(hdata, pos) - h.block_flags, pos = load_vint(hdata, pos) - - if h.block_flags & RAR5_BLOCK_FLAG_EXTRA_DATA: - h.block_extra_size, pos = load_vint(hdata, pos) - if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: - h.add_size, pos = load_vint(hdata, pos) - - h.compress_size = h.add_size - - if h.block_flags & RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN: - h.flags |= RAR_SKIP_IF_UNKNOWN - if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: - h.flags |= RAR_LONG_BLOCK - return h, pos - - def _parse_main_block(self, h, hdata, pos): - h.main_flags, pos = load_vint(hdata, pos) - if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR: - h.main_volume_number = load_vint(hdata, pos) - - h.flags |= RAR_MAIN_NEWNUMBERING - if h.main_flags & RAR5_MAIN_FLAG_SOLID: - h.flags |= RAR_MAIN_SOLID - if h.main_flags & RAR5_MAIN_FLAG_ISVOL: - h.flags |= RAR_MAIN_VOLUME - if h.main_flags & RAR5_MAIN_FLAG_RECOVERY: - h.flags |= RAR_MAIN_RECOVERY - if self._hdrenc_main: - h.flags |= RAR_MAIN_PASSWORD - if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR == 0: - h.flags |= RAR_MAIN_FIRSTVOLUME - - return h - - def _parse_file_block(self, h, hdata, pos): - h.file_flags, pos = load_vint(hdata, pos) - h.file_size, pos = load_vint(hdata, pos) - h.mode, pos = load_vint(hdata, pos) - - if h.file_flags & RAR5_FILE_FLAG_HAS_MTIME: - h.mtime, pos = load_unixtime(hdata, pos) - h.date_time = h.mtime.timetuple()[:6] - if h.file_flags & RAR5_FILE_FLAG_HAS_CRC32: - h.CRC, pos = load_le32(hdata, pos) - h._md_class = CRC32Context - h._md_expect = h.CRC - - h.file_compress_flags, pos = load_vint(hdata, pos) - h.file_host_os, pos = load_vint(hdata, pos) - h.orig_filename, pos = load_vstr(hdata, pos) - h.filename = h.orig_filename.decode('utf8', 'replace') - - # use compatible values - if h.file_host_os == RAR5_OS_WINDOWS: - h.host_os = RAR_OS_WIN32 - else: - h.host_os = RAR_OS_UNIX - h.compress_type = RAR_M0 + ((h.file_compress_flags >> 7) & 7) - - if h.block_extra_size: - # allow 1 byte of garbage - while pos < len(hdata) - 1: - xsize, pos = load_vint(hdata, pos) - xdata, pos = load_bytes(hdata, xsize, pos) - self._process_file_extra(h, xdata) - - if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE: - h.flags |= RAR_FILE_SPLIT_BEFORE - if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_AFTER: - h.flags |= RAR_FILE_SPLIT_AFTER - if h.file_flags & RAR5_FILE_FLAG_ISDIR: - h.flags |= RAR_FILE_DIRECTORY - if h.file_compress_flags & RAR5_COMPR_SOLID: - h.flags |= RAR_FILE_SOLID - - return h - - def _parse_endarc_block(self, h, hdata, pos): - h.endarc_flags, pos = load_vint(hdata, pos) - if h.endarc_flags & RAR5_ENDARC_FLAG_NEXT_VOL: - h.flags |= RAR_ENDARC_NEXT_VOLUME - return h - - def _parse_encryption_block(self, h, hdata, pos): - h.encryption_algo, pos = load_vint(hdata, pos) - h.encryption_flags, pos = load_vint(hdata, pos) - h.encryption_kdf_count, pos = load_byte(hdata, pos) - h.encryption_salt, pos = load_bytes(hdata, 16, pos) - if h.encryption_flags & RAR5_ENC_FLAG_HAS_CHECKVAL: - h.encryption_check_value = load_bytes(hdata, 12, pos) - if h.encryption_algo != RAR5_XENC_CIPHER_AES256: - raise BadRarFile('Unsupported header encryption cipher') - self._hdrenc_main = h - return h - - # file extra record - def _process_file_extra(self, h, xdata): - xtype, pos = load_vint(xdata, 0) - if xtype == RAR5_XFILE_TIME: - self._parse_file_xtime(h, xdata, pos) - elif xtype == RAR5_XFILE_ENCRYPTION: - self._parse_file_encryption(h, xdata, pos) - elif xtype == RAR5_XFILE_HASH: - self._parse_file_hash(h, xdata, pos) - elif xtype == RAR5_XFILE_VERSION: - self._parse_file_version(h, xdata, pos) - elif xtype == RAR5_XFILE_REDIR: - self._parse_file_redir(h, xdata, pos) - elif xtype == RAR5_XFILE_OWNER: - self._parse_file_owner(h, xdata, pos) - elif xtype == RAR5_XFILE_SERVICE: - pass - else: - pass - - # extra block for file time record - def _parse_file_xtime(self, h, xdata, pos): - tflags, pos = load_vint(xdata, pos) - ldr = load_windowstime - if tflags & RAR5_XTIME_UNIXTIME: - ldr = load_unixtime - if tflags & RAR5_XTIME_HAS_MTIME: - h.mtime, pos = ldr(xdata, pos) - h.date_time = h.mtime.timetuple()[:6] - if tflags & RAR5_XTIME_HAS_CTIME: - h.ctime, pos = ldr(xdata, pos) - if tflags & RAR5_XTIME_HAS_ATIME: - h.atime, pos = ldr(xdata, pos) - - # just remember encryption info - def _parse_file_encryption(self, h, xdata, pos): - algo, pos = load_vint(xdata, pos) - flags, pos = load_vint(xdata, pos) - kdf_count, pos = load_byte(xdata, pos) - salt, pos = load_bytes(xdata, 16, pos) - iv, pos = load_bytes(xdata, 16, pos) - checkval = None - if flags & RAR5_XENC_CHECKVAL: - checkval, pos = load_bytes(xdata, 12, pos) - if flags & RAR5_XENC_TWEAKED: - h._md_expect = None - h._md_class = NoHashContext - - h.file_encryption = (algo, flags, kdf_count, salt, iv, checkval) - h.flags |= RAR_FILE_PASSWORD - - def _parse_file_hash(self, h, xdata, pos): - hash_type, pos = load_vint(xdata, pos) - if hash_type == RAR5_XHASH_BLAKE2SP: - h.blake2sp_hash, pos = load_bytes(xdata, 32, pos) - if _have_blake2 and (h.file_encryption[1] & RAR5_XENC_TWEAKED) == 0: - h._md_class = Blake2SP - h._md_expect = h.blake2sp_hash - - def _parse_file_version(self, h, xdata, pos): - flags, pos = load_vint(xdata, pos) - version, pos = load_vint(xdata, pos) - h.file_version = (flags, version) - - def _parse_file_redir(self, h, xdata, pos): - redir_type, pos = load_vint(xdata, pos) - redir_flags, pos = load_vint(xdata, pos) - redir_name, pos = load_vstr(xdata, pos) - redir_name = redir_name.decode('utf8', 'replace') - h.file_redir = (redir_type, redir_flags, redir_name) - - def _parse_file_owner(self, h, xdata, pos): - user_name = group_name = user_id = group_id = None - - flags, pos = load_vint(xdata, pos) - if flags & RAR5_XOWNER_UNAME: - user_name, pos = load_vstr(xdata, pos) - if flags & RAR5_XOWNER_GNAME: - group_name, pos = load_vstr(xdata, pos) - if flags & RAR5_XOWNER_UID: - user_id, pos = load_vint(xdata, pos) - if flags & RAR5_XOWNER_GID: - group_id, pos = load_vint(xdata, pos) - - h.file_owner = (user_name, group_name, user_id, group_id) - - def process_entry(self, fd, item): - if item.block_type == RAR5_BLOCK_FILE: - # use only first part - if (item.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE) == 0: - self._info_map[item.filename] = item - self._info_list.append(item) - elif len(self._info_list) > 0: - # final crc is in last block - old = self._info_list[-1] - old.CRC = item.CRC - old._md_expect = item._md_expect - old.blake2sp_hash = item.blake2sp_hash - old.compress_size += item.compress_size - elif item.block_type == RAR5_BLOCK_SERVICE: - if item.filename == 'CMT': - self._load_comment(fd, item) - - def _load_comment(self, fd, item): - if item.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): - return None - if item.compress_type != RAR_M0: - return None - - if item.flags & RAR_FILE_PASSWORD: - algo, ___flags, kdf_count, salt, iv, ___checkval = item.file_encryption - if algo != RAR5_XENC_CIPHER_AES256: - return None - key = self._gen_key(kdf_count, salt) - f = HeaderDecrypt(fd, key, iv) - cmt = f.read(item.file_size) - else: - # archive comment - with self._open_clear(item) as cmtstream: - cmt = cmtstream.read() - - # rar bug? - appends zero to comment - cmt = cmt.split(ZERO, 1)[0] - self.comment = cmt.decode('utf8') - - def _open_hack(self, inf, psw): - # len, type, blk_flags, flags - main_hdr = b'\x03\x01\x00\x00' - endarc_hdr = b'\x03\x05\x00\x00' - main_hdr = S_LONG.pack(rar_crc32(main_hdr)) + main_hdr - endarc_hdr = S_LONG.pack(rar_crc32(endarc_hdr)) + endarc_hdr - return self._open_hack_core(inf, psw, RAR5_ID + main_hdr, endarc_hdr) - -## -## Utility classes -## - -class UnicodeFilename(object): - """Handle RAR3 unicode filename decompression. - """ - def __init__(self, name, encdata): - self.std_name = bytearray(name) - self.encdata = bytearray(encdata) - self.pos = self.encpos = 0 - self.buf = bytearray() - self.failed = 0 - - def enc_byte(self): - """Copy encoded byte.""" - try: - c = self.encdata[self.encpos] - self.encpos += 1 - return c - except IndexError: - self.failed = 1 - return 0 - - def std_byte(self): - """Copy byte from 8-bit representation.""" - try: - return self.std_name[self.pos] - except IndexError: - self.failed = 1 - return ord('?') - - def put(self, lo, hi): - """Copy 16-bit value to result.""" - self.buf.append(lo) - self.buf.append(hi) - self.pos += 1 - - def decode(self): - """Decompress compressed UTF16 value.""" - hi = self.enc_byte() - flagbits = 0 - while self.encpos < len(self.encdata): - if flagbits == 0: - flags = self.enc_byte() - flagbits = 8 - flagbits -= 2 - t = (flags >> flagbits) & 3 - if t == 0: - self.put(self.enc_byte(), 0) - elif t == 1: - self.put(self.enc_byte(), hi) - elif t == 2: - self.put(self.enc_byte(), self.enc_byte()) - else: - n = self.enc_byte() - if n & 0x80: - c = self.enc_byte() - for _ in range((n & 0x7f) + 2): - lo = (self.std_byte() + c) & 0xFF - self.put(lo, hi) - else: - for _ in range(n + 2): - self.put(self.std_byte(), 0) - return self.buf.decode("utf-16le", "replace") - - -class RarExtFile(RawIOBase): - """Base class for file-like object that :meth:`RarFile.open` returns. - - Provides public methods and common crc checking. - - Behaviour: - - no short reads - .read() and .readinfo() read as much as requested. - - no internal buffer, use io.BufferedReader for that. - """ - - #: Filename of the archive entry - name = None - - def __init__(self, parser, inf): - super(RarExtFile, self).__init__() - - # standard io.* properties - self.name = inf.filename - self.mode = 'rb' - - self._parser = parser - self._inf = inf - self._fd = None - self._remain = 0 - self._returncode = 0 - - self._md_context = None - - self._open() - - def _open(self): - if self._fd: - self._fd.close() - md_class = self._inf._md_class or NoHashContext - self._md_context = md_class() - self._fd = None - self._remain = self._inf.file_size - - def read(self, cnt=None): - """Read all or specified amount of data from archive entry.""" - - # sanitize cnt - if cnt is None or cnt < 0: - cnt = self._remain - elif cnt > self._remain: - cnt = self._remain - if cnt == 0: - return EMPTY - - # actual read - data = self._read(cnt) - if data: - self._md_context.update(data) - self._remain -= len(data) - if len(data) != cnt: - raise BadRarFile("Failed the read enough data") - - # done? - if not data or self._remain == 0: - # self.close() - self._check() - return data - - def _check(self): - """Check final CRC.""" - final = self._md_context.digest() - exp = self._inf._md_expect - if exp is None: - return - if final is None: - return - if self._returncode: - check_returncode(self, '') - if self._remain != 0: - raise BadRarFile("Failed the read enough data") - if final != exp: - raise BadRarFile("Corrupt file - CRC check failed: %s - exp=%r got=%r" % ( - self._inf.filename, exp, final)) - - def _read(self, cnt): - """Actual read that gets sanitized cnt.""" - - def close(self): - """Close open resources.""" - - super(RarExtFile, self).close() - - if self._fd: - self._fd.close() - self._fd = None - - def __del__(self): - """Hook delete to make sure tempfile is removed.""" - self.close() - - def readinto(self, buf): - """Zero-copy read directly into buffer. - - Returns bytes read. - """ - raise NotImplementedError('readinto') - - def tell(self): - """Return current reading position in uncompressed data.""" - return self._inf.file_size - self._remain - - def seek(self, ofs, whence=0): - """Seek in data. - - On uncompressed files, the seeking works by actual - seeks so it's fast. On compresses files its slow - - forward seeking happends by reading ahead, - backwards by re-opening and decompressing from the start. - """ - - # disable crc check when seeking - self._md_context = NoHashContext() - - fsize = self._inf.file_size - cur_ofs = self.tell() - - if whence == 0: # seek from beginning of file - new_ofs = ofs - elif whence == 1: # seek from current position - new_ofs = cur_ofs + ofs - elif whence == 2: # seek from end of file - new_ofs = fsize + ofs - else: - raise ValueError('Invalid value for whence') - - # sanity check - if new_ofs < 0: - new_ofs = 0 - elif new_ofs > fsize: - new_ofs = fsize - - # do the actual seek - if new_ofs >= cur_ofs: - self._skip(new_ofs - cur_ofs) - else: - # reopen and seek - self._open() - self._skip(new_ofs) - return self.tell() - - def _skip(self, cnt): - """Read and discard data""" - while cnt > 0: - if cnt > 8192: - buf = self.read(8192) - else: - buf = self.read(cnt) - if not buf: - break - cnt -= len(buf) - - def readable(self): - """Returns True""" - return True - - def writable(self): - """Returns False. - - Writing is not supported.""" - return False - - def seekable(self): - """Returns True. - - Seeking is supported, although it's slow on compressed files. - """ - return True - - def readall(self): - """Read all remaining data""" - # avoid RawIOBase default impl - return self.read() - - -class PipeReader(RarExtFile): - """Read data from pipe, handle tempfile cleanup.""" - - def __init__(self, rf, inf, cmd, tempfile=None): - self._cmd = cmd - self._proc = None - self._tempfile = tempfile - super(PipeReader, self).__init__(rf, inf) - - def _close_proc(self): - if not self._proc: - return - if self._proc.stdout: - self._proc.stdout.close() - if self._proc.stdin: - self._proc.stdin.close() - if self._proc.stderr: - self._proc.stderr.close() - self._proc.wait() - self._returncode = self._proc.returncode - self._proc = None - - def _open(self): - super(PipeReader, self)._open() - - # stop old process - self._close_proc() - - # launch new process - self._returncode = 0 - self._proc = custom_popen(self._cmd) - self._fd = self._proc.stdout - - # avoid situation where unrar waits on stdin - if self._proc.stdin: - self._proc.stdin.close() - - def _read(self, cnt): - """Read from pipe.""" - - # normal read is usually enough - data = self._fd.read(cnt) - if len(data) == cnt or not data: - return data - - # short read, try looping - buf = [data] - cnt -= len(data) - while cnt > 0: - data = self._fd.read(cnt) - if not data: - break - cnt -= len(data) - buf.append(data) - return EMPTY.join(buf) - - def close(self): - """Close open resources.""" - - self._close_proc() - super(PipeReader, self).close() - - if self._tempfile: - try: - os.unlink(self._tempfile) - except OSError: - pass - self._tempfile = None - - def readinto(self, buf): - """Zero-copy read directly into buffer.""" - cnt = len(buf) - if cnt > self._remain: - cnt = self._remain - vbuf = memoryview(buf) - res = got = 0 - while got < cnt: - res = self._fd.readinto(vbuf[got : cnt]) - if not res: - break - self._md_context.update(vbuf[got : got + res]) - self._remain -= res - got += res - return got - - -class DirectReader(RarExtFile): - """Read uncompressed data directly from archive. - """ - _cur = None - _cur_avail = None - _volfile = None - - def _open(self): - super(DirectReader, self)._open() - - self._volfile = self._inf.volume_file - self._fd = XFile(self._volfile, 0) - self._fd.seek(self._inf.header_offset, 0) - self._cur = self._parser._parse_header(self._fd) - self._cur_avail = self._cur.add_size - - def _skip(self, cnt): - """RAR Seek, skipping through rar files to get to correct position - """ - - while cnt > 0: - # next vol needed? - if self._cur_avail == 0: - if not self._open_next(): - break - - # fd is in read pos, do the read - if cnt > self._cur_avail: - cnt -= self._cur_avail - self._remain -= self._cur_avail - self._cur_avail = 0 - else: - self._fd.seek(cnt, 1) - self._cur_avail -= cnt - self._remain -= cnt - cnt = 0 - - def _read(self, cnt): - """Read from potentially multi-volume archive.""" - - buf = [] - while cnt > 0: - # next vol needed? - if self._cur_avail == 0: - if not self._open_next(): - break - - # fd is in read pos, do the read - if cnt > self._cur_avail: - data = self._fd.read(self._cur_avail) - else: - data = self._fd.read(cnt) - if not data: - break - - # got some data - cnt -= len(data) - self._cur_avail -= len(data) - buf.append(data) - - if len(buf) == 1: - return buf[0] - return EMPTY.join(buf) - - def _open_next(self): - """Proceed to next volume.""" - - # is the file split over archives? - if (self._cur.flags & RAR_FILE_SPLIT_AFTER) == 0: - return False - - if self._fd: - self._fd.close() - self._fd = None - - # open next part - self._volfile = self._parser._next_volname(self._volfile) - fd = open(self._volfile, "rb", 0) - self._fd = fd - sig = fd.read(len(self._parser._expect_sig)) - if sig != self._parser._expect_sig: - raise BadRarFile("Invalid signature") - - # loop until first file header - while 1: - cur = self._parser._parse_header(fd) - if not cur: - raise BadRarFile("Unexpected EOF") - if cur.type in (RAR_BLOCK_MARK, RAR_BLOCK_MAIN): - if cur.add_size: - fd.seek(cur.add_size, 1) - continue - if cur.orig_filename != self._inf.orig_filename: - raise BadRarFile("Did not found file entry") - self._cur = cur - self._cur_avail = cur.add_size - return True - - def readinto(self, buf): - """Zero-copy read directly into buffer.""" - got = 0 - vbuf = memoryview(buf) - while got < len(buf): - # next vol needed? - if self._cur_avail == 0: - if not self._open_next(): - break - - # length for next read - cnt = len(buf) - got - if cnt > self._cur_avail: - cnt = self._cur_avail - - # read into temp view - res = self._fd.readinto(vbuf[got : got + cnt]) - if not res: - break - self._md_context.update(vbuf[got : got + res]) - self._cur_avail -= res - self._remain -= res - got += res - return got - - -class HeaderDecrypt(object): - """File-like object that decrypts from another file""" - def __init__(self, f, key, iv): - self.f = f - self.ciph = AES_CBC_Decrypt(key, iv) - self.buf = EMPTY - - def tell(self): - """Current file pos - works only on block boundaries.""" - return self.f.tell() - - def read(self, cnt=None): - """Read and decrypt.""" - if cnt > 8 * 1024: - raise BadRarFile('Bad count to header decrypt - wrong password?') - - # consume old data - if cnt <= len(self.buf): - res = self.buf[:cnt] - self.buf = self.buf[cnt:] - return res - res = self.buf - self.buf = EMPTY - cnt -= len(res) - - # decrypt new data - blklen = 16 - while cnt > 0: - enc = self.f.read(blklen) - if len(enc) < blklen: - break - dec = self.ciph.decrypt(enc) - if cnt >= len(dec): - res += dec - cnt -= len(dec) - else: - res += dec[:cnt] - self.buf = dec[cnt:] - cnt = 0 - - return res - - -# handle (filename|filelike) object -class XFile(object): - """Input may be filename or file object. - """ - __slots__ = ('_fd', '_need_close') - - def __init__(self, xfile, bufsize=1024): - if is_filelike(xfile): - self._need_close = False - self._fd = xfile - self._fd.seek(0) - else: - self._need_close = True - self._fd = open(xfile, 'rb', bufsize) - - def read(self, n=None): - """Read from file.""" - return self._fd.read(n) - - def tell(self): - """Return file pos.""" - return self._fd.tell() - - def seek(self, ofs, whence=0): - """Move file pos.""" - return self._fd.seek(ofs, whence) - - def readinto(self, dst): - """Read into buffer.""" - return self._fd.readinto(dst) - - def close(self): - """Close file object.""" - if self._need_close: - self._fd.close() - - def __enter__(self): - return self - - def __exit__(self, typ, val, tb): - self.close() - - -class NoHashContext(object): - """No-op hash function.""" - def __init__(self, data=None): - """Initialize""" - def update(self, data): - """Update data""" - def digest(self): - """Final hash""" - def hexdigest(self): - """Hexadecimal digest.""" - - -class CRC32Context(object): - """Hash context that uses CRC32.""" - __slots__ = ['_crc'] - - def __init__(self, data=None): - self._crc = 0 - if data: - self.update(data) - - def update(self, data): - """Process data.""" - self._crc = rar_crc32(data, self._crc) - - def digest(self): - """Final hash.""" - return self._crc - - def hexdigest(self): - """Hexadecimal digest.""" - return '%08x' % self.digest() - - -class Blake2SP(object): - """Blake2sp hash context. - """ - __slots__ = ['_thread', '_buf', '_cur', '_digest'] - digest_size = 32 - block_size = 64 - parallelism = 8 - - def __init__(self, data=None): - self._buf = b'' - self._cur = 0 - self._digest = None - self._thread = [] - - for i in range(self.parallelism): - ctx = self._blake2s(i, 0, i == (self.parallelism - 1)) - self._thread.append(ctx) - - if data: - self.update(data) - - def _blake2s(self, ofs, depth, is_last): - return blake2s(node_offset=ofs, node_depth=depth, last_node=is_last, - depth=2, inner_size=32, fanout=self.parallelism) - - def _add_block(self, blk): - self._thread[self._cur].update(blk) - self._cur = (self._cur + 1) % self.parallelism - - def update(self, data): - """Hash data. - """ - view = memoryview(data) - bs = self.block_size - if self._buf: - need = bs - len(self._buf) - if len(view) < need: - self._buf += view.tobytes() - return - self._add_block(self._buf + view[:need].tobytes()) - view = view[need:] - while len(view) >= bs: - self._add_block(view[:bs]) - view = view[bs:] - self._buf = view.tobytes() - - def digest(self): - """Return final digest value. - """ - if self._digest is None: - if self._buf: - self._add_block(self._buf) - self._buf = EMPTY - ctx = self._blake2s(0, 1, True) - for t in self._thread: - ctx.update(t.digest()) - self._digest = ctx.digest() - return self._digest - - def hexdigest(self): - """Hexadecimal digest.""" - return tohex(self.digest()) - -## -## Utility functions -## - -S_LONG = Struct(' len(buf): - raise BadRarFile('cannot load byte') - return S_BYTE.unpack_from(buf, pos)[0], end - -def load_le32(buf, pos): - """Load little-endian 32-bit integer""" - end = pos + 4 - if end > len(buf): - raise BadRarFile('cannot load le32') - return S_LONG.unpack_from(buf, pos)[0], pos + 4 - -def load_bytes(buf, num, pos): - """Load sequence of bytes""" - end = pos + num - if end > len(buf): - raise BadRarFile('cannot load bytes') - return buf[pos : end], end - -def load_vstr(buf, pos): - """Load bytes prefixed by vint length""" - slen, pos = load_vint(buf, pos) - return load_bytes(buf, slen, pos) - -def load_dostime(buf, pos): - """Load LE32 dos timestamp""" - stamp, pos = load_le32(buf, pos) - tup = parse_dos_time(stamp) - return to_datetime(tup), pos - -def load_unixtime(buf, pos): - """Load LE32 unix timestamp""" - secs, pos = load_le32(buf, pos) - dt = datetime.fromtimestamp(secs, UTC) - return dt, pos - -def load_windowstime(buf, pos): - """Load LE64 windows timestamp""" - # unix epoch (1970) in seconds from windows epoch (1601) - unix_epoch = 11644473600 - val1, pos = load_le32(buf, pos) - val2, pos = load_le32(buf, pos) - secs, n1secs = divmod((val2 << 32) | val1, 10000000) - dt = datetime.fromtimestamp(secs - unix_epoch, UTC) - dt = dt.replace(microsecond=n1secs // 10) - return dt, pos - -# new-style next volume -def _next_newvol(volfile): - i = len(volfile) - 1 - while i >= 0: - if volfile[i] >= '0' and volfile[i] <= '9': - return _inc_volname(volfile, i) - i -= 1 - raise BadRarName("Cannot construct volume name: " + volfile) - -# old-style next volume -def _next_oldvol(volfile): - # rar -> r00 - if volfile[-4:].lower() == '.rar': - return volfile[:-2] + '00' - return _inc_volname(volfile, len(volfile) - 1) - -# increase digits with carry, otherwise just increment char -def _inc_volname(volfile, i): - fn = list(volfile) - while i >= 0: - if fn[i] != '9': - fn[i] = chr(ord(fn[i]) + 1) - break - fn[i] = '0' - i -= 1 - return ''.join(fn) - -# rar3 extended time fields -def _parse_ext_time(h, data, pos): - # flags and rest of data can be missing - flags = 0 - if pos + 2 <= len(data): - flags = S_SHORT.unpack_from(data, pos)[0] - pos += 2 - - mtime, pos = _parse_xtime(flags >> 3 * 4, data, pos, h.mtime) - h.ctime, pos = _parse_xtime(flags >> 2 * 4, data, pos) - h.atime, pos = _parse_xtime(flags >> 1 * 4, data, pos) - h.arctime, pos = _parse_xtime(flags >> 0 * 4, data, pos) - if mtime: - h.mtime = mtime - h.date_time = mtime.timetuple()[:6] - return pos - -# rar3 one extended time field -def _parse_xtime(flag, data, pos, basetime=None): - res = None - if flag & 8: - if not basetime: - basetime, pos = load_dostime(data, pos) - - # load second fractions - rem = 0 - cnt = flag & 3 - for _ in range(cnt): - b, pos = load_byte(data, pos) - rem = (b << 16) | (rem >> 8) - - # convert 100ns units to microseconds - usec = rem // 10 - if usec > 1000000: - usec = 999999 - - # dostime has room for 30 seconds only, correct if needed - if flag & 4 and basetime.second < 59: - res = basetime.replace(microsecond=usec, second=basetime.second + 1) - else: - res = basetime.replace(microsecond=usec) - return res, pos - -def is_filelike(obj): - """Filename or file object? - """ - if isinstance(obj, str) or isinstance(obj, unicode): - return False - res = True - for a in ('read', 'tell', 'seek'): - res = res and hasattr(obj, a) - if not res: - raise ValueError("Invalid object passed as file") - return True - -def rar3_s2k(psw, salt): - """String-to-key hash for RAR3. - """ - if not isinstance(psw, unicode): - psw = psw.decode('utf8') - seed = psw.encode('utf-16le') + salt - iv = EMPTY - h = sha1() - for i in range(16): - for j in range(0x4000): - cnt = S_LONG.pack(i * 0x4000 + j) - h.update(seed + cnt[:3]) - if j == 0: - iv += h.digest()[19:20] - key_be = h.digest()[:16] - key_le = pack("LLLL", key_be)) - return key_le, iv - -def rar3_decompress(vers, meth, data, declen=0, flags=0, crc=0, psw=None, salt=None): - """Decompress blob of compressed data. - - Used for data with non-standard header - eg. comments. - """ - # already uncompressed? - if meth == RAR_M0 and (flags & RAR_FILE_PASSWORD) == 0: - return data - - # take only necessary flags - flags = flags & (RAR_FILE_PASSWORD | RAR_FILE_SALT | RAR_FILE_DICTMASK) - flags |= RAR_LONG_BLOCK - - # file header - fname = b'data' - date = 0 - mode = 0x20 - fhdr = S_FILE_HDR.pack(len(data), declen, RAR_OS_MSDOS, crc, - date, vers, meth, len(fname), mode) - fhdr += fname - if flags & RAR_FILE_SALT: - if not salt: - return EMPTY - fhdr += salt - - # full header - hlen = S_BLK_HDR.size + len(fhdr) - hdr = S_BLK_HDR.pack(0, RAR_BLOCK_FILE, flags, hlen) + fhdr - hcrc = rar_crc32(hdr[2:]) & 0xFFFF - hdr = S_BLK_HDR.pack(hcrc, RAR_BLOCK_FILE, flags, hlen) + fhdr - - # archive main header - mh = S_BLK_HDR.pack(0x90CF, RAR_BLOCK_MAIN, 0, 13) + ZERO * (2 + 4) - - # decompress via temp rar - tmpfd, tmpname = mkstemp(suffix='.rar') - tmpf = os.fdopen(tmpfd, "wb") - try: - tmpf.write(RAR_ID + mh + hdr + data) - tmpf.close() - - cmd = [UNRAR_TOOL] + list(OPEN_ARGS) - add_password_arg(cmd, psw, (flags & RAR_FILE_PASSWORD)) - cmd.append(tmpname) - - p = custom_popen(cmd) - return p.communicate()[0] - finally: - tmpf.close() - os.unlink(tmpname) - -def to_datetime(t): - """Convert 6-part time tuple into datetime object. - """ - if t is None: - return None - - # extract values - year, mon, day, h, m, s = t - - # assume the values are valid - try: - return datetime(year, mon, day, h, m, s) - except ValueError: - pass - - # sanitize invalid values - mday = (0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) - if mon < 1: - mon = 1 - if mon > 12: - mon = 12 - if day < 1: - day = 1 - if day > mday[mon]: - day = mday[mon] - if h > 23: - h = 23 - if m > 59: - m = 59 - if s > 59: - s = 59 - if mon == 2 and day == 29: - try: - return datetime(year, mon, day, h, m, s) - except ValueError: - day = 28 - return datetime(year, mon, day, h, m, s) - -def parse_dos_time(stamp): - """Parse standard 32-bit DOS timestamp. - """ - sec, stamp = stamp & 0x1F, stamp >> 5 - mn, stamp = stamp & 0x3F, stamp >> 6 - hr, stamp = stamp & 0x1F, stamp >> 5 - day, stamp = stamp & 0x1F, stamp >> 5 - mon, stamp = stamp & 0x0F, stamp >> 4 - yr = (stamp & 0x7F) + 1980 - return (yr, mon, day, hr, mn, sec * 2) - -def custom_popen(cmd): - """Disconnect cmd from parent fds, read only from stdout. - """ - # needed for py2exe - creationflags = 0 - if sys.platform == 'win32': - creationflags = 0x08000000 # CREATE_NO_WINDOW - - # run command - try: - p = Popen(cmd, bufsize=0, stdout=PIPE, stdin=PIPE, stderr=STDOUT, - creationflags=creationflags) - except OSError as ex: - if ex.errno == errno.ENOENT: - raise RarCannotExec("Unrar not installed? (rarfile.UNRAR_TOOL=%r)" % UNRAR_TOOL) - raise - return p - -def custom_check(cmd, ignore_retcode=False): - """Run command, collect output, raise error if needed. - """ - p = custom_popen(cmd) - out, _ = p.communicate() - if p.returncode and not ignore_retcode: - raise RarExecError("Check-run failed") - return out - -def add_password_arg(cmd, psw, ___required=False): - """Append password switch to commandline. - """ - if UNRAR_TOOL == ALT_TOOL: - return - if psw is not None: - cmd.append('-p' + psw) - else: - cmd.append('-p-') - -def check_returncode(p, out): - """Raise exception according to unrar exit code. - """ - code = p.returncode - if code == 0: - return - - # map return code to exception class, codes from rar.txt - errmap = [None, - RarWarning, RarFatalError, RarCRCError, RarLockedArchiveError, # 1..4 - RarWriteError, RarOpenError, RarUserError, RarMemoryError, # 5..8 - RarCreateError, RarNoFilesError, RarWrongPassword] # 9..11 - if UNRAR_TOOL == ALT_TOOL: - errmap = [None] - if code > 0 and code < len(errmap): - exc = errmap[code] - elif code == 255: - exc = RarUserBreak - elif code < 0: - exc = RarSignalExit - else: - exc = RarUnknownError - - # format message - if out: - msg = "%s [%d]: %s" % (exc.__doc__, p.returncode, out) - else: - msg = "%s [%d]" % (exc.__doc__, p.returncode) - - raise exc(msg) - -def hmac_sha256(key, data): - """HMAC-SHA256""" - return HMAC(key, data, sha256).digest() - -def membuf_tempfile(memfile): - memfile.seek(0, 0) - - tmpfd, tmpname = mkstemp(suffix='.rar') - tmpf = os.fdopen(tmpfd, "wb") - - try: - while True: - buf = memfile.read(BSIZE) - if not buf: - break - tmpf.write(buf) - tmpf.close() - except: - tmpf.close() - os.unlink(tmpname) - raise - return tmpname - -class XTempFile(object): - __slots__ = ('_tmpfile', '_filename') - - def __init__(self, rarfile): - if is_filelike(rarfile): - self._tmpfile = membuf_tempfile(rarfile) - self._filename = self._tmpfile - else: - self._tmpfile = None - self._filename = rarfile - - def __enter__(self): - return self._filename - - def __exit__(self, exc_type, exc_value, tb): - if self._tmpfile: - try: - os.unlink(self._tmpfile) - except OSError: - pass - self._tmpfile = None - -# -# Check if unrar works -# - -ORIG_UNRAR_TOOL = UNRAR_TOOL -ORIG_OPEN_ARGS = OPEN_ARGS -ORIG_EXTRACT_ARGS = EXTRACT_ARGS -ORIG_TEST_ARGS = TEST_ARGS - -def _check_unrar_tool(): - global UNRAR_TOOL, OPEN_ARGS, EXTRACT_ARGS, TEST_ARGS - try: - # does UNRAR_TOOL work? - custom_check([ORIG_UNRAR_TOOL], True) - - UNRAR_TOOL = ORIG_UNRAR_TOOL - OPEN_ARGS = ORIG_OPEN_ARGS - EXTRACT_ARGS = ORIG_EXTRACT_ARGS - TEST_ARGS = ORIG_TEST_ARGS - except RarCannotExec: - try: - # does ALT_TOOL work? - custom_check([ALT_TOOL] + list(ALT_CHECK_ARGS), True) - # replace config - UNRAR_TOOL = ALT_TOOL - OPEN_ARGS = ALT_OPEN_ARGS - EXTRACT_ARGS = ALT_EXTRACT_ARGS - TEST_ARGS = ALT_TEST_ARGS - except RarCannotExec: - # no usable tool, only uncompressed archives work - pass - -_check_unrar_tool() From a4dbf11dca5bc5456ca2473afbb428ca1bbc5db9 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:36:31 +0000 Subject: [PATCH 45/51] Delete __init__.py --- apps/templates/nzbget-mp4/scripts/rarfile/__init__.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/rarfile/__init__.py diff --git a/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py b/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py +++ /dev/null @@ -1 +0,0 @@ - From a52ded07be36cdf100065fbaee85847fdc0450ff Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:36:49 +0000 Subject: [PATCH 46/51] Delete installer.sh --- .../nzbget-mp4/installer/installer.sh | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/installer/installer.sh diff --git a/apps/templates/nzbget-mp4/installer/installer.sh b/apps/templates/nzbget-mp4/installer/installer.sh deleted file mode 100644 index 8c26cf9..0000000 --- a/apps/templates/nzbget-mp4/installer/installer.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -apk update -apk upgrade -apk add --no-cache git -apk add build-base gcc wget diffutils perl -apk add curl -git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp -cp -r /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator -rm -rf /config/scripts/MP4_Automator/tmp -git unstage -apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev -pip install --upgrade PIP -pip install requests -pip install requests[security] -pip install requests-cache -pip install babelfish -pip install "guessit<2" -pip install "subliminal<2" -pip install qtfaststart -# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 -pip uninstall -y stevedore -pip install stevedore==1.19.1 -#Remove default NZBGetPostProcess script settings, and replace with our own -rm /config/scripts/MP4_Automator/NZBGetPostProcess.py -cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py -#Build ffmpeg -cd /config -. /config/ffmpeg-build/web-install.sh -#Set script file permissions -chmod 777 -R /config/scripts From b8fe69ab0d4144970d2484aaf17b108169b01ff5 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:02 +0000 Subject: [PATCH 47/51] Delete web-install.sh --- .../nzbget-mp4/ffmpeg-build/web-install.sh | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh b/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh deleted file mode 100644 index ffe4158..0000000 --- a/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Helper script to download and run the build-ffmpeg script. - -make_dir () { - if [ ! -d $1 ]; then - if ! mkdir $1; then - printf "\n Failed to create dir %s" "$1"; - exit 1 - fi - fi -} - -command_exists() { - if ! [[ -x $(command -v "$1") ]]; then - return 1 - fi - - return 0 -} - -TARGET='ffmpeg-build' - -if ! command_exists "curl"; then - echo "curl not installed."; - exit 1 -fi - -echo "ffmpeg-build-script-downloader v0.1" -echo "=========================================" -echo "" - -echo "First we create the ffmpeg build directory $TARGET" -make_dir $TARGET -cd $TARGET - -echo "Now we download and execute the build script" -echo "" - -bash build-ffmpeg --build - From 6706b384debad37457ec96b55522eabc4c70126d Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:14 +0000 Subject: [PATCH 48/51] Delete build-ffmpeg --- .../nzbget-mp4/ffmpeg-build/build-ffmpeg | 400 ------------------ 1 file changed, 400 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg b/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg deleted file mode 100644 index 9aad5c4..0000000 --- a/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg +++ /dev/null @@ -1,400 +0,0 @@ -#!/bin/bash - -# https://github.com/markus-perl/ffmpeg-build-script - -VERSION=1.1 -CWD=$(pwd) -PACKAGES="$CWD/packages" -WORKSPACE="$CWD/workspace" -CC=clang -LDFLAGS="-L${WORKSPACE}/lib -lm" -CFLAGS="-I${WORKSPACE}/include" -PKG_CONFIG_PATH="${WORKSPACE}/lib/pkgconfig" -ADDITIONAL_CONFIGURE_OPTIONS="" - -# Speed up the process -# Env Var NUMJOBS overrides automatic detection -if [[ -n $NUMJOBS ]]; then - MJOBS=$NUMJOBS -elif [[ -f /proc/cpuinfo ]]; then - MJOBS=$(grep -c processor /proc/cpuinfo) -elif [[ "$OSTYPE" == "darwin"* ]]; then - MJOBS=$(sysctl -n machdep.cpu.thread_count) - ADDITIONAL_CONFIGURE_OPTIONS="--enable-videotoolbox" -else - MJOBS=4 -fi - -make_dir () { - if [ ! -d $1 ]; then - if ! mkdir $1; then - printf "\n Failed to create dir %s" "$1"; - exit 1 - fi - fi -} - -remove_dir () { - if [ -d $1 ]; then - rm -r "$1" - fi -} - -download () { - - DOWNLOAD_PATH=$PACKAGES; - - if [ ! -z "$3" ]; then - mkdir -p $PACKAGES/$3 - DOWNLOAD_PATH=$PACKAGES/$3 - fi; - - if [ ! -f "$DOWNLOAD_PATH/$2" ]; then - - echo "Downloading $1" - curl -L --silent -o "$DOWNLOAD_PATH/$2" "$1" - - EXITCODE=$? - if [ $EXITCODE -ne 0 ]; then - echo "" - echo "Failed to download $1. Exitcode $EXITCODE. Retrying in 10 seconds"; - sleep 10 - curl -L --silent -o "$DOWNLOAD_PATH/$2" "$1" - fi - - EXITCODE=$? - if [ $EXITCODE -ne 0 ]; then - echo "" - echo "Failed to download $1. Exitcode $EXITCODE"; - exit 1 - fi - - echo "... Done" - - if ! tar -xvf "$DOWNLOAD_PATH/$2" -C "$DOWNLOAD_PATH" 2>/dev/null >/dev/null; then - echo "Failed to extract $2"; - exit 1 - fi - - fi -} - -execute () { - echo "$ $*" - - OUTPUT=$($@ 2>&1) - - if [ $? -ne 0 ]; then - echo "$OUTPUT" - echo "" - echo "Failed to Execute $*" >&2 - exit 1 - fi -} - -build () { - echo "" - echo "building $1" - echo "=======================" - - if [ -f "$PACKAGES/$1.done" ]; then - echo "$1 already built. Remove $PACKAGES/$1.done lockfile to rebuild it." - return 1 - fi - - return 0 -} - -command_exists() { - if ! [[ -x $(command -v "$1") ]]; then - return 1 - fi - - return 0 -} - - -build_done () { - touch "$PACKAGES/$1.done" -} - -echo "ffmpeg-build-script v$VERSION" -echo "=========================" -echo "" - -case "$1" in -"--cleanup") - remove_dir $PACKAGES - remove_dir $WORKSPACE - echo "Cleanup done." - echo "" - exit 0 - ;; -"--build") - - ;; -*) - echo "Usage: $0" - echo " --build: start building process" - echo " --cleanup: remove all working dirs" - echo " --help: show this help" - echo "" - exit 0 - ;; -esac - -echo "Using $MJOBS make jobs simultaneously." - -make_dir $PACKAGES -make_dir $WORKSPACE - -export PATH=${WORKSPACE}/bin:$PATH - -if ! command_exists "make"; then - echo "make not installed."; - exit 1 -fi - -if ! command_exists "g++"; then - echo "g++ not installed."; - exit 1 -fi - -if ! command_exists "curl"; then - echo "curl not installed."; - exit 1 -fi - -if build "yasm"; then - download "http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz" "yasm-1.3.0.tar.gz" - cd $PACKAGES/yasm-1.3.0 || exit - execute ./configure --prefix=${WORKSPACE} - execute make -j $MJOBS - execute make install - build_done "yasm" -fi - -if build "nasm"; then - download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.13.03.tar.gz" "nasm.tar.gz" - cd $PACKAGES/nasm-2.13.03 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "nasm" -fi - -if build "opencore"; then - download "http://downloads.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-0.1.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Fopencore-amr%2F&ts=1442256558&use_mirror=netassist" "opencore-amr-0.1.5.tar.gz" - cd $PACKAGES/opencore-amr-0.1.5 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "opencore" -fi - -if build "libvpx"; then - download "https://github.com/webmproject/libvpx/archive/v1.7.0.tar.gz" "libvpx-1.7.0.tar.gz" - cd $PACKAGES/libvpx-*0 || exit - - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "Applying Darwin patch" - sed "s/,--version-script//g" build/make/Makefile > build/make/Makefile.patched - sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched > build/make/Makefile - fi - - execute ./configure --prefix=${WORKSPACE} --disable-unit-tests --disable-shared - execute make -j $MJOBS - execute make install - build_done "libvpx" -fi - -if build "lame"; then - download "http://kent.dl.sourceforge.net/project/lame/lame/3.100/lame-3.100.tar.gz" "lame-3.100.tar.gz" - cd $PACKAGES/lame-3.100 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "lame" -fi - -if build "xvidcore"; then - download "http://downloads.xvid.org/downloads/xvidcore-1.3.4.tar.gz" "xvidcore-1.3.4.tar.gz" - cd $PACKAGES/xvidcore || exit - cd build/generic || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - - if [[ -f ${WORKSPACE}/lib/libxvidcore.4.dylib ]]; then - execute rm "${WORKSPACE}/lib/libxvidcore.4.dylib" - fi - - build_done "xvidcore" -fi - -if build "x264"; then - download "http://ftp.videolan.org/pub/x264/snapshots/x264-snapshot-20190204-2245-stable.tar.bz2" "last_x264.tar.bz2" - cd $PACKAGES/x264-snapshot-* || exit - - if [[ "$OSTYPE" == "linux-gnu" ]]; then - execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic CXXFLAGS="-fPIC" - else - execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic - fi - - execute make -j $MJOBS - execute make install - execute make install-lib-static - build_done "x264" -fi - -if build "libogg"; then - download "http://downloads.xiph.org/releases/ogg/libogg-1.3.3.tar.gz" "libogg-1.3.3.tar.gz" - cd $PACKAGES/libogg-1.3.3 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "libogg" -fi - -if build "libvorbis"; then - download "http://downloads.xiph.org/releases/vorbis/libvorbis-1.3.6.tar.gz" "libvorbis-1.3.6.tar.gz" - cd $PACKAGES/libvorbis-1.3.6 || exit - execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest - execute make -j $MJOBS - execute make install - build_done "libvorbis" -fi - -if build "libtheora"; then - download "http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.gz" "libtheora-1.1.1.tar.bz" - cd $PACKAGES/libtheora-1.1.1 || exit - sed "s/-fforce-addr//g" configure > configure.patched - chmod +x configure.patched - mv configure.patched configure - execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --with-vorbis-libraries=${WORKSPACE}/lib --with-vorbis-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm - execute make -j $MJOBS - execute make install - build_done "libtheora" -fi - -if build "pkg-config"; then - download "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" "pkg-config-0.29.2.tar.gz" - cd $PACKAGES/pkg-config-0.29.2 || exit - execute ./configure --silent --prefix=${WORKSPACE} --with-pc-path=${WORKSPACE}/lib/pkgconfig --with-internal-glib - execute make -j $MJOBS - execute make install - build_done "pkg-config" -fi - -if build "cmake"; then - download "https://cmake.org/files/v3.11/cmake-3.11.3.tar.gz" "cmake-3.11.3.tar.gz" - cd $PACKAGES/cmake-3.11.3 || exit - rm Modules/FindJava.cmake - perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt - perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt - execute ./configure --prefix=${WORKSPACE} - execute make -j $MJOBS - execute make install - build_done "cmake" -fi - -if build "vid_stab"; then - download "https://codeload.github.com/georgmartius/vid.stab/legacy.tar.gz/release-0.98b" "vid.stab-0.98b-transcode-1.1-binary-x86_64.tgz" - cd $PACKAGES/georgmartius-vid* || exit - perl -p -i -e "s/vidstab SHARED/vidstab STATIC/" CMakeLists.txt - execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} . - execute make install - build_done "vid_stab" -fi - -if build "x265"; then - download "https://bitbucket.org/multicoreware/x265/downloads/x265_3.0.tar.gz" "x265-3.0.tar.gz" - cd $PACKAGES/x265_* || exit - cd source || exit - execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} -DENABLE_SHARED:bool=off . - execute make -j $MJOBS - execute make install - sed "s/-lx265/-lx265 -lstdc++/g" "$WORKSPACE/lib/pkgconfig/x265.pc" > "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" - mv "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" "$WORKSPACE/lib/pkgconfig/x265.pc" - build_done "x265" -fi - -if build "fdk_aac"; then - download "http://downloads.sourceforge.net/project/opencore-amr/fdk-aac/fdk-aac-0.1.6.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Ffdk-aac%2F&ts=1457561564&use_mirror=kent" "fdk-aac-0.1.6.tar.gz" - cd $PACKAGES/fdk-aac-0.1.6 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "fdk_aac" -fi - - -build "ffmpeg" -download "http://ffmpeg.org/releases/ffmpeg-4.1.tar.bz2" "ffmpeg-snapshot.tar.bz2" -cd $PACKAGES/ffmpeg-4.1 || exit -./configure $ADDITIONAL_CONFIGURE_OPTIONS \ - --pkgconfigdir="$WORKSPACE/lib/pkgconfig" \ - --prefix=${WORKSPACE} \ - --pkg-config-flags="--static" \ - --extra-cflags="-I$WORKSPACE/include" \ - --extra-ldflags="-L$WORKSPACE/lib" \ - --extra-libs="-lpthread -lm" \ - --enable-static \ - --disable-debug \ - --disable-shared \ - --disable-ffplay \ - --disable-doc \ - --enable-gpl \ - --enable-version3 \ - --enable-nonfree \ - --enable-pthreads \ - --enable-libvpx \ - --enable-libmp3lame \ - --enable-libtheora \ - --enable-libvorbis \ - --enable-libx264 \ - --enable-libx265 \ - --enable-runtime-cpudetect \ - --enable-libfdk-aac \ - --enable-avfilter \ - --enable-libopencore_amrwb \ - --enable-libopencore_amrnb \ - --enable-filters \ - --enable-libvidstab - -execute make -j $MJOBS -execute make install - -INSTALL_FOLDER="/usr/bin" -if [[ "$OSTYPE" == "darwin"* ]]; then -INSTALL_FOLDER="/usr/local/bin" -fi - -echo "" -echo "Building done. The binary can be found here: $WORKSPACE/bin/ffmpeg" -echo "" - - -if [[ $AUTOINSTALL == "yes" ]]; then - if command_exists "sudo"; then - sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" - sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" - echo "Done. ffmpeg is now installed to your system" - fi -elif [[ ! $SKIPINSTALL == "yes" ]]; then - if command_exists "sudo"; then - - read -r -p "Install the binary to your $INSTALL_FOLDER folder? [Y/n] " response - - case $response in - [yY][eE][sS]|[yY]) - sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" - sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" - echo "Done. ffmpeg is now installed to your system" - ;; - esac - fi -fi - -exit 0 From 9bd40af054169b5faa404119127a2042c6d9ec0f Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:26 +0000 Subject: [PATCH 49/51] Delete 30-config --- .../nzbget-mp4/cont-init.d/30-config | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/cont-init.d/30-config diff --git a/apps/templates/nzbget-mp4/cont-init.d/30-config b/apps/templates/nzbget-mp4/cont-init.d/30-config deleted file mode 100644 index deaaacd..0000000 --- a/apps/templates/nzbget-mp4/cont-init.d/30-config +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/with-contenv bash - -# delete lock file if found -[[ -f /downloads/nzbget.lock ]] && \ - rm /downloads/nzbget.lock - -# check if config file exists in /config -[[ ! -f /config/nzbget.conf ]] && \ - cp /defaults/nzbget.conf /config/nzbget.conf - -# permissions -chown 1000:1000 \ - /downloads -chown 1000:1000 -R \ - /app/nzbget \ - /config -chmod u+rw \ - /config/nzbget.conf - -chmod 777 -R \ - /config -chmod 777 -R \ - /app/nzbget -chmod 777 -R \ - /downloads - -exec /config/installer/installer.sh From 2916e91fc9d76f37f9f1634608cbe16ddb48b23b Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:37 +0000 Subject: [PATCH 50/51] Delete autoProcess.ini --- .../nzbget-mp4/MP4_Automator/autoProcess.ini | 143 ------------------ 1 file changed, 143 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini diff --git a/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini b/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini deleted file mode 100644 index 83b18aa..0000000 --- a/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini +++ /dev/null @@ -1,143 +0,0 @@ -[SickBeard] -host = sickbeard -port = 8081 -username = -password = -web_root = -ssl = False -api_key = - -[Sonarr] -host = sonarr -port = 8989 -web_root = -ssl = False -apikey = - -[Radarr] -host = radarr -port = 7878 -web_root = -ssl = False -apikey = - -[MP4] -ffmpeg = /config/ffmpeg-build/workspace/bin/ffmpeg -ffprobe = /config/ffmpeg-build/workspace/bin/ffprobe -threads = 1 -output_directory = -copy_to = -move_to = -output_extension = mp4 -output_format = mp4 -delete_original = True -relocate_moov = True -video-codec = h264,x264 -video-bitrate = -video-crf = 18 -video-max-width = -h264-max-level = 4.1 -use-qsv-decoder-with-encoder = True -ios-audio = libfdk_aac -ios-first-track-only = False -ios-audio-filter = dynaudnorm -max-audio-channels = -audio-codec = ac3,mp3,dts,dca,aac,libfdk_aac -audio-language = eng -audio-default-language = eng -audio-channel-bitrate = 256 -audio-filter = -subtitle-codec = srt -subtitle-language = eng -subtitle-default-language = -subtitle-encoding = -fullpathguess = True -convert-mp4 = True -tagfile = True -tag-language = en -download-artwork = Poster -download-subs = True -embed-subs = False -sub-providers = addic7ed,podnapisi,thesubdb,opensubtitles -permissions = 0777 -post-process = False -pix-fmt = -aac_adtstoasc = False -postopts = -preset,slower -preopts = -audio-copy-original = False -enable_dxva2_gpu_decode = False -ios-move-last = False -use-hevc-qsv-decoder = False -embed-only-internal-subs = False -audio-first-track-of-language = False -video-profile = - -[CouchPotato] -host = couchpotato -port = 5050 -username = -password = -web_root = -ssl = False -apikey = -delay = 65 -method = renamer -delete_failed = False - -[uTorrent] -convert = -couchpotato-label = couchpotato -sickbeard-label = sickbeard -sonarr-label = sonarr -bypass-label = bypass -sickrage-label = sickrage -webui = False -action_before = stop -action_after = removedata -host = http://utorrent:8080/ -username = -password = -output_directory = -radarr-label = radarr - -[Deluge] -host = deluge -username = -convert = True -password = -sonarr-label = sonarr -radarr-label = radarr -bypass-label = bypass -sickbeard-label = sickbeard -port = 12569 -sickrage-label = sickrage -couchpotato-label = couchpotato -output_directory = -remove = true - -[SABNZBD] -convert = True -sickrage-category = sickrage -sonarr-category = sonarr -radarr-category = radarr -bypass-category = bypass -couchpotato-category = couchpotato -sickbeard-category = sickbeard -output_directory = - -[Sickrage] -host = sickrage -port = 8081 -username = -password = -web_root = -ssl = False -api_key = - -[Plex] -host = plex -port = 32400 -refresh = False -token = - From 43d7dbdb86711f0bbebbf198a27bf30e57627b39 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:47 +0000 Subject: [PATCH 51/51] Delete TEMPLATEPPScript --- .../nzbget-mp4/MP4_Automator/TEMPLATEPPScript | 244 ------------------ 1 file changed, 244 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript diff --git a/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript b/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript deleted file mode 100644 index 6eb7055..0000000 --- a/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# -############################################################################## -### NZBGET POST-PROCESSING SCRIPT ### - -# Modified to enable multiple bypass categories, -# as per: https://github.com/mdhiggins/sickbeard_mp4_automator/issues/509 -# -# Converts files and passes them to Sonarr for further processing. -# -# NOTE: This script requires Python to be installed on your system. - -############################################################################## -### OPTIONS ### - -# Change to full path to MP4 Automator folder. No quotes and a trailing / -#MP4_FOLDER=~/sickbeard_mp4_automator/ - -# Convert file before passing to destination (True, False) -#SHOULDCONVERT=False - -# Category for Couchpotato -#CP_CAT=Couchpotato - -# Category for Sonarr -#SONARR_CAT=Sonarr - -# Category for Radarr -#RADARR_CAT=Radarr - -# Category for Sickbeard -#SICKBEARD_CAT=Sickbeard - -# Category for Sickrage -#SICKRAGE_CAT=Sickrage - -# Category list (comma seperated) for bypassing any further processing but still converting -#BYPASS_CAT=tv,movies - -# Custom output_directory setting -#OUTPUT_DIR= - -### NZBGET POST-PROCESSING SCRIPT ### -############################################################################## - -import os -import sys -import re -import json -import traceback - -# Sanity checks for path string -MP4folder = os.environ['NZBPO_MP4_FOLDER'].strip() -MP4folder = MP4folder.replace('"', '') -MP4folder = MP4folder.replace("'", "") -MP4folder = MP4folder.replace("\\", "/") -if not(MP4folder.endswith("/")): - MP4folder += "/" -#DEBUG#print MP4folder+" the original is "+os.environ['NZBPO_MP4_FOLDER'] - -output_dir = None -if 'NZBPO_OUTPUT_DIR' in os.environ: - output_dir = os.environ['NZBPO_OUTPUT_DIR'].strip() - if len(output_dir) > 0: - output_dir = output_dir.replace('"', '') - output_dir = output_dir.replace("'", "") - output_dir = output_dir.replace("\\", "/") - if not(output_dir.endswith("/")): - output_dir += "/" - #DEBUG#print Overriding output directory - -sys.path.append(MP4folder) -try: - from readSettings import ReadSettings - from mkvtomp4 import MkvtoMp4 - from autoprocess import autoProcessMovie, autoProcessTV, autoProcessTVSR, sonarr, radarr - import logging - from logging.config import fileConfig -except ImportError: - print("[ERROR] Wrong path to sickbeard_mp4_automator: " + os.environ['NZBPO_MP4_FOLDER']) - print("[ERROR] %s" % traceback.print_exc()) - sys.exit(0) - -# Setup Logging -logpath = '/var/log/sickbeard_mp4_automator' -if os.name == 'nt': - logpath = MP4folder -elif not os.path.isdir(logpath): - try: - os.mkdir(logpath) - except: - logpath = MP4folder -configPath = os.path.abspath(os.path.join(MP4folder, 'logging.ini')).replace("\\", "\\\\") -logPath = os.path.abspath(os.path.join(logpath, 'index.log')).replace("\\", "\\\\") -fileConfig(configPath, defaults={'logfilename': logPath}) -log = logging.getLogger("NZBGetPostProcess") - -# Determine if conversion will take place -shouldConvert = (os.environ['NZBPO_SHOULDCONVERT'].lower() in ("yes", "true", "t", "1")) - -if 'NZBOP_SCRIPTDIR' in os.environ and not os.environ['NZBOP_VERSION'][0:5] < '11.0': - log.info("Script triggered from NZBGet (11.0 or later).") - - path = os.environ['NZBPP_DIRECTORY'] # Path to NZB directory - nzb = os.environ['NZBPP_NZBFILENAME'] # Original NZB name - category = os.environ['NZBPP_CATEGORY'] # NZB Category to determine destination - #DEBUG#print "Category is %s." % category - - couchcat = os.environ['NZBPO_CP_CAT'].lower() - sonarrcat = os.environ['NZBPO_SONARR_CAT'].lower() - radarrcat = os.environ['NZBPO_RADARR_CAT'].lower() - sickbeardcat = os.environ['NZBPO_SICKBEARD_CAT'].lower() - sickragecat = os.environ['NZBPO_SICKRAGE_CAT'].lower() - bypass = os.environ['NZBPO_BYPASS_CAT'].lower().replace(' ','').split(',') - - categories = [sickbeardcat, couchcat, sonarrcat, radarrcat, sickragecat] - - log.debug("Path: %s" % path) - log.debug("NZB: %s" % nzb) - log.debug("Category: %s" % category) - log.debug("Categories: %s" % categories) - - # NZBGet argv: all passed as environment variables. - clientAgent = "nzbget" - # Exit codes used by NZBGet - POSTPROCESS_PARCHECK = 92 - POSTPROCESS_SUCCESS = 93 - POSTPROCESS_ERROR = 94 - POSTPROCESS_NONE = 95 - - # Check nzbget.conf options - status = 0 - - if os.environ['NZBOP_UNPACK'] != 'yes': - log.error("Please enable option \"Unpack\" in nzbget configuration file, exiting.") - sys.exit(POSTPROCESS_NONE) - - # Check par status - if os.environ['NZBPP_PARSTATUS'] == '3': - log.error("Par-check successful, but Par-repair disabled, exiting") - sys.exit(POSTPROCESS_NONE) - - if os.environ['NZBPP_PARSTATUS'] == '1': - log.error("Par-check failed, setting status \"failed\".") - status = 1 - sys.exit(POSTPROCESS_NONE) - - # Check unpack status - if os.environ['NZBPP_UNPACKSTATUS'] == '1': - log.error("Unpack failed, setting status \"failed\".") - status = 1 - sys.exit(POSTPROCESS_NONE) - - if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] != '2': - # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check - - for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): - for file in filenames: - fileExtension = os.path.splitext(file)[1] - - if fileExtension in ['.par2']: - log.error("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\".") - status = 1 - break - - if os.path.isfile(os.path.join(os.environ['NZBPP_DIRECTORY'], "_brokenlog.txt")) and not status == 1: - log.error("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting.") - status = 1 - - if not status == 1: - log.error("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful.") - - # Check if destination directory exists (important for reprocessing of history items) - if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): - log.error("Post-Process: Nothing to post-process: destination directory ", os.environ['NZBPP_DIRECTORY'], "doesn't exist.") - status = 1 - sys.exit(POSTPROCESS_NONE) - - # Make sure one of the appropriate categories is set - if category.lower() not in categories and category.lower() not in bypass: - log.error("Post-Process: No valid category detected. Category was %s." % (category)) - status = 1 - sys.exit(POSTPROCESS_NONE) - - # Make sure there are no duplicate categories - if len(categories) != len(set(categories)): - log.error("Duplicate category detected. Category names must be unique.") - status = 1 - sys.exit(POSTPROCESS_NONE) - - # All checks done, now launching the script. - settings = ReadSettings(MP4folder, "autoProcess.ini") - - if shouldConvert: - if output_dir: - settings.output_dir = output_dir - converter = MkvtoMp4(settings, logger=log) - for r, d, f in os.walk(path): - for files in f: - inputfile = os.path.join(r, files) - #DEBUG#print inputfile - #Ignores files under 50MB - if os.path.getsize(inputfile) > 50000000: - if MkvtoMp4(settings, logger=log).validSource(inputfile): - try: - output = converter.process(inputfile) - log.info("Successfully processed %s." % inputfile) - except: - log.exception("File processing failed.") - if converter.output_dir: - path = converter.output_dir - if (category.lower() == categories[0]): - #DEBUG#print "Sickbeard Processing Activated" - autoProcessTV.processEpisode(path, settings, nzb) - sys.exit(POSTPROCESS_SUCCESS) - elif (category.lower() == categories[1]): - #DEBUG#print "CouchPotato Processing Activated" - autoProcessMovie.process(path, settings, nzb, status) - sys.exit(POSTPROCESS_SUCCESS) - elif (category.lower() == categories[2]): - #DEBUG#print "Sonarr Processing Activated" - success = sonarr.processEpisode(path, settings, True) - if success: - sys.exit(POSTPROCESS_SUCCESS) - else: - sys.exit(POSTPROCESS_ERROR) - elif (category.lower() == categories[3]): - #DEBUG#print "Radarr Processing Activated" - success = radarr.processMovie(path, settings, True) - if success: - sys.exit(POSTPROCESS_SUCCESS) - else: - sys.exit(POSTPROCESS_ERROR) - elif (category.lower() == categories[4]): - #DEBUG#print "Sickrage Processing Activated" - autoProcessTVSR.processEpisode(path, settings, nzb) - sys.exit(POSTPROCESS_SUCCESS) - elif (category.lower() in bypass): - #DEBUG#print "Bypass Further Processing" - sys.exit(POSTPROCESS_NONE) - -else: - log.error("This script can only be called from NZBGet (11.0 or later).") - sys.exit(0)