From 386850d3fd91ead9794b837ec59fc8e10f6a491e Mon Sep 17 00:00:00 2001 From: Admin9705 <24727006+Admin9705@users.noreply.github.com> Date: Sun, 24 Feb 2019 18:08:54 -0500 Subject: [PATCH 001/185] Update _core.yml --- apps/_core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/_core.yml b/apps/_core.yml index c21e982..3051063 100644 --- a/apps/_core.yml +++ b/apps/_core.yml @@ -113,7 +113,7 @@ # FOR AUTHENTICATION - name: 'Script Execute Part I' - shell: 'bash /opt/plexguide/menu/pgshield/drop.sh' + shell: 'bash /opt/pgshield/drop.sh' - name: 'Script Execute Part II' shell: 'cat /var/plexguide/auth.var' From 7d01bc708e8b7928b2745f9200c2882c42c96096 Mon Sep 17 00:00:00 2001 From: Robert Baker Date: Thu, 28 Feb 2019 17:42:37 -0700 Subject: [PATCH 002/185] fix --- apps/dockergc.yml | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 apps/dockergc.yml diff --git a/apps/dockergc.yml b/apps/dockergc.yml new file mode 100644 index 0000000..b8a1ed6 --- /dev/null +++ b/apps/dockergc.yml @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Title: PlexGuide (Reference Title File) +# Author(s): Admin9705 +# URL: https://plexguide.com - http://github.plexguide.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'dockergc' + intport: '1' + extport: '1' + image: 'clockworksoul/docker-gc-cron' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # MANDATORY DOCKERGC ########################################################## + - name: Install docker-gc-exclude + template: + src: /opt/communityapps/apps/templates/docker-gc-exclude + dest: /opt/appdata/{{pgrole}}/docker-gc-exclude + force: yes + mode: 0775 + owner: '1000' + group: '1000' + + # LABELS ###################################################################### + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - /opt/appdata/dockergc/docker-gc-exclude:/etc/docker-gc-exclude + - /var/run/docker.sock:/var/run/docker.sock + - /etc/localtime:/etc/localtime:ro + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + CLEAN_UP_VOLUMES: 1 + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: + traefik.enabled: 'false' From 0f8d989b2a749b606d40d21cd7c6a0238f13a0df Mon Sep 17 00:00:00 2001 From: Robert Baker Date: Thu, 28 Feb 2019 17:57:37 -0700 Subject: [PATCH 003/185] mass update urls --- README.md | 14 +++++++------- apps/_appsgen.sh | 2 +- apps/_core.yml | 2 +- apps/airsonic.yml | 2 +- apps/alltube.yml | 2 +- apps/bazarr.yml | 2 +- apps/beets.yml | 2 +- apps/bitwarden.yml | 2 +- apps/booksonic.yml | 2 +- apps/cadvisor.yml | 2 +- apps/cloudcmd.yml | 2 +- apps/ddclient.yml | 2 +- apps/deezloaderremix.yml | 2 +- apps/deluge.yml | 2 +- apps/delugevpn.yml | 2 +- apps/dockergc.yml | 2 +- apps/duplicati.yml | 2 +- apps/embystats.yml | 2 +- apps/gazee.yml | 2 +- apps/gitea.yml | 2 +- apps/handbrake.yml | 2 +- apps/handbrake2.yml | 2 +- apps/headphones.yml | 2 +- apps/heimdall.yml | 2 +- apps/home-assistant.yml | 2 +- apps/htpcmanager.yml | 2 +- apps/image/_image.sh | 2 +- apps/jdownloader2.yml | 2 +- apps/kitana.yml | 2 +- apps/logarr.yml | 2 +- apps/makemkv.yml | 2 +- apps/mcmyadmin.yml | 2 +- apps/medusa.yml | 2 +- apps/mellow.yml | 2 +- apps/mkvtoolnix.yml | 2 +- apps/monitorr.yml | 2 +- apps/muximux.yml | 2 +- apps/mylar.yml | 2 +- apps/nextcloud.yml | 2 +- apps/nowshowing.yml | 2 +- apps/organizr.yml | 2 +- apps/pyload.yml | 2 +- apps/radarr4k.yml | 2 +- apps/radarrhdr.yml | 2 +- apps/resilio.yml | 4 ++-- apps/sharesite.yml | 4 ++-- apps/sonarr4k.yml | 2 +- apps/sonarrhdr.yml | 2 +- apps/speedtest.yml | 2 +- apps/synclounge.yml | 2 +- apps/syncthing.yml | 2 +- apps/templates/broken/nzbthrottle.yml | 2 +- apps/thelounge.yml | 2 +- apps/ubooquity.yml | 2 +- apps/xteve.yml | 2 +- 55 files changed, 63 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 0718163..640ed34 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ 📂 [**Click Here**](https://controlpanel.newshosting.com/signup/index.php?promo=partners&a_aid=5a65169240efd&a_bid=5ecfe99b) - NZB's with from NewsHost - PG Members Receive a 58% Discount

- - - - + + + +

_**Table of Contents**_ @@ -20,10 +20,10 @@ _**Table of Contents**_ **PG Box Community:** PG Community is a PG repo that collects and stores accepted user container writeups for community usage install. User can add programs such as Radarr4k or other unique programs that others may want to utlize. -

+

Manage PlexGuide - AnyTime, Anywhere!

-Forum Node for Community Box: [ CLICK HERE ](https://plexguide.com/forums/pg-app-community-box.191/) +Forum Node for Community Box: [ CLICK HERE ](https://pgblitz.com/forums/pg-app-community-box.191/) **WARNING: Wiki Under Construction** @@ -41,5 +41,5 @@ PG Box Community Edition is dead simple to use! Have a container in mind? Push c 📂 [**Click Here**](http://usenetserver.com/partners/?a_aid=5a65169240efd&a_bid=5725b6ed) - NZB's from USENET Server - PG Members Receive a 60% Discount

- +

diff --git a/apps/_appsgen.sh b/apps/_appsgen.sh index 8a84e0a..0ede57f 100644 --- a/apps/_appsgen.sh +++ b/apps/_appsgen.sh @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ diff --git a/apps/_core.yml b/apps/_core.yml index 3051063..c2a694a 100644 --- a/apps/_core.yml +++ b/apps/_core.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/airsonic.yml b/apps/airsonic.yml index a014171..e39a80e 100644 --- a/apps/airsonic.yml +++ b/apps/airsonic.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/alltube.yml b/apps/alltube.yml index 550f5cc..4e7c4ba 100644 --- a/apps/alltube.yml +++ b/apps/alltube.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/bazarr.yml b/apps/bazarr.yml index 1fac08d..5da069c 100644 --- a/apps/bazarr.yml +++ b/apps/bazarr.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/beets.yml b/apps/beets.yml index 6427030..f0f731c 100644 --- a/apps/beets.yml +++ b/apps/beets.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/bitwarden.yml b/apps/bitwarden.yml index dd1bd07..4abece6 100644 --- a/apps/bitwarden.yml +++ b/apps/bitwarden.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/booksonic.yml b/apps/booksonic.yml index 7866260..ca33126 100644 --- a/apps/booksonic.yml +++ b/apps/booksonic.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/cadvisor.yml b/apps/cadvisor.yml index 8db8d59..86216d4 100644 --- a/apps/cadvisor.yml +++ b/apps/cadvisor.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/cloudcmd.yml b/apps/cloudcmd.yml index bd301f3..a291f45 100644 --- a/apps/cloudcmd.yml +++ b/apps/cloudcmd.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/ddclient.yml b/apps/ddclient.yml index 06fdd62..c02ea03 100644 --- a/apps/ddclient.yml +++ b/apps/ddclient.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/deezloaderremix.yml b/apps/deezloaderremix.yml index bfb9be1..40664d5 100644 --- a/apps/deezloaderremix.yml +++ b/apps/deezloaderremix.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/deluge.yml b/apps/deluge.yml index 08f8e0d..c4e208b 100644 --- a/apps/deluge.yml +++ b/apps/deluge.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/delugevpn.yml b/apps/delugevpn.yml index 7c2c007..aafa066 100644 --- a/apps/delugevpn.yml +++ b/apps/delugevpn.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/dockergc.yml b/apps/dockergc.yml index b8a1ed6..0444951 100644 --- a/apps/dockergc.yml +++ b/apps/dockergc.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/duplicati.yml b/apps/duplicati.yml index 886b259..2f92fdc 100644 --- a/apps/duplicati.yml +++ b/apps/duplicati.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/embystats.yml b/apps/embystats.yml index 567ed11..2aa1f51 100644 --- a/apps/embystats.yml +++ b/apps/embystats.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/gazee.yml b/apps/gazee.yml index 53c40e9..f1c1cf0 100644 --- a/apps/gazee.yml +++ b/apps/gazee.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/gitea.yml b/apps/gitea.yml index 15aba27..553cdf4 100644 --- a/apps/gitea.yml +++ b/apps/gitea.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/handbrake.yml b/apps/handbrake.yml index 31dd645..55b4839 100644 --- a/apps/handbrake.yml +++ b/apps/handbrake.yml @@ -2,7 +2,7 @@ # # Title: Handbrake for PlexGuide # Author(s): timekills -# URL: https://plexguide.com - https://github.com/timekills +# URL: https://pgblitz.com - https://github.com/timekills # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/handbrake2.yml b/apps/handbrake2.yml index ccaeefa..9c1d8d7 100644 --- a/apps/handbrake2.yml +++ b/apps/handbrake2.yml @@ -2,7 +2,7 @@ # # Title: Handbrake for PlexGuide Community (user/password) # Author(s): timekills -# URL: https://plexguide.com - https://github.com/timekills +# URL: https://pgblitz.com - https://github.com/timekills # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/headphones.yml b/apps/headphones.yml index a8804fc..3042931 100644 --- a/apps/headphones.yml +++ b/apps/headphones.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/heimdall.yml b/apps/heimdall.yml index 412a1c2..352cb47 100644 --- a/apps/heimdall.yml +++ b/apps/heimdall.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/home-assistant.yml b/apps/home-assistant.yml index 0d7dd3b..181f958 100644 --- a/apps/home-assistant.yml +++ b/apps/home-assistant.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/htpcmanager.yml b/apps/htpcmanager.yml index 70ad6dd..cb705ad 100644 --- a/apps/htpcmanager.yml +++ b/apps/htpcmanager.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/image/_image.sh b/apps/image/_image.sh index f806158..645270c 100644 --- a/apps/image/_image.sh +++ b/apps/image/_image.sh @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ diff --git a/apps/jdownloader2.yml b/apps/jdownloader2.yml index 7bbae15..c2d743b 100644 --- a/apps/jdownloader2.yml +++ b/apps/jdownloader2.yml @@ -2,7 +2,7 @@ # # Title: Jdownloader2 for PlexGuide (OAuth security) # Author(s): timekills -# URL: https://plexguide.com - https://github.com/timekills/jdownloader2-for-Plexguide +# URL: https://pgblitz.com - https://github.com/timekills/jdownloader2-for-Plexguide # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/kitana.yml b/apps/kitana.yml index 1e349c7..1e90ee9 100644 --- a/apps/kitana.yml +++ b/apps/kitana.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/logarr.yml b/apps/logarr.yml index b0d7f29..2758a38 100644 --- a/apps/logarr.yml +++ b/apps/logarr.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/makemkv.yml b/apps/makemkv.yml index c768211..de2a059 100644 --- a/apps/makemkv.yml +++ b/apps/makemkv.yml @@ -2,7 +2,7 @@ # # Title: MakeMKV for PlexGuide # Author(s): rla999 -# URL: https://plexguide.com - https://github.com/rla999/makemkv-for-plexguide/ +# URL: https://pgblitz.com - https://github.com/rla999/makemkv-for-plexguide/ # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/mcmyadmin.yml b/apps/mcmyadmin.yml index ce1ecca..8e55ebd 100644 --- a/apps/mcmyadmin.yml +++ b/apps/mcmyadmin.yml @@ -2,7 +2,7 @@ # # Title: McMyadmin (v2) # Author(s): timekills -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/medusa.yml b/apps/medusa.yml index 5569455..e6b97bd 100644 --- a/apps/medusa.yml +++ b/apps/medusa.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/mellow.yml b/apps/mellow.yml index f08cd72..33ed8eb 100644 --- a/apps/mellow.yml +++ b/apps/mellow.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/mkvtoolnix.yml b/apps/mkvtoolnix.yml index 0c2bb30..e39bbd3 100644 --- a/apps/mkvtoolnix.yml +++ b/apps/mkvtoolnix.yml @@ -2,7 +2,7 @@ # # Title: MKVToolNix # Author(s): jlesage/timekills -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 # Docker: Docker image is available at: https://hub.docker.com/r/metabrainz/picard-website ################################################################################ diff --git a/apps/monitorr.yml b/apps/monitorr.yml index 56dc800..7f0595f 100644 --- a/apps/monitorr.yml +++ b/apps/monitorr.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/muximux.yml b/apps/muximux.yml index ceffacc..bbbba75 100644 --- a/apps/muximux.yml +++ b/apps/muximux.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/mylar.yml b/apps/mylar.yml index dacd214..f0aab4c 100644 --- a/apps/mylar.yml +++ b/apps/mylar.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/nextcloud.yml b/apps/nextcloud.yml index 1b23a12..8a70c64 100644 --- a/apps/nextcloud.yml +++ b/apps/nextcloud.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/nowshowing.yml b/apps/nowshowing.yml index 4f63f74..7e5184c 100644 --- a/apps/nowshowing.yml +++ b/apps/nowshowing.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/organizr.yml b/apps/organizr.yml index 47563ff..5214dd7 100644 --- a/apps/organizr.yml +++ b/apps/organizr.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/pyload.yml b/apps/pyload.yml index 4a65137..afadbc8 100644 --- a/apps/pyload.yml +++ b/apps/pyload.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/radarr4k.yml b/apps/radarr4k.yml index 0afb988..49ea3e7 100644 --- a/apps/radarr4k.yml +++ b/apps/radarr4k.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/radarrhdr.yml b/apps/radarrhdr.yml index 214f6ea..af0d124 100644 --- a/apps/radarrhdr.yml +++ b/apps/radarrhdr.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/resilio.yml b/apps/resilio.yml index ff4b576..f9c146d 100644 --- a/apps/resilio.yml +++ b/apps/resilio.yml @@ -1,8 +1,8 @@ #!/bin/bash # -# GitHub: https://github.com/Admin9705/PlexGuide.com-The-Awesome-Plex-Server +# GitHub: https://github.com/PGBlitz/PGBlitz.com # Author: Admin9705 -# URL: https://plexguide.com +# URL: https://pgblitz.com # # PlexGuide Copyright (C) 2018 PlexGuide.com # Licensed under GNU General Public License v3.0 GPL-3 (in short) diff --git a/apps/sharesite.yml b/apps/sharesite.yml index af4f066..f34a9df 100644 --- a/apps/sharesite.yml +++ b/apps/sharesite.yml @@ -1,8 +1,8 @@ #!/bin/bash # -# GitHub: https://github.com/Admin9705/PlexGuide.com-The-Awesome-Plex-Server +# GitHub: https://github.com/PGBlitz/PGBlitz.com # Author: Admin9705 -# URL: https://plexguide.com +# URL: https://pgblitz.com # # PlexGuide Copyright (C) 2018 PlexGuide.com # Licensed under GNU General Public License v3.0 GPL-3 (in short) diff --git a/apps/sonarr4k.yml b/apps/sonarr4k.yml index acb0e40..65add11 100644 --- a/apps/sonarr4k.yml +++ b/apps/sonarr4k.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/sonarrhdr.yml b/apps/sonarrhdr.yml index 04e59fe..770126c 100644 --- a/apps/sonarrhdr.yml +++ b/apps/sonarrhdr.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/speedtest.yml b/apps/speedtest.yml index dc516a0..5da91d0 100644 --- a/apps/speedtest.yml +++ b/apps/speedtest.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/synclounge.yml b/apps/synclounge.yml index 2aa5d17..eb3557e 100644 --- a/apps/synclounge.yml +++ b/apps/synclounge.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/syncthing.yml b/apps/syncthing.yml index 5ce7764..43b54d0 100644 --- a/apps/syncthing.yml +++ b/apps/syncthing.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/templates/broken/nzbthrottle.yml b/apps/templates/broken/nzbthrottle.yml index 861ac01..9523ccb 100644 --- a/apps/templates/broken/nzbthrottle.yml +++ b/apps/templates/broken/nzbthrottle.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/thelounge.yml b/apps/thelounge.yml index 898d53e..c34ea0e 100644 --- a/apps/thelounge.yml +++ b/apps/thelounge.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/ubooquity.yml b/apps/ubooquity.yml index 7151b91..422c1d1 100644 --- a/apps/ubooquity.yml +++ b/apps/ubooquity.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): Admin9705 -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- diff --git a/apps/xteve.yml b/apps/xteve.yml index 90aa350..5f97bf0 100644 --- a/apps/xteve.yml +++ b/apps/xteve.yml @@ -2,7 +2,7 @@ # # Title: PlexGuide (Reference Title File) # Author(s): TechButton -# URL: https://plexguide.com - http://github.plexguide.com +# URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 ################################################################################ --- From a1da81f21abfce6477758ecdc3c3faf17a4de1eb Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 28 Feb 2019 20:03:30 -0500 Subject: [PATCH 004/185] Initial Transmissionbt commit --- apps/transmission.yml | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 apps/transmission.yml diff --git a/apps/transmission.yml b/apps/transmission.yml new file mode 100644 index 0000000..9fa0bc2 --- /dev/null +++ b/apps/transmission.yml @@ -0,0 +1,111 @@ +#!/bin/bash +# +# Title: Transmissionbt (with OpenVPN, WebProxy, RSS) +# Author(s): Eric Petit, Josh Elsasser, Bryan Varner, Charles Kerr, Kristian Haugene, nning +# URL: https://transmissionbt.com/ - https://github.com/haugene/docker-transmission-openvpn +# GNU: General Public License v2.0, v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'transmission' + pgrole2: 'transmission-rss' + intport: '9091' + extport: '9091' + intport2: '9092' + extport2: '9092' + intport3: '8888' + extport3: '8888' + image: 'haugene/transmission-openvpn:latest' + image2: 'haugene/transmission-rss:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + + - name: 'Setting {{pgrole}} ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + OPENVPN_PROVIDER: 'see https://git.io/fpCSF' + OPENVPN_CONFIG: 'see https://git.io/fpCSF' + OPENVPN_USERNAME: 'vpnuser' + OPENVPN_PASSWORD: 'vpnpass' + OPENVPN_OPTS: '--inactive 3600 --ping 10 --ping-exit 60' + WEBPROXY_ENABLED: 'true' + WEBPROXY_PORT: '{{extport3}}' + LOCAL_NETWORK: 172.18.0.0/24 + TRANSMISSION_WEB_HOME: '/config/webui-theme' + TRANSMISSION_WEB_UI: 'combustion' + TRANSMISSION_BLOCKLIST_ENABLED: 'true' + TRANSMISSION_BLOCKLIST_URL: 'http://john.bitsurge.net/public/biglist.p2p.gz' + TRANSMISSION_DHT_ENABLED: 'false' + TRANSMISSION_DOWNLOAD_DIR: '/mnt/downloads/{{pgrole}}' + TRANSMISSION_INCOMPLETE_DIR: '/mnt/incomplete/{{pgrole}}' + TRANSMISSION_RPC_PORT: '{{intport2}}' + TRANSMISSION_WATCH_DIR: '/config/watch' + TRANSMISSION_WATCH_DIR_ENABLED: 'false' + + - name: 'Setting {{pgrole2}} ENV' + set_fact: + pg_env2: + RSS_URL: 'http://your.rss' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' + - '{{ports.stdout}}{{extport3}}:{{intport3}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + capabilities: + - NET-ADMIN + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' + + - name: 'Deploying {{pgrole2}}' + docker_container: + name: '{{pgrole2}}' + image: '{{image2}}' + pull: yes + env: '{{pg_env2}}' + links: + - '{{pgrole}}:transmission' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole2}}' + state: started \ No newline at end of file From b08e161fad58095dfc87924151944aa495a6302c Mon Sep 17 00:00:00 2001 From: Robert Baker Date: Thu, 28 Feb 2019 18:10:06 -0700 Subject: [PATCH 005/185] mass update name (not breaking, no paths changed) --- README.md | 2 +- apps/_appsgen.sh | 2 +- apps/_core.yml | 2 +- apps/airsonic.yml | 2 +- apps/alltube.yml | 2 +- apps/bazarr.yml | 2 +- apps/beets.yml | 2 +- apps/bitwarden.yml | 2 +- apps/booksonic.yml | 2 +- apps/cadvisor.yml | 2 +- apps/cloudcmd.yml | 2 +- apps/ddclient.yml | 2 +- apps/deezloaderremix.yml | 2 +- apps/deluge.yml | 2 +- apps/delugevpn.yml | 2 +- apps/dockergc.yml | 2 +- apps/duplicati.yml | 2 +- apps/embystats.yml | 2 +- apps/gazee.yml | 2 +- apps/gitea.yml | 2 +- apps/handbrake.yml | 2 +- apps/handbrake2.yml | 2 +- apps/headphones.yml | 2 +- apps/heimdall.yml | 2 +- apps/home-assistant.yml | 2 +- apps/htpcmanager.yml | 2 +- apps/image/_image.sh | 2 +- apps/jd2-openvpn.yml | 2 +- apps/jdownloader2.yml | 2 +- apps/kitana.yml | 2 +- apps/logarr.yml | 2 +- apps/makemkv.yml | 2 +- apps/medusa.yml | 2 +- apps/mellow.yml | 2 +- apps/monitorr.yml | 2 +- apps/muximux.yml | 2 +- apps/mylar.yml | 2 +- apps/nextcloud.yml | 2 +- apps/nowshowing.yml | 2 +- apps/organizr.yml | 2 +- apps/pyload.yml | 2 +- apps/radarr4k.yml | 2 +- apps/radarrhdr.yml | 2 +- apps/resilio.yml | 2 +- apps/sharesite.yml | 2 +- apps/sonarr4k.yml | 2 +- apps/sonarrhdr.yml | 2 +- apps/speedtest.yml | 2 +- apps/synclounge.yml | 2 +- apps/syncthing.yml | 2 +- apps/templates/broken/nzbthrottle.yml | 2 +- apps/thelounge.yml | 2 +- apps/ubooquity.yml | 2 +- apps/xteve.yml | 2 +- 54 files changed, 54 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 640ed34..cbab32b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ _**Table of Contents**_ **PG Box Community:** PG Community is a PG repo that collects and stores accepted user container writeups for community usage install. User can add programs such as Radarr4k or other unique programs that others may want to utlize.

-

Manage PlexGuide - AnyTime, Anywhere!

+

Manage PGBlitz - AnyTime, Anywhere!

Forum Node for Community Box: [ CLICK HERE ](https://pgblitz.com/forums/pg-app-community-box.191/) diff --git a/apps/_appsgen.sh b/apps/_appsgen.sh index 0ede57f..aa87c16 100644 --- a/apps/_appsgen.sh +++ b/apps/_appsgen.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/_core.yml b/apps/_core.yml index c2a694a..28f7047 100644 --- a/apps/_core.yml +++ b/apps/_core.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/airsonic.yml b/apps/airsonic.yml index e39a80e..25f9cf7 100644 --- a/apps/airsonic.yml +++ b/apps/airsonic.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/alltube.yml b/apps/alltube.yml index 4e7c4ba..be82b40 100644 --- a/apps/alltube.yml +++ b/apps/alltube.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/bazarr.yml b/apps/bazarr.yml index 5da069c..3aded00 100644 --- a/apps/bazarr.yml +++ b/apps/bazarr.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/beets.yml b/apps/beets.yml index f0f731c..90a3ce0 100644 --- a/apps/beets.yml +++ b/apps/beets.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/bitwarden.yml b/apps/bitwarden.yml index 4abece6..7870044 100644 --- a/apps/bitwarden.yml +++ b/apps/bitwarden.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/booksonic.yml b/apps/booksonic.yml index ca33126..ead0354 100644 --- a/apps/booksonic.yml +++ b/apps/booksonic.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/cadvisor.yml b/apps/cadvisor.yml index 86216d4..1f7c1f2 100644 --- a/apps/cadvisor.yml +++ b/apps/cadvisor.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/cloudcmd.yml b/apps/cloudcmd.yml index a291f45..799a4f8 100644 --- a/apps/cloudcmd.yml +++ b/apps/cloudcmd.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/ddclient.yml b/apps/ddclient.yml index c02ea03..625bb84 100644 --- a/apps/ddclient.yml +++ b/apps/ddclient.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/deezloaderremix.yml b/apps/deezloaderremix.yml index 40664d5..6399ca9 100644 --- a/apps/deezloaderremix.yml +++ b/apps/deezloaderremix.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/deluge.yml b/apps/deluge.yml index c4e208b..63476a3 100644 --- a/apps/deluge.yml +++ b/apps/deluge.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/delugevpn.yml b/apps/delugevpn.yml index aafa066..55976a6 100644 --- a/apps/delugevpn.yml +++ b/apps/delugevpn.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/dockergc.yml b/apps/dockergc.yml index 0444951..0f83a64 100644 --- a/apps/dockergc.yml +++ b/apps/dockergc.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/duplicati.yml b/apps/duplicati.yml index 2f92fdc..d14c2fd 100644 --- a/apps/duplicati.yml +++ b/apps/duplicati.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/embystats.yml b/apps/embystats.yml index 2aa1f51..41651f9 100644 --- a/apps/embystats.yml +++ b/apps/embystats.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/gazee.yml b/apps/gazee.yml index f1c1cf0..ba2a63c 100644 --- a/apps/gazee.yml +++ b/apps/gazee.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/gitea.yml b/apps/gitea.yml index 553cdf4..54324fa 100644 --- a/apps/gitea.yml +++ b/apps/gitea.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/handbrake.yml b/apps/handbrake.yml index 55b4839..991120f 100644 --- a/apps/handbrake.yml +++ b/apps/handbrake.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: Handbrake for PlexGuide +# Title: Handbrake for PGBlitz # Author(s): timekills # URL: https://pgblitz.com - https://github.com/timekills # GNU: General Public License v3.0 diff --git a/apps/handbrake2.yml b/apps/handbrake2.yml index 9c1d8d7..6e3e0cb 100644 --- a/apps/handbrake2.yml +++ b/apps/handbrake2.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: Handbrake for PlexGuide Community (user/password) +# Title: Handbrake for PGBlitz Community (user/password) # Author(s): timekills # URL: https://pgblitz.com - https://github.com/timekills # GNU: General Public License v3.0 diff --git a/apps/headphones.yml b/apps/headphones.yml index 3042931..53312f0 100644 --- a/apps/headphones.yml +++ b/apps/headphones.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/heimdall.yml b/apps/heimdall.yml index 352cb47..77fcf5f 100644 --- a/apps/heimdall.yml +++ b/apps/heimdall.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/home-assistant.yml b/apps/home-assistant.yml index 181f958..b6f94a3 100644 --- a/apps/home-assistant.yml +++ b/apps/home-assistant.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/htpcmanager.yml b/apps/htpcmanager.yml index cb705ad..207bcc0 100644 --- a/apps/htpcmanager.yml +++ b/apps/htpcmanager.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/image/_image.sh b/apps/image/_image.sh index 645270c..ee4ecdf 100644 --- a/apps/image/_image.sh +++ b/apps/image/_image.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/jd2-openvpn.yml b/apps/jd2-openvpn.yml index 5af7044..292946c 100644 --- a/apps/jd2-openvpn.yml +++ b/apps/jd2-openvpn.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: Jdownloader2 for PlexGuide (OAuth security) with OpenVPN integration +# Title: Jdownloader2 for PGBlitz (OAuth security) with OpenVPN integration # Author(s): timekills (mod. h1f0x) # URL: https://hub.docker.com/r/h1f0x/jd2-openvpn # GNU: General Public License v3.0 diff --git a/apps/jdownloader2.yml b/apps/jdownloader2.yml index c2d743b..b3e56b8 100644 --- a/apps/jdownloader2.yml +++ b/apps/jdownloader2.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: Jdownloader2 for PlexGuide (OAuth security) +# Title: Jdownloader2 for PGBlitz (OAuth security) # Author(s): timekills # URL: https://pgblitz.com - https://github.com/timekills/jdownloader2-for-Plexguide # GNU: General Public License v3.0 diff --git a/apps/kitana.yml b/apps/kitana.yml index 1e90ee9..1591b0c 100644 --- a/apps/kitana.yml +++ b/apps/kitana.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/logarr.yml b/apps/logarr.yml index 2758a38..e98dc4e 100644 --- a/apps/logarr.yml +++ b/apps/logarr.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/makemkv.yml b/apps/makemkv.yml index de2a059..70d4c8e 100644 --- a/apps/makemkv.yml +++ b/apps/makemkv.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: MakeMKV for PlexGuide +# Title: MakeMKV for PGBlitz # Author(s): rla999 # URL: https://pgblitz.com - https://github.com/rla999/makemkv-for-plexguide/ # GNU: General Public License v3.0 diff --git a/apps/medusa.yml b/apps/medusa.yml index e6b97bd..091841f 100644 --- a/apps/medusa.yml +++ b/apps/medusa.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/mellow.yml b/apps/mellow.yml index 33ed8eb..8460dad 100644 --- a/apps/mellow.yml +++ b/apps/mellow.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/monitorr.yml b/apps/monitorr.yml index 7f0595f..7c6cbe2 100644 --- a/apps/monitorr.yml +++ b/apps/monitorr.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/muximux.yml b/apps/muximux.yml index bbbba75..fce2d5f 100644 --- a/apps/muximux.yml +++ b/apps/muximux.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/mylar.yml b/apps/mylar.yml index f0aab4c..1f737bf 100644 --- a/apps/mylar.yml +++ b/apps/mylar.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/nextcloud.yml b/apps/nextcloud.yml index 8a70c64..3c75ef8 100644 --- a/apps/nextcloud.yml +++ b/apps/nextcloud.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/nowshowing.yml b/apps/nowshowing.yml index 7e5184c..9303c25 100644 --- a/apps/nowshowing.yml +++ b/apps/nowshowing.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/organizr.yml b/apps/organizr.yml index 5214dd7..70530a1 100644 --- a/apps/organizr.yml +++ b/apps/organizr.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/pyload.yml b/apps/pyload.yml index afadbc8..ee21ab0 100644 --- a/apps/pyload.yml +++ b/apps/pyload.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/radarr4k.yml b/apps/radarr4k.yml index 49ea3e7..7c3a2cd 100644 --- a/apps/radarr4k.yml +++ b/apps/radarr4k.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/radarrhdr.yml b/apps/radarrhdr.yml index af0d124..8c468d7 100644 --- a/apps/radarrhdr.yml +++ b/apps/radarrhdr.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/resilio.yml b/apps/resilio.yml index f9c146d..9af7405 100644 --- a/apps/resilio.yml +++ b/apps/resilio.yml @@ -4,7 +4,7 @@ # Author: Admin9705 # URL: https://pgblitz.com # -# PlexGuide Copyright (C) 2018 PlexGuide.com +# PGBlitz Copyright (C) 2018 PGBlitz.com # Licensed under GNU General Public License v3.0 GPL-3 (in short) # # You may copy, distribute and modify the software as long as you track diff --git a/apps/sharesite.yml b/apps/sharesite.yml index f34a9df..91056a1 100644 --- a/apps/sharesite.yml +++ b/apps/sharesite.yml @@ -4,7 +4,7 @@ # Author: Admin9705 # URL: https://pgblitz.com # -# PlexGuide Copyright (C) 2018 PlexGuide.com +# PGBlitz Copyright (C) 2018 PGBlitz.com # Licensed under GNU General Public License v3.0 GPL-3 (in short) # # You may copy, distribute and modify the software as long as you track diff --git a/apps/sonarr4k.yml b/apps/sonarr4k.yml index 65add11..fe4fbaf 100644 --- a/apps/sonarr4k.yml +++ b/apps/sonarr4k.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/sonarrhdr.yml b/apps/sonarrhdr.yml index 770126c..0541208 100644 --- a/apps/sonarrhdr.yml +++ b/apps/sonarrhdr.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/speedtest.yml b/apps/speedtest.yml index 5da91d0..1c29241 100644 --- a/apps/speedtest.yml +++ b/apps/speedtest.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/synclounge.yml b/apps/synclounge.yml index eb3557e..9523a6d 100644 --- a/apps/synclounge.yml +++ b/apps/synclounge.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/syncthing.yml b/apps/syncthing.yml index 43b54d0..cff6db7 100644 --- a/apps/syncthing.yml +++ b/apps/syncthing.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/templates/broken/nzbthrottle.yml b/apps/templates/broken/nzbthrottle.yml index 9523ccb..27f3a7d 100644 --- a/apps/templates/broken/nzbthrottle.yml +++ b/apps/templates/broken/nzbthrottle.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/thelounge.yml b/apps/thelounge.yml index c34ea0e..1fbefb0 100644 --- a/apps/thelounge.yml +++ b/apps/thelounge.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/ubooquity.yml b/apps/ubooquity.yml index 422c1d1..fea2b84 100644 --- a/apps/ubooquity.yml +++ b/apps/ubooquity.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): Admin9705 # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 diff --git a/apps/xteve.yml b/apps/xteve.yml index 5f97bf0..e222fe8 100644 --- a/apps/xteve.yml +++ b/apps/xteve.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PlexGuide (Reference Title File) +# Title: PGBlitz (Reference Title File) # Author(s): TechButton # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 From 4e5925502bd5c62e07fa645902225ad41705866c Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 28 Feb 2019 21:22:27 -0500 Subject: [PATCH 006/185] Fixed typo --- apps/transmission.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index 9fa0bc2..da2021a 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -87,7 +87,7 @@ env: '{{pg_env}}' restart_policy: unless-stopped capabilities: - - NET-ADMIN + - NET_ADMIN networks: - name: plexguide aliases: From 9cda9e101184342c6346de6dca82314013ffa258 Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 28 Feb 2019 23:19:35 -0500 Subject: [PATCH 007/185] Removed alt. port, added rss template, stopped container when finished, misc. --- apps/transmission.yml | 66 ++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index da2021a..f7787c5 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -16,10 +16,8 @@ pgrole2: 'transmission-rss' intport: '9091' extport: '9091' - intport2: '9092' - extport2: '9092' - intport3: '8888' - extport3: '8888' + intport2: '8888' + extport2: '8888' image: 'haugene/transmission-openvpn:latest' image2: 'haugene/transmission-rss:latest' @@ -27,6 +25,16 @@ - name: 'Including cron job' include_tasks: '/opt/communityapps/apps/_core.yml' + - name: Checking for existing rss config + stat: + path: "/opt/appdata/{{pgrole}}/rss/transmission-rss.conf" + register: rsscheck + + #- name: Checking for existing blocklist + # stat: + # path: "/opt/appdata/{{pgrole}}/home/biglist.p2p" + # register: blcheck + # LABELS ###################################################################### - name: 'Adding Traefik' set_fact: @@ -44,34 +52,42 @@ - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' + - name: 'Setting {{pgrole2}} Volumes' + set_facts: + pg_volumes2: + - '/opt/appdata/{{pgrole}}/rss:/etc/transmission-rss.conf:ro' + - name: 'Setting {{pgrole}} ENV' set_fact: pg_env: PUID: '1000' PGID: '1000' - OPENVPN_PROVIDER: 'see https://git.io/fpCSF' - OPENVPN_CONFIG: 'see https://git.io/fpCSF' + OPENVPN_PROVIDER: 'see available configs at https://git.io/fpCSF' + OPENVPN_CONFIG: 'see available configs at https://git.io/fpCSF' OPENVPN_USERNAME: 'vpnuser' OPENVPN_PASSWORD: 'vpnpass' OPENVPN_OPTS: '--inactive 3600 --ping 10 --ping-exit 60' + CREATE_TUN_DEVICE: 'true' WEBPROXY_ENABLED: 'true' - WEBPROXY_PORT: '{{extport3}}' - LOCAL_NETWORK: 172.18.0.0/24 + WEBPROXY_PORT: '{{extport2}}' + LOCAL_NETWORK: 192.168.0.0/24 TRANSMISSION_WEB_HOME: '/config/webui-theme' - TRANSMISSION_WEB_UI: 'combustion' + TRANSMISSION_WEB_UI: 'transmission-web-control' TRANSMISSION_BLOCKLIST_ENABLED: 'true' TRANSMISSION_BLOCKLIST_URL: 'http://john.bitsurge.net/public/biglist.p2p.gz' TRANSMISSION_DHT_ENABLED: 'false' TRANSMISSION_DOWNLOAD_DIR: '/mnt/downloads/{{pgrole}}' TRANSMISSION_INCOMPLETE_DIR: '/mnt/incomplete/{{pgrole}}' - TRANSMISSION_RPC_PORT: '{{intport2}}' + TRANSMISSION_RPC_PORT: '{{intport}}' TRANSMISSION_WATCH_DIR: '/config/watch' TRANSMISSION_WATCH_DIR_ENABLED: 'false' + TRANSMISSION_HOME: '/config/home' - name: 'Setting {{pgrole2}} ENV' set_fact: pg_env2: - RSS_URL: 'http://your.rss' + TRANSMISSION_DOWNLOAD_DIR: '/mnt/downloads/{{pgrole}}' + RSS_URL: 'hxxp://your.rss' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' @@ -82,10 +98,11 @@ published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{ports.stdout}}{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: unless-stopped + devices: + - '/dev/net/tun:/dev/net/tun:rwm' capabilities: - NET_ADMIN networks: @@ -95,17 +112,32 @@ state: started labels: '{{pg_labels}}' + #- name: 'Downloading blocklist' + # block: + # shell: "curl -s http://john.bitsurge.net/public/biglist.p2p.gz | gunzip > /config/home/biglist.p2p" + # when: not blcheck.stat.exists + + - name: 'Creating RSS feed configuration' + block: + shell: "touch /etc/transmission-rss.conf && chown 1000:1000 /etc/transmission-rss.conf && echo see https://git.io/fhAm2 to configure > /etc/transmission-rss.conf" + when: not rsscheck.stat.exists + - name: 'Deploying {{pgrole2}}' docker_container: name: '{{pgrole2}}' image: '{{image2}}' pull: yes + volumes: '{{pg_volumes2}}' env: '{{pg_env2}}' links: - '{{pgrole}}:transmission' restart_policy: unless-stopped - networks: - - name: plexguide - aliases: - - '{{pgrole2}}' - state: started \ No newline at end of file + state: started + + - name: 'Wait for {{pgrole}} to initialize' + - wait_for: timeout=30 + + - name: 'Stopping {{pgrole}}. Go configure ENV values.' + docker_container: + name: '{{pgrole}}' + state: stopped \ No newline at end of file From 6c41135d2cf55a3b04ed82f7a65609e2725dd323 Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 28 Feb 2019 23:25:09 -0500 Subject: [PATCH 008/185] Fixed yaml typo --- apps/transmission.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index f7787c5..b8fe665 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -135,7 +135,7 @@ state: started - name: 'Wait for {{pgrole}} to initialize' - - wait_for: timeout=30 + wait_for: timeout=30 - name: 'Stopping {{pgrole}}. Go configure ENV values.' docker_container: From 4cf9eb93e0c2bbc723b67c104f340ef7b9d33790 Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 28 Feb 2019 23:31:13 -0500 Subject: [PATCH 009/185] Fix for real this time... --- apps/transmission.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index b8fe665..df565e8 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -52,7 +52,7 @@ - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' - - name: 'Setting {{pgrole2}} Volumes' + - name: 'Setting additional volumes' set_facts: pg_volumes2: - '/opt/appdata/{{pgrole}}/rss:/etc/transmission-rss.conf:ro' From 361d6e4dc312af9aebb4dbc8b286873964c2b198 Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 28 Feb 2019 23:33:44 -0500 Subject: [PATCH 010/185] Third time's a charm... --- apps/transmission.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index df565e8..393cf6a 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -52,8 +52,8 @@ - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' - - name: 'Setting additional volumes' - set_facts: + - name: 'Setting {{pgrole2}} Volumes' + set_fact: pg_volumes2: - '/opt/appdata/{{pgrole}}/rss:/etc/transmission-rss.conf:ro' From ec742ab4a08c39488366549f464406ed25192231 Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 28 Feb 2019 23:39:27 -0500 Subject: [PATCH 011/185] Fix RSS config step --- apps/transmission.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index 393cf6a..dac9fbd 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -117,9 +117,10 @@ # shell: "curl -s http://john.bitsurge.net/public/biglist.p2p.gz | gunzip > /config/home/biglist.p2p" # when: not blcheck.stat.exists - - name: 'Creating RSS feed configuration' + - name: 'RSS feed configuration' block: - shell: "touch /etc/transmission-rss.conf && chown 1000:1000 /etc/transmission-rss.conf && echo see https://git.io/fhAm2 to configure > /etc/transmission-rss.conf" + - name: 'Creating configuration' + shell: "touch /etc/transmission-rss.conf && chown 1000:1000 /etc/transmission-rss.conf && echo see https://git.io/fhAm2 to configure > /etc/transmission-rss.conf" when: not rsscheck.stat.exists - name: 'Deploying {{pgrole2}}' From 6cb6447e9fa6b9f854a8c5fa35a1723994a9ec0c Mon Sep 17 00:00:00 2001 From: Cody Date: Fri, 1 Mar 2019 00:01:47 -0500 Subject: [PATCH 012/185] Fixed RSS deployment --- apps/transmission.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index dac9fbd..bd8cc85 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -25,10 +25,15 @@ - name: 'Including cron job' include_tasks: '/opt/communityapps/apps/_core.yml' + - name: Checking for existing rss folder + stat: + path: "/opt/appdata/{{pgrole}}/rss" + register: rsscheck + - name: Checking for existing rss config stat: path: "/opt/appdata/{{pgrole}}/rss/transmission-rss.conf" - register: rsscheck + register: cfgcheck #- name: Checking for existing blocklist # stat: @@ -117,12 +122,18 @@ # shell: "curl -s http://john.bitsurge.net/public/biglist.p2p.gz | gunzip > /config/home/biglist.p2p" # when: not blcheck.stat.exists - - name: 'RSS feed configuration' + - name: 'RSS feed configuration - Folder' block: - - name: 'Creating configuration' - shell: "touch /etc/transmission-rss.conf && chown 1000:1000 /etc/transmission-rss.conf && echo see https://git.io/fhAm2 to configure > /etc/transmission-rss.conf" + - name: 'Creating RSS folder' + shell: "mkdir /opt/appdata/{{pgrole}}/rss" when: not rsscheck.stat.exists + - name: 'RSS feed configuration - Config' + block: + - name: 'Creating configuration file' + shell: "touch /opt/appdata/{{pgrole}}/rss/transmission-rss.conf && chown -R 1000:1000 /opt/appdata/{{pgrole}}/rss/ && echo see https://git.io/fhAm2 to configure > /opt/appdata/{{pgrole}}/rss/transmission-rss.conf" + when: not cfgcheck.stat.exists + - name: 'Deploying {{pgrole2}}' docker_container: name: '{{pgrole2}}' From 1196a0b53fee6f9f1b2eeeb2712c28ae36e2b790 Mon Sep 17 00:00:00 2001 From: Cody Date: Fri, 1 Mar 2019 00:13:00 -0500 Subject: [PATCH 013/185] Fix for volume mnt --- apps/transmission.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index bd8cc85..f2fb9c0 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -60,7 +60,7 @@ - name: 'Setting {{pgrole2}} Volumes' set_fact: pg_volumes2: - - '/opt/appdata/{{pgrole}}/rss:/etc/transmission-rss.conf:ro' + - '/opt/appdata/{{pgrole}}/rss/transmission-rss.conf:/etc/transmission-rss.conf:ro' - name: 'Setting {{pgrole}} ENV' set_fact: @@ -92,7 +92,7 @@ set_fact: pg_env2: TRANSMISSION_DOWNLOAD_DIR: '/mnt/downloads/{{pgrole}}' - RSS_URL: 'hxxp://your.rss' + RSS_URL: 'hxxp://unused' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' From 530efdb441563950f3188500a75ff12660ac5cb3 Mon Sep 17 00:00:00 2001 From: Cody Date: Mon, 4 Mar 2019 01:12:44 -0500 Subject: [PATCH 014/185] Update transmission.yml Removed commented blocks. --- apps/transmission.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/apps/transmission.yml b/apps/transmission.yml index f2fb9c0..67c1086 100644 --- a/apps/transmission.yml +++ b/apps/transmission.yml @@ -35,11 +35,6 @@ path: "/opt/appdata/{{pgrole}}/rss/transmission-rss.conf" register: cfgcheck - #- name: Checking for existing blocklist - # stat: - # path: "/opt/appdata/{{pgrole}}/home/biglist.p2p" - # register: blcheck - # LABELS ###################################################################### - name: 'Adding Traefik' set_fact: @@ -117,11 +112,6 @@ state: started labels: '{{pg_labels}}' - #- name: 'Downloading blocklist' - # block: - # shell: "curl -s http://john.bitsurge.net/public/biglist.p2p.gz | gunzip > /config/home/biglist.p2p" - # when: not blcheck.stat.exists - - name: 'RSS feed configuration - Folder' block: - name: 'Creating RSS folder' @@ -152,4 +142,4 @@ - name: 'Stopping {{pgrole}}. Go configure ENV values.' docker_container: name: '{{pgrole}}' - state: stopped \ No newline at end of file + state: stopped From 8966982b55eb8365e2a564402f183c2a6b084c50 Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Mon, 4 Mar 2019 19:54:21 -0500 Subject: [PATCH 015/185] Initial MariaDB commit. --- apps/mariadb.yml | 68 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 apps/mariadb.yml diff --git a/apps/mariadb.yml b/apps/mariadb.yml new file mode 100644 index 0000000..ed9ee65 --- /dev/null +++ b/apps/mariadb.yml @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Title: MariaDB +# Author(s): MariaDB Foundation +# URL: https://mariadb.com - https://github.com/docker-library/mariadb +# GNU: General Public License v2.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: +# CORE (MANDATORY) DO NOT CHANGE ########################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: "mariadb" + intport: "3306" + extport: "6603" + image: "mariadb:latest" + + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + +# LABELS ################################################################ + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/var/lib/mysql' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + MYSQL_ROOT_PASSWORD: plexguide + + # MAIN DEPLOYMENT ############################################################## + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' + + # POST DEPLOYMENT ############################################################## + + - name: 'Post Deployment Notes' + debug: + msg: |- + * Login Information * root:plexguide * This should be changed * \ No newline at end of file From 45d2a91e566a652623a4b66d429c80768819d907 Mon Sep 17 00:00:00 2001 From: Cody Date: Mon, 4 Mar 2019 20:16:06 -0500 Subject: [PATCH 016/185] Removed forced https form traefik --- .vscode/settings.json | 3 +++ apps/medusa.yml | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..b22aab5 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "git.enableCommitSigning": true +} \ No newline at end of file diff --git a/apps/medusa.yml b/apps/medusa.yml index 091841f..ce96bfb 100644 --- a/apps/medusa.yml +++ b/apps/medusa.yml @@ -29,7 +29,6 @@ - name: 'Adding Traefik' set_fact: pg_labels: - traefik.protocol: 'https' traefik.frontend.auth.forward.address: '{{gauth}}' traefik.enable: 'true' traefik.port: '{{intport}}' From 530015e3a49982cb86943adc88a6b96c210958d1 Mon Sep 17 00:00:00 2001 From: Cody Date: Mon, 4 Mar 2019 20:23:08 -0500 Subject: [PATCH 017/185] Delete settings.json --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index b22aab5..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "git.enableCommitSigning": true -} \ No newline at end of file From a060b0cd42dc2f6f8ae87899231f1a6de959396d Mon Sep 17 00:00:00 2001 From: Robert Baker Date: Mon, 4 Mar 2019 22:00:42 -0700 Subject: [PATCH 018/185] deluge port fix --- apps/deluge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/deluge.yml b/apps/deluge.yml index 63476a3..1450fe4 100644 --- a/apps/deluge.yml +++ b/apps/deluge.yml @@ -68,7 +68,7 @@ published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{ports.stdout}}{{extport3}}:{{intport3}}' + - '{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: unless-stopped From a14be686b34cb50fc2a017e4d7732b39f6e98da8 Mon Sep 17 00:00:00 2001 From: Cody Date: Fri, 8 Mar 2019 18:21:43 -0500 Subject: [PATCH 019/185] FlexGit initial commit --- apps/flexget.yml | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 apps/flexget.yml diff --git a/apps/flexget.yml b/apps/flexget.yml new file mode 100644 index 0000000..cadd05c --- /dev/null +++ b/apps/flexget.yml @@ -0,0 +1,82 @@ +#!/bin/bash +# +# Title: FlexGet +# Author(s): paranoidi, Chase Sterling, cpoppema +# URL: https://flexget.com/ | https://github.com/cpoppema/docker-flexget +# GNU: MIT +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'flexget' + intport: '5050' + extport: '6060' + image: 'cpoppema/docker-flexget:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' + - '/mnt/downloads/{{pgrole}}:/downloads' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + + + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' + + # CONFIGURATION ############################################################# + #- name: Checking for existing app data + # stat: + # path: "/opt/appdata/{{pgrole}}/config.yml" + # register: cfcheck + # + #- name: "Configuring {{pgrole}} for first time use" + # block: + # - name: Creating config file + # shell: 'touch /opt/appdata/{{pgrole}}/config.yml' + # - name: Populate example config + # shell: 'echo -e ''web_server:\n bind: 0.0.0.0\n port: {{intport}}\n\ntasks:\n example-task:\n rss: hxxp://example.com/feed.xml\n download: /download\n series:\n - some series 1\n - some series 2'' > /opt/appdata/{{pgrole}}/config.yml' + # - name: Fix permissions + # shell: 'chown {{PUID}}:{{PGID}} /opt/appdata/{{pgrole}}/config.yml' + # when: not cfcheck.stat.exists \ No newline at end of file From 4e43c5e98903618823ce3c56c573b3e880dd8576 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:04:22 +0000 Subject: [PATCH 020/185] Create nzbget-mp4.yml NzbGet, with build in sickbeard_mp4_automator and ffmpeg build script Built for PGBlitz version 8.5.6 Should be fine for versions above this too --- apps/nzbget-mp4.yml | 402 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100644 apps/nzbget-mp4.yml diff --git a/apps/nzbget-mp4.yml b/apps/nzbget-mp4.yml new file mode 100644 index 0000000..87a8703 --- /dev/null +++ b/apps/nzbget-mp4.yml @@ -0,0 +1,402 @@ +#!/bin/bash +# +# NzbGet, with build in sickbeard_mp4_automator and ffmpeg build script +# Built for PGBlitz version 8.5.6 +# Should be fine for versions above this too +# +# Title: PGBlitz (Reference Title File) +# Author(s): Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################# +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'nzbget' + intport: '6789' + extport: '6789' + image: 'linuxserver/nzbget' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + - name: 'Including folders' + include_tasks: '/opt/coreapps/apps/_downloaders.yml' + + - name: Create nzb folder + file: 'path={{item}} state=directory mode=0775 owner=1000 group=1000' + with_items: + - '{{path.stdout}}/nzb' +# force: yes + + - name: 'Including plugins' + include_tasks: '/opt/coreapps/apps/_plugins.yml' + + - name: 'Checking for existing app data' + stat: + path: /opt/appdata/{{pgrole}}/nzbget.conf + register: confcheck + + # EXTRAS FOR MP4 MODS ########################################################## + - name: 'Create {{pgrole}} mp4 directories' + file: 'path={{item}} state=directory mode=0775 owner=1000 group=1000 recurse=yes' + with_items: + - '/opt/appdata/{{pgrole}}' + - '/opt/appdata/{{pgrole}}/cont-init.d' + - '/opt/appdata/{{pgrole}}/installer' + - '/opt/appdata/{{pgrole}}/scripts' + - '/opt/appdata/{{pgrole}}/scripts/MP4_Automator' + - '/opt/appdata/{{pgrole}}/ffmpeg-build' + - '/opt/appdata/{{pgrole}}/services.d' + + - name: 'Copy custom init scripts into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/cont-init.d/30-config + dest: /opt/appdata/{{pgrole}}/cont-init.d + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy custom install scripts into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/installer/installer.sh + dest: /opt/appdata/{{pgrole}}/installer + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy custom mp4 config into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini + dest: /opt/appdata/{{pgrole}}/scripts/MP4_Automator + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy custom NZBGetPostProcess script config into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript + dest: /opt/appdata/{{pgrole}}/ + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy ffmpeg build script into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh + dest: /opt/appdata/{{pgrole}}/ffmpeg-build + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy ffmpeg build script into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg + dest: /opt/appdata/{{pgrole}}/ffmpeg-build + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Copy custom init scripts into directory for {{pgrole}}' + copy: + src: /opt/communityapps/apps/templates/nzbget-mp4/services.d/run + dest: /opt/appdata/{{pgrole}}/services.d + directory_mode: yes + force: yes + owner: 1000 + group: 1000 + mode: 0755 + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/tmp:/tmp' + - '/opt/appdata/{{pgrole}}/cont-init.d:/etc/cont-init.d' + - '/opt/appdata/{{pgrole}}/services.d:/etc/services.d/nzbget' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + LC_ALL: 'C' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' + + # CONFIGURATION ############################################################# + - name: 'Waiting for {{pgrole}} to initialize' + wait_for: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + state: present + delay: 5 + + - name: 'Stopping {{pgrole}}' + docker_container: + name: '{{pgrole}}' + state: stopped + + - name: Set Main Location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^MainDir\s*=.*' + line: 'MainDir=/config' + state: present + + - name: Set download location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^DestDir\s*=.*' + line: 'DestDir={{path.stdout}}/downloads/{{pgrole}}' + state: present + + - name: Set incomplete location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^InterDir\s*=.*' + line: 'InterDir={{path.stdout}}/incomplete/{{pgrole}}' + state: present + + - name: Set TempDir + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^TempDir\s*=.*' + line: 'TempDir=/tmp' + state: present + + - name: Set NzbDir Location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^NzbDir\s*=.*' + line: 'NzbDir={{path.stdout}}/nzb' + state: present + + - name: ScriptDir + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^ScriptDir\s*=.*' + line: 'ScriptDir=${MainDir}/scripts' + state: present + + # FIRST TIME CONFIGURATION ############################################################# + - name: 'Configuring {{pgrole}} for first time use' + block: + - name: Lowercase & Set Movie Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category1.Name\s*=.*' + line: 'Category1.Name=movies' + state: present + + - name: Set Location of Movies + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category1.DestDir\s*=.*' + line: 'Category1.DestDir=' + state: present + + - name: Set postprocess of Movies + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category1.Extensions\s*=.*' + line: 'Category1.Extensions=MP4_Automator/NZBGetPostProcess.py' + state: present + + - name: Lowercase & Set TV Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category2.Name\s*=.*' + line: 'Category2.Name=tv' + state: present + + - name: Set Location of TV + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category2.DestDir\s*=.*' + line: 'Category2.DestDir=' + state: present + + - name: Set postprocess of TV + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category2.Extensions\s*=.*' + line: 'Category2.Extensions=MP4_Automator/NZBGetPostProcess.py' + state: present + + - name: Lowercase & Set Music Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category3.Name\s*=.*' + line: 'Category3.Name=music' + state: present + + - name: Set Location of Music + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category3.DestDir\s*=.*' + line: 'Category3.DestDir=' + state: present + + - name: Lowercase & Set EBook Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category4.Name\s*=.*' + line: 'Category4.Name=ebooks' + state: present + + - name: Set Location of EBooks + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Category4.DestDir\s*=.*' + line: 'Category4.DestDir=' + state: present + + - name: Lowercase & Set abook Category + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: 'Category5.Name\s*=.*' + line: 'Category5.Name=abooks' + state: present + + - name: Set Location of aBooks + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: 'Category5.DestDir\s*=.*' + line: 'Category5.DestDir=' + state: present + + - name: Set Global Extensions + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^Extensions\s*=.*' + line: 'Extensions=unzip.py' + state: present + + - name: Set ScriptOrder + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^ScriptOrder\s*=.*' + line: 'ScriptOrder=unzip.py, MP4_Automator/NZBGetPostProcess.py' + state: present + + - name: Set mp4 script location + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^MP4_Automator/NZBGetPostProcess.py:MP4_FOLDER\s*=.*' + line: 'MP4_Automator/NZBGetPostProcess.py:MP4_FOLDER=/config/scripts/MP4_Automator/' + state: present + + - name: Set mp4 conversion true + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^MP4_Automator/NZBGetPostProcess.py:SHOULDCONVERT\s*=.*' + line: 'MP4_Automator/NZBGetPostProcess.py:SHOULDCONVERT=True' + state: present + + - name: Default User + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^ControlUsername\s*=.*' + line: 'ControlUsername=' + state: present + + - name: Default Password + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: 'ControlPassword\s*=.*' + line: 'ControlPassword=' + state: present + + - name: DirectUnpack Set to On + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^DirectUnpack\s*=.*' + line: 'DirectUnpack=yes' + state: present + + - name: HealthCheck + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^HealthCheck\s*=.*' + line: 'HealthCheck=Delete' + state: present + + - name: Set DiskSpace + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '^DiskSpace\s*=.*' + line: 'DiskSpace=25000' + state: present + + - name: Remove Generic Task + lineinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + regexp: '{{ item.regexp }}' + state: absent + with_items: + - { regexp: '.Task1\.Time\=' } + - { regexp: '.Task1\.WeekDays\=' } + - { regexp: '.Task1\.Command\=' } + - { regexp: '.Task1\.Param\=' } + + - name: Unpause + blockinfile: + path: '/opt/appdata/{{pgrole}}/nzbget.conf' + block: | + Task1.Time=*,*:00,*:30 + Task1.WeekDays=1-7 + Task1.Command=UnpauseDownload + Task1.Param= + insertafter: '^### SCHEDULER' + when: not confcheck.stat.exists + + - name: Restart {{pgrole}} + docker_container: + name: '{{pgrole}}' + state: started From 01f7ad55c91bd4f3bbf70acfbb7035e427938a16 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:05:31 +0000 Subject: [PATCH 021/185] Create 30-config --- .../nzbget-mp4/cont-init.d/30-config | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 apps/templates/nzbget-mp4/cont-init.d/30-config diff --git a/apps/templates/nzbget-mp4/cont-init.d/30-config b/apps/templates/nzbget-mp4/cont-init.d/30-config new file mode 100644 index 0000000..deaaacd --- /dev/null +++ b/apps/templates/nzbget-mp4/cont-init.d/30-config @@ -0,0 +1,27 @@ +#!/usr/bin/with-contenv bash + +# delete lock file if found +[[ -f /downloads/nzbget.lock ]] && \ + rm /downloads/nzbget.lock + +# check if config file exists in /config +[[ ! -f /config/nzbget.conf ]] && \ + cp /defaults/nzbget.conf /config/nzbget.conf + +# permissions +chown 1000:1000 \ + /downloads +chown 1000:1000 -R \ + /app/nzbget \ + /config +chmod u+rw \ + /config/nzbget.conf + +chmod 777 -R \ + /config +chmod 777 -R \ + /app/nzbget +chmod 777 -R \ + /downloads + +exec /config/installer/installer.sh From 5ac3779584da722357f48511b8e942c7fdcc4a8a Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:06:08 +0000 Subject: [PATCH 022/185] Create build-ffmpeg --- .../nzbget-mp4/ffmpeg-build/build-ffmpeg | 400 ++++++++++++++++++ 1 file changed, 400 insertions(+) create mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg b/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg new file mode 100644 index 0000000..9aad5c4 --- /dev/null +++ b/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg @@ -0,0 +1,400 @@ +#!/bin/bash + +# https://github.com/markus-perl/ffmpeg-build-script + +VERSION=1.1 +CWD=$(pwd) +PACKAGES="$CWD/packages" +WORKSPACE="$CWD/workspace" +CC=clang +LDFLAGS="-L${WORKSPACE}/lib -lm" +CFLAGS="-I${WORKSPACE}/include" +PKG_CONFIG_PATH="${WORKSPACE}/lib/pkgconfig" +ADDITIONAL_CONFIGURE_OPTIONS="" + +# Speed up the process +# Env Var NUMJOBS overrides automatic detection +if [[ -n $NUMJOBS ]]; then + MJOBS=$NUMJOBS +elif [[ -f /proc/cpuinfo ]]; then + MJOBS=$(grep -c processor /proc/cpuinfo) +elif [[ "$OSTYPE" == "darwin"* ]]; then + MJOBS=$(sysctl -n machdep.cpu.thread_count) + ADDITIONAL_CONFIGURE_OPTIONS="--enable-videotoolbox" +else + MJOBS=4 +fi + +make_dir () { + if [ ! -d $1 ]; then + if ! mkdir $1; then + printf "\n Failed to create dir %s" "$1"; + exit 1 + fi + fi +} + +remove_dir () { + if [ -d $1 ]; then + rm -r "$1" + fi +} + +download () { + + DOWNLOAD_PATH=$PACKAGES; + + if [ ! -z "$3" ]; then + mkdir -p $PACKAGES/$3 + DOWNLOAD_PATH=$PACKAGES/$3 + fi; + + if [ ! -f "$DOWNLOAD_PATH/$2" ]; then + + echo "Downloading $1" + curl -L --silent -o "$DOWNLOAD_PATH/$2" "$1" + + EXITCODE=$? + if [ $EXITCODE -ne 0 ]; then + echo "" + echo "Failed to download $1. Exitcode $EXITCODE. Retrying in 10 seconds"; + sleep 10 + curl -L --silent -o "$DOWNLOAD_PATH/$2" "$1" + fi + + EXITCODE=$? + if [ $EXITCODE -ne 0 ]; then + echo "" + echo "Failed to download $1. Exitcode $EXITCODE"; + exit 1 + fi + + echo "... Done" + + if ! tar -xvf "$DOWNLOAD_PATH/$2" -C "$DOWNLOAD_PATH" 2>/dev/null >/dev/null; then + echo "Failed to extract $2"; + exit 1 + fi + + fi +} + +execute () { + echo "$ $*" + + OUTPUT=$($@ 2>&1) + + if [ $? -ne 0 ]; then + echo "$OUTPUT" + echo "" + echo "Failed to Execute $*" >&2 + exit 1 + fi +} + +build () { + echo "" + echo "building $1" + echo "=======================" + + if [ -f "$PACKAGES/$1.done" ]; then + echo "$1 already built. Remove $PACKAGES/$1.done lockfile to rebuild it." + return 1 + fi + + return 0 +} + +command_exists() { + if ! [[ -x $(command -v "$1") ]]; then + return 1 + fi + + return 0 +} + + +build_done () { + touch "$PACKAGES/$1.done" +} + +echo "ffmpeg-build-script v$VERSION" +echo "=========================" +echo "" + +case "$1" in +"--cleanup") + remove_dir $PACKAGES + remove_dir $WORKSPACE + echo "Cleanup done." + echo "" + exit 0 + ;; +"--build") + + ;; +*) + echo "Usage: $0" + echo " --build: start building process" + echo " --cleanup: remove all working dirs" + echo " --help: show this help" + echo "" + exit 0 + ;; +esac + +echo "Using $MJOBS make jobs simultaneously." + +make_dir $PACKAGES +make_dir $WORKSPACE + +export PATH=${WORKSPACE}/bin:$PATH + +if ! command_exists "make"; then + echo "make not installed."; + exit 1 +fi + +if ! command_exists "g++"; then + echo "g++ not installed."; + exit 1 +fi + +if ! command_exists "curl"; then + echo "curl not installed."; + exit 1 +fi + +if build "yasm"; then + download "http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz" "yasm-1.3.0.tar.gz" + cd $PACKAGES/yasm-1.3.0 || exit + execute ./configure --prefix=${WORKSPACE} + execute make -j $MJOBS + execute make install + build_done "yasm" +fi + +if build "nasm"; then + download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.13.03.tar.gz" "nasm.tar.gz" + cd $PACKAGES/nasm-2.13.03 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "nasm" +fi + +if build "opencore"; then + download "http://downloads.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-0.1.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Fopencore-amr%2F&ts=1442256558&use_mirror=netassist" "opencore-amr-0.1.5.tar.gz" + cd $PACKAGES/opencore-amr-0.1.5 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "opencore" +fi + +if build "libvpx"; then + download "https://github.com/webmproject/libvpx/archive/v1.7.0.tar.gz" "libvpx-1.7.0.tar.gz" + cd $PACKAGES/libvpx-*0 || exit + + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Applying Darwin patch" + sed "s/,--version-script//g" build/make/Makefile > build/make/Makefile.patched + sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched > build/make/Makefile + fi + + execute ./configure --prefix=${WORKSPACE} --disable-unit-tests --disable-shared + execute make -j $MJOBS + execute make install + build_done "libvpx" +fi + +if build "lame"; then + download "http://kent.dl.sourceforge.net/project/lame/lame/3.100/lame-3.100.tar.gz" "lame-3.100.tar.gz" + cd $PACKAGES/lame-3.100 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "lame" +fi + +if build "xvidcore"; then + download "http://downloads.xvid.org/downloads/xvidcore-1.3.4.tar.gz" "xvidcore-1.3.4.tar.gz" + cd $PACKAGES/xvidcore || exit + cd build/generic || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + + if [[ -f ${WORKSPACE}/lib/libxvidcore.4.dylib ]]; then + execute rm "${WORKSPACE}/lib/libxvidcore.4.dylib" + fi + + build_done "xvidcore" +fi + +if build "x264"; then + download "http://ftp.videolan.org/pub/x264/snapshots/x264-snapshot-20190204-2245-stable.tar.bz2" "last_x264.tar.bz2" + cd $PACKAGES/x264-snapshot-* || exit + + if [[ "$OSTYPE" == "linux-gnu" ]]; then + execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic CXXFLAGS="-fPIC" + else + execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic + fi + + execute make -j $MJOBS + execute make install + execute make install-lib-static + build_done "x264" +fi + +if build "libogg"; then + download "http://downloads.xiph.org/releases/ogg/libogg-1.3.3.tar.gz" "libogg-1.3.3.tar.gz" + cd $PACKAGES/libogg-1.3.3 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "libogg" +fi + +if build "libvorbis"; then + download "http://downloads.xiph.org/releases/vorbis/libvorbis-1.3.6.tar.gz" "libvorbis-1.3.6.tar.gz" + cd $PACKAGES/libvorbis-1.3.6 || exit + execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest + execute make -j $MJOBS + execute make install + build_done "libvorbis" +fi + +if build "libtheora"; then + download "http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.gz" "libtheora-1.1.1.tar.bz" + cd $PACKAGES/libtheora-1.1.1 || exit + sed "s/-fforce-addr//g" configure > configure.patched + chmod +x configure.patched + mv configure.patched configure + execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --with-vorbis-libraries=${WORKSPACE}/lib --with-vorbis-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm + execute make -j $MJOBS + execute make install + build_done "libtheora" +fi + +if build "pkg-config"; then + download "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" "pkg-config-0.29.2.tar.gz" + cd $PACKAGES/pkg-config-0.29.2 || exit + execute ./configure --silent --prefix=${WORKSPACE} --with-pc-path=${WORKSPACE}/lib/pkgconfig --with-internal-glib + execute make -j $MJOBS + execute make install + build_done "pkg-config" +fi + +if build "cmake"; then + download "https://cmake.org/files/v3.11/cmake-3.11.3.tar.gz" "cmake-3.11.3.tar.gz" + cd $PACKAGES/cmake-3.11.3 || exit + rm Modules/FindJava.cmake + perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt + perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt + execute ./configure --prefix=${WORKSPACE} + execute make -j $MJOBS + execute make install + build_done "cmake" +fi + +if build "vid_stab"; then + download "https://codeload.github.com/georgmartius/vid.stab/legacy.tar.gz/release-0.98b" "vid.stab-0.98b-transcode-1.1-binary-x86_64.tgz" + cd $PACKAGES/georgmartius-vid* || exit + perl -p -i -e "s/vidstab SHARED/vidstab STATIC/" CMakeLists.txt + execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} . + execute make install + build_done "vid_stab" +fi + +if build "x265"; then + download "https://bitbucket.org/multicoreware/x265/downloads/x265_3.0.tar.gz" "x265-3.0.tar.gz" + cd $PACKAGES/x265_* || exit + cd source || exit + execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} -DENABLE_SHARED:bool=off . + execute make -j $MJOBS + execute make install + sed "s/-lx265/-lx265 -lstdc++/g" "$WORKSPACE/lib/pkgconfig/x265.pc" > "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" + mv "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" "$WORKSPACE/lib/pkgconfig/x265.pc" + build_done "x265" +fi + +if build "fdk_aac"; then + download "http://downloads.sourceforge.net/project/opencore-amr/fdk-aac/fdk-aac-0.1.6.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Ffdk-aac%2F&ts=1457561564&use_mirror=kent" "fdk-aac-0.1.6.tar.gz" + cd $PACKAGES/fdk-aac-0.1.6 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "fdk_aac" +fi + + +build "ffmpeg" +download "http://ffmpeg.org/releases/ffmpeg-4.1.tar.bz2" "ffmpeg-snapshot.tar.bz2" +cd $PACKAGES/ffmpeg-4.1 || exit +./configure $ADDITIONAL_CONFIGURE_OPTIONS \ + --pkgconfigdir="$WORKSPACE/lib/pkgconfig" \ + --prefix=${WORKSPACE} \ + --pkg-config-flags="--static" \ + --extra-cflags="-I$WORKSPACE/include" \ + --extra-ldflags="-L$WORKSPACE/lib" \ + --extra-libs="-lpthread -lm" \ + --enable-static \ + --disable-debug \ + --disable-shared \ + --disable-ffplay \ + --disable-doc \ + --enable-gpl \ + --enable-version3 \ + --enable-nonfree \ + --enable-pthreads \ + --enable-libvpx \ + --enable-libmp3lame \ + --enable-libtheora \ + --enable-libvorbis \ + --enable-libx264 \ + --enable-libx265 \ + --enable-runtime-cpudetect \ + --enable-libfdk-aac \ + --enable-avfilter \ + --enable-libopencore_amrwb \ + --enable-libopencore_amrnb \ + --enable-filters \ + --enable-libvidstab + +execute make -j $MJOBS +execute make install + +INSTALL_FOLDER="/usr/bin" +if [[ "$OSTYPE" == "darwin"* ]]; then +INSTALL_FOLDER="/usr/local/bin" +fi + +echo "" +echo "Building done. The binary can be found here: $WORKSPACE/bin/ffmpeg" +echo "" + + +if [[ $AUTOINSTALL == "yes" ]]; then + if command_exists "sudo"; then + sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" + sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" + echo "Done. ffmpeg is now installed to your system" + fi +elif [[ ! $SKIPINSTALL == "yes" ]]; then + if command_exists "sudo"; then + + read -r -p "Install the binary to your $INSTALL_FOLDER folder? [Y/n] " response + + case $response in + [yY][eE][sS]|[yY]) + sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" + sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" + echo "Done. ffmpeg is now installed to your system" + ;; + esac + fi +fi + +exit 0 From df88651991858f9c3beea93a52ef0e21f8f654c1 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:06:28 +0000 Subject: [PATCH 023/185] Create web-install.sh --- .../nzbget-mp4/ffmpeg-build/web-install.sh | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh b/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh new file mode 100644 index 0000000..ffe4158 --- /dev/null +++ b/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Helper script to download and run the build-ffmpeg script. + +make_dir () { + if [ ! -d $1 ]; then + if ! mkdir $1; then + printf "\n Failed to create dir %s" "$1"; + exit 1 + fi + fi +} + +command_exists() { + if ! [[ -x $(command -v "$1") ]]; then + return 1 + fi + + return 0 +} + +TARGET='ffmpeg-build' + +if ! command_exists "curl"; then + echo "curl not installed."; + exit 1 +fi + +echo "ffmpeg-build-script-downloader v0.1" +echo "=========================================" +echo "" + +echo "First we create the ffmpeg build directory $TARGET" +make_dir $TARGET +cd $TARGET + +echo "Now we download and execute the build script" +echo "" + +bash build-ffmpeg --build + From f2a87711f0a184d218ce9184e8588f0c36dfa22b Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:07:02 +0000 Subject: [PATCH 024/185] Create installer.sh --- .../ffmpeg-build/installer/installer.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh b/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh new file mode 100644 index 0000000..2fe24f5 --- /dev/null +++ b/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh @@ -0,0 +1,30 @@ +#!/bin/bash +apk update +apk upgrade +apk add --no-cache git +apk add build-base gcc wget diffutils perl +apk add curl +git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp +mv /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator/ +rm -rf /config/scripts/MP4_Automator/tmp +git unstage +apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev +pip install --upgrade PIP +pip install requests +pip install requests[security] +pip install requests-cache +pip install babelfish +pip install "guessit<2" +pip install "subliminal<2" +pip install qtfaststart +# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 +pip uninstall -y stevedore +pip install stevedore==1.19.1 +#Remove default NZBGetPostProcess script settings, and replace with our own +rm /config/scripts/MP4_Automator/NZBGetPostProcess.py +cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py +#Build ffmpeg +cd /config +. /config/ffmpeg-build/web-install.sh +#Set script file permissions +chmod 777 -R /config/scripts From 7e7e531d5e0b8eb90a2f446d7045709ea2dae677 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:11:27 +0000 Subject: [PATCH 025/185] remove file in wrong location --- .../ffmpeg-build/installer/installer.sh | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh b/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh deleted file mode 100644 index 2fe24f5..0000000 --- a/apps/templates/nzbget-mp4/ffmpeg-build/installer/installer.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -apk update -apk upgrade -apk add --no-cache git -apk add build-base gcc wget diffutils perl -apk add curl -git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp -mv /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator/ -rm -rf /config/scripts/MP4_Automator/tmp -git unstage -apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev -pip install --upgrade PIP -pip install requests -pip install requests[security] -pip install requests-cache -pip install babelfish -pip install "guessit<2" -pip install "subliminal<2" -pip install qtfaststart -# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 -pip uninstall -y stevedore -pip install stevedore==1.19.1 -#Remove default NZBGetPostProcess script settings, and replace with our own -rm /config/scripts/MP4_Automator/NZBGetPostProcess.py -cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py -#Build ffmpeg -cd /config -. /config/ffmpeg-build/web-install.sh -#Set script file permissions -chmod 777 -R /config/scripts From a4c0e34d2d34f75842ad4276e9eee7a85ec2c674 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:12:14 +0000 Subject: [PATCH 026/185] Create installer.sh --- .../nzbget-mp4/installer/installer.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 apps/templates/nzbget-mp4/installer/installer.sh diff --git a/apps/templates/nzbget-mp4/installer/installer.sh b/apps/templates/nzbget-mp4/installer/installer.sh new file mode 100644 index 0000000..2fe24f5 --- /dev/null +++ b/apps/templates/nzbget-mp4/installer/installer.sh @@ -0,0 +1,30 @@ +#!/bin/bash +apk update +apk upgrade +apk add --no-cache git +apk add build-base gcc wget diffutils perl +apk add curl +git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp +mv /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator/ +rm -rf /config/scripts/MP4_Automator/tmp +git unstage +apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev +pip install --upgrade PIP +pip install requests +pip install requests[security] +pip install requests-cache +pip install babelfish +pip install "guessit<2" +pip install "subliminal<2" +pip install qtfaststart +# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 +pip uninstall -y stevedore +pip install stevedore==1.19.1 +#Remove default NZBGetPostProcess script settings, and replace with our own +rm /config/scripts/MP4_Automator/NZBGetPostProcess.py +cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py +#Build ffmpeg +cd /config +. /config/ffmpeg-build/web-install.sh +#Set script file permissions +chmod 777 -R /config/scripts From 6a57cbc8a013f11f93534f48dc73c2eaf7e212a3 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:12:56 +0000 Subject: [PATCH 027/185] Create autoProcess.ini --- .../nzbget-mp4/MP4_Automator/autoProcess.ini | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini diff --git a/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini b/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini new file mode 100644 index 0000000..83b18aa --- /dev/null +++ b/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini @@ -0,0 +1,143 @@ +[SickBeard] +host = sickbeard +port = 8081 +username = +password = +web_root = +ssl = False +api_key = + +[Sonarr] +host = sonarr +port = 8989 +web_root = +ssl = False +apikey = + +[Radarr] +host = radarr +port = 7878 +web_root = +ssl = False +apikey = + +[MP4] +ffmpeg = /config/ffmpeg-build/workspace/bin/ffmpeg +ffprobe = /config/ffmpeg-build/workspace/bin/ffprobe +threads = 1 +output_directory = +copy_to = +move_to = +output_extension = mp4 +output_format = mp4 +delete_original = True +relocate_moov = True +video-codec = h264,x264 +video-bitrate = +video-crf = 18 +video-max-width = +h264-max-level = 4.1 +use-qsv-decoder-with-encoder = True +ios-audio = libfdk_aac +ios-first-track-only = False +ios-audio-filter = dynaudnorm +max-audio-channels = +audio-codec = ac3,mp3,dts,dca,aac,libfdk_aac +audio-language = eng +audio-default-language = eng +audio-channel-bitrate = 256 +audio-filter = +subtitle-codec = srt +subtitle-language = eng +subtitle-default-language = +subtitle-encoding = +fullpathguess = True +convert-mp4 = True +tagfile = True +tag-language = en +download-artwork = Poster +download-subs = True +embed-subs = False +sub-providers = addic7ed,podnapisi,thesubdb,opensubtitles +permissions = 0777 +post-process = False +pix-fmt = +aac_adtstoasc = False +postopts = -preset,slower +preopts = +audio-copy-original = False +enable_dxva2_gpu_decode = False +ios-move-last = False +use-hevc-qsv-decoder = False +embed-only-internal-subs = False +audio-first-track-of-language = False +video-profile = + +[CouchPotato] +host = couchpotato +port = 5050 +username = +password = +web_root = +ssl = False +apikey = +delay = 65 +method = renamer +delete_failed = False + +[uTorrent] +convert = +couchpotato-label = couchpotato +sickbeard-label = sickbeard +sonarr-label = sonarr +bypass-label = bypass +sickrage-label = sickrage +webui = False +action_before = stop +action_after = removedata +host = http://utorrent:8080/ +username = +password = +output_directory = +radarr-label = radarr + +[Deluge] +host = deluge +username = +convert = True +password = +sonarr-label = sonarr +radarr-label = radarr +bypass-label = bypass +sickbeard-label = sickbeard +port = 12569 +sickrage-label = sickrage +couchpotato-label = couchpotato +output_directory = +remove = true + +[SABNZBD] +convert = True +sickrage-category = sickrage +sonarr-category = sonarr +radarr-category = radarr +bypass-category = bypass +couchpotato-category = couchpotato +sickbeard-category = sickbeard +output_directory = + +[Sickrage] +host = sickrage +port = 8081 +username = +password = +web_root = +ssl = False +api_key = + +[Plex] +host = plex +port = 32400 +refresh = False +token = + From a67d15da03bad5a98c285db1526661e1c3b4303a Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:13:32 +0000 Subject: [PATCH 028/185] Create TEMPLATEPPScript --- .../nzbget-mp4/MP4_Automator/TEMPLATEPPScript | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript diff --git a/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript b/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript new file mode 100644 index 0000000..6eb7055 --- /dev/null +++ b/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# +############################################################################## +### NZBGET POST-PROCESSING SCRIPT ### + +# Modified to enable multiple bypass categories, +# as per: https://github.com/mdhiggins/sickbeard_mp4_automator/issues/509 +# +# Converts files and passes them to Sonarr for further processing. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### + +# Change to full path to MP4 Automator folder. No quotes and a trailing / +#MP4_FOLDER=~/sickbeard_mp4_automator/ + +# Convert file before passing to destination (True, False) +#SHOULDCONVERT=False + +# Category for Couchpotato +#CP_CAT=Couchpotato + +# Category for Sonarr +#SONARR_CAT=Sonarr + +# Category for Radarr +#RADARR_CAT=Radarr + +# Category for Sickbeard +#SICKBEARD_CAT=Sickbeard + +# Category for Sickrage +#SICKRAGE_CAT=Sickrage + +# Category list (comma seperated) for bypassing any further processing but still converting +#BYPASS_CAT=tv,movies + +# Custom output_directory setting +#OUTPUT_DIR= + +### NZBGET POST-PROCESSING SCRIPT ### +############################################################################## + +import os +import sys +import re +import json +import traceback + +# Sanity checks for path string +MP4folder = os.environ['NZBPO_MP4_FOLDER'].strip() +MP4folder = MP4folder.replace('"', '') +MP4folder = MP4folder.replace("'", "") +MP4folder = MP4folder.replace("\\", "/") +if not(MP4folder.endswith("/")): + MP4folder += "/" +#DEBUG#print MP4folder+" the original is "+os.environ['NZBPO_MP4_FOLDER'] + +output_dir = None +if 'NZBPO_OUTPUT_DIR' in os.environ: + output_dir = os.environ['NZBPO_OUTPUT_DIR'].strip() + if len(output_dir) > 0: + output_dir = output_dir.replace('"', '') + output_dir = output_dir.replace("'", "") + output_dir = output_dir.replace("\\", "/") + if not(output_dir.endswith("/")): + output_dir += "/" + #DEBUG#print Overriding output directory + +sys.path.append(MP4folder) +try: + from readSettings import ReadSettings + from mkvtomp4 import MkvtoMp4 + from autoprocess import autoProcessMovie, autoProcessTV, autoProcessTVSR, sonarr, radarr + import logging + from logging.config import fileConfig +except ImportError: + print("[ERROR] Wrong path to sickbeard_mp4_automator: " + os.environ['NZBPO_MP4_FOLDER']) + print("[ERROR] %s" % traceback.print_exc()) + sys.exit(0) + +# Setup Logging +logpath = '/var/log/sickbeard_mp4_automator' +if os.name == 'nt': + logpath = MP4folder +elif not os.path.isdir(logpath): + try: + os.mkdir(logpath) + except: + logpath = MP4folder +configPath = os.path.abspath(os.path.join(MP4folder, 'logging.ini')).replace("\\", "\\\\") +logPath = os.path.abspath(os.path.join(logpath, 'index.log')).replace("\\", "\\\\") +fileConfig(configPath, defaults={'logfilename': logPath}) +log = logging.getLogger("NZBGetPostProcess") + +# Determine if conversion will take place +shouldConvert = (os.environ['NZBPO_SHOULDCONVERT'].lower() in ("yes", "true", "t", "1")) + +if 'NZBOP_SCRIPTDIR' in os.environ and not os.environ['NZBOP_VERSION'][0:5] < '11.0': + log.info("Script triggered from NZBGet (11.0 or later).") + + path = os.environ['NZBPP_DIRECTORY'] # Path to NZB directory + nzb = os.environ['NZBPP_NZBFILENAME'] # Original NZB name + category = os.environ['NZBPP_CATEGORY'] # NZB Category to determine destination + #DEBUG#print "Category is %s." % category + + couchcat = os.environ['NZBPO_CP_CAT'].lower() + sonarrcat = os.environ['NZBPO_SONARR_CAT'].lower() + radarrcat = os.environ['NZBPO_RADARR_CAT'].lower() + sickbeardcat = os.environ['NZBPO_SICKBEARD_CAT'].lower() + sickragecat = os.environ['NZBPO_SICKRAGE_CAT'].lower() + bypass = os.environ['NZBPO_BYPASS_CAT'].lower().replace(' ','').split(',') + + categories = [sickbeardcat, couchcat, sonarrcat, radarrcat, sickragecat] + + log.debug("Path: %s" % path) + log.debug("NZB: %s" % nzb) + log.debug("Category: %s" % category) + log.debug("Categories: %s" % categories) + + # NZBGet argv: all passed as environment variables. + clientAgent = "nzbget" + # Exit codes used by NZBGet + POSTPROCESS_PARCHECK = 92 + POSTPROCESS_SUCCESS = 93 + POSTPROCESS_ERROR = 94 + POSTPROCESS_NONE = 95 + + # Check nzbget.conf options + status = 0 + + if os.environ['NZBOP_UNPACK'] != 'yes': + log.error("Please enable option \"Unpack\" in nzbget configuration file, exiting.") + sys.exit(POSTPROCESS_NONE) + + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '3': + log.error("Par-check successful, but Par-repair disabled, exiting") + sys.exit(POSTPROCESS_NONE) + + if os.environ['NZBPP_PARSTATUS'] == '1': + log.error("Par-check failed, setting status \"failed\".") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + log.error("Unpack failed, setting status \"failed\".") + status = 1 + sys.exit(POSTPROCESS_NONE) + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] != '2': + # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check + + for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): + for file in filenames: + fileExtension = os.path.splitext(file)[1] + + if fileExtension in ['.par2']: + log.error("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\".") + status = 1 + break + + if os.path.isfile(os.path.join(os.environ['NZBPP_DIRECTORY'], "_brokenlog.txt")) and not status == 1: + log.error("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting.") + status = 1 + + if not status == 1: + log.error("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful.") + + # Check if destination directory exists (important for reprocessing of history items) + if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + log.error("Post-Process: Nothing to post-process: destination directory ", os.environ['NZBPP_DIRECTORY'], "doesn't exist.") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Make sure one of the appropriate categories is set + if category.lower() not in categories and category.lower() not in bypass: + log.error("Post-Process: No valid category detected. Category was %s." % (category)) + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Make sure there are no duplicate categories + if len(categories) != len(set(categories)): + log.error("Duplicate category detected. Category names must be unique.") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # All checks done, now launching the script. + settings = ReadSettings(MP4folder, "autoProcess.ini") + + if shouldConvert: + if output_dir: + settings.output_dir = output_dir + converter = MkvtoMp4(settings, logger=log) + for r, d, f in os.walk(path): + for files in f: + inputfile = os.path.join(r, files) + #DEBUG#print inputfile + #Ignores files under 50MB + if os.path.getsize(inputfile) > 50000000: + if MkvtoMp4(settings, logger=log).validSource(inputfile): + try: + output = converter.process(inputfile) + log.info("Successfully processed %s." % inputfile) + except: + log.exception("File processing failed.") + if converter.output_dir: + path = converter.output_dir + if (category.lower() == categories[0]): + #DEBUG#print "Sickbeard Processing Activated" + autoProcessTV.processEpisode(path, settings, nzb) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() == categories[1]): + #DEBUG#print "CouchPotato Processing Activated" + autoProcessMovie.process(path, settings, nzb, status) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() == categories[2]): + #DEBUG#print "Sonarr Processing Activated" + success = sonarr.processEpisode(path, settings, True) + if success: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_ERROR) + elif (category.lower() == categories[3]): + #DEBUG#print "Radarr Processing Activated" + success = radarr.processMovie(path, settings, True) + if success: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_ERROR) + elif (category.lower() == categories[4]): + #DEBUG#print "Sickrage Processing Activated" + autoProcessTVSR.processEpisode(path, settings, nzb) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() in bypass): + #DEBUG#print "Bypass Further Processing" + sys.exit(POSTPROCESS_NONE) + +else: + log.error("This script can only be called from NZBGet (11.0 or later).") + sys.exit(0) From 068c47edd10e88f369568de78777bae27e1ce895 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:14:14 +0000 Subject: [PATCH 029/185] Create run --- apps/templates/nzbget-mp4/services.d/run | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 apps/templates/nzbget-mp4/services.d/run diff --git a/apps/templates/nzbget-mp4/services.d/run b/apps/templates/nzbget-mp4/services.d/run new file mode 100644 index 0000000..ad735b4 --- /dev/null +++ b/apps/templates/nzbget-mp4/services.d/run @@ -0,0 +1,7 @@ +#!/usr/bin/with-contenv bash + +umask 000 + +exec \ + s6-envuidgid -nB 1000:1000 /app/nzbget/nzbget -s -c /config/nzbget.conf \ + -o OutputMode=log From 53b8e77fb7d5a23ff61a87108ea9c3971a058e2e Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:49:14 +0000 Subject: [PATCH 030/185] changed role name and ports --- apps/nzbget-mp4.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/nzbget-mp4.yml b/apps/nzbget-mp4.yml index 87a8703..b9650cd 100644 --- a/apps/nzbget-mp4.yml +++ b/apps/nzbget-mp4.yml @@ -17,17 +17,17 @@ - name: 'Set Known Facts' set_fact: - pgrole: 'nzbget' - intport: '6789' - extport: '6789' + pgrole: 'nzbget-mp4' + intport: '6790' + extport: '6790' image: 'linuxserver/nzbget' # CORE (MANDATORY) ############################################################ - name: 'Including cron job' - include_tasks: '/opt/coreapps/apps/_core.yml' + include_tasks: '/opt/communityapps/apps/_core.yml' - name: 'Including folders' - include_tasks: '/opt/coreapps/apps/_downloaders.yml' + include_tasks: '/opt/communityapps/apps/_downloaders.yml' - name: Create nzb folder file: 'path={{item}} state=directory mode=0775 owner=1000 group=1000' @@ -36,7 +36,7 @@ # force: yes - name: 'Including plugins' - include_tasks: '/opt/coreapps/apps/_plugins.yml' + include_tasks: '/opt/communityapps/apps/_plugins.yml' - name: 'Checking for existing app data' stat: From bc8627c4a01f79fd05cb72443e24f3b579095b46 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:51:44 +0000 Subject: [PATCH 031/185] Create nzbget-mp4 --- apps/image/nzbget-mp4 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 apps/image/nzbget-mp4 diff --git a/apps/image/nzbget-mp4 b/apps/image/nzbget-mp4 new file mode 100644 index 0000000..d8f3f35 --- /dev/null +++ b/apps/image/nzbget-mp4 @@ -0,0 +1,2 @@ +linuxserver/nzbget +linuxserver/nzbget:testing From 080ce649962fd0e95e771cdec4d3536539cc527d Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:54:20 +0000 Subject: [PATCH 032/185] Create __init__.py --- apps/templates/nzbget-mp4/scripts/rarfile/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 apps/templates/nzbget-mp4/scripts/rarfile/__init__.py diff --git a/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py b/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py @@ -0,0 +1 @@ + From 01be43661f2d50ff15fa262f93f0d75326e8edcf Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:54:59 +0000 Subject: [PATCH 033/185] Create rarfile.py --- .../nzbget-mp4/scripts/rarfile/rarfile.py | 2931 +++++++++++++++++ 1 file changed, 2931 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py diff --git a/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py b/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py new file mode 100644 index 0000000..6a3a647 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py @@ -0,0 +1,2931 @@ +# rarfile.py +# +# Copyright (c) 2005-2016 Marko Kreen +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +r"""RAR archive reader. + +This is Python module for Rar archive reading. The interface +is made as :mod:`zipfile`-like as possible. + +Basic logic: + - Parse archive structure with Python. + - Extract non-compressed files with Python + - Extract compressed files with unrar. + - Optionally write compressed data to temp file to speed up unrar, + otherwise it needs to scan whole archive on each execution. + +Example:: + + import rarfile + + rf = rarfile.RarFile('myarchive.rar') + for f in rf.infolist(): + print f.filename, f.file_size + if f.filename == 'README': + print(rf.read(f)) + +Archive files can also be accessed via file-like object returned +by :meth:`RarFile.open`:: + + import rarfile + + with rarfile.RarFile('archive.rar') as rf: + with rf.open('README') as f: + for ln in f: + print(ln.strip()) + +There are few module-level parameters to tune behaviour, +here they are with defaults, and reason to change it:: + + import rarfile + + # Set to full path of unrar.exe if it is not in PATH + rarfile.UNRAR_TOOL = "unrar" + + # Set to '\\' to be more compatible with old rarfile + rarfile.PATH_SEP = '/' + +For more details, refer to source. + +""" + +from __future__ import division, print_function + +## +## Imports and compat - support both Python 2.x and 3.x +## + +import sys +import os +import errno +import struct + +from struct import pack, unpack, Struct +from binascii import crc32, hexlify +from tempfile import mkstemp +from subprocess import Popen, PIPE, STDOUT +from io import RawIOBase +from hashlib import sha1, sha256 +from hmac import HMAC +from datetime import datetime, timedelta, tzinfo + +# fixed offset timezone, for UTC +try: + from datetime import timezone +except ImportError: + class timezone(tzinfo): + """Compat timezone.""" + __slots__ = ('_ofs', '_name') + _DST = timedelta(0) + + def __init__(self, offset, name): + super(timezone, self).__init__() + self._ofs, self._name = offset, name + + def utcoffset(self, dt): + return self._ofs + + def tzname(self, dt): + return self._name + + def dst(self, dt): + return self._DST + +# only needed for encryped headers +try: + try: + from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf import pbkdf2 + + class AES_CBC_Decrypt(object): + """Decrypt API""" + def __init__(self, key, iv): + ciph = Cipher(algorithms.AES(key), modes.CBC(iv), default_backend()) + self.decrypt = ciph.decryptor().update + + def pbkdf2_sha256(password, salt, iters): + """PBKDF2 with HMAC-SHA256""" + ctx = pbkdf2.PBKDF2HMAC(hashes.SHA256(), 32, salt, iters, default_backend()) + return ctx.derive(password) + + except ImportError: + from Crypto.Cipher import AES + from Crypto.Protocol import KDF + + class AES_CBC_Decrypt(object): + """Decrypt API""" + def __init__(self, key, iv): + self.decrypt = AES.new(key, AES.MODE_CBC, iv).decrypt + + def pbkdf2_sha256(password, salt, iters): + """PBKDF2 with HMAC-SHA256""" + return KDF.PBKDF2(password, salt, 32, iters, hmac_sha256) + + _have_crypto = 1 +except ImportError: + _have_crypto = 0 + +try: + from pyblake2 import blake2s + _have_blake2 = True +except ImportError: + _have_blake2 = False + +# compat with 2.x +if sys.hexversion < 0x3000000: + def rar_crc32(data, prev=0): + """CRC32 with unsigned values. + """ + if (prev > 0) and (prev & 0x80000000): + prev -= (1 << 32) + res = crc32(data, prev) + if res < 0: + res += (1 << 32) + return res + tohex = hexlify + _byte_code = ord +else: # pragma: no cover + def tohex(data): + """Return hex string.""" + return hexlify(data).decode('ascii') + rar_crc32 = crc32 + unicode = str + _byte_code = int # noqa + + +__version__ = '3.0' + +# export only interesting items +__all__ = ['is_rarfile', 'RarInfo', 'RarFile', 'RarExtFile'] + +## +## Module configuration. Can be tuned after importing. +## + +#: default fallback charset +DEFAULT_CHARSET = "windows-1252" + +#: list of encodings to try, with fallback to DEFAULT_CHARSET if none succeed +TRY_ENCODINGS = ('utf8', 'utf-16le') + +#: 'unrar', 'rar' or full path to either one +UNRAR_TOOL = "unrar" + +#: Command line args to use for opening file for reading. +OPEN_ARGS = ('p', '-inul') + +#: Command line args to use for extracting file to disk. +EXTRACT_ARGS = ('x', '-y', '-idq') + +#: args for testrar() +TEST_ARGS = ('t', '-idq') + +# +# Allow use of tool that is not compatible with unrar. +# +# By default use 'bsdtar' which is 'tar' program that +# sits on top of libarchive. +# +# Problems with libarchive RAR backend: +# - Does not support solid archives. +# - Does not support password-protected archives. +# + +ALT_TOOL = 'bsdtar' +ALT_OPEN_ARGS = ('-x', '--to-stdout', '-f') +ALT_EXTRACT_ARGS = ('-x', '-f') +ALT_TEST_ARGS = ('-t', '-f') +ALT_CHECK_ARGS = ('--help',) + +#: whether to speed up decompression by using tmp archive +USE_EXTRACT_HACK = 1 + +#: limit the filesize for tmp archive usage +HACK_SIZE_LIMIT = 20 * 1024 * 1024 + +#: Separator for path name components. RAR internally uses '\\'. +#: Use '/' to be similar with zipfile. +PATH_SEP = '/' + +## +## rar constants +## + +# block types +RAR_BLOCK_MARK = 0x72 # r +RAR_BLOCK_MAIN = 0x73 # s +RAR_BLOCK_FILE = 0x74 # t +RAR_BLOCK_OLD_COMMENT = 0x75 # u +RAR_BLOCK_OLD_EXTRA = 0x76 # v +RAR_BLOCK_OLD_SUB = 0x77 # w +RAR_BLOCK_OLD_RECOVERY = 0x78 # x +RAR_BLOCK_OLD_AUTH = 0x79 # y +RAR_BLOCK_SUB = 0x7a # z +RAR_BLOCK_ENDARC = 0x7b # { + +# flags for RAR_BLOCK_MAIN +RAR_MAIN_VOLUME = 0x0001 +RAR_MAIN_COMMENT = 0x0002 +RAR_MAIN_LOCK = 0x0004 +RAR_MAIN_SOLID = 0x0008 +RAR_MAIN_NEWNUMBERING = 0x0010 +RAR_MAIN_AUTH = 0x0020 +RAR_MAIN_RECOVERY = 0x0040 +RAR_MAIN_PASSWORD = 0x0080 +RAR_MAIN_FIRSTVOLUME = 0x0100 +RAR_MAIN_ENCRYPTVER = 0x0200 + +# flags for RAR_BLOCK_FILE +RAR_FILE_SPLIT_BEFORE = 0x0001 +RAR_FILE_SPLIT_AFTER = 0x0002 +RAR_FILE_PASSWORD = 0x0004 +RAR_FILE_COMMENT = 0x0008 +RAR_FILE_SOLID = 0x0010 +RAR_FILE_DICTMASK = 0x00e0 +RAR_FILE_DICT64 = 0x0000 +RAR_FILE_DICT128 = 0x0020 +RAR_FILE_DICT256 = 0x0040 +RAR_FILE_DICT512 = 0x0060 +RAR_FILE_DICT1024 = 0x0080 +RAR_FILE_DICT2048 = 0x00a0 +RAR_FILE_DICT4096 = 0x00c0 +RAR_FILE_DIRECTORY = 0x00e0 +RAR_FILE_LARGE = 0x0100 +RAR_FILE_UNICODE = 0x0200 +RAR_FILE_SALT = 0x0400 +RAR_FILE_VERSION = 0x0800 +RAR_FILE_EXTTIME = 0x1000 +RAR_FILE_EXTFLAGS = 0x2000 + +# flags for RAR_BLOCK_ENDARC +RAR_ENDARC_NEXT_VOLUME = 0x0001 +RAR_ENDARC_DATACRC = 0x0002 +RAR_ENDARC_REVSPACE = 0x0004 +RAR_ENDARC_VOLNR = 0x0008 + +# flags common to all blocks +RAR_SKIP_IF_UNKNOWN = 0x4000 +RAR_LONG_BLOCK = 0x8000 + +# Host OS types +RAR_OS_MSDOS = 0 +RAR_OS_OS2 = 1 +RAR_OS_WIN32 = 2 +RAR_OS_UNIX = 3 +RAR_OS_MACOS = 4 +RAR_OS_BEOS = 5 + +# Compression methods - '0'..'5' +RAR_M0 = 0x30 +RAR_M1 = 0x31 +RAR_M2 = 0x32 +RAR_M3 = 0x33 +RAR_M4 = 0x34 +RAR_M5 = 0x35 + +# +# RAR5 constants +# + +RAR5_BLOCK_MAIN = 1 +RAR5_BLOCK_FILE = 2 +RAR5_BLOCK_SERVICE = 3 +RAR5_BLOCK_ENCRYPTION = 4 +RAR5_BLOCK_ENDARC = 5 + +RAR5_BLOCK_FLAG_EXTRA_DATA = 0x01 +RAR5_BLOCK_FLAG_DATA_AREA = 0x02 +RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN = 0x04 +RAR5_BLOCK_FLAG_SPLIT_BEFORE = 0x08 +RAR5_BLOCK_FLAG_SPLIT_AFTER = 0x10 +RAR5_BLOCK_FLAG_DEPENDS_PREV = 0x20 +RAR5_BLOCK_FLAG_KEEP_WITH_PARENT = 0x40 + +RAR5_MAIN_FLAG_ISVOL = 0x01 +RAR5_MAIN_FLAG_HAS_VOLNR = 0x02 +RAR5_MAIN_FLAG_SOLID = 0x04 +RAR5_MAIN_FLAG_RECOVERY = 0x08 +RAR5_MAIN_FLAG_LOCKED = 0x10 + +RAR5_FILE_FLAG_ISDIR = 0x01 +RAR5_FILE_FLAG_HAS_MTIME = 0x02 +RAR5_FILE_FLAG_HAS_CRC32 = 0x04 +RAR5_FILE_FLAG_UNKNOWN_SIZE = 0x08 + +RAR5_COMPR_SOLID = 0x40 + +RAR5_ENC_FLAG_HAS_CHECKVAL = 0x01 + +RAR5_ENDARC_FLAG_NEXT_VOL = 0x01 + +RAR5_XFILE_ENCRYPTION = 1 +RAR5_XFILE_HASH = 2 +RAR5_XFILE_TIME = 3 +RAR5_XFILE_VERSION = 4 +RAR5_XFILE_REDIR = 5 +RAR5_XFILE_OWNER = 6 +RAR5_XFILE_SERVICE = 7 + +RAR5_XTIME_UNIXTIME = 0x01 +RAR5_XTIME_HAS_MTIME = 0x02 +RAR5_XTIME_HAS_CTIME = 0x04 +RAR5_XTIME_HAS_ATIME = 0x08 + +RAR5_XENC_CIPHER_AES256 = 0 + +RAR5_XENC_CHECKVAL = 0x01 +RAR5_XENC_TWEAKED = 0x02 + +RAR5_XHASH_BLAKE2SP = 0 + +RAR5_XREDIR_UNIX_SYMLINK = 1 +RAR5_XREDIR_WINDOWS_SYMLINK = 2 +RAR5_XREDIR_WINDOWS_JUNCTION = 3 +RAR5_XREDIR_HARD_LINK = 4 +RAR5_XREDIR_FILE_COPY = 5 + +RAR5_XREDIR_ISDIR = 0x01 + +RAR5_XOWNER_UNAME = 0x01 +RAR5_XOWNER_GNAME = 0x02 +RAR5_XOWNER_UID = 0x04 +RAR5_XOWNER_GID = 0x08 + +RAR5_OS_WINDOWS = 0 +RAR5_OS_UNIX = 1 + +## +## internal constants +## + +RAR_ID = b"Rar!\x1a\x07\x00" +RAR5_ID = b"Rar!\x1a\x07\x01\x00" +ZERO = b'\0' +EMPTY = b'' +UTC = timezone(timedelta(0), 'UTC') +BSIZE = 32 * 1024 + +def _get_rar_version(xfile): + '''Check quickly whether file is rar archive. + ''' + with XFile(xfile) as fd: + buf = fd.read(len(RAR5_ID)) + if buf.startswith(RAR_ID): + return 3 + elif buf.startswith(RAR5_ID): + return 5 + return 0 + +## +## Public interface +## + +def is_rarfile(xfile): + '''Check quickly whether file is rar archive. + ''' + return _get_rar_version(xfile) > 0 + +class Error(Exception): + """Base class for rarfile errors.""" + +class BadRarFile(Error): + """Incorrect data in archive.""" + +class NotRarFile(Error): + """The file is not RAR archive.""" + +class BadRarName(Error): + """Cannot guess multipart name components.""" + +class NoRarEntry(Error): + """File not found in RAR""" + +class PasswordRequired(Error): + """File requires password""" + +class NeedFirstVolume(Error): + """Need to start from first volume.""" + +class NoCrypto(Error): + """Cannot parse encrypted headers - no crypto available.""" + +class RarExecError(Error): + """Problem reported by unrar/rar.""" + +class RarWarning(RarExecError): + """Non-fatal error""" + +class RarFatalError(RarExecError): + """Fatal error""" + +class RarCRCError(RarExecError): + """CRC error during unpacking""" + +class RarLockedArchiveError(RarExecError): + """Must not modify locked archive""" + +class RarWriteError(RarExecError): + """Write error""" + +class RarOpenError(RarExecError): + """Open error""" + +class RarUserError(RarExecError): + """User error""" + +class RarMemoryError(RarExecError): + """Memory error""" + +class RarCreateError(RarExecError): + """Create error""" + +class RarNoFilesError(RarExecError): + """No files that match pattern were found""" + +class RarUserBreak(RarExecError): + """User stop""" + +class RarWrongPassword(RarExecError): + """Incorrect password""" + +class RarUnknownError(RarExecError): + """Unknown exit code""" + +class RarSignalExit(RarExecError): + """Unrar exited with signal""" + +class RarCannotExec(RarExecError): + """Executable not found.""" + + +class RarInfo(object): + r'''An entry in rar archive. + + RAR3 extended timestamps are :class:`datetime.datetime` objects without timezone. + RAR5 extended timestamps are :class:`datetime.datetime` objects with UTC timezone. + + Attributes: + + filename + File name with relative path. + Path separator is '/'. Always unicode string. + + date_time + File modification timestamp. As tuple of (year, month, day, hour, minute, second). + RAR5 allows archives where it is missing, it's None then. + + file_size + Uncompressed size. + + compress_size + Compressed size. + + compress_type + Compression method: one of :data:`RAR_M0` .. :data:`RAR_M5` constants. + + extract_version + Minimal Rar version needed for decompressing. As (major*10 + minor), + so 2.9 is 29. + + RAR3: 10, 20, 29 + + RAR5 does not have such field in archive, it's simply set to 50. + + host_os + Host OS type, one of RAR_OS_* constants. + + RAR3: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`, :data:`RAR_OS_MSDOS`, + :data:`RAR_OS_OS2`, :data:`RAR_OS_BEOS`. + + RAR5: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`. + + mode + File attributes. May be either dos-style or unix-style, depending on host_os. + + mtime + File modification time. Same value as :attr:`date_time` + but as :class:`datetime.datetime` object with extended precision. + + ctime + Optional time field: creation time. As :class:`datetime.datetime` object. + + atime + Optional time field: last access time. As :class:`datetime.datetime` object. + + arctime + Optional time field: archival time. As :class:`datetime.datetime` object. + (RAR3-only) + + CRC + CRC-32 of uncompressed file, unsigned int. + + RAR5: may be None. + + blake2sp_hash + Blake2SP hash over decompressed data. (RAR5-only) + + comment + Optional file comment field. Unicode string. (RAR3-only) + + file_redir + If not None, file is link of some sort. Contains tuple of (type, flags, target). + (RAR5-only) + + Type is one of constants: + + :data:`RAR5_XREDIR_UNIX_SYMLINK` + unix symlink to target. + :data:`RAR5_XREDIR_WINDOWS_SYMLINK` + windows symlink to target. + :data:`RAR5_XREDIR_WINDOWS_JUNCTION` + windows junction. + :data:`RAR5_XREDIR_HARD_LINK` + hard link to target. + :data:`RAR5_XREDIR_FILE_COPY` + current file is copy of another archive entry. + + Flags may contain :data:`RAR5_XREDIR_ISDIR` bit. + + volume + Volume nr, starting from 0. + + volume_file + Volume file name, where file starts. + + ''' + + # zipfile-compatible fields + filename = None + file_size = None + compress_size = None + date_time = None + comment = None + CRC = None + volume = None + orig_filename = None + + # optional extended time fields, datetime() objects. + mtime = None + ctime = None + atime = None + + extract_version = None + mode = None + host_os = None + compress_type = None + + # rar3-only fields + comment = None + arctime = None + + # rar5-only fields + blake2sp_hash = None + file_redir = None + + # internal fields + flags = 0 + type = None + + def isdir(self): + """Returns True if entry is a directory. + """ + if self.type == RAR_BLOCK_FILE: + return (self.flags & RAR_FILE_DIRECTORY) == RAR_FILE_DIRECTORY + return False + + def needs_password(self): + """Returns True if data is stored password-protected. + """ + if self.type == RAR_BLOCK_FILE: + return (self.flags & RAR_FILE_PASSWORD) > 0 + return False + + +class RarFile(object): + '''Parse RAR structure, provide access to files in archive. + ''' + + #: Archive comment. Unicode string or None. + comment = None + + def __init__(self, rarfile, mode="r", charset=None, info_callback=None, + crc_check=True, errors="stop"): + """Open and parse a RAR archive. + + Parameters: + + rarfile + archive file name + mode + only 'r' is supported. + charset + fallback charset to use, if filenames are not already Unicode-enabled. + info_callback + debug callback, gets to see all archive entries. + crc_check + set to False to disable CRC checks + errors + Either "stop" to quietly stop parsing on errors, + or "strict" to raise errors. Default is "stop". + """ + self._rarfile = rarfile + self._charset = charset or DEFAULT_CHARSET + self._info_callback = info_callback + self._crc_check = crc_check + self._password = None + self._file_parser = None + + if errors == "stop": + self._strict = False + elif errors == "strict": + self._strict = True + else: + raise ValueError("Invalid value for 'errors' parameter.") + + if mode != "r": + raise NotImplementedError("RarFile supports only mode=r") + + self._parse() + + def __enter__(self): + return self + + def __exit__(self, typ, value, traceback): + self.close() + + def setpassword(self, password): + '''Sets the password to use when extracting.''' + self._password = password + if self._file_parser: + if self._file_parser.has_header_encryption(): + self._file_parser = None + if not self._file_parser: + self._parse() + else: + self._file_parser.setpassword(self._password) + + def needs_password(self): + '''Returns True if any archive entries require password for extraction.''' + return self._file_parser.needs_password() + + def namelist(self): + '''Return list of filenames in archive.''' + return [f.filename for f in self.infolist()] + + def infolist(self): + '''Return RarInfo objects for all files/directories in archive.''' + return self._file_parser.infolist() + + def volumelist(self): + '''Returns filenames of archive volumes. + + In case of single-volume archive, the list contains + just the name of main archive file. + ''' + return self._file_parser.volumelist() + + def getinfo(self, fname): + '''Return RarInfo for file. + ''' + return self._file_parser.getinfo(fname) + + def open(self, fname, mode='r', psw=None): + '''Returns file-like object (:class:`RarExtFile`), + from where the data can be read. + + The object implements :class:`io.RawIOBase` interface, so it can + be further wrapped with :class:`io.BufferedReader` + and :class:`io.TextIOWrapper`. + + On older Python where io module is not available, it implements + only .read(), .seek(), .tell() and .close() methods. + + The object is seekable, although the seeking is fast only on + uncompressed files, on compressed files the seeking is implemented + by reading ahead and/or restarting the decompression. + + Parameters: + + fname + file name or RarInfo instance. + mode + must be 'r' + psw + password to use for extracting. + ''' + + if mode != 'r': + raise NotImplementedError("RarFile.open() supports only mode=r") + + # entry lookup + inf = self.getinfo(fname) + if inf.isdir(): + raise TypeError("Directory does not have any data: " + inf.filename) + + # check password + if inf.needs_password(): + psw = psw or self._password + if psw is None: + raise PasswordRequired("File %s requires password" % inf.filename) + else: + psw = None + + return self._file_parser.open(inf, psw) + + def read(self, fname, psw=None): + """Return uncompressed data for archive entry. + + For longer files using :meth:`RarFile.open` may be better idea. + + Parameters: + + fname + filename or RarInfo instance + psw + password to use for extracting. + """ + + with self.open(fname, 'r', psw) as f: + return f.read() + + def close(self): + """Release open resources.""" + pass + + def printdir(self): + """Print archive file list to stdout.""" + for f in self.infolist(): + print(f.filename) + + def extract(self, member, path=None, pwd=None): + """Extract single file into current directory. + + Parameters: + + member + filename or :class:`RarInfo` instance + path + optional destination path + pwd + optional password to use + """ + if isinstance(member, RarInfo): + fname = member.filename + else: + fname = member + self._extract([fname], path, pwd) + + def extractall(self, path=None, members=None, pwd=None): + """Extract all files into current directory. + + Parameters: + + path + optional destination path + members + optional filename or :class:`RarInfo` instance list to extract + pwd + optional password to use + """ + fnlist = [] + if members is not None: + for m in members: + if isinstance(m, RarInfo): + fnlist.append(m.filename) + else: + fnlist.append(m) + self._extract(fnlist, path, pwd) + + def testrar(self): + """Let 'unrar' test the archive. + """ + cmd = [UNRAR_TOOL] + list(TEST_ARGS) + add_password_arg(cmd, self._password) + cmd.append('--') + with XTempFile(self._rarfile) as rarfile: + cmd.append(rarfile) + p = custom_popen(cmd) + output = p.communicate()[0] + check_returncode(p, output) + + def strerror(self): + """Return error string if parsing failed, + or None if no problems. + """ + if not self._file_parser: + return "Not a RAR file" + return self._file_parser.strerror() + + ## + ## private methods + ## + + def _parse(self): + ver = _get_rar_version(self._rarfile) + if ver == 3: + p3 = RAR3Parser(self._rarfile, self._password, self._crc_check, + self._charset, self._strict, self._info_callback) + self._file_parser = p3 # noqa + elif ver == 5: + p5 = RAR5Parser(self._rarfile, self._password, self._crc_check, + self._charset, self._strict, self._info_callback) + self._file_parser = p5 # noqa + else: + raise BadRarFile("Not a RAR file") + + self._file_parser.parse() + self.comment = self._file_parser.comment + + # call unrar to extract a file + def _extract(self, fnlist, path=None, psw=None): + cmd = [UNRAR_TOOL] + list(EXTRACT_ARGS) + + # pasoword + psw = psw or self._password + add_password_arg(cmd, psw) + cmd.append('--') + + # rar file + with XTempFile(self._rarfile) as rarfn: + cmd.append(rarfn) + + # file list + for fn in fnlist: + if os.sep != PATH_SEP: + fn = fn.replace(PATH_SEP, os.sep) + cmd.append(fn) + + # destination path + if path is not None: + cmd.append(path + os.sep) + + # call + p = custom_popen(cmd) + output = p.communicate()[0] + check_returncode(p, output) + +# +# File format parsing +# + +class CommonParser(object): + """Shared parser parts.""" + _main = None + _hdrenc_main = None + _needs_password = False + _fd = None + _expect_sig = None + _parse_error = None + _password = None + comment = None + + def __init__(self, rarfile, password, crc_check, charset, strict, info_cb): + self._rarfile = rarfile + self._password = password + self._crc_check = crc_check + self._charset = charset + self._strict = strict + self._info_callback = info_cb + self._info_list = [] + self._info_map = {} + self._vol_list = [] + + def has_header_encryption(self): + """Returns True if headers are encrypted + """ + if self._hdrenc_main: + return True + if self._main: + if self._main.flags & RAR_MAIN_PASSWORD: + return True + return False + + def setpassword(self, psw): + """Set cached password.""" + self._password = psw + + def volumelist(self): + """Volume files""" + return self._vol_list + + def needs_password(self): + """Is password required""" + return self._needs_password + + def strerror(self): + """Last error""" + return self._parse_error + + def infolist(self): + """List of RarInfo records. + """ + return self._info_list + + def getinfo(self, fname): + """Return RarInfo for filename + """ + # accept both ways here + if PATH_SEP == '/': + fname2 = fname.replace("\\", "/") + else: + fname2 = fname.replace("/", "\\") + + try: + return self._info_map[fname] + except KeyError: + try: + return self._info_map[fname2] + except KeyError: + raise NoRarEntry("No such file: %s" % fname) + + # read rar + def parse(self): + """Process file.""" + self._fd = None + try: + self._parse_real() + finally: + if self._fd: + self._fd.close() + self._fd = None + + def _parse_real(self): + fd = XFile(self._rarfile) + self._fd = fd + sig = fd.read(len(self._expect_sig)) + if sig != self._expect_sig: + if isinstance(self._rarfile, (str, unicode)): + raise NotRarFile("Not a Rar archive: {}".format(self._rarfile)) + raise NotRarFile("Not a Rar archive") + + volume = 0 # first vol (.rar) is 0 + more_vols = False + endarc = False + volfile = self._rarfile + self._vol_list = [self._rarfile] + while 1: + if endarc: + h = None # don't read past ENDARC + else: + h = self._parse_header(fd) + if not h: + if more_vols: + volume += 1 + fd.close() + try: + volfile = self._next_volname(volfile) + fd = XFile(volfile) + except IOError: + self._set_error("Cannot open next volume: %s", volfile) + break + self._fd = fd + sig = fd.read(len(self._expect_sig)) + if sig != self._expect_sig: + self._set_error("Invalid volume sig: %s", volfile) + break + more_vols = False + endarc = False + self._vol_list.append(volfile) + continue + break + h.volume = volume + h.volume_file = volfile + + if h.type == RAR_BLOCK_MAIN and not self._main: + self._main = h + if h.flags & RAR_MAIN_NEWNUMBERING: + # RAR 2.x does not set FIRSTVOLUME, + # so check it only if NEWNUMBERING is used + if (h.flags & RAR_MAIN_FIRSTVOLUME) == 0: + raise NeedFirstVolume("Need to start from first volume") + if h.flags & RAR_MAIN_PASSWORD: + self._needs_password = True + if not self._password: + break + elif h.type == RAR_BLOCK_ENDARC: + more_vols = (h.flags & RAR_ENDARC_NEXT_VOLUME) > 0 + endarc = True + elif h.type == RAR_BLOCK_FILE: + # RAR 2.x does not write RAR_BLOCK_ENDARC + if h.flags & RAR_FILE_SPLIT_AFTER: + more_vols = True + # RAR 2.x does not set RAR_MAIN_FIRSTVOLUME + if volume == 0 and h.flags & RAR_FILE_SPLIT_BEFORE: + raise NeedFirstVolume("Need to start from first volume") + + if h.needs_password(): + self._needs_password = True + + # store it + self.process_entry(fd, h) + + if self._info_callback: + self._info_callback(h) + + # go to next header + if h.add_size > 0: + fd.seek(h.data_offset + h.add_size, 0) + + def process_entry(self, fd, item): + """Examine item, add into lookup cache.""" + raise NotImplementedError() + + def _decrypt_header(self, fd): + raise NotImplementedError('_decrypt_header') + + def _parse_block_header(self, fd): + raise NotImplementedError('_parse_block_header') + + def _open_hack(self, inf, psw): + raise NotImplementedError('_open_hack') + + # read single header + def _parse_header(self, fd): + try: + # handle encrypted headers + if (self._main and self._main.flags & RAR_MAIN_PASSWORD) or self._hdrenc_main: + if not self._password: + return + fd = self._decrypt_header(fd) + + # now read actual header + return self._parse_block_header(fd) + except struct.error: + self._set_error('Broken header in RAR file') + return None + + # given current vol name, construct next one + def _next_volname(self, volfile): + if is_filelike(volfile): + raise IOError("Working on single FD") + if self._main.flags & RAR_MAIN_NEWNUMBERING: + return _next_newvol(volfile) + return _next_oldvol(volfile) + + def _set_error(self, msg, *args): + if args: + msg = msg % args + self._parse_error = msg + if self._strict: + raise BadRarFile(msg) + + def open(self, inf, psw): + """Return stream object for file data.""" + + if inf.file_redir: + # cannot leave to unrar as it expects copied file to exist + if inf.file_redir[0] in (RAR5_XREDIR_FILE_COPY, RAR5_XREDIR_HARD_LINK): + inf = self.getinfo(inf.file_redir[2]) + if not inf: + raise BadRarFile('cannot find copied file') + + if inf.flags & RAR_FILE_SPLIT_BEFORE: + raise NeedFirstVolume("Partial file, please start from first volume: " + inf.filename) + + # is temp write usable? + use_hack = 1 + if not self._main: + use_hack = 0 + elif self._main._must_disable_hack(): + use_hack = 0 + elif inf._must_disable_hack(): + use_hack = 0 + elif is_filelike(self._rarfile): + pass + elif inf.file_size > HACK_SIZE_LIMIT: + use_hack = 0 + elif not USE_EXTRACT_HACK: + use_hack = 0 + + # now extract + if inf.compress_type == RAR_M0 and (inf.flags & RAR_FILE_PASSWORD) == 0 and inf.file_redir is None: + return self._open_clear(inf) + elif use_hack: + return self._open_hack(inf, psw) + elif is_filelike(self._rarfile): + return self._open_unrar_membuf(self._rarfile, inf, psw) + else: + return self._open_unrar(self._rarfile, inf, psw) + + def _open_clear(self, inf): + return DirectReader(self, inf) + + def _open_hack_core(self, inf, psw, prefix, suffix): + + size = inf.compress_size + inf.header_size + rf = XFile(inf.volume_file, 0) + rf.seek(inf.header_offset) + + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + + try: + tmpf.write(prefix) + while size > 0: + if size > BSIZE: + buf = rf.read(BSIZE) + else: + buf = rf.read(size) + if not buf: + raise BadRarFile('read failed: ' + inf.filename) + tmpf.write(buf) + size -= len(buf) + tmpf.write(suffix) + tmpf.close() + rf.close() + except: + rf.close() + tmpf.close() + os.unlink(tmpname) + raise + + return self._open_unrar(tmpname, inf, psw, tmpname) + + # write in-memory archive to temp file - needed for solid archives + def _open_unrar_membuf(self, memfile, inf, psw): + tmpname = membuf_tempfile(memfile) + return self._open_unrar(tmpname, inf, psw, tmpname, force_file=True) + + # extract using unrar + def _open_unrar(self, rarfile, inf, psw=None, tmpfile=None, force_file=False): + cmd = [UNRAR_TOOL] + list(OPEN_ARGS) + add_password_arg(cmd, psw) + cmd.append("--") + cmd.append(rarfile) + + # not giving filename avoids encoding related problems + if not tmpfile or force_file: + fn = inf.filename + if PATH_SEP != os.sep: + fn = fn.replace(PATH_SEP, os.sep) + cmd.append(fn) + + # read from unrar pipe + return PipeReader(self, inf, cmd, tmpfile) + +# +# RAR3 format +# + +class Rar3Info(RarInfo): + """RAR3 specific fields.""" + extract_version = 15 + salt = None + add_size = 0 + header_crc = None + header_size = None + header_offset = None + data_offset = None + _md_class = None + _md_expect = None + + # make sure some rar5 fields are always present + file_redir = None + blake2sp_hash = None + + def _must_disable_hack(self): + if self.type == RAR_BLOCK_FILE: + if self.flags & RAR_FILE_PASSWORD: + return True + elif self.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): + return True + elif self.type == RAR_BLOCK_MAIN: + if self.flags & (RAR_MAIN_SOLID | RAR_MAIN_PASSWORD): + return True + return False + + +class RAR3Parser(CommonParser): + """Parse RAR3 file format. + """ + _expect_sig = RAR_ID + _last_aes_key = (None, None, None) # (salt, key, iv) + + def _decrypt_header(self, fd): + if not _have_crypto: + raise NoCrypto('Cannot parse encrypted headers - no crypto') + salt = fd.read(8) + if self._last_aes_key[0] == salt: + key, iv = self._last_aes_key[1:] + else: + key, iv = rar3_s2k(self._password, salt) + self._last_aes_key = (salt, key, iv) + return HeaderDecrypt(fd, key, iv) + + # common header + def _parse_block_header(self, fd): + h = Rar3Info() + h.header_offset = fd.tell() + + # read and parse base header + buf = fd.read(S_BLK_HDR.size) + if not buf: + return None + t = S_BLK_HDR.unpack_from(buf) + h.header_crc, h.type, h.flags, h.header_size = t + + # read full header + if h.header_size > S_BLK_HDR.size: + hdata = buf + fd.read(h.header_size - S_BLK_HDR.size) + else: + hdata = buf + h.data_offset = fd.tell() + + # unexpected EOF? + if len(hdata) != h.header_size: + self._set_error('Unexpected EOF when reading header') + return None + + pos = S_BLK_HDR.size + + # block has data assiciated with it? + if h.flags & RAR_LONG_BLOCK: + h.add_size, pos = load_le32(hdata, pos) + else: + h.add_size = 0 + + # parse interesting ones, decide header boundaries for crc + if h.type == RAR_BLOCK_MARK: + return h + elif h.type == RAR_BLOCK_MAIN: + pos += 6 + if h.flags & RAR_MAIN_ENCRYPTVER: + pos += 1 + crc_pos = pos + if h.flags & RAR_MAIN_COMMENT: + self._parse_subblocks(h, hdata, pos) + elif h.type == RAR_BLOCK_FILE: + pos = self._parse_file_header(h, hdata, pos - 4) + crc_pos = pos + if h.flags & RAR_FILE_COMMENT: + pos = self._parse_subblocks(h, hdata, pos) + elif h.type == RAR_BLOCK_SUB: + pos = self._parse_file_header(h, hdata, pos - 4) + crc_pos = h.header_size + elif h.type == RAR_BLOCK_OLD_AUTH: + pos += 8 + crc_pos = pos + elif h.type == RAR_BLOCK_OLD_EXTRA: + pos += 7 + crc_pos = pos + else: + crc_pos = h.header_size + + # check crc + if h.type == RAR_BLOCK_OLD_SUB: + crcdat = hdata[2:] + fd.read(h.add_size) + else: + crcdat = hdata[2:crc_pos] + + calc_crc = rar_crc32(crcdat) & 0xFFFF + + # return good header + if h.header_crc == calc_crc: + return h + + # header parsing failed. + self._set_error('Header CRC error (%02x): exp=%x got=%x (xlen = %d)', + h.type, h.header_crc, calc_crc, len(crcdat)) + + # instead panicing, send eof + return None + + # read file-specific header + def _parse_file_header(self, h, hdata, pos): + fld = S_FILE_HDR.unpack_from(hdata, pos) + pos += S_FILE_HDR.size + + h.compress_size = fld[0] + h.file_size = fld[1] + h.host_os = fld[2] + h.CRC = fld[3] + h.date_time = parse_dos_time(fld[4]) + h.mtime = to_datetime(h.date_time) + h.extract_version = fld[5] + h.compress_type = fld[6] + name_size = fld[7] + h.mode = fld[8] + + h._md_class = CRC32Context + h._md_expect = h.CRC + + if h.flags & RAR_FILE_LARGE: + h1, pos = load_le32(hdata, pos) + h2, pos = load_le32(hdata, pos) + h.compress_size |= h1 << 32 + h.file_size |= h2 << 32 + h.add_size = h.compress_size + + name, pos = load_bytes(hdata, name_size, pos) + if h.flags & RAR_FILE_UNICODE: + nul = name.find(ZERO) + h.orig_filename = name[:nul] + u = UnicodeFilename(h.orig_filename, name[nul + 1:]) + h.filename = u.decode() + + # if parsing failed fall back to simple name + if u.failed: + h.filename = self._decode(h.orig_filename) + else: + h.orig_filename = name + h.filename = self._decode(name) + + # change separator, if requested + if PATH_SEP != '\\': + h.filename = h.filename.replace('\\', PATH_SEP) + + if h.flags & RAR_FILE_SALT: + h.salt, pos = load_bytes(hdata, 8, pos) + else: + h.salt = None + + # optional extended time stamps + if h.flags & RAR_FILE_EXTTIME: + pos = _parse_ext_time(h, hdata, pos) + else: + h.mtime = h.atime = h.ctime = h.arctime = None + + return pos + + # find old-style comment subblock + def _parse_subblocks(self, h, hdata, pos): + while pos < len(hdata): + # ordinary block header + t = S_BLK_HDR.unpack_from(hdata, pos) + ___scrc, stype, sflags, slen = t + pos_next = pos + slen + pos += S_BLK_HDR.size + + # corrupt header + if pos_next < pos: + break + + # followed by block-specific header + if stype == RAR_BLOCK_OLD_COMMENT and pos + S_COMMENT_HDR.size <= pos_next: + declen, ver, meth, crc = S_COMMENT_HDR.unpack_from(hdata, pos) + pos += S_COMMENT_HDR.size + data = hdata[pos : pos_next] + cmt = rar3_decompress(ver, meth, data, declen, sflags, + crc, self._password) + if not self._crc_check: + h.comment = self._decode_comment(cmt) + elif rar_crc32(cmt) & 0xFFFF == crc: + h.comment = self._decode_comment(cmt) + + pos = pos_next + return pos + + def _read_comment_v3(self, inf, psw=None): + + # read data + with XFile(inf.volume_file) as rf: + rf.seek(inf.data_offset) + data = rf.read(inf.compress_size) + + # decompress + cmt = rar3_decompress(inf.extract_version, inf.compress_type, data, + inf.file_size, inf.flags, inf.CRC, psw, inf.salt) + + # check crc + if self._crc_check: + crc = rar_crc32(cmt) + if crc != inf.CRC: + return None + + return self._decode_comment(cmt) + + def _decode(self, val): + for c in TRY_ENCODINGS: + try: + return val.decode(c) + except UnicodeError: + pass + return val.decode(self._charset, 'replace') + + def _decode_comment(self, val): + return self._decode(val) + + def process_entry(self, fd, item): + if item.type == RAR_BLOCK_FILE: + # use only first part + if (item.flags & RAR_FILE_SPLIT_BEFORE) == 0: + self._info_map[item.filename] = item + self._info_list.append(item) + elif len(self._info_list) > 0: + # final crc is in last block + old = self._info_list[-1] + old.CRC = item.CRC + old._md_expect = item._md_expect + old.compress_size += item.compress_size + + # parse new-style comment + if item.type == RAR_BLOCK_SUB and item.filename == 'CMT': + if item.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): + pass + elif item.flags & RAR_FILE_SOLID: + # file comment + cmt = self._read_comment_v3(item, self._password) + if len(self._info_list) > 0: + old = self._info_list[-1] + old.comment = cmt + else: + # archive comment + cmt = self._read_comment_v3(item, self._password) + self.comment = cmt + + if item.type == RAR_BLOCK_MAIN: + if item.flags & RAR_MAIN_COMMENT: + self.comment = item.comment + if item.flags & RAR_MAIN_PASSWORD: + self._needs_password = True + + # put file compressed data into temporary .rar archive, and run + # unrar on that, thus avoiding unrar going over whole archive + def _open_hack(self, inf, psw): + # create main header: crc, type, flags, size, res1, res2 + prefix = RAR_ID + S_BLK_HDR.pack(0x90CF, 0x73, 0, 13) + ZERO * (2 + 4) + return self._open_hack_core(inf, psw, prefix, EMPTY) + +# +# RAR5 format +# + +class Rar5Info(RarInfo): + """Shared fields for RAR5 records. + """ + extract_version = 50 + header_crc = None + header_size = None + header_offset = None + data_offset = None + + # type=all + block_type = None + block_flags = None + add_size = 0 + block_extra_size = 0 + + # type=MAIN + volume_number = None + _md_class = None + _md_expect = None + + def _must_disable_hack(self): + return False + + +class Rar5BaseFile(Rar5Info): + """Shared sturct for file & service record. + """ + type = -1 + file_flags = None + file_encryption = (0, 0, 0, EMPTY, EMPTY, EMPTY) + file_compress_flags = None + file_redir = None + file_owner = None + file_version = None + blake2sp_hash = None + + def _must_disable_hack(self): + if self.flags & RAR_FILE_PASSWORD: + return True + if self.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): + return True + if self.file_compress_flags & RAR5_COMPR_SOLID: + return True + if self.file_redir: + return True + return False + + +class Rar5FileInfo(Rar5BaseFile): + """RAR5 file record. + """ + type = RAR_BLOCK_FILE + + +class Rar5ServiceInfo(Rar5BaseFile): + """RAR5 service record. + """ + type = RAR_BLOCK_SUB + + +class Rar5MainInfo(Rar5Info): + """RAR5 archive main record. + """ + type = RAR_BLOCK_MAIN + main_flags = None + main_volume_number = None + + def _must_disable_hack(self): + if self.main_flags & RAR5_MAIN_FLAG_SOLID: + return True + return False + + +class Rar5EncryptionInfo(Rar5Info): + """RAR5 archive header encryption record. + """ + type = RAR5_BLOCK_ENCRYPTION + encryption_algo = None + encryption_flags = None + encryption_kdf_count = None + encryption_salt = None + encryption_check_value = None + + def needs_password(self): + return True + + +class Rar5EndArcInfo(Rar5Info): + """RAR5 end of archive record. + """ + type = RAR_BLOCK_ENDARC + endarc_flags = None + + +class RAR5Parser(CommonParser): + """Parse RAR5 format. + """ + _expect_sig = RAR5_ID + _hdrenc_main = None + + # AES encrypted headers + _last_aes256_key = (-1, None, None) # (kdf_count, salt, key) + + def _gen_key(self, kdf_count, salt): + if self._last_aes256_key[:2] == (kdf_count, salt): + return self._last_aes256_key[2] + if kdf_count > 24: + raise BadRarFile('Too large kdf_count') + psw = self._password + if isinstance(psw, unicode): + psw = psw.encode('utf8') + key = pbkdf2_sha256(psw, salt, 1 << kdf_count) + self._last_aes256_key = (kdf_count, salt, key) + return key + + def _decrypt_header(self, fd): + if not _have_crypto: + raise NoCrypto('Cannot parse encrypted headers - no crypto') + h = self._hdrenc_main + key = self._gen_key(h.encryption_kdf_count, h.encryption_salt) + iv = fd.read(16) + return HeaderDecrypt(fd, key, iv) + + # common header + def _parse_block_header(self, fd): + header_offset = fd.tell() + + preload = 4 + 3 + start_bytes = fd.read(preload) + header_crc, pos = load_le32(start_bytes, 0) + hdrlen, pos = load_vint(start_bytes, pos) + if hdrlen > 2 * 1024 * 1024: + return None + header_size = pos + hdrlen + + # read full header, check for EOF + hdata = start_bytes + fd.read(header_size - len(start_bytes)) + if len(hdata) != header_size: + self._set_error('Unexpected EOF when reading header') + return None + data_offset = fd.tell() + + calc_crc = rar_crc32(memoryview(hdata)[4:]) + if header_crc != calc_crc: + # header parsing failed. + self._set_error('Header CRC error: exp=%x got=%x (xlen = %d)', + header_crc, calc_crc, len(hdata)) + return None + + block_type, pos = load_vint(hdata, pos) + + if block_type == RAR5_BLOCK_MAIN: + h, pos = self._parse_block_common(Rar5MainInfo(), hdata) + h = self._parse_main_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_FILE: + h, pos = self._parse_block_common(Rar5FileInfo(), hdata) + h = self._parse_file_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_SERVICE: + h, pos = self._parse_block_common(Rar5ServiceInfo(), hdata) + h = self._parse_file_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_ENCRYPTION: + h, pos = self._parse_block_common(Rar5EncryptionInfo(), hdata) + h = self._parse_encryption_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_ENDARC: + h, pos = self._parse_block_common(Rar5EndArcInfo(), hdata) + h = self._parse_endarc_block(h, hdata, pos) + else: + h = None + if h: + h.header_offset = header_offset + h.data_offset = data_offset + return h + + def _parse_block_common(self, h, hdata): + h.header_crc, pos = load_le32(hdata, 0) + hdrlen, pos = load_vint(hdata, pos) + h.header_size = hdrlen + pos + h.block_type, pos = load_vint(hdata, pos) + h.block_flags, pos = load_vint(hdata, pos) + + if h.block_flags & RAR5_BLOCK_FLAG_EXTRA_DATA: + h.block_extra_size, pos = load_vint(hdata, pos) + if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: + h.add_size, pos = load_vint(hdata, pos) + + h.compress_size = h.add_size + + if h.block_flags & RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN: + h.flags |= RAR_SKIP_IF_UNKNOWN + if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: + h.flags |= RAR_LONG_BLOCK + return h, pos + + def _parse_main_block(self, h, hdata, pos): + h.main_flags, pos = load_vint(hdata, pos) + if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR: + h.main_volume_number = load_vint(hdata, pos) + + h.flags |= RAR_MAIN_NEWNUMBERING + if h.main_flags & RAR5_MAIN_FLAG_SOLID: + h.flags |= RAR_MAIN_SOLID + if h.main_flags & RAR5_MAIN_FLAG_ISVOL: + h.flags |= RAR_MAIN_VOLUME + if h.main_flags & RAR5_MAIN_FLAG_RECOVERY: + h.flags |= RAR_MAIN_RECOVERY + if self._hdrenc_main: + h.flags |= RAR_MAIN_PASSWORD + if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR == 0: + h.flags |= RAR_MAIN_FIRSTVOLUME + + return h + + def _parse_file_block(self, h, hdata, pos): + h.file_flags, pos = load_vint(hdata, pos) + h.file_size, pos = load_vint(hdata, pos) + h.mode, pos = load_vint(hdata, pos) + + if h.file_flags & RAR5_FILE_FLAG_HAS_MTIME: + h.mtime, pos = load_unixtime(hdata, pos) + h.date_time = h.mtime.timetuple()[:6] + if h.file_flags & RAR5_FILE_FLAG_HAS_CRC32: + h.CRC, pos = load_le32(hdata, pos) + h._md_class = CRC32Context + h._md_expect = h.CRC + + h.file_compress_flags, pos = load_vint(hdata, pos) + h.file_host_os, pos = load_vint(hdata, pos) + h.orig_filename, pos = load_vstr(hdata, pos) + h.filename = h.orig_filename.decode('utf8', 'replace') + + # use compatible values + if h.file_host_os == RAR5_OS_WINDOWS: + h.host_os = RAR_OS_WIN32 + else: + h.host_os = RAR_OS_UNIX + h.compress_type = RAR_M0 + ((h.file_compress_flags >> 7) & 7) + + if h.block_extra_size: + # allow 1 byte of garbage + while pos < len(hdata) - 1: + xsize, pos = load_vint(hdata, pos) + xdata, pos = load_bytes(hdata, xsize, pos) + self._process_file_extra(h, xdata) + + if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE: + h.flags |= RAR_FILE_SPLIT_BEFORE + if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_AFTER: + h.flags |= RAR_FILE_SPLIT_AFTER + if h.file_flags & RAR5_FILE_FLAG_ISDIR: + h.flags |= RAR_FILE_DIRECTORY + if h.file_compress_flags & RAR5_COMPR_SOLID: + h.flags |= RAR_FILE_SOLID + + return h + + def _parse_endarc_block(self, h, hdata, pos): + h.endarc_flags, pos = load_vint(hdata, pos) + if h.endarc_flags & RAR5_ENDARC_FLAG_NEXT_VOL: + h.flags |= RAR_ENDARC_NEXT_VOLUME + return h + + def _parse_encryption_block(self, h, hdata, pos): + h.encryption_algo, pos = load_vint(hdata, pos) + h.encryption_flags, pos = load_vint(hdata, pos) + h.encryption_kdf_count, pos = load_byte(hdata, pos) + h.encryption_salt, pos = load_bytes(hdata, 16, pos) + if h.encryption_flags & RAR5_ENC_FLAG_HAS_CHECKVAL: + h.encryption_check_value = load_bytes(hdata, 12, pos) + if h.encryption_algo != RAR5_XENC_CIPHER_AES256: + raise BadRarFile('Unsupported header encryption cipher') + self._hdrenc_main = h + return h + + # file extra record + def _process_file_extra(self, h, xdata): + xtype, pos = load_vint(xdata, 0) + if xtype == RAR5_XFILE_TIME: + self._parse_file_xtime(h, xdata, pos) + elif xtype == RAR5_XFILE_ENCRYPTION: + self._parse_file_encryption(h, xdata, pos) + elif xtype == RAR5_XFILE_HASH: + self._parse_file_hash(h, xdata, pos) + elif xtype == RAR5_XFILE_VERSION: + self._parse_file_version(h, xdata, pos) + elif xtype == RAR5_XFILE_REDIR: + self._parse_file_redir(h, xdata, pos) + elif xtype == RAR5_XFILE_OWNER: + self._parse_file_owner(h, xdata, pos) + elif xtype == RAR5_XFILE_SERVICE: + pass + else: + pass + + # extra block for file time record + def _parse_file_xtime(self, h, xdata, pos): + tflags, pos = load_vint(xdata, pos) + ldr = load_windowstime + if tflags & RAR5_XTIME_UNIXTIME: + ldr = load_unixtime + if tflags & RAR5_XTIME_HAS_MTIME: + h.mtime, pos = ldr(xdata, pos) + h.date_time = h.mtime.timetuple()[:6] + if tflags & RAR5_XTIME_HAS_CTIME: + h.ctime, pos = ldr(xdata, pos) + if tflags & RAR5_XTIME_HAS_ATIME: + h.atime, pos = ldr(xdata, pos) + + # just remember encryption info + def _parse_file_encryption(self, h, xdata, pos): + algo, pos = load_vint(xdata, pos) + flags, pos = load_vint(xdata, pos) + kdf_count, pos = load_byte(xdata, pos) + salt, pos = load_bytes(xdata, 16, pos) + iv, pos = load_bytes(xdata, 16, pos) + checkval = None + if flags & RAR5_XENC_CHECKVAL: + checkval, pos = load_bytes(xdata, 12, pos) + if flags & RAR5_XENC_TWEAKED: + h._md_expect = None + h._md_class = NoHashContext + + h.file_encryption = (algo, flags, kdf_count, salt, iv, checkval) + h.flags |= RAR_FILE_PASSWORD + + def _parse_file_hash(self, h, xdata, pos): + hash_type, pos = load_vint(xdata, pos) + if hash_type == RAR5_XHASH_BLAKE2SP: + h.blake2sp_hash, pos = load_bytes(xdata, 32, pos) + if _have_blake2 and (h.file_encryption[1] & RAR5_XENC_TWEAKED) == 0: + h._md_class = Blake2SP + h._md_expect = h.blake2sp_hash + + def _parse_file_version(self, h, xdata, pos): + flags, pos = load_vint(xdata, pos) + version, pos = load_vint(xdata, pos) + h.file_version = (flags, version) + + def _parse_file_redir(self, h, xdata, pos): + redir_type, pos = load_vint(xdata, pos) + redir_flags, pos = load_vint(xdata, pos) + redir_name, pos = load_vstr(xdata, pos) + redir_name = redir_name.decode('utf8', 'replace') + h.file_redir = (redir_type, redir_flags, redir_name) + + def _parse_file_owner(self, h, xdata, pos): + user_name = group_name = user_id = group_id = None + + flags, pos = load_vint(xdata, pos) + if flags & RAR5_XOWNER_UNAME: + user_name, pos = load_vstr(xdata, pos) + if flags & RAR5_XOWNER_GNAME: + group_name, pos = load_vstr(xdata, pos) + if flags & RAR5_XOWNER_UID: + user_id, pos = load_vint(xdata, pos) + if flags & RAR5_XOWNER_GID: + group_id, pos = load_vint(xdata, pos) + + h.file_owner = (user_name, group_name, user_id, group_id) + + def process_entry(self, fd, item): + if item.block_type == RAR5_BLOCK_FILE: + # use only first part + if (item.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE) == 0: + self._info_map[item.filename] = item + self._info_list.append(item) + elif len(self._info_list) > 0: + # final crc is in last block + old = self._info_list[-1] + old.CRC = item.CRC + old._md_expect = item._md_expect + old.blake2sp_hash = item.blake2sp_hash + old.compress_size += item.compress_size + elif item.block_type == RAR5_BLOCK_SERVICE: + if item.filename == 'CMT': + self._load_comment(fd, item) + + def _load_comment(self, fd, item): + if item.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): + return None + if item.compress_type != RAR_M0: + return None + + if item.flags & RAR_FILE_PASSWORD: + algo, ___flags, kdf_count, salt, iv, ___checkval = item.file_encryption + if algo != RAR5_XENC_CIPHER_AES256: + return None + key = self._gen_key(kdf_count, salt) + f = HeaderDecrypt(fd, key, iv) + cmt = f.read(item.file_size) + else: + # archive comment + with self._open_clear(item) as cmtstream: + cmt = cmtstream.read() + + # rar bug? - appends zero to comment + cmt = cmt.split(ZERO, 1)[0] + self.comment = cmt.decode('utf8') + + def _open_hack(self, inf, psw): + # len, type, blk_flags, flags + main_hdr = b'\x03\x01\x00\x00' + endarc_hdr = b'\x03\x05\x00\x00' + main_hdr = S_LONG.pack(rar_crc32(main_hdr)) + main_hdr + endarc_hdr = S_LONG.pack(rar_crc32(endarc_hdr)) + endarc_hdr + return self._open_hack_core(inf, psw, RAR5_ID + main_hdr, endarc_hdr) + +## +## Utility classes +## + +class UnicodeFilename(object): + """Handle RAR3 unicode filename decompression. + """ + def __init__(self, name, encdata): + self.std_name = bytearray(name) + self.encdata = bytearray(encdata) + self.pos = self.encpos = 0 + self.buf = bytearray() + self.failed = 0 + + def enc_byte(self): + """Copy encoded byte.""" + try: + c = self.encdata[self.encpos] + self.encpos += 1 + return c + except IndexError: + self.failed = 1 + return 0 + + def std_byte(self): + """Copy byte from 8-bit representation.""" + try: + return self.std_name[self.pos] + except IndexError: + self.failed = 1 + return ord('?') + + def put(self, lo, hi): + """Copy 16-bit value to result.""" + self.buf.append(lo) + self.buf.append(hi) + self.pos += 1 + + def decode(self): + """Decompress compressed UTF16 value.""" + hi = self.enc_byte() + flagbits = 0 + while self.encpos < len(self.encdata): + if flagbits == 0: + flags = self.enc_byte() + flagbits = 8 + flagbits -= 2 + t = (flags >> flagbits) & 3 + if t == 0: + self.put(self.enc_byte(), 0) + elif t == 1: + self.put(self.enc_byte(), hi) + elif t == 2: + self.put(self.enc_byte(), self.enc_byte()) + else: + n = self.enc_byte() + if n & 0x80: + c = self.enc_byte() + for _ in range((n & 0x7f) + 2): + lo = (self.std_byte() + c) & 0xFF + self.put(lo, hi) + else: + for _ in range(n + 2): + self.put(self.std_byte(), 0) + return self.buf.decode("utf-16le", "replace") + + +class RarExtFile(RawIOBase): + """Base class for file-like object that :meth:`RarFile.open` returns. + + Provides public methods and common crc checking. + + Behaviour: + - no short reads - .read() and .readinfo() read as much as requested. + - no internal buffer, use io.BufferedReader for that. + """ + + #: Filename of the archive entry + name = None + + def __init__(self, parser, inf): + super(RarExtFile, self).__init__() + + # standard io.* properties + self.name = inf.filename + self.mode = 'rb' + + self._parser = parser + self._inf = inf + self._fd = None + self._remain = 0 + self._returncode = 0 + + self._md_context = None + + self._open() + + def _open(self): + if self._fd: + self._fd.close() + md_class = self._inf._md_class or NoHashContext + self._md_context = md_class() + self._fd = None + self._remain = self._inf.file_size + + def read(self, cnt=None): + """Read all or specified amount of data from archive entry.""" + + # sanitize cnt + if cnt is None or cnt < 0: + cnt = self._remain + elif cnt > self._remain: + cnt = self._remain + if cnt == 0: + return EMPTY + + # actual read + data = self._read(cnt) + if data: + self._md_context.update(data) + self._remain -= len(data) + if len(data) != cnt: + raise BadRarFile("Failed the read enough data") + + # done? + if not data or self._remain == 0: + # self.close() + self._check() + return data + + def _check(self): + """Check final CRC.""" + final = self._md_context.digest() + exp = self._inf._md_expect + if exp is None: + return + if final is None: + return + if self._returncode: + check_returncode(self, '') + if self._remain != 0: + raise BadRarFile("Failed the read enough data") + if final != exp: + raise BadRarFile("Corrupt file - CRC check failed: %s - exp=%r got=%r" % ( + self._inf.filename, exp, final)) + + def _read(self, cnt): + """Actual read that gets sanitized cnt.""" + + def close(self): + """Close open resources.""" + + super(RarExtFile, self).close() + + if self._fd: + self._fd.close() + self._fd = None + + def __del__(self): + """Hook delete to make sure tempfile is removed.""" + self.close() + + def readinto(self, buf): + """Zero-copy read directly into buffer. + + Returns bytes read. + """ + raise NotImplementedError('readinto') + + def tell(self): + """Return current reading position in uncompressed data.""" + return self._inf.file_size - self._remain + + def seek(self, ofs, whence=0): + """Seek in data. + + On uncompressed files, the seeking works by actual + seeks so it's fast. On compresses files its slow + - forward seeking happends by reading ahead, + backwards by re-opening and decompressing from the start. + """ + + # disable crc check when seeking + self._md_context = NoHashContext() + + fsize = self._inf.file_size + cur_ofs = self.tell() + + if whence == 0: # seek from beginning of file + new_ofs = ofs + elif whence == 1: # seek from current position + new_ofs = cur_ofs + ofs + elif whence == 2: # seek from end of file + new_ofs = fsize + ofs + else: + raise ValueError('Invalid value for whence') + + # sanity check + if new_ofs < 0: + new_ofs = 0 + elif new_ofs > fsize: + new_ofs = fsize + + # do the actual seek + if new_ofs >= cur_ofs: + self._skip(new_ofs - cur_ofs) + else: + # reopen and seek + self._open() + self._skip(new_ofs) + return self.tell() + + def _skip(self, cnt): + """Read and discard data""" + while cnt > 0: + if cnt > 8192: + buf = self.read(8192) + else: + buf = self.read(cnt) + if not buf: + break + cnt -= len(buf) + + def readable(self): + """Returns True""" + return True + + def writable(self): + """Returns False. + + Writing is not supported.""" + return False + + def seekable(self): + """Returns True. + + Seeking is supported, although it's slow on compressed files. + """ + return True + + def readall(self): + """Read all remaining data""" + # avoid RawIOBase default impl + return self.read() + + +class PipeReader(RarExtFile): + """Read data from pipe, handle tempfile cleanup.""" + + def __init__(self, rf, inf, cmd, tempfile=None): + self._cmd = cmd + self._proc = None + self._tempfile = tempfile + super(PipeReader, self).__init__(rf, inf) + + def _close_proc(self): + if not self._proc: + return + if self._proc.stdout: + self._proc.stdout.close() + if self._proc.stdin: + self._proc.stdin.close() + if self._proc.stderr: + self._proc.stderr.close() + self._proc.wait() + self._returncode = self._proc.returncode + self._proc = None + + def _open(self): + super(PipeReader, self)._open() + + # stop old process + self._close_proc() + + # launch new process + self._returncode = 0 + self._proc = custom_popen(self._cmd) + self._fd = self._proc.stdout + + # avoid situation where unrar waits on stdin + if self._proc.stdin: + self._proc.stdin.close() + + def _read(self, cnt): + """Read from pipe.""" + + # normal read is usually enough + data = self._fd.read(cnt) + if len(data) == cnt or not data: + return data + + # short read, try looping + buf = [data] + cnt -= len(data) + while cnt > 0: + data = self._fd.read(cnt) + if not data: + break + cnt -= len(data) + buf.append(data) + return EMPTY.join(buf) + + def close(self): + """Close open resources.""" + + self._close_proc() + super(PipeReader, self).close() + + if self._tempfile: + try: + os.unlink(self._tempfile) + except OSError: + pass + self._tempfile = None + + def readinto(self, buf): + """Zero-copy read directly into buffer.""" + cnt = len(buf) + if cnt > self._remain: + cnt = self._remain + vbuf = memoryview(buf) + res = got = 0 + while got < cnt: + res = self._fd.readinto(vbuf[got : cnt]) + if not res: + break + self._md_context.update(vbuf[got : got + res]) + self._remain -= res + got += res + return got + + +class DirectReader(RarExtFile): + """Read uncompressed data directly from archive. + """ + _cur = None + _cur_avail = None + _volfile = None + + def _open(self): + super(DirectReader, self)._open() + + self._volfile = self._inf.volume_file + self._fd = XFile(self._volfile, 0) + self._fd.seek(self._inf.header_offset, 0) + self._cur = self._parser._parse_header(self._fd) + self._cur_avail = self._cur.add_size + + def _skip(self, cnt): + """RAR Seek, skipping through rar files to get to correct position + """ + + while cnt > 0: + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # fd is in read pos, do the read + if cnt > self._cur_avail: + cnt -= self._cur_avail + self._remain -= self._cur_avail + self._cur_avail = 0 + else: + self._fd.seek(cnt, 1) + self._cur_avail -= cnt + self._remain -= cnt + cnt = 0 + + def _read(self, cnt): + """Read from potentially multi-volume archive.""" + + buf = [] + while cnt > 0: + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # fd is in read pos, do the read + if cnt > self._cur_avail: + data = self._fd.read(self._cur_avail) + else: + data = self._fd.read(cnt) + if not data: + break + + # got some data + cnt -= len(data) + self._cur_avail -= len(data) + buf.append(data) + + if len(buf) == 1: + return buf[0] + return EMPTY.join(buf) + + def _open_next(self): + """Proceed to next volume.""" + + # is the file split over archives? + if (self._cur.flags & RAR_FILE_SPLIT_AFTER) == 0: + return False + + if self._fd: + self._fd.close() + self._fd = None + + # open next part + self._volfile = self._parser._next_volname(self._volfile) + fd = open(self._volfile, "rb", 0) + self._fd = fd + sig = fd.read(len(self._parser._expect_sig)) + if sig != self._parser._expect_sig: + raise BadRarFile("Invalid signature") + + # loop until first file header + while 1: + cur = self._parser._parse_header(fd) + if not cur: + raise BadRarFile("Unexpected EOF") + if cur.type in (RAR_BLOCK_MARK, RAR_BLOCK_MAIN): + if cur.add_size: + fd.seek(cur.add_size, 1) + continue + if cur.orig_filename != self._inf.orig_filename: + raise BadRarFile("Did not found file entry") + self._cur = cur + self._cur_avail = cur.add_size + return True + + def readinto(self, buf): + """Zero-copy read directly into buffer.""" + got = 0 + vbuf = memoryview(buf) + while got < len(buf): + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # length for next read + cnt = len(buf) - got + if cnt > self._cur_avail: + cnt = self._cur_avail + + # read into temp view + res = self._fd.readinto(vbuf[got : got + cnt]) + if not res: + break + self._md_context.update(vbuf[got : got + res]) + self._cur_avail -= res + self._remain -= res + got += res + return got + + +class HeaderDecrypt(object): + """File-like object that decrypts from another file""" + def __init__(self, f, key, iv): + self.f = f + self.ciph = AES_CBC_Decrypt(key, iv) + self.buf = EMPTY + + def tell(self): + """Current file pos - works only on block boundaries.""" + return self.f.tell() + + def read(self, cnt=None): + """Read and decrypt.""" + if cnt > 8 * 1024: + raise BadRarFile('Bad count to header decrypt - wrong password?') + + # consume old data + if cnt <= len(self.buf): + res = self.buf[:cnt] + self.buf = self.buf[cnt:] + return res + res = self.buf + self.buf = EMPTY + cnt -= len(res) + + # decrypt new data + blklen = 16 + while cnt > 0: + enc = self.f.read(blklen) + if len(enc) < blklen: + break + dec = self.ciph.decrypt(enc) + if cnt >= len(dec): + res += dec + cnt -= len(dec) + else: + res += dec[:cnt] + self.buf = dec[cnt:] + cnt = 0 + + return res + + +# handle (filename|filelike) object +class XFile(object): + """Input may be filename or file object. + """ + __slots__ = ('_fd', '_need_close') + + def __init__(self, xfile, bufsize=1024): + if is_filelike(xfile): + self._need_close = False + self._fd = xfile + self._fd.seek(0) + else: + self._need_close = True + self._fd = open(xfile, 'rb', bufsize) + + def read(self, n=None): + """Read from file.""" + return self._fd.read(n) + + def tell(self): + """Return file pos.""" + return self._fd.tell() + + def seek(self, ofs, whence=0): + """Move file pos.""" + return self._fd.seek(ofs, whence) + + def readinto(self, dst): + """Read into buffer.""" + return self._fd.readinto(dst) + + def close(self): + """Close file object.""" + if self._need_close: + self._fd.close() + + def __enter__(self): + return self + + def __exit__(self, typ, val, tb): + self.close() + + +class NoHashContext(object): + """No-op hash function.""" + def __init__(self, data=None): + """Initialize""" + def update(self, data): + """Update data""" + def digest(self): + """Final hash""" + def hexdigest(self): + """Hexadecimal digest.""" + + +class CRC32Context(object): + """Hash context that uses CRC32.""" + __slots__ = ['_crc'] + + def __init__(self, data=None): + self._crc = 0 + if data: + self.update(data) + + def update(self, data): + """Process data.""" + self._crc = rar_crc32(data, self._crc) + + def digest(self): + """Final hash.""" + return self._crc + + def hexdigest(self): + """Hexadecimal digest.""" + return '%08x' % self.digest() + + +class Blake2SP(object): + """Blake2sp hash context. + """ + __slots__ = ['_thread', '_buf', '_cur', '_digest'] + digest_size = 32 + block_size = 64 + parallelism = 8 + + def __init__(self, data=None): + self._buf = b'' + self._cur = 0 + self._digest = None + self._thread = [] + + for i in range(self.parallelism): + ctx = self._blake2s(i, 0, i == (self.parallelism - 1)) + self._thread.append(ctx) + + if data: + self.update(data) + + def _blake2s(self, ofs, depth, is_last): + return blake2s(node_offset=ofs, node_depth=depth, last_node=is_last, + depth=2, inner_size=32, fanout=self.parallelism) + + def _add_block(self, blk): + self._thread[self._cur].update(blk) + self._cur = (self._cur + 1) % self.parallelism + + def update(self, data): + """Hash data. + """ + view = memoryview(data) + bs = self.block_size + if self._buf: + need = bs - len(self._buf) + if len(view) < need: + self._buf += view.tobytes() + return + self._add_block(self._buf + view[:need].tobytes()) + view = view[need:] + while len(view) >= bs: + self._add_block(view[:bs]) + view = view[bs:] + self._buf = view.tobytes() + + def digest(self): + """Return final digest value. + """ + if self._digest is None: + if self._buf: + self._add_block(self._buf) + self._buf = EMPTY + ctx = self._blake2s(0, 1, True) + for t in self._thread: + ctx.update(t.digest()) + self._digest = ctx.digest() + return self._digest + + def hexdigest(self): + """Hexadecimal digest.""" + return tohex(self.digest()) + +## +## Utility functions +## + +S_LONG = Struct(' len(buf): + raise BadRarFile('cannot load byte') + return S_BYTE.unpack_from(buf, pos)[0], end + +def load_le32(buf, pos): + """Load little-endian 32-bit integer""" + end = pos + 4 + if end > len(buf): + raise BadRarFile('cannot load le32') + return S_LONG.unpack_from(buf, pos)[0], pos + 4 + +def load_bytes(buf, num, pos): + """Load sequence of bytes""" + end = pos + num + if end > len(buf): + raise BadRarFile('cannot load bytes') + return buf[pos : end], end + +def load_vstr(buf, pos): + """Load bytes prefixed by vint length""" + slen, pos = load_vint(buf, pos) + return load_bytes(buf, slen, pos) + +def load_dostime(buf, pos): + """Load LE32 dos timestamp""" + stamp, pos = load_le32(buf, pos) + tup = parse_dos_time(stamp) + return to_datetime(tup), pos + +def load_unixtime(buf, pos): + """Load LE32 unix timestamp""" + secs, pos = load_le32(buf, pos) + dt = datetime.fromtimestamp(secs, UTC) + return dt, pos + +def load_windowstime(buf, pos): + """Load LE64 windows timestamp""" + # unix epoch (1970) in seconds from windows epoch (1601) + unix_epoch = 11644473600 + val1, pos = load_le32(buf, pos) + val2, pos = load_le32(buf, pos) + secs, n1secs = divmod((val2 << 32) | val1, 10000000) + dt = datetime.fromtimestamp(secs - unix_epoch, UTC) + dt = dt.replace(microsecond=n1secs // 10) + return dt, pos + +# new-style next volume +def _next_newvol(volfile): + i = len(volfile) - 1 + while i >= 0: + if volfile[i] >= '0' and volfile[i] <= '9': + return _inc_volname(volfile, i) + i -= 1 + raise BadRarName("Cannot construct volume name: " + volfile) + +# old-style next volume +def _next_oldvol(volfile): + # rar -> r00 + if volfile[-4:].lower() == '.rar': + return volfile[:-2] + '00' + return _inc_volname(volfile, len(volfile) - 1) + +# increase digits with carry, otherwise just increment char +def _inc_volname(volfile, i): + fn = list(volfile) + while i >= 0: + if fn[i] != '9': + fn[i] = chr(ord(fn[i]) + 1) + break + fn[i] = '0' + i -= 1 + return ''.join(fn) + +# rar3 extended time fields +def _parse_ext_time(h, data, pos): + # flags and rest of data can be missing + flags = 0 + if pos + 2 <= len(data): + flags = S_SHORT.unpack_from(data, pos)[0] + pos += 2 + + mtime, pos = _parse_xtime(flags >> 3 * 4, data, pos, h.mtime) + h.ctime, pos = _parse_xtime(flags >> 2 * 4, data, pos) + h.atime, pos = _parse_xtime(flags >> 1 * 4, data, pos) + h.arctime, pos = _parse_xtime(flags >> 0 * 4, data, pos) + if mtime: + h.mtime = mtime + h.date_time = mtime.timetuple()[:6] + return pos + +# rar3 one extended time field +def _parse_xtime(flag, data, pos, basetime=None): + res = None + if flag & 8: + if not basetime: + basetime, pos = load_dostime(data, pos) + + # load second fractions + rem = 0 + cnt = flag & 3 + for _ in range(cnt): + b, pos = load_byte(data, pos) + rem = (b << 16) | (rem >> 8) + + # convert 100ns units to microseconds + usec = rem // 10 + if usec > 1000000: + usec = 999999 + + # dostime has room for 30 seconds only, correct if needed + if flag & 4 and basetime.second < 59: + res = basetime.replace(microsecond=usec, second=basetime.second + 1) + else: + res = basetime.replace(microsecond=usec) + return res, pos + +def is_filelike(obj): + """Filename or file object? + """ + if isinstance(obj, str) or isinstance(obj, unicode): + return False + res = True + for a in ('read', 'tell', 'seek'): + res = res and hasattr(obj, a) + if not res: + raise ValueError("Invalid object passed as file") + return True + +def rar3_s2k(psw, salt): + """String-to-key hash for RAR3. + """ + if not isinstance(psw, unicode): + psw = psw.decode('utf8') + seed = psw.encode('utf-16le') + salt + iv = EMPTY + h = sha1() + for i in range(16): + for j in range(0x4000): + cnt = S_LONG.pack(i * 0x4000 + j) + h.update(seed + cnt[:3]) + if j == 0: + iv += h.digest()[19:20] + key_be = h.digest()[:16] + key_le = pack("LLLL", key_be)) + return key_le, iv + +def rar3_decompress(vers, meth, data, declen=0, flags=0, crc=0, psw=None, salt=None): + """Decompress blob of compressed data. + + Used for data with non-standard header - eg. comments. + """ + # already uncompressed? + if meth == RAR_M0 and (flags & RAR_FILE_PASSWORD) == 0: + return data + + # take only necessary flags + flags = flags & (RAR_FILE_PASSWORD | RAR_FILE_SALT | RAR_FILE_DICTMASK) + flags |= RAR_LONG_BLOCK + + # file header + fname = b'data' + date = 0 + mode = 0x20 + fhdr = S_FILE_HDR.pack(len(data), declen, RAR_OS_MSDOS, crc, + date, vers, meth, len(fname), mode) + fhdr += fname + if flags & RAR_FILE_SALT: + if not salt: + return EMPTY + fhdr += salt + + # full header + hlen = S_BLK_HDR.size + len(fhdr) + hdr = S_BLK_HDR.pack(0, RAR_BLOCK_FILE, flags, hlen) + fhdr + hcrc = rar_crc32(hdr[2:]) & 0xFFFF + hdr = S_BLK_HDR.pack(hcrc, RAR_BLOCK_FILE, flags, hlen) + fhdr + + # archive main header + mh = S_BLK_HDR.pack(0x90CF, RAR_BLOCK_MAIN, 0, 13) + ZERO * (2 + 4) + + # decompress via temp rar + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + try: + tmpf.write(RAR_ID + mh + hdr + data) + tmpf.close() + + cmd = [UNRAR_TOOL] + list(OPEN_ARGS) + add_password_arg(cmd, psw, (flags & RAR_FILE_PASSWORD)) + cmd.append(tmpname) + + p = custom_popen(cmd) + return p.communicate()[0] + finally: + tmpf.close() + os.unlink(tmpname) + +def to_datetime(t): + """Convert 6-part time tuple into datetime object. + """ + if t is None: + return None + + # extract values + year, mon, day, h, m, s = t + + # assume the values are valid + try: + return datetime(year, mon, day, h, m, s) + except ValueError: + pass + + # sanitize invalid values + mday = (0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) + if mon < 1: + mon = 1 + if mon > 12: + mon = 12 + if day < 1: + day = 1 + if day > mday[mon]: + day = mday[mon] + if h > 23: + h = 23 + if m > 59: + m = 59 + if s > 59: + s = 59 + if mon == 2 and day == 29: + try: + return datetime(year, mon, day, h, m, s) + except ValueError: + day = 28 + return datetime(year, mon, day, h, m, s) + +def parse_dos_time(stamp): + """Parse standard 32-bit DOS timestamp. + """ + sec, stamp = stamp & 0x1F, stamp >> 5 + mn, stamp = stamp & 0x3F, stamp >> 6 + hr, stamp = stamp & 0x1F, stamp >> 5 + day, stamp = stamp & 0x1F, stamp >> 5 + mon, stamp = stamp & 0x0F, stamp >> 4 + yr = (stamp & 0x7F) + 1980 + return (yr, mon, day, hr, mn, sec * 2) + +def custom_popen(cmd): + """Disconnect cmd from parent fds, read only from stdout. + """ + # needed for py2exe + creationflags = 0 + if sys.platform == 'win32': + creationflags = 0x08000000 # CREATE_NO_WINDOW + + # run command + try: + p = Popen(cmd, bufsize=0, stdout=PIPE, stdin=PIPE, stderr=STDOUT, + creationflags=creationflags) + except OSError as ex: + if ex.errno == errno.ENOENT: + raise RarCannotExec("Unrar not installed? (rarfile.UNRAR_TOOL=%r)" % UNRAR_TOOL) + raise + return p + +def custom_check(cmd, ignore_retcode=False): + """Run command, collect output, raise error if needed. + """ + p = custom_popen(cmd) + out, _ = p.communicate() + if p.returncode and not ignore_retcode: + raise RarExecError("Check-run failed") + return out + +def add_password_arg(cmd, psw, ___required=False): + """Append password switch to commandline. + """ + if UNRAR_TOOL == ALT_TOOL: + return + if psw is not None: + cmd.append('-p' + psw) + else: + cmd.append('-p-') + +def check_returncode(p, out): + """Raise exception according to unrar exit code. + """ + code = p.returncode + if code == 0: + return + + # map return code to exception class, codes from rar.txt + errmap = [None, + RarWarning, RarFatalError, RarCRCError, RarLockedArchiveError, # 1..4 + RarWriteError, RarOpenError, RarUserError, RarMemoryError, # 5..8 + RarCreateError, RarNoFilesError, RarWrongPassword] # 9..11 + if UNRAR_TOOL == ALT_TOOL: + errmap = [None] + if code > 0 and code < len(errmap): + exc = errmap[code] + elif code == 255: + exc = RarUserBreak + elif code < 0: + exc = RarSignalExit + else: + exc = RarUnknownError + + # format message + if out: + msg = "%s [%d]: %s" % (exc.__doc__, p.returncode, out) + else: + msg = "%s [%d]" % (exc.__doc__, p.returncode) + + raise exc(msg) + +def hmac_sha256(key, data): + """HMAC-SHA256""" + return HMAC(key, data, sha256).digest() + +def membuf_tempfile(memfile): + memfile.seek(0, 0) + + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + + try: + while True: + buf = memfile.read(BSIZE) + if not buf: + break + tmpf.write(buf) + tmpf.close() + except: + tmpf.close() + os.unlink(tmpname) + raise + return tmpname + +class XTempFile(object): + __slots__ = ('_tmpfile', '_filename') + + def __init__(self, rarfile): + if is_filelike(rarfile): + self._tmpfile = membuf_tempfile(rarfile) + self._filename = self._tmpfile + else: + self._tmpfile = None + self._filename = rarfile + + def __enter__(self): + return self._filename + + def __exit__(self, exc_type, exc_value, tb): + if self._tmpfile: + try: + os.unlink(self._tmpfile) + except OSError: + pass + self._tmpfile = None + +# +# Check if unrar works +# + +ORIG_UNRAR_TOOL = UNRAR_TOOL +ORIG_OPEN_ARGS = OPEN_ARGS +ORIG_EXTRACT_ARGS = EXTRACT_ARGS +ORIG_TEST_ARGS = TEST_ARGS + +def _check_unrar_tool(): + global UNRAR_TOOL, OPEN_ARGS, EXTRACT_ARGS, TEST_ARGS + try: + # does UNRAR_TOOL work? + custom_check([ORIG_UNRAR_TOOL], True) + + UNRAR_TOOL = ORIG_UNRAR_TOOL + OPEN_ARGS = ORIG_OPEN_ARGS + EXTRACT_ARGS = ORIG_EXTRACT_ARGS + TEST_ARGS = ORIG_TEST_ARGS + except RarCannotExec: + try: + # does ALT_TOOL work? + custom_check([ALT_TOOL] + list(ALT_CHECK_ARGS), True) + # replace config + UNRAR_TOOL = ALT_TOOL + OPEN_ARGS = ALT_OPEN_ARGS + EXTRACT_ARGS = ALT_EXTRACT_ARGS + TEST_ARGS = ALT_TEST_ARGS + except RarCannotExec: + # no usable tool, only uncompressed archives work + pass + +_check_unrar_tool() From 470e8c621ba703a40e5c5f0c01b8d03aead5e97d Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:55:39 +0000 Subject: [PATCH 034/185] Create DeleteSamples.py --- .../nzbget-mp4/scripts/DeleteSamples.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/DeleteSamples.py diff --git a/apps/templates/nzbget-mp4/scripts/DeleteSamples.py b/apps/templates/nzbget-mp4/scripts/DeleteSamples.py new file mode 100644 index 0000000..395628f --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/DeleteSamples.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +def is_sample(filePath, inputName, maxSampleSize, SampleIDs): + # 200 MB in bytes + SIZE_CUTOFF = int(maxSampleSize) * 1024 * 1024 + if os.path.getsize(filePath) < SIZE_CUTOFF: + if 'SizeOnly' in SampleIDs: + return True + # Ignore 'sample' in files unless 'sample' in Torrent Name + for ident in SampleIDs: + if ident.lower() in filePath.lower() and not ident.lower() in inputName.lower(): + return True + # Return False if none of these were met. + return False + +if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + +if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + +print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) +status = 0 +if os.environ.has_key('NZBPP_TOTALSTATUS'): + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + +else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." + print "Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." + print "Please check your Par-check/repair settings for future downloads." + +# Check if destination directory exists (important for reprocessing of history items) +if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + +# All checks done, now launching the script. +if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + +mediaContainer = os.environ['NZBPO_MEDIAEXTENSIONS'].split(',') +SampleIDs = os.environ['NZBPO_SAMPLEIDS'].split(',') +for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): + for file in filenames: + filePath = os.path.join(dirpath, file) + fileName, fileExtension = os.path.splitext(file) + if fileExtension in mediaContainer or ".*" in mediaContainer : # If the file is a video file + if is_sample(filePath, os.environ['NZBPP_NZBNAME'], os.environ['NZBPO_MAXSAMPLESIZE'], SampleIDs): # Ignore samples + print "Deleting sample file: ", filePath + try: + os.unlink(filePath) + except: + print "Error: unable to delete file", filePath + sys.exit(NZBGET_POSTPROCESS_ERROR) +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From bfeef308c55b3903eec671025493e600ad21e624 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:56:01 +0000 Subject: [PATCH 035/185] Create flatten.py --- apps/templates/nzbget-mp4/scripts/flatten.py | 106 +++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/flatten.py diff --git a/apps/templates/nzbget-mp4/scripts/flatten.py b/apps/templates/nzbget-mp4/scripts/flatten.py new file mode 100644 index 0000000..73dd0d5 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/flatten.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys +import shutil + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + +if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + +print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) +status = 0 +if os.environ.has_key('NZBPP_TOTALSTATUS'): + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + +else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "[ERROR] Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "[ERROR] Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." + print "[ERROR] Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." + print "[WARNING] Please check your Par-check/repair settings for future downloads." + +# Check if destination directory exists (important for reprocessing of history items) +if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "[ERROR] Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + +# All checks done, now launching the script. +if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + +def removeEmptyFolders(path, removeRoot=True): + #Function to remove empty folders + if not os.path.isdir(path): + return + + # remove empty subfolders + print "[INFO] Checking for empty folders in:%s" % path + files = os.listdir(path) + if len(files): + for f in files: + fullpath = os.path.join(path, f) + if os.path.isdir(fullpath): + removeEmptyFolders(fullpath) + + # if folder empty, delete it + files = os.listdir(path) + if len(files) == 0 and removeRoot: + print "[INFO] Removing empty folder:%s" % path + os.rmdir(path) + +directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) +if os.environ['NZBPO_DESTINATIONDIRECTORY'] and os.path.isdir(os.environ['NZBPO_DESTINATIONDIRECTORY']): + destination = os.environ['NZBPO_DESTINATIONDIRECTORY'] + if os.environ['NZBPO_APPENDCATEGORIES'] == 'yes': + destination = os.path.join(destination, os.environ['NZBPP_CATEGORY']) +else: + destination = directory +print "Flattening directory: %s" % (directory) +for dirpath, dirnames, filenames in os.walk(directory): + for fileName in filenames: + outputFile = os.path.join(dirpath, fileName) + if dirpath == directory: + continue + target = os.path.join(destination, fileName) + try: + shutil.move(outputFile, target) + except: + print "[ERROR] Could not flatten %s" % outputFile +removeEmptyFolders(directory) # Cleanup empty directories +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From d64d8fccdc910ffc02a9ae5cb90e76a234bf5258 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:56:18 +0000 Subject: [PATCH 036/185] Create hash.py --- apps/templates/nzbget-mp4/scripts/hash.py | 167 ++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/hash.py diff --git a/apps/templates/nzbget-mp4/scripts/hash.py b/apps/templates/nzbget-mp4/scripts/hash.py new file mode 100644 index 0000000..f2fbec1 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/hash.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Original Author: clinton-hall +# https://github.com/clinton-hall/GetScripts/blob/master/SafeRename.py +# +# Modified By: desimaniac (No Acknowledgement of Source Above) +################################################################################ +import os +import re +import shutil +import sys + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +############################################################ +# EXTENSION STUFF +############################################################ + +def do_check(): + if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + + if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + + print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) + + status = 0 + if 'NZBPP_TOTALSTATUS' in os.environ: + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "[ERROR] Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "[ERROR] Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. " \ + "Setting status \"failed\"." + print "[ERROR] Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is " \ + "ok so handle as though download successful." + print "[WARNING] Please check your Par-check/repair settings for future downloads." + + # Check if destination directory exists (important for reprocessing of history items) + if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "[ERROR] Nothing to post-process: destination directory", os.environ[ + 'NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + + # All checks done, now launching the script. + if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + + +def get_file_name(path): + try: + file_name = os.path.basename(path) + extensions = re.findall(r'\.([^.]+)', file_name) + ext = '.'.join(extensions) + name = file_name.replace(".%s" % ext, '') + return name, ext + except Exception: + pass + return None + + +def is_file_hash(file_name): + hash_regexp = [ + r'^[a-fA-F0-9]{40}$', + r'^[a-fA-F0-9]{32}$', + r'^[a-f0-9]{128}$', + r'^[a-zA-Z0-9]{42}$' + ] + for hash in hash_regexp: + if re.match(hash, file_name): + return True + return False + + +def find_files(folder, extension=None, depth=None): + file_list = [] + start_count = folder.count(os.sep) + for path, subdirs, files in os.walk(folder, topdown=True): + for name in files: + if depth and path.count(os.sep) - start_count >= depth: + del subdirs[:] + continue + file = os.path.join(path, name) + if not extension: + file_list.append(file) + else: + if file.lower().endswith(extension.lower()): + file_list.append(file) + + return sorted(file_list, key=lambda x: x.count(os.path.sep), reverse=True) + + +############################################################ +# MAIN +############################################################ + +# do checks +do_check() + +# retrieve required variables +directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) +nzb_name = os.environ['NZBPP_NZBFILENAME'] +if nzb_name is None: + print("[ERROR] Unable to retrieve NZBPP_NZBFILENAME") + sys.exit(NZBGET_POSTPROCESS_ERROR) +nzb_name = nzb_name.replace('.nzb', '') + +print("[INFO] Using \"%s\" for hashed filenames" % nzb_name) +print("[INFO] Scanning \"%s\" for hashed filenames" % directory) + +# scan for files +found_files = find_files(directory) +if not found_files: + print("[INFO] No files were found in \"%s\"" % directory) + sys.exit(NZBGET_POSTPROCESS_NONE) +else: + print("[INFO] Found %d files to check for hashed filenames" % len(found_files)) + # loop files checking for file hash + moved_files = 0 + for found_file_path in found_files: + # set variable + dir_name = os.path.dirname(found_file_path) + file_name, file_ext = get_file_name(found_file_path) + + # is this a file hash + if is_file_hash(file_name): + new_file_path = os.path.join(dir_name, "%s.%s" % (nzb_name, file_ext)) + print("[INFO] Moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) + try: + shutil.move(found_file_path, new_file_path) + moved_files += 1 + except Exception: + print("[ERROR] Failed moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) + + print("[INFO] Finished processing \"%s\", moved %d files" % (directory, moved_files)) + +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From 4ca7a8e0d67c1133c2c8e1456ff4383b8658dab7 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:56:40 +0000 Subject: [PATCH 037/185] Create reverse_name.py --- .../nzbget-mp4/scripts/reverse_name.py | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/reverse_name.py diff --git a/apps/templates/nzbget-mp4/scripts/reverse_name.py b/apps/templates/nzbget-mp4/scripts/reverse_name.py new file mode 100644 index 0000000..98819d0 --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/reverse_name.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys +import re +import locale + +reverse_list = [r"\.\d{2}e\d{2}s\.", r"\.p0612\.", r"\.[pi]0801\.", r"\.p027\.", r"\.[pi]675\.", r"\.[pi]084\.", r"\.p063\.", r"\b[45]62[xh]\.", r"\.yarulb\.", r"\.vtd[hp]\.", + r'\.(?:ld[.-]?)?bew\.', r"\.pir.?(shv|dov|bew|dvd|db|rb)\.", r"\brdvd\.", r"\.vts\.", r"\.reneercs\.", r"\.dcv\.", r"\b(pir|mac)dh\b", r"\.reporp\.", r"\.kcaper\.", + r"\.lanretni\.", r"\b3ca\b", r"\bcaa\b", r"\b3pm\b", r"\.cstn\.", r"\.5r\.", r"\brcs\b"] +reverse_pattern = re.compile('|'.join(reverse_list), flags=re.IGNORECASE) +season_pattern = re.compile(r"(.*\.\d{2}e\d{2}s\.)(.*)", flags=re.IGNORECASE) +word_pattern = re.compile(r"([^A-Z0-9]*[A-Z0-9]+)") +char_replace = [[r"(\w)1\.(\w)",r"\1i\2"] +] +garbage_name = re.compile(r"^[a-zA-Z0-9]{2,}$") +media_list = [r"\.s\d{2}e\d{2}\.", r"\.2160p\.", r"\.1080[pi]\.", r"\.720p\.", r"\.576[pi]\.", r"\.480[pi]\.", r"\.360p\.", r"\.[xh]26[45]\b", r"\.bluray\.", r"\.[hp]dtv\.", + r'\.web(?:[.-]?dl)?\.', r"\.(vhs|vod|dvd|web|bd|br).?rip\.", r"\.dvdr\b", r"\.stv\.", r"\.screener\.", r"\.vcd\.", r"\bhd(cam|rip)\b", r"\.proper\.", r"\.repack\.", + r"\.internal\.", r"\bac3\b", r"\baac\b", r"\bmp3\b", r"\.ntsc\.", r"\.pal\.", r"\.secam\.", r"\bdivx\b", r"\bxvid\b", r"\.r5\.", r"\.scr\."] +media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) +media_extentions = [".mkv", ".mp4", ".avi", ".wmv", ".divx", ".xvid"] + +if 'nt' == os.name: + import ctypes + + class WinEnv: + def __init__(self): + pass + + @staticmethod + def get_environment_variable(name): + name = unicode(name) # ensures string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + env_value = None + if n: + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + env_value = buf.value + return env_value + + def __getitem__(self, key): + return self.get_environment_variable(key) + + def get(self, key, default=None): + r = self.get_environment_variable(key) + return r if r is not None else default + + evn = WinEnv() +else: + class LinuxEnv(object): + def __init__(self, environ): + self.environ = environ + + def __getitem__(self, key): + v = self.environ.get(key) + try: + return v.decode(SYS_ENCODING) if isinstance(v, str) else v + except (UnicodeDecodeError, UnicodeEncodeError): + return v + + def get(self, key, default=None): + v = self[key] + return v if v is not None else default + + evn = LinuxEnv(os.environ) + +SYS_ENCODING = None + +try: + locale.setlocale(locale.LC_ALL, '') +except (locale.Error, IOError): + pass +try: + SYS_ENCODING = locale.getpreferredencoding() +except (locale.Error, IOError): + pass + +if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + SYS_ENCODING = 'UTF-8' + + +class ek: + def __init__(self): + pass + + @staticmethod + def fix_string_encoding(x): + if str == type(x): + try: + return x.decode(SYS_ENCODING) + except UnicodeDecodeError: + return None + elif unicode == type(x): + return x + return None + + @staticmethod + def fix_out_encoding(x): + if isinstance(x, basestring): + return ek.fix_string_encoding(x) + return x + + @staticmethod + def fix_list_encoding(x): + if type(x) not in (list, tuple): + return x + return filter(lambda i: None is not i, map(ek.fix_out_encoding, x)) + + @staticmethod + def encode_item(x): + try: + return x.encode(SYS_ENCODING) + except UnicodeEncodeError: + return x.encode(SYS_ENCODING, 'ignore') + + @staticmethod + def win_encode_unicode(x): + if isinstance(x, str): + try: + return x.decode('UTF-8') + except UnicodeDecodeError: + return x + return x + + @staticmethod + def ek(func, *args, **kwargs): + if 'nt' == os.name: + # convert all str parameter values to unicode + args = tuple([x if not isinstance(x, str) else ek.win_encode_unicode(x) for x in args]) + kwargs = {k: x if not isinstance(x, str) else ek.win_encode_unicode(x) for k, x in + kwargs.iteritems()} + func_result = func(*args, **kwargs) + else: + func_result = func(*[ek.encode_item(x) if type(x) == str else x for x in args], **kwargs) + + if type(func_result) in (list, tuple): + return ek.fix_list_encoding(func_result) + elif str == type(func_result): + return ek.fix_string_encoding(func_result) + return func_result + + +class logger: + INFO = 'INFO' + DETAIL = 'DETAIL' + ERROR = 'ERROR' + WARNING = 'WARNING' + + @staticmethod + def log(message, msg_type=INFO): + print('[%s] %s' % (msg_type, message)) + + +def tryInt(s, s_default=0): + try: + return int(s) + except: + return s_default + +# NZBGet V11+ +# Check if the script is called from nzbget 11.0 or later +nzbget_version = evn.get('NZBOP_VERSION', '0.1') +nzbget_version = tryInt(nzbget_version[:nzbget_version.find(".")]) +if nzbget_version >= 11: + logger.log("Script triggered from NZBGet (11.0 or later).") + + # NZBGet argv: all passed as environment variables. + clientAgent = "nzbget" + # Exit codes used by NZBGet + POSTPROCESS_PARCHECK=92 + POSTPROCESS_SUCCESS=93 + POSTPROCESS_ERROR=94 + POSTPROCESS_NONE=95 + + # Check nzbget.conf options + status = 0 + + if evn['NZBOP_UNPACK'] != 'yes': + logger.log("Please enable option \"Unpack\" in nzbget configuration file, exiting") + sys.exit(POSTPROCESS_NONE) + + parstatus = evn['NZBPP_PARSTATUS'] + + # Check par status + if parstatus == '3': + logger.log("Par-check successful, but Par-repair disabled, exiting") + sys.exit(POSTPROCESS_NONE) + + if parstatus == '1': + logger.log("Par-check failed, setting status \"failed\"") + status = 1 + sys.exit(POSTPROCESS_NONE) + + unpackstatus = evn['NZBPP_UNPACKSTATUS'] + + # Check unpack status + if unpackstatus == '1': + logger.log("Unpack failed, setting status \"failed\"") + status = 1 + sys.exit(POSTPROCESS_NONE) + + directory = evn['NZBPP_DIRECTORY'] + + if unpackstatus == '0' and parstatus != '2': + # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check + + for dirpath, dirnames, filenames in ek.ek(os.walk, directory): + for file in filenames: + fileExtension = ek.ek(os.path.splitext, file)[1] + + if fileExtension in ['.par2']: + logger.log("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\"g") + status = 1 + break + + if ek.ek(os.path.isfile, ek.ek(os.path.join, directory, "_brokenlog.txt")) and not status == 1: + logger.log("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting") + status = 1 + + if not status == 1: + logger.log("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful") + + # Check if destination directory exists (important for reprocessing of history items) + if not ek.ek(os.path.isdir, directory): + logger.log("Post-Process: Nothing to post-process: destination directory %s doesn't exist" % directory) + status = 1 + + # All checks done, now launching the script. + + rd = False + videos = 0 + old_name = None + new_name = None + base_name = ek.ek(os.path.basename, directory) + for dirpath, dirnames, filenames in ek.ek(os.walk, directory): + for file in filenames: + + filePath = ek.ek(os.path.join, dirpath, file) + fileName, fileExtension = ek.ek(os.path.splitext, file) + dirname = ek.ek(os.path.dirname, filePath) + + if reverse_pattern.search(fileName) is not None: + na_parts = season_pattern.search(fileName) + if na_parts is not None: + word_p = word_pattern.findall(na_parts.group(2)) + new_words = "" + for wp in word_p: + if wp[0] == ".": + new_words += "." + new_words += re.sub(r"\W","",wp) + for cr in char_replace: + new_words = re.sub(cr[0],cr[1],new_words) + new_filename = new_words[::-1] + na_parts.group(1)[::-1] + else: + new_filename = fileName[::-1] + logger.log("reversing filename from: %s to %s" % (fileName, new_filename)) + try: + ek.ek(os.rename, filePath, ek.ek(os.path.join, dirpath, new_filename + fileExtension)) + rd = True + except Exception,e: + logger.log(e, logger.ERROR) + logger.log("Error: unable to rename file %s" % file, logger.ERROR) + pass + elif (fileExtension.lower() in media_extentions) and (garbage_name.search(fileName) is not None) and (media_pattern.search(base_name) is not None): + videos += 1 + old_name = filePath + new_name = ek.ek(os.path.join, dirname, '%s%s' % (base_name, fileExtension)) + + if not rd and videos == 1 and old_name is not None and new_name is not None: + logger.log("renaming the File %s to the Dirname %s" % (ek.ek(os.path.basename, old_name), base_name)) + try: + ek.ek(os.rename, old_name, new_name) + rd = True + except Exception,e: + logger.log(e, logger.ERROR) + logger.log("Error unable to rename file %s" % old_name, logger.ERROR) + pass + + if rd: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_NONE) + +else: + logger.log("This script can only be called from NZBGet (11.0 or later).", logger.ERROR) + sys.exit(0) From 007070dcbb4f36df14861f00160a164d8cf93777 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 16:57:00 +0000 Subject: [PATCH 038/185] Create unzip.py --- apps/templates/nzbget-mp4/scripts/unzip.py | 294 +++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 apps/templates/nzbget-mp4/scripts/unzip.py diff --git a/apps/templates/nzbget-mp4/scripts/unzip.py b/apps/templates/nzbget-mp4/scripts/unzip.py new file mode 100644 index 0000000..521a50f --- /dev/null +++ b/apps/templates/nzbget-mp4/scripts/unzip.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# +############################################################################## +### NZBGET SCAN SCRIPT ### + +# Unzips zipped nzbs. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### +### NZBGET SCAN SCRIPT ### +############################################################################## + +import os, zipfile, tarfile, gzip, pickle, datetime, re, struct, locale +import rarfile.rarfile as rarfile + +from gzip import FEXTRA, FNAME + +if 'nt' == os.name: + import ctypes + + class WinEnv: + def __init__(self): + pass + + @staticmethod + def get_environment_variable(name): + name = unicode(name) # ensures string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + result = None + if n: + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + result = buf.value + return result + + def __getitem__(self, key): + return self.get_environment_variable(key) + + def get(self, key, default=None): + r = self.get_environment_variable(key) + return r if r is not None else default + + env_var = WinEnv() +else: + class LinuxEnv(object): + def __init__(self, environ): + self.environ = environ + + def __getitem__(self, key): + v = self.environ.get(key) + try: + return v.decode(SYS_ENCODING) if isinstance(v, str) else v + except (UnicodeDecodeError, UnicodeEncodeError): + return v + + def get(self, key, default=None): + v = self[key] + return v if v is not None else default + + env_var = LinuxEnv(os.environ) + + +SYS_ENCODING = None +try: + locale.setlocale(locale.LC_ALL, '') +except (locale.Error, IOError): + pass +try: + SYS_ENCODING = locale.getpreferredencoding() +except (locale.Error, IOError): + pass +if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + SYS_ENCODING = 'UTF-8' + + +filename = env_var.get('NZBNP_FILENAME') +if re.search(r"\.tar\.gz$", filename, flags=re.I) is None: + ext = os.path.splitext(filename)[1].lower() +else: + ext = '.tar.gz' +cat = env_var.get('NZBNP_CATEGORY') +dir = env_var.get('NZBNP_DIRECTORY') +prio = env_var.get('NZBNP_PRIORITY') +top = env_var.get('NZBNP_TOP') +pause = env_var.get('NZBNP_PAUSED') +if 'NZBNP_DUPEKEY' in os.environ: + dupekey = env_var.get('NZBNP_DUPEKEY') + dupescore = env_var.get('NZBNP_DUPESCORE') + dupemode = env_var.get('NZBNP_DUPEMODE') +else: + dupekey = None + dupescore = None + dupemode = None + +tmp_zipinfo = os.path.join(os.environ.get('NZBOP_TEMPDIR'), r'nzbget\unzip_scan\info') +nzb_list = [] + +def read_gzip_info(gzipfile): + gf = gzipfile.fileobj + pos = gf.tell() + + # Read archive size + gf.seek(-4, 2) + size = struct.unpack('/dev/null >/dev/null; then + echo "Failed to extract $2"; + exit 1 + fi + + fi +} + +execute () { + echo "$ $*" + + OUTPUT=$($@ 2>&1) + + if [ $? -ne 0 ]; then + echo "$OUTPUT" + echo "" + echo "Failed to Execute $*" >&2 + exit 1 + fi +} + +build () { + echo "" + echo "building $1" + echo "=======================" + + if [ -f "$PACKAGES/$1.done" ]; then + echo "$1 already built. Remove $PACKAGES/$1.done lockfile to rebuild it." + return 1 + fi + + return 0 +} + +command_exists() { + if ! [[ -x $(command -v "$1") ]]; then + return 1 + fi + + return 0 +} + + +build_done () { + touch "$PACKAGES/$1.done" +} + +echo "ffmpeg-build-script v$VERSION" +echo "=========================" +echo "" + +case "$1" in +"--cleanup") + remove_dir $PACKAGES + remove_dir $WORKSPACE + echo "Cleanup done." + echo "" + exit 0 + ;; +"--build") + + ;; +*) + echo "Usage: $0" + echo " --build: start building process" + echo " --cleanup: remove all working dirs" + echo " --help: show this help" + echo "" + exit 0 + ;; +esac + +echo "Using $MJOBS make jobs simultaneously." + +make_dir $PACKAGES +make_dir $WORKSPACE + +export PATH=${WORKSPACE}/bin:$PATH + +if ! command_exists "make"; then + echo "make not installed."; + exit 1 +fi + +if ! command_exists "g++"; then + echo "g++ not installed."; + exit 1 +fi + +if ! command_exists "curl"; then + echo "curl not installed."; + exit 1 +fi + +if build "yasm"; then + download "http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz" "yasm-1.3.0.tar.gz" + cd $PACKAGES/yasm-1.3.0 || exit + execute ./configure --prefix=${WORKSPACE} + execute make -j $MJOBS + execute make install + build_done "yasm" +fi + +if build "nasm"; then + download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.13.03.tar.gz" "nasm.tar.gz" + cd $PACKAGES/nasm-2.13.03 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "nasm" +fi + +if build "opencore"; then + download "http://downloads.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-0.1.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Fopencore-amr%2F&ts=1442256558&use_mirror=netassist" "opencore-amr-0.1.5.tar.gz" + cd $PACKAGES/opencore-amr-0.1.5 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "opencore" +fi + +if build "libvpx"; then + download "https://github.com/webmproject/libvpx/archive/v1.7.0.tar.gz" "libvpx-1.7.0.tar.gz" + cd $PACKAGES/libvpx-*0 || exit + + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Applying Darwin patch" + sed "s/,--version-script//g" build/make/Makefile > build/make/Makefile.patched + sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched > build/make/Makefile + fi + + execute ./configure --prefix=${WORKSPACE} --disable-unit-tests --disable-shared + execute make -j $MJOBS + execute make install + build_done "libvpx" +fi + +if build "lame"; then + download "http://kent.dl.sourceforge.net/project/lame/lame/3.100/lame-3.100.tar.gz" "lame-3.100.tar.gz" + cd $PACKAGES/lame-3.100 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "lame" +fi + +if build "xvidcore"; then + download "http://downloads.xvid.org/downloads/xvidcore-1.3.4.tar.gz" "xvidcore-1.3.4.tar.gz" + cd $PACKAGES/xvidcore || exit + cd build/generic || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + + if [[ -f ${WORKSPACE}/lib/libxvidcore.4.dylib ]]; then + execute rm "${WORKSPACE}/lib/libxvidcore.4.dylib" + fi + + build_done "xvidcore" +fi + +if build "x264"; then + download "http://ftp.videolan.org/pub/x264/snapshots/x264-snapshot-20190204-2245-stable.tar.bz2" "last_x264.tar.bz2" + cd $PACKAGES/x264-snapshot-* || exit + + if [[ "$OSTYPE" == "linux-gnu" ]]; then + execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic CXXFLAGS="-fPIC" + else + execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic + fi + + execute make -j $MJOBS + execute make install + execute make install-lib-static + build_done "x264" +fi + +if build "libogg"; then + download "http://downloads.xiph.org/releases/ogg/libogg-1.3.3.tar.gz" "libogg-1.3.3.tar.gz" + cd $PACKAGES/libogg-1.3.3 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "libogg" +fi + +if build "libvorbis"; then + download "http://downloads.xiph.org/releases/vorbis/libvorbis-1.3.6.tar.gz" "libvorbis-1.3.6.tar.gz" + cd $PACKAGES/libvorbis-1.3.6 || exit + execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest + execute make -j $MJOBS + execute make install + build_done "libvorbis" +fi + +if build "libtheora"; then + download "http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.gz" "libtheora-1.1.1.tar.bz" + cd $PACKAGES/libtheora-1.1.1 || exit + sed "s/-fforce-addr//g" configure > configure.patched + chmod +x configure.patched + mv configure.patched configure + execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --with-vorbis-libraries=${WORKSPACE}/lib --with-vorbis-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm + execute make -j $MJOBS + execute make install + build_done "libtheora" +fi + +if build "pkg-config"; then + download "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" "pkg-config-0.29.2.tar.gz" + cd $PACKAGES/pkg-config-0.29.2 || exit + execute ./configure --silent --prefix=${WORKSPACE} --with-pc-path=${WORKSPACE}/lib/pkgconfig --with-internal-glib + execute make -j $MJOBS + execute make install + build_done "pkg-config" +fi + +if build "cmake"; then + download "https://cmake.org/files/v3.11/cmake-3.11.3.tar.gz" "cmake-3.11.3.tar.gz" + cd $PACKAGES/cmake-3.11.3 || exit + rm Modules/FindJava.cmake + perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt + perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt + execute ./configure --prefix=${WORKSPACE} + execute make -j $MJOBS + execute make install + build_done "cmake" +fi + +if build "vid_stab"; then + download "https://codeload.github.com/georgmartius/vid.stab/legacy.tar.gz/release-0.98b" "vid.stab-0.98b-transcode-1.1-binary-x86_64.tgz" + cd $PACKAGES/georgmartius-vid* || exit + perl -p -i -e "s/vidstab SHARED/vidstab STATIC/" CMakeLists.txt + execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} . + execute make install + build_done "vid_stab" +fi + +if build "x265"; then + download "https://bitbucket.org/multicoreware/x265/downloads/x265_3.0.tar.gz" "x265-3.0.tar.gz" + cd $PACKAGES/x265_* || exit + cd source || exit + execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} -DENABLE_SHARED:bool=off . + execute make -j $MJOBS + execute make install + sed "s/-lx265/-lx265 -lstdc++/g" "$WORKSPACE/lib/pkgconfig/x265.pc" > "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" + mv "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" "$WORKSPACE/lib/pkgconfig/x265.pc" + build_done "x265" +fi + +if build "fdk_aac"; then + download "http://downloads.sourceforge.net/project/opencore-amr/fdk-aac/fdk-aac-0.1.6.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Ffdk-aac%2F&ts=1457561564&use_mirror=kent" "fdk-aac-0.1.6.tar.gz" + cd $PACKAGES/fdk-aac-0.1.6 || exit + execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "fdk_aac" +fi + + +build "ffmpeg" +download "http://ffmpeg.org/releases/ffmpeg-4.1.tar.bz2" "ffmpeg-snapshot.tar.bz2" +cd $PACKAGES/ffmpeg-4.1 || exit +./configure $ADDITIONAL_CONFIGURE_OPTIONS \ + --pkgconfigdir="$WORKSPACE/lib/pkgconfig" \ + --prefix=${WORKSPACE} \ + --pkg-config-flags="--static" \ + --extra-cflags="-I$WORKSPACE/include" \ + --extra-ldflags="-L$WORKSPACE/lib" \ + --extra-libs="-lpthread -lm" \ + --enable-static \ + --disable-debug \ + --disable-shared \ + --disable-ffplay \ + --disable-doc \ + --enable-gpl \ + --enable-version3 \ + --enable-nonfree \ + --enable-pthreads \ + --enable-libvpx \ + --enable-libmp3lame \ + --enable-libtheora \ + --enable-libvorbis \ + --enable-libx264 \ + --enable-libx265 \ + --enable-runtime-cpudetect \ + --enable-libfdk-aac \ + --enable-avfilter \ + --enable-libopencore_amrwb \ + --enable-libopencore_amrnb \ + --enable-filters \ + --enable-libvidstab + +execute make -j $MJOBS +execute make install + +INSTALL_FOLDER="/usr/bin" +if [[ "$OSTYPE" == "darwin"* ]]; then +INSTALL_FOLDER="/usr/local/bin" +fi + +echo "" +echo "Building done. The binary can be found here: $WORKSPACE/bin/ffmpeg" +echo "" + + +if [[ $AUTOINSTALL == "yes" ]]; then + if command_exists "sudo"; then + sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" + sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" + echo "Done. ffmpeg is now installed to your system" + fi +elif [[ ! $SKIPINSTALL == "yes" ]]; then + if command_exists "sudo"; then + + read -r -p "Install the binary to your $INSTALL_FOLDER folder? [Y/n] " response + + case $response in + [yY][eE][sS]|[yY]) + sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" + sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" + echo "Done. ffmpeg is now installed to your system" + ;; + esac + fi +fi + +exit 0 From c15975c0af634b5953b234adbbb3416d5dcf483f Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:15:12 +0000 Subject: [PATCH 045/185] Create web-install.sh --- .../nzgbet/ffmpeg-build/web-install.sh | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 apps/templates/nzgbet/ffmpeg-build/web-install.sh diff --git a/apps/templates/nzgbet/ffmpeg-build/web-install.sh b/apps/templates/nzgbet/ffmpeg-build/web-install.sh new file mode 100644 index 0000000..ffe4158 --- /dev/null +++ b/apps/templates/nzgbet/ffmpeg-build/web-install.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Helper script to download and run the build-ffmpeg script. + +make_dir () { + if [ ! -d $1 ]; then + if ! mkdir $1; then + printf "\n Failed to create dir %s" "$1"; + exit 1 + fi + fi +} + +command_exists() { + if ! [[ -x $(command -v "$1") ]]; then + return 1 + fi + + return 0 +} + +TARGET='ffmpeg-build' + +if ! command_exists "curl"; then + echo "curl not installed."; + exit 1 +fi + +echo "ffmpeg-build-script-downloader v0.1" +echo "=========================================" +echo "" + +echo "First we create the ffmpeg build directory $TARGET" +make_dir $TARGET +cd $TARGET + +echo "Now we download and execute the build script" +echo "" + +bash build-ffmpeg --build + From e4f0b4aec7162c3a963c9117377d2279dd235cc6 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:16:32 +0000 Subject: [PATCH 046/185] Create installer.sh --- apps/templates/nzgbet/installer/installer.sh | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 apps/templates/nzgbet/installer/installer.sh diff --git a/apps/templates/nzgbet/installer/installer.sh b/apps/templates/nzgbet/installer/installer.sh new file mode 100644 index 0000000..8c26cf9 --- /dev/null +++ b/apps/templates/nzgbet/installer/installer.sh @@ -0,0 +1,30 @@ +#!/bin/bash +apk update +apk upgrade +apk add --no-cache git +apk add build-base gcc wget diffutils perl +apk add curl +git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp +cp -r /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator +rm -rf /config/scripts/MP4_Automator/tmp +git unstage +apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev +pip install --upgrade PIP +pip install requests +pip install requests[security] +pip install requests-cache +pip install babelfish +pip install "guessit<2" +pip install "subliminal<2" +pip install qtfaststart +# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 +pip uninstall -y stevedore +pip install stevedore==1.19.1 +#Remove default NZBGetPostProcess script settings, and replace with our own +rm /config/scripts/MP4_Automator/NZBGetPostProcess.py +cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py +#Build ffmpeg +cd /config +. /config/ffmpeg-build/web-install.sh +#Set script file permissions +chmod 777 -R /config/scripts From c3834b613886a341720b2be0bbc95c84d35413ab Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:17:23 +0000 Subject: [PATCH 047/185] Create autoProcess.ini --- .../nzgbet/mp4_automator/autoProcess.ini | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 apps/templates/nzgbet/mp4_automator/autoProcess.ini diff --git a/apps/templates/nzgbet/mp4_automator/autoProcess.ini b/apps/templates/nzgbet/mp4_automator/autoProcess.ini new file mode 100644 index 0000000..83b18aa --- /dev/null +++ b/apps/templates/nzgbet/mp4_automator/autoProcess.ini @@ -0,0 +1,143 @@ +[SickBeard] +host = sickbeard +port = 8081 +username = +password = +web_root = +ssl = False +api_key = + +[Sonarr] +host = sonarr +port = 8989 +web_root = +ssl = False +apikey = + +[Radarr] +host = radarr +port = 7878 +web_root = +ssl = False +apikey = + +[MP4] +ffmpeg = /config/ffmpeg-build/workspace/bin/ffmpeg +ffprobe = /config/ffmpeg-build/workspace/bin/ffprobe +threads = 1 +output_directory = +copy_to = +move_to = +output_extension = mp4 +output_format = mp4 +delete_original = True +relocate_moov = True +video-codec = h264,x264 +video-bitrate = +video-crf = 18 +video-max-width = +h264-max-level = 4.1 +use-qsv-decoder-with-encoder = True +ios-audio = libfdk_aac +ios-first-track-only = False +ios-audio-filter = dynaudnorm +max-audio-channels = +audio-codec = ac3,mp3,dts,dca,aac,libfdk_aac +audio-language = eng +audio-default-language = eng +audio-channel-bitrate = 256 +audio-filter = +subtitle-codec = srt +subtitle-language = eng +subtitle-default-language = +subtitle-encoding = +fullpathguess = True +convert-mp4 = True +tagfile = True +tag-language = en +download-artwork = Poster +download-subs = True +embed-subs = False +sub-providers = addic7ed,podnapisi,thesubdb,opensubtitles +permissions = 0777 +post-process = False +pix-fmt = +aac_adtstoasc = False +postopts = -preset,slower +preopts = +audio-copy-original = False +enable_dxva2_gpu_decode = False +ios-move-last = False +use-hevc-qsv-decoder = False +embed-only-internal-subs = False +audio-first-track-of-language = False +video-profile = + +[CouchPotato] +host = couchpotato +port = 5050 +username = +password = +web_root = +ssl = False +apikey = +delay = 65 +method = renamer +delete_failed = False + +[uTorrent] +convert = +couchpotato-label = couchpotato +sickbeard-label = sickbeard +sonarr-label = sonarr +bypass-label = bypass +sickrage-label = sickrage +webui = False +action_before = stop +action_after = removedata +host = http://utorrent:8080/ +username = +password = +output_directory = +radarr-label = radarr + +[Deluge] +host = deluge +username = +convert = True +password = +sonarr-label = sonarr +radarr-label = radarr +bypass-label = bypass +sickbeard-label = sickbeard +port = 12569 +sickrage-label = sickrage +couchpotato-label = couchpotato +output_directory = +remove = true + +[SABNZBD] +convert = True +sickrage-category = sickrage +sonarr-category = sonarr +radarr-category = radarr +bypass-category = bypass +couchpotato-category = couchpotato +sickbeard-category = sickbeard +output_directory = + +[Sickrage] +host = sickrage +port = 8081 +username = +password = +web_root = +ssl = False +api_key = + +[Plex] +host = plex +port = 32400 +refresh = False +token = + From 1f2f5abb1dc82d2fc80a18b02c6dd30b3538f607 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:17:44 +0000 Subject: [PATCH 048/185] Create TEMPLATEPPScript --- .../nzgbet/mp4_automator/TEMPLATEPPScript | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript diff --git a/apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript b/apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript new file mode 100644 index 0000000..6eb7055 --- /dev/null +++ b/apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# +############################################################################## +### NZBGET POST-PROCESSING SCRIPT ### + +# Modified to enable multiple bypass categories, +# as per: https://github.com/mdhiggins/sickbeard_mp4_automator/issues/509 +# +# Converts files and passes them to Sonarr for further processing. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### + +# Change to full path to MP4 Automator folder. No quotes and a trailing / +#MP4_FOLDER=~/sickbeard_mp4_automator/ + +# Convert file before passing to destination (True, False) +#SHOULDCONVERT=False + +# Category for Couchpotato +#CP_CAT=Couchpotato + +# Category for Sonarr +#SONARR_CAT=Sonarr + +# Category for Radarr +#RADARR_CAT=Radarr + +# Category for Sickbeard +#SICKBEARD_CAT=Sickbeard + +# Category for Sickrage +#SICKRAGE_CAT=Sickrage + +# Category list (comma seperated) for bypassing any further processing but still converting +#BYPASS_CAT=tv,movies + +# Custom output_directory setting +#OUTPUT_DIR= + +### NZBGET POST-PROCESSING SCRIPT ### +############################################################################## + +import os +import sys +import re +import json +import traceback + +# Sanity checks for path string +MP4folder = os.environ['NZBPO_MP4_FOLDER'].strip() +MP4folder = MP4folder.replace('"', '') +MP4folder = MP4folder.replace("'", "") +MP4folder = MP4folder.replace("\\", "/") +if not(MP4folder.endswith("/")): + MP4folder += "/" +#DEBUG#print MP4folder+" the original is "+os.environ['NZBPO_MP4_FOLDER'] + +output_dir = None +if 'NZBPO_OUTPUT_DIR' in os.environ: + output_dir = os.environ['NZBPO_OUTPUT_DIR'].strip() + if len(output_dir) > 0: + output_dir = output_dir.replace('"', '') + output_dir = output_dir.replace("'", "") + output_dir = output_dir.replace("\\", "/") + if not(output_dir.endswith("/")): + output_dir += "/" + #DEBUG#print Overriding output directory + +sys.path.append(MP4folder) +try: + from readSettings import ReadSettings + from mkvtomp4 import MkvtoMp4 + from autoprocess import autoProcessMovie, autoProcessTV, autoProcessTVSR, sonarr, radarr + import logging + from logging.config import fileConfig +except ImportError: + print("[ERROR] Wrong path to sickbeard_mp4_automator: " + os.environ['NZBPO_MP4_FOLDER']) + print("[ERROR] %s" % traceback.print_exc()) + sys.exit(0) + +# Setup Logging +logpath = '/var/log/sickbeard_mp4_automator' +if os.name == 'nt': + logpath = MP4folder +elif not os.path.isdir(logpath): + try: + os.mkdir(logpath) + except: + logpath = MP4folder +configPath = os.path.abspath(os.path.join(MP4folder, 'logging.ini')).replace("\\", "\\\\") +logPath = os.path.abspath(os.path.join(logpath, 'index.log')).replace("\\", "\\\\") +fileConfig(configPath, defaults={'logfilename': logPath}) +log = logging.getLogger("NZBGetPostProcess") + +# Determine if conversion will take place +shouldConvert = (os.environ['NZBPO_SHOULDCONVERT'].lower() in ("yes", "true", "t", "1")) + +if 'NZBOP_SCRIPTDIR' in os.environ and not os.environ['NZBOP_VERSION'][0:5] < '11.0': + log.info("Script triggered from NZBGet (11.0 or later).") + + path = os.environ['NZBPP_DIRECTORY'] # Path to NZB directory + nzb = os.environ['NZBPP_NZBFILENAME'] # Original NZB name + category = os.environ['NZBPP_CATEGORY'] # NZB Category to determine destination + #DEBUG#print "Category is %s." % category + + couchcat = os.environ['NZBPO_CP_CAT'].lower() + sonarrcat = os.environ['NZBPO_SONARR_CAT'].lower() + radarrcat = os.environ['NZBPO_RADARR_CAT'].lower() + sickbeardcat = os.environ['NZBPO_SICKBEARD_CAT'].lower() + sickragecat = os.environ['NZBPO_SICKRAGE_CAT'].lower() + bypass = os.environ['NZBPO_BYPASS_CAT'].lower().replace(' ','').split(',') + + categories = [sickbeardcat, couchcat, sonarrcat, radarrcat, sickragecat] + + log.debug("Path: %s" % path) + log.debug("NZB: %s" % nzb) + log.debug("Category: %s" % category) + log.debug("Categories: %s" % categories) + + # NZBGet argv: all passed as environment variables. + clientAgent = "nzbget" + # Exit codes used by NZBGet + POSTPROCESS_PARCHECK = 92 + POSTPROCESS_SUCCESS = 93 + POSTPROCESS_ERROR = 94 + POSTPROCESS_NONE = 95 + + # Check nzbget.conf options + status = 0 + + if os.environ['NZBOP_UNPACK'] != 'yes': + log.error("Please enable option \"Unpack\" in nzbget configuration file, exiting.") + sys.exit(POSTPROCESS_NONE) + + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '3': + log.error("Par-check successful, but Par-repair disabled, exiting") + sys.exit(POSTPROCESS_NONE) + + if os.environ['NZBPP_PARSTATUS'] == '1': + log.error("Par-check failed, setting status \"failed\".") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + log.error("Unpack failed, setting status \"failed\".") + status = 1 + sys.exit(POSTPROCESS_NONE) + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] != '2': + # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check + + for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): + for file in filenames: + fileExtension = os.path.splitext(file)[1] + + if fileExtension in ['.par2']: + log.error("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\".") + status = 1 + break + + if os.path.isfile(os.path.join(os.environ['NZBPP_DIRECTORY'], "_brokenlog.txt")) and not status == 1: + log.error("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting.") + status = 1 + + if not status == 1: + log.error("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful.") + + # Check if destination directory exists (important for reprocessing of history items) + if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + log.error("Post-Process: Nothing to post-process: destination directory ", os.environ['NZBPP_DIRECTORY'], "doesn't exist.") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Make sure one of the appropriate categories is set + if category.lower() not in categories and category.lower() not in bypass: + log.error("Post-Process: No valid category detected. Category was %s." % (category)) + status = 1 + sys.exit(POSTPROCESS_NONE) + + # Make sure there are no duplicate categories + if len(categories) != len(set(categories)): + log.error("Duplicate category detected. Category names must be unique.") + status = 1 + sys.exit(POSTPROCESS_NONE) + + # All checks done, now launching the script. + settings = ReadSettings(MP4folder, "autoProcess.ini") + + if shouldConvert: + if output_dir: + settings.output_dir = output_dir + converter = MkvtoMp4(settings, logger=log) + for r, d, f in os.walk(path): + for files in f: + inputfile = os.path.join(r, files) + #DEBUG#print inputfile + #Ignores files under 50MB + if os.path.getsize(inputfile) > 50000000: + if MkvtoMp4(settings, logger=log).validSource(inputfile): + try: + output = converter.process(inputfile) + log.info("Successfully processed %s." % inputfile) + except: + log.exception("File processing failed.") + if converter.output_dir: + path = converter.output_dir + if (category.lower() == categories[0]): + #DEBUG#print "Sickbeard Processing Activated" + autoProcessTV.processEpisode(path, settings, nzb) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() == categories[1]): + #DEBUG#print "CouchPotato Processing Activated" + autoProcessMovie.process(path, settings, nzb, status) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() == categories[2]): + #DEBUG#print "Sonarr Processing Activated" + success = sonarr.processEpisode(path, settings, True) + if success: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_ERROR) + elif (category.lower() == categories[3]): + #DEBUG#print "Radarr Processing Activated" + success = radarr.processMovie(path, settings, True) + if success: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_ERROR) + elif (category.lower() == categories[4]): + #DEBUG#print "Sickrage Processing Activated" + autoProcessTVSR.processEpisode(path, settings, nzb) + sys.exit(POSTPROCESS_SUCCESS) + elif (category.lower() in bypass): + #DEBUG#print "Bypass Further Processing" + sys.exit(POSTPROCESS_NONE) + +else: + log.error("This script can only be called from NZBGet (11.0 or later).") + sys.exit(0) From b31e90f6b162a857d6b440c6590e7bb966460649 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:18:23 +0000 Subject: [PATCH 049/185] Create __init__.py --- apps/templates/nzgbet/scripts/rarfile/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 apps/templates/nzgbet/scripts/rarfile/__init__.py diff --git a/apps/templates/nzgbet/scripts/rarfile/__init__.py b/apps/templates/nzgbet/scripts/rarfile/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/apps/templates/nzgbet/scripts/rarfile/__init__.py @@ -0,0 +1 @@ + From b596ffa4a8f7f9246473149a1f419887e48c9af6 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:18:52 +0000 Subject: [PATCH 050/185] Create rarfile.py --- .../nzgbet/scripts/rarfile/rarfile.py | 2932 +++++++++++++++++ 1 file changed, 2932 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/rarfile/rarfile.py diff --git a/apps/templates/nzgbet/scripts/rarfile/rarfile.py b/apps/templates/nzgbet/scripts/rarfile/rarfile.py new file mode 100644 index 0000000..c6e0d44 --- /dev/null +++ b/apps/templates/nzgbet/scripts/rarfile/rarfile.py @@ -0,0 +1,2932 @@ +# rarfile.py +# +# Copyright (c) 2005-2016 Marko Kreen +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +r"""RAR archive reader. + +This is Python module for Rar archive reading. The interface +is made as :mod:`zipfile`-like as possible. + +Basic logic: + - Parse archive structure with Python. + - Extract non-compressed files with Python + - Extract compressed files with unrar. + - Optionally write compressed data to temp file to speed up unrar, + otherwise it needs to scan whole archive on each execution. + +Example:: + + import rarfile + + rf = rarfile.RarFile('myarchive.rar') + for f in rf.infolist(): + print f.filename, f.file_size + if f.filename == 'README': + print(rf.read(f)) + +Archive files can also be accessed via file-like object returned +by :meth:`RarFile.open`:: + + import rarfile + + with rarfile.RarFile('archive.rar') as rf: + with rf.open('README') as f: + for ln in f: + print(ln.strip()) + +There are few module-level parameters to tune behaviour, +here they are with defaults, and reason to change it:: + + import rarfile + + # Set to full path of unrar.exe if it is not in PATH + rarfile.UNRAR_TOOL = "unrar" + + # Set to '\\' to be more compatible with old rarfile + rarfile.PATH_SEP = '/' + +For more details, refer to source. + +""" + +from __future__ import division, print_function + +## +## Imports and compat - support both Python 2.x and 3.x +## + +import sys +import os +import errno +import struct + +from struct import pack, unpack, Struct +from binascii import crc32, hexlify +from tempfile import mkstemp +from subprocess import Popen, PIPE, STDOUT +from io import RawIOBase +from hashlib import sha1, sha256 +from hmac import HMAC +from datetime import datetime, timedelta, tzinfo + +# fixed offset timezone, for UTC +try: + from datetime import timezone +except ImportError: + class timezone(tzinfo): + """Compat timezone.""" + __slots__ = ('_ofs', '_name') + _DST = timedelta(0) + + def __init__(self, offset, name): + super(timezone, self).__init__() + self._ofs, self._name = offset, name + + def utcoffset(self, dt): + return self._ofs + + def tzname(self, dt): + return self._name + + def dst(self, dt): + return self._DST + +# only needed for encryped headers +try: + try: + from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf import pbkdf2 + + class AES_CBC_Decrypt(object): + """Decrypt API""" + def __init__(self, key, iv): + ciph = Cipher(algorithms.AES(key), modes.CBC(iv), default_backend()) + self.decrypt = ciph.decryptor().update + + def pbkdf2_sha256(password, salt, iters): + """PBKDF2 with HMAC-SHA256""" + ctx = pbkdf2.PBKDF2HMAC(hashes.SHA256(), 32, salt, iters, default_backend()) + return ctx.derive(password) + + except ImportError: + from Crypto.Cipher import AES + from Crypto.Protocol import KDF + + class AES_CBC_Decrypt(object): + """Decrypt API""" + def __init__(self, key, iv): + self.decrypt = AES.new(key, AES.MODE_CBC, iv).decrypt + + def pbkdf2_sha256(password, salt, iters): + """PBKDF2 with HMAC-SHA256""" + return KDF.PBKDF2(password, salt, 32, iters, hmac_sha256) + + _have_crypto = 1 +except ImportError: + _have_crypto = 0 + +try: + from pyblake2 import blake2s + _have_blake2 = True +except ImportError: + _have_blake2 = False + +# compat with 2.x +if sys.hexversion < 0x3000000: + def rar_crc32(data, prev=0): + """CRC32 with unsigned values. + """ + if (prev > 0) and (prev & 0x80000000): + prev -= (1 << 32) + res = crc32(data, prev) + if res < 0: + res += (1 << 32) + return res + tohex = hexlify + _byte_code = ord +else: # pragma: no cover + def tohex(data): + """Return hex string.""" + return hexlify(data).decode('ascii') + rar_crc32 = crc32 + unicode = str + _byte_code = int # noqa + + +__version__ = '3.0' + +# export only interesting items +__all__ = ['is_rarfile', 'RarInfo', 'RarFile', 'RarExtFile'] + +## +## Module configuration. Can be tuned after importing. +## + +#: default fallback charset +DEFAULT_CHARSET = "windows-1252" + +#: list of encodings to try, with fallback to DEFAULT_CHARSET if none succeed +TRY_ENCODINGS = ('utf8', 'utf-16le') + +#: 'unrar', 'rar' or full path to either one +UNRAR_TOOL = "unrar" + +#: Command line args to use for opening file for reading. +OPEN_ARGS = ('p', '-inul') + +#: Command line args to use for extracting file to disk. +EXTRACT_ARGS = ('x', '-y', '-idq') + +#: args for testrar() +TEST_ARGS = ('t', '-idq') + +# +# Allow use of tool that is not compatible with unrar. +# +# By default use 'bsdtar' which is 'tar' program that +# sits on top of libarchive. +# +# Problems with libarchive RAR backend: +# - Does not support solid archives. +# - Does not support password-protected archives. +# + +ALT_TOOL = 'bsdtar' +ALT_OPEN_ARGS = ('-x', '--to-stdout', '-f') +ALT_EXTRACT_ARGS = ('-x', '-f') +ALT_TEST_ARGS = ('-t', '-f') +ALT_CHECK_ARGS = ('--help',) + +#: whether to speed up decompression by using tmp archive +USE_EXTRACT_HACK = 1 + +#: limit the filesize for tmp archive usage +HACK_SIZE_LIMIT = 20 * 1024 * 1024 + +#: Separator for path name components. RAR internally uses '\\'. +#: Use '/' to be similar with zipfile. +PATH_SEP = '/' + +## +## rar constants +## + +# block types +RAR_BLOCK_MARK = 0x72 # r +RAR_BLOCK_MAIN = 0x73 # s +RAR_BLOCK_FILE = 0x74 # t +RAR_BLOCK_OLD_COMMENT = 0x75 # u +RAR_BLOCK_OLD_EXTRA = 0x76 # v +RAR_BLOCK_OLD_SUB = 0x77 # w +RAR_BLOCK_OLD_RECOVERY = 0x78 # x +RAR_BLOCK_OLD_AUTH = 0x79 # y +RAR_BLOCK_SUB = 0x7a # z +RAR_BLOCK_ENDARC = 0x7b # { + +# flags for RAR_BLOCK_MAIN +RAR_MAIN_VOLUME = 0x0001 +RAR_MAIN_COMMENT = 0x0002 +RAR_MAIN_LOCK = 0x0004 +RAR_MAIN_SOLID = 0x0008 +RAR_MAIN_NEWNUMBERING = 0x0010 +RAR_MAIN_AUTH = 0x0020 +RAR_MAIN_RECOVERY = 0x0040 +RAR_MAIN_PASSWORD = 0x0080 +RAR_MAIN_FIRSTVOLUME = 0x0100 +RAR_MAIN_ENCRYPTVER = 0x0200 + +# flags for RAR_BLOCK_FILE +RAR_FILE_SPLIT_BEFORE = 0x0001 +RAR_FILE_SPLIT_AFTER = 0x0002 +RAR_FILE_PASSWORD = 0x0004 +RAR_FILE_COMMENT = 0x0008 +RAR_FILE_SOLID = 0x0010 +RAR_FILE_DICTMASK = 0x00e0 +RAR_FILE_DICT64 = 0x0000 +RAR_FILE_DICT128 = 0x0020 +RAR_FILE_DICT256 = 0x0040 +RAR_FILE_DICT512 = 0x0060 +RAR_FILE_DICT1024 = 0x0080 +RAR_FILE_DICT2048 = 0x00a0 +RAR_FILE_DICT4096 = 0x00c0 +RAR_FILE_DIRECTORY = 0x00e0 +RAR_FILE_LARGE = 0x0100 +RAR_FILE_UNICODE = 0x0200 +RAR_FILE_SALT = 0x0400 +RAR_FILE_VERSION = 0x0800 +RAR_FILE_EXTTIME = 0x1000 +RAR_FILE_EXTFLAGS = 0x2000 + +# flags for RAR_BLOCK_ENDARC +RAR_ENDARC_NEXT_VOLUME = 0x0001 +RAR_ENDARC_DATACRC = 0x0002 +RAR_ENDARC_REVSPACE = 0x0004 +RAR_ENDARC_VOLNR = 0x0008 + +# flags common to all blocks +RAR_SKIP_IF_UNKNOWN = 0x4000 +RAR_LONG_BLOCK = 0x8000 + +# Host OS types +RAR_OS_MSDOS = 0 +RAR_OS_OS2 = 1 +RAR_OS_WIN32 = 2 +RAR_OS_UNIX = 3 +RAR_OS_MACOS = 4 +RAR_OS_BEOS = 5 + +# Compression methods - '0'..'5' +RAR_M0 = 0x30 +RAR_M1 = 0x31 +RAR_M2 = 0x32 +RAR_M3 = 0x33 +RAR_M4 = 0x34 +RAR_M5 = 0x35 + +# +# RAR5 constants +# + +RAR5_BLOCK_MAIN = 1 +RAR5_BLOCK_FILE = 2 +RAR5_BLOCK_SERVICE = 3 +RAR5_BLOCK_ENCRYPTION = 4 +RAR5_BLOCK_ENDARC = 5 + +RAR5_BLOCK_FLAG_EXTRA_DATA = 0x01 +RAR5_BLOCK_FLAG_DATA_AREA = 0x02 +RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN = 0x04 +RAR5_BLOCK_FLAG_SPLIT_BEFORE = 0x08 +RAR5_BLOCK_FLAG_SPLIT_AFTER = 0x10 +RAR5_BLOCK_FLAG_DEPENDS_PREV = 0x20 +RAR5_BLOCK_FLAG_KEEP_WITH_PARENT = 0x40 + +RAR5_MAIN_FLAG_ISVOL = 0x01 +RAR5_MAIN_FLAG_HAS_VOLNR = 0x02 +RAR5_MAIN_FLAG_SOLID = 0x04 +RAR5_MAIN_FLAG_RECOVERY = 0x08 +RAR5_MAIN_FLAG_LOCKED = 0x10 + +RAR5_FILE_FLAG_ISDIR = 0x01 +RAR5_FILE_FLAG_HAS_MTIME = 0x02 +RAR5_FILE_FLAG_HAS_CRC32 = 0x04 +RAR5_FILE_FLAG_UNKNOWN_SIZE = 0x08 + +RAR5_COMPR_SOLID = 0x40 + +RAR5_ENC_FLAG_HAS_CHECKVAL = 0x01 + +RAR5_ENDARC_FLAG_NEXT_VOL = 0x01 + +RAR5_XFILE_ENCRYPTION = 1 +RAR5_XFILE_HASH = 2 +RAR5_XFILE_TIME = 3 +RAR5_XFILE_VERSION = 4 +RAR5_XFILE_REDIR = 5 +RAR5_XFILE_OWNER = 6 +RAR5_XFILE_SERVICE = 7 + +RAR5_XTIME_UNIXTIME = 0x01 +RAR5_XTIME_HAS_MTIME = 0x02 +RAR5_XTIME_HAS_CTIME = 0x04 +RAR5_XTIME_HAS_ATIME = 0x08 + +RAR5_XENC_CIPHER_AES256 = 0 + +RAR5_XENC_CHECKVAL = 0x01 +RAR5_XENC_TWEAKED = 0x02 + +RAR5_XHASH_BLAKE2SP = 0 + +RAR5_XREDIR_UNIX_SYMLINK = 1 +RAR5_XREDIR_WINDOWS_SYMLINK = 2 +RAR5_XREDIR_WINDOWS_JUNCTION = 3 +RAR5_XREDIR_HARD_LINK = 4 +RAR5_XREDIR_FILE_COPY = 5 + +RAR5_XREDIR_ISDIR = 0x01 + +RAR5_XOWNER_UNAME = 0x01 +RAR5_XOWNER_GNAME = 0x02 +RAR5_XOWNER_UID = 0x04 +RAR5_XOWNER_GID = 0x08 + +RAR5_OS_WINDOWS = 0 +RAR5_OS_UNIX = 1 + +## +## internal constants +## + +RAR_ID = b"Rar!\x1a\x07\x00" +RAR5_ID = b"Rar!\x1a\x07\x01\x00" +ZERO = b'\0' +EMPTY = b'' +UTC = timezone(timedelta(0), 'UTC') +BSIZE = 32 * 1024 + +def _get_rar_version(xfile): + '''Check quickly whether file is rar archive. + ''' + with XFile(xfile) as fd: + buf = fd.read(len(RAR5_ID)) + if buf.startswith(RAR_ID): + return 3 + elif buf.startswith(RAR5_ID): + return 5 + return 0 + +## +## Public interface +## + +def is_rarfile(xfile): + '''Check quickly whether file is rar archive. + ''' + return _get_rar_version(xfile) > 0 + +class Error(Exception): + """Base class for rarfile errors.""" + +class BadRarFile(Error): + """Incorrect data in archive.""" + +class NotRarFile(Error): + """The file is not RAR archive.""" + +class BadRarName(Error): + """Cannot guess multipart name components.""" + +class NoRarEntry(Error): + """File not found in RAR""" + +class PasswordRequired(Error): + """File requires password""" + +class NeedFirstVolume(Error): + """Need to start from first volume.""" + +class NoCrypto(Error): + """Cannot parse encrypted headers - no crypto available.""" + +class RarExecError(Error): + """Problem reported by unrar/rar.""" + +class RarWarning(RarExecError): + """Non-fatal error""" + +class RarFatalError(RarExecError): + """Fatal error""" + +class RarCRCError(RarExecError): + """CRC error during unpacking""" + +class RarLockedArchiveError(RarExecError): + """Must not modify locked archive""" + +class RarWriteError(RarExecError): + """Write error""" + +class RarOpenError(RarExecError): + """Open error""" + +class RarUserError(RarExecError): + """User error""" + +class RarMemoryError(RarExecError): + """Memory error""" + +class RarCreateError(RarExecError): + """Create error""" + +class RarNoFilesError(RarExecError): + """No files that match pattern were found""" + +class RarUserBreak(RarExecError): + """User stop""" + +class RarWrongPassword(RarExecError): + """Incorrect password""" + +class RarUnknownError(RarExecError): + """Unknown exit code""" + +class RarSignalExit(RarExecError): + """Unrar exited with signal""" + +class RarCannotExec(RarExecError): + """Executable not found.""" + + +class RarInfo(object): + r'''An entry in rar archive. + + RAR3 extended timestamps are :class:`datetime.datetime` objects without timezone. + RAR5 extended timestamps are :class:`datetime.datetime` objects with UTC timezone. + + Attributes: + + filename + File name with relative path. + Path separator is '/'. Always unicode string. + + date_time + File modification timestamp. As tuple of (year, month, day, hour, minute, second). + RAR5 allows archives where it is missing, it's None then. + + file_size + Uncompressed size. + + compress_size + Compressed size. + + compress_type + Compression method: one of :data:`RAR_M0` .. :data:`RAR_M5` constants. + + extract_version + Minimal Rar version needed for decompressing. As (major*10 + minor), + so 2.9 is 29. + + RAR3: 10, 20, 29 + + RAR5 does not have such field in archive, it's simply set to 50. + + host_os + Host OS type, one of RAR_OS_* constants. + + RAR3: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`, :data:`RAR_OS_MSDOS`, + :data:`RAR_OS_OS2`, :data:`RAR_OS_BEOS`. + + RAR5: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`. + + mode + File attributes. May be either dos-style or unix-style, depending on host_os. + + mtime + File modification time. Same value as :attr:`date_time` + but as :class:`datetime.datetime` object with extended precision. + + ctime + Optional time field: creation time. As :class:`datetime.datetime` object. + + atime + Optional time field: last access time. As :class:`datetime.datetime` object. + + arctime + Optional time field: archival time. As :class:`datetime.datetime` object. + (RAR3-only) + + CRC + CRC-32 of uncompressed file, unsigned int. + + RAR5: may be None. + + blake2sp_hash + Blake2SP hash over decompressed data. (RAR5-only) + + comment + Optional file comment field. Unicode string. (RAR3-only) + + file_redir + If not None, file is link of some sort. Contains tuple of (type, flags, target). + (RAR5-only) + + Type is one of constants: + + :data:`RAR5_XREDIR_UNIX_SYMLINK` + unix symlink to target. + :data:`RAR5_XREDIR_WINDOWS_SYMLINK` + windows symlink to target. + :data:`RAR5_XREDIR_WINDOWS_JUNCTION` + windows junction. + :data:`RAR5_XREDIR_HARD_LINK` + hard link to target. + :data:`RAR5_XREDIR_FILE_COPY` + current file is copy of another archive entry. + + Flags may contain :data:`RAR5_XREDIR_ISDIR` bit. + + volume + Volume nr, starting from 0. + + volume_file + Volume file name, where file starts. + + ''' + + # zipfile-compatible fields + filename = None + file_size = None + compress_size = None + date_time = None + comment = None + CRC = None + volume = None + orig_filename = None + + # optional extended time fields, datetime() objects. + mtime = None + ctime = None + atime = None + + extract_version = None + mode = None + host_os = None + compress_type = None + + # rar3-only fields + comment = None + arctime = None + + # rar5-only fields + blake2sp_hash = None + file_redir = None + + # internal fields + flags = 0 + type = None + + def isdir(self): + """Returns True if entry is a directory. + """ + if self.type == RAR_BLOCK_FILE: + return (self.flags & RAR_FILE_DIRECTORY) == RAR_FILE_DIRECTORY + return False + + def needs_password(self): + """Returns True if data is stored password-protected. + """ + if self.type == RAR_BLOCK_FILE: + return (self.flags & RAR_FILE_PASSWORD) > 0 + return False + + +class RarFile(object): + '''Parse RAR structure, provide access to files in archive. + ''' + + #: Archive comment. Unicode string or None. + comment = None + + def __init__(self, rarfile, mode="r", charset=None, info_callback=None, + crc_check=True, errors="stop"): + """Open and parse a RAR archive. + + Parameters: + + rarfile + archive file name + mode + only 'r' is supported. + charset + fallback charset to use, if filenames are not already Unicode-enabled. + info_callback + debug callback, gets to see all archive entries. + crc_check + set to False to disable CRC checks + errors + Either "stop" to quietly stop parsing on errors, + or "strict" to raise errors. Default is "stop". + """ + self._rarfile = rarfile + self._charset = charset or DEFAULT_CHARSET + self._info_callback = info_callback + self._crc_check = crc_check + self._password = None + self._file_parser = None + + if errors == "stop": + self._strict = False + elif errors == "strict": + self._strict = True + else: + raise ValueError("Invalid value for 'errors' parameter.") + + if mode != "r": + raise NotImplementedError("RarFile supports only mode=r") + + self._parse() + + def __enter__(self): + return self + + def __exit__(self, typ, value, traceback): + self.close() + + def setpassword(self, password): + '''Sets the password to use when extracting.''' + self._password = password + if self._file_parser: + if self._file_parser.has_header_encryption(): + self._file_parser = None + if not self._file_parser: + self._parse() + else: + self._file_parser.setpassword(self._password) + + def needs_password(self): + '''Returns True if any archive entries require password for extraction.''' + return self._file_parser.needs_password() + + def namelist(self): + '''Return list of filenames in archive.''' + return [f.filename for f in self.infolist()] + + def infolist(self): + '''Return RarInfo objects for all files/directories in archive.''' + return self._file_parser.infolist() + + def volumelist(self): + '''Returns filenames of archive volumes. + + In case of single-volume archive, the list contains + just the name of main archive file. + ''' + return self._file_parser.volumelist() + + def getinfo(self, fname): + '''Return RarInfo for file. + ''' + return self._file_parser.getinfo(fname) + + def open(self, fname, mode='r', psw=None): + '''Returns file-like object (:class:`RarExtFile`), + from where the data can be read. + + The object implements :class:`io.RawIOBase` interface, so it can + be further wrapped with :class:`io.BufferedReader` + and :class:`io.TextIOWrapper`. + + On older Python where io module is not available, it implements + only .read(), .seek(), .tell() and .close() methods. + + The object is seekable, although the seeking is fast only on + uncompressed files, on compressed files the seeking is implemented + by reading ahead and/or restarting the decompression. + + Parameters: + + fname + file name or RarInfo instance. + mode + must be 'r' + psw + password to use for extracting. + ''' + + if mode != 'r': + raise NotImplementedError("RarFile.open() supports only mode=r") + + # entry lookup + inf = self.getinfo(fname) + if inf.isdir(): + raise TypeError("Directory does not have any data: " + inf.filename) + + # check password + if inf.needs_password(): + psw = psw or self._password + if psw is None: + raise PasswordRequired("File %s requires password" % inf.filename) + else: + psw = None + + return self._file_parser.open(inf, psw) + + def read(self, fname, psw=None): + """Return uncompressed data for archive entry. + + For longer files using :meth:`RarFile.open` may be better idea. + + Parameters: + + fname + filename or RarInfo instance + psw + password to use for extracting. + """ + + with self.open(fname, 'r', psw) as f: + return f.read() + + def close(self): + """Release open resources.""" + pass + + def printdir(self): + """Print archive file list to stdout.""" + for f in self.infolist(): + print(f.filename) + + def extract(self, member, path=None, pwd=None): + """Extract single file into current directory. + + Parameters: + + member + filename or :class:`RarInfo` instance + path + optional destination path + pwd + optional password to use + """ + if isinstance(member, RarInfo): + fname = member.filename + else: + fname = member + self._extract([fname], path, pwd) + + def extractall(self, path=None, members=None, pwd=None): + """Extract all files into current directory. + + Parameters: + + path + optional destination path + members + optional filename or :class:`RarInfo` instance list to extract + pwd + optional password to use + """ + fnlist = [] + if members is not None: + for m in members: + if isinstance(m, RarInfo): + fnlist.append(m.filename) + else: + fnlist.append(m) + self._extract(fnlist, path, pwd) + + def testrar(self): + """Let 'unrar' test the archive. + """ + cmd = [UNRAR_TOOL] + list(TEST_ARGS) + add_password_arg(cmd, self._password) + cmd.append('--') + with XTempFile(self._rarfile) as rarfile: + cmd.append(rarfile) + p = custom_popen(cmd) + output = p.communicate()[0] + check_returncode(p, output) + + def strerror(self): + """Return error string if parsing failed, + or None if no problems. + """ + if not self._file_parser: + return "Not a RAR file" + return self._file_parser.strerror() + + ## + ## private methods + ## + + def _parse(self): + ver = _get_rar_version(self._rarfile) + if ver == 3: + p3 = RAR3Parser(self._rarfile, self._password, self._crc_check, + self._charset, self._strict, self._info_callback) + self._file_parser = p3 # noqa + elif ver == 5: + p5 = RAR5Parser(self._rarfile, self._password, self._crc_check, + self._charset, self._strict, self._info_callback) + self._file_parser = p5 # noqa + else: + raise BadRarFile("Not a RAR file") + + self._file_parser.parse() + self.comment = self._file_parser.comment + + # call unrar to extract a file + def _extract(self, fnlist, path=None, psw=None): + cmd = [UNRAR_TOOL] + list(EXTRACT_ARGS) + + # pasoword + psw = psw or self._password + add_password_arg(cmd, psw) + cmd.append('--') + + # rar file + with XTempFile(self._rarfile) as rarfn: + cmd.append(rarfn) + + # file list + for fn in fnlist: + if os.sep != PATH_SEP: + fn = fn.replace(PATH_SEP, os.sep) + cmd.append(fn) + + # destination path + if path is not None: + cmd.append(path + os.sep) + + # call + p = custom_popen(cmd) + output = p.communicate()[0] + check_returncode(p, output) + +# +# File format parsing +# + +class CommonParser(object): + """Shared parser parts.""" + _main = None + _hdrenc_main = None + _needs_password = False + _fd = None + _expect_sig = None + _parse_error = None + _password = None + comment = None + + def __init__(self, rarfile, password, crc_check, charset, strict, info_cb): + self._rarfile = rarfile + self._password = password + self._crc_check = crc_check + self._charset = charset + self._strict = strict + self._info_callback = info_cb + self._info_list = [] + self._info_map = {} + self._vol_list = [] + + def has_header_encryption(self): + """Returns True if headers are encrypted + """ + if self._hdrenc_main: + return True + if self._main: + if self._main.flags & RAR_MAIN_PASSWORD: + return True + return False + + def setpassword(self, psw): + """Set cached password.""" + self._password = psw + + def volumelist(self): + """Volume files""" + return self._vol_list + + def needs_password(self): + """Is password required""" + return self._needs_password + + def strerror(self): + """Last error""" + return self._parse_error + + def infolist(self): + """List of RarInfo records. + """ + return self._info_list + + def getinfo(self, fname): + """Return RarInfo for filename + """ + # accept both ways here + if PATH_SEP == '/': + fname2 = fname.replace("\\", "/") + else: + fname2 = fname.replace("/", "\\") + + try: + return self._info_map[fname] + except KeyError: + try: + return self._info_map[fname2] + except KeyError: + raise NoRarEntry("No such file: %s" % fname) + + # read rar + def parse(self): + """Process file.""" + self._fd = None + try: + self._parse_real() + finally: + if self._fd: + self._fd.close() + self._fd = None + + def _parse_real(self): + fd = XFile(self._rarfile) + self._fd = fd + sig = fd.read(len(self._expect_sig)) + if sig != self._expect_sig: + if isinstance(self._rarfile, (str, unicode)): + raise NotRarFile("Not a Rar archive: {}".format(self._rarfile)) + raise NotRarFile("Not a Rar archive") + + volume = 0 # first vol (.rar) is 0 + more_vols = False + endarc = False + volfile = self._rarfile + self._vol_list = [self._rarfile] + while 1: + if endarc: + h = None # don't read past ENDARC + else: + h = self._parse_header(fd) + if not h: + if more_vols: + volume += 1 + fd.close() + try: + volfile = self._next_volname(volfile) + fd = XFile(volfile) + except IOError: + self._set_error("Cannot open next volume: %s", volfile) + break + self._fd = fd + sig = fd.read(len(self._expect_sig)) + if sig != self._expect_sig: + self._set_error("Invalid volume sig: %s", volfile) + break + more_vols = False + endarc = False + self._vol_list.append(volfile) + continue + break + h.volume = volume + h.volume_file = volfile + + if h.type == RAR_BLOCK_MAIN and not self._main: + self._main = h + if h.flags & RAR_MAIN_NEWNUMBERING: + # RAR 2.x does not set FIRSTVOLUME, + # so check it only if NEWNUMBERING is used + if (h.flags & RAR_MAIN_FIRSTVOLUME) == 0: + raise NeedFirstVolume("Need to start from first volume") + if h.flags & RAR_MAIN_PASSWORD: + self._needs_password = True + if not self._password: + break + elif h.type == RAR_BLOCK_ENDARC: + more_vols = (h.flags & RAR_ENDARC_NEXT_VOLUME) > 0 + endarc = True + elif h.type == RAR_BLOCK_FILE: + # RAR 2.x does not write RAR_BLOCK_ENDARC + if h.flags & RAR_FILE_SPLIT_AFTER: + more_vols = True + # RAR 2.x does not set RAR_MAIN_FIRSTVOLUME + if volume == 0 and h.flags & RAR_FILE_SPLIT_BEFORE: + raise NeedFirstVolume("Need to start from first volume") + + if h.needs_password(): + self._needs_password = True + + # store it + self.process_entry(fd, h) + + if self._info_callback: + self._info_callback(h) + + # go to next header + if h.add_size > 0: + fd.seek(h.data_offset + h.add_size, 0) + + def process_entry(self, fd, item): + """Examine item, add into lookup cache.""" + raise NotImplementedError() + + def _decrypt_header(self, fd): + raise NotImplementedError('_decrypt_header') + + def _parse_block_header(self, fd): + raise NotImplementedError('_parse_block_header') + + def _open_hack(self, inf, psw): + raise NotImplementedError('_open_hack') + + # read single header + def _parse_header(self, fd): + try: + # handle encrypted headers + if (self._main and self._main.flags & RAR_MAIN_PASSWORD) or self._hdrenc_main: + if not self._password: + return + fd = self._decrypt_header(fd) + + # now read actual header + return self._parse_block_header(fd) + except struct.error: + self._set_error('Broken header in RAR file') + return None + + # given current vol name, construct next one + def _next_volname(self, volfile): + if is_filelike(volfile): + raise IOError("Working on single FD") + if self._main.flags & RAR_MAIN_NEWNUMBERING: + return _next_newvol(volfile) + return _next_oldvol(volfile) + + def _set_error(self, msg, *args): + if args: + msg = msg % args + self._parse_error = msg + if self._strict: + raise BadRarFile(msg) + + def open(self, inf, psw): + """Return stream object for file data.""" + + if inf.file_redir: + # cannot leave to unrar as it expects copied file to exist + if inf.file_redir[0] in (RAR5_XREDIR_FILE_COPY, RAR5_XREDIR_HARD_LINK): + inf = self.getinfo(inf.file_redir[2]) + if not inf: + raise BadRarFile('cannot find copied file') + + if inf.flags & RAR_FILE_SPLIT_BEFORE: + raise NeedFirstVolume("Partial file, please start from first volume: " + inf.filename) + + # is temp write usable? + use_hack = 1 + if not self._main: + use_hack = 0 + elif self._main._must_disable_hack(): + use_hack = 0 + elif inf._must_disable_hack(): + use_hack = 0 + elif is_filelike(self._rarfile): + pass + elif inf.file_size > HACK_SIZE_LIMIT: + use_hack = 0 + elif not USE_EXTRACT_HACK: + use_hack = 0 + + # now extract + if inf.compress_type == RAR_M0 and (inf.flags & RAR_FILE_PASSWORD) == 0 and inf.file_redir is None: + return self._open_clear(inf) + elif use_hack: + return self._open_hack(inf, psw) + elif is_filelike(self._rarfile): + return self._open_unrar_membuf(self._rarfile, inf, psw) + else: + return self._open_unrar(self._rarfile, inf, psw) + + def _open_clear(self, inf): + return DirectReader(self, inf) + + def _open_hack_core(self, inf, psw, prefix, suffix): + + size = inf.compress_size + inf.header_size + rf = XFile(inf.volume_file, 0) + rf.seek(inf.header_offset) + + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + + try: + tmpf.write(prefix) + while size > 0: + if size > BSIZE: + buf = rf.read(BSIZE) + else: + buf = rf.read(size) + if not buf: + raise BadRarFile('read failed: ' + inf.filename) + tmpf.write(buf) + size -= len(buf) + tmpf.write(suffix) + tmpf.close() + rf.close() + except: + rf.close() + tmpf.close() + os.unlink(tmpname) + raise + + return self._open_unrar(tmpname, inf, psw, tmpname) + + # write in-memory archive to temp file - needed for solid archives + def _open_unrar_membuf(self, memfile, inf, psw): + tmpname = membuf_tempfile(memfile) + return self._open_unrar(tmpname, inf, psw, tmpname, force_file=True) + + # extract using unrar + def _open_unrar(self, rarfile, inf, psw=None, tmpfile=None, force_file=False): + cmd = [UNRAR_TOOL] + list(OPEN_ARGS) + add_password_arg(cmd, psw) + cmd.append("--") + cmd.append(rarfile) + + # not giving filename avoids encoding related problems + if not tmpfile or force_file: + fn = inf.filename + if PATH_SEP != os.sep: + fn = fn.replace(PATH_SEP, os.sep) + cmd.append(fn) + + # read from unrar pipe + return PipeReader(self, inf, cmd, tmpfile) + +# +# RAR3 format +# + +class Rar3Info(RarInfo): + """RAR3 specific fields.""" + extract_version = 15 + salt = None + add_size = 0 + header_crc = None + header_size = None + header_offset = None + data_offset = None + _md_class = None + _md_expect = None + + # make sure some rar5 fields are always present + file_redir = None + blake2sp_hash = None + + def _must_disable_hack(self): + if self.type == RAR_BLOCK_FILE: + if self.flags & RAR_FILE_PASSWORD: + return True + elif self.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): + return True + elif self.type == RAR_BLOCK_MAIN: + if self.flags & (RAR_MAIN_SOLID | RAR_MAIN_PASSWORD): + return True + return False + + +class RAR3Parser(CommonParser): + """Parse RAR3 file format. + """ + _expect_sig = RAR_ID + _last_aes_key = (None, None, None) # (salt, key, iv) + + def _decrypt_header(self, fd): + if not _have_crypto: + raise NoCrypto('Cannot parse encrypted headers - no crypto') + salt = fd.read(8) + if self._last_aes_key[0] == salt: + key, iv = self._last_aes_key[1:] + else: + key, iv = rar3_s2k(self._password, salt) + self._last_aes_key = (salt, key, iv) + return HeaderDecrypt(fd, key, iv) + + # common header + def _parse_block_header(self, fd): + h = Rar3Info() + h.header_offset = fd.tell() + + # read and parse base header + buf = fd.read(S_BLK_HDR.size) + if not buf: + return None + t = S_BLK_HDR.unpack_from(buf) + h.header_crc, h.type, h.flags, h.header_size = t + + # read full header + if h.header_size > S_BLK_HDR.size: + hdata = buf + fd.read(h.header_size - S_BLK_HDR.size) + else: + hdata = buf + h.data_offset = fd.tell() + + # unexpected EOF? + if len(hdata) != h.header_size: + self._set_error('Unexpected EOF when reading header') + return None + + pos = S_BLK_HDR.size + + # block has data assiciated with it? + if h.flags & RAR_LONG_BLOCK: + h.add_size, pos = load_le32(hdata, pos) + else: + h.add_size = 0 + + # parse interesting ones, decide header boundaries for crc + if h.type == RAR_BLOCK_MARK: + return h + elif h.type == RAR_BLOCK_MAIN: + pos += 6 + if h.flags & RAR_MAIN_ENCRYPTVER: + pos += 1 + crc_pos = pos + if h.flags & RAR_MAIN_COMMENT: + self._parse_subblocks(h, hdata, pos) + elif h.type == RAR_BLOCK_FILE: + pos = self._parse_file_header(h, hdata, pos - 4) + crc_pos = pos + if h.flags & RAR_FILE_COMMENT: + pos = self._parse_subblocks(h, hdata, pos) + elif h.type == RAR_BLOCK_SUB: + pos = self._parse_file_header(h, hdata, pos - 4) + crc_pos = h.header_size + elif h.type == RAR_BLOCK_OLD_AUTH: + pos += 8 + crc_pos = pos + elif h.type == RAR_BLOCK_OLD_EXTRA: + pos += 7 + crc_pos = pos + else: + crc_pos = h.header_size + + # check crc + if h.type == RAR_BLOCK_OLD_SUB: + crcdat = hdata[2:] + fd.read(h.add_size) + else: + crcdat = hdata[2:crc_pos] + + calc_crc = rar_crc32(crcdat) & 0xFFFF + + # return good header + if h.header_crc == calc_crc: + return h + + # header parsing failed. + self._set_error('Header CRC error (%02x): exp=%x got=%x (xlen = %d)', + h.type, h.header_crc, calc_crc, len(crcdat)) + + # instead panicing, send eof + return None + + # read file-specific header + def _parse_file_header(self, h, hdata, pos): + fld = S_FILE_HDR.unpack_from(hdata, pos) + pos += S_FILE_HDR.size + + h.compress_size = fld[0] + h.file_size = fld[1] + h.host_os = fld[2] + h.CRC = fld[3] + h.date_time = parse_dos_time(fld[4]) + h.mtime = to_datetime(h.date_time) + h.extract_version = fld[5] + h.compress_type = fld[6] + name_size = fld[7] + h.mode = fld[8] + + h._md_class = CRC32Context + h._md_expect = h.CRC + + if h.flags & RAR_FILE_LARGE: + h1, pos = load_le32(hdata, pos) + h2, pos = load_le32(hdata, pos) + h.compress_size |= h1 << 32 + h.file_size |= h2 << 32 + h.add_size = h.compress_size + + name, pos = load_bytes(hdata, name_size, pos) + if h.flags & RAR_FILE_UNICODE: + nul = name.find(ZERO) + h.orig_filename = name[:nul] + u = UnicodeFilename(h.orig_filename, name[nul + 1:]) + h.filename = u.decode() + + # if parsing failed fall back to simple name + if u.failed: + h.filename = self._decode(h.orig_filename) + else: + h.orig_filename = name + h.filename = self._decode(name) + + # change separator, if requested + if PATH_SEP != '\\': + h.filename = h.filename.replace('\\', PATH_SEP) + + if h.flags & RAR_FILE_SALT: + h.salt, pos = load_bytes(hdata, 8, pos) + else: + h.salt = None + + # optional extended time stamps + if h.flags & RAR_FILE_EXTTIME: + pos = _parse_ext_time(h, hdata, pos) + else: + h.mtime = h.atime = h.ctime = h.arctime = None + + return pos + + # find old-style comment subblock + def _parse_subblocks(self, h, hdata, pos): + while pos < len(hdata): + # ordinary block header + t = S_BLK_HDR.unpack_from(hdata, pos) + ___scrc, stype, sflags, slen = t + pos_next = pos + slen + pos += S_BLK_HDR.size + + # corrupt header + if pos_next < pos: + break + + # followed by block-specific header + if stype == RAR_BLOCK_OLD_COMMENT and pos + S_COMMENT_HDR.size <= pos_next: + declen, ver, meth, crc = S_COMMENT_HDR.unpack_from(hdata, pos) + pos += S_COMMENT_HDR.size + data = hdata[pos : pos_next] + cmt = rar3_decompress(ver, meth, data, declen, sflags, + crc, self._password) + if not self._crc_check: + h.comment = self._decode_comment(cmt) + elif rar_crc32(cmt) & 0xFFFF == crc: + h.comment = self._decode_comment(cmt) + + pos = pos_next + return pos + + def _read_comment_v3(self, inf, psw=None): + + # read data + with XFile(inf.volume_file) as rf: + rf.seek(inf.data_offset) + data = rf.read(inf.compress_size) + + # decompress + cmt = rar3_decompress(inf.extract_version, inf.compress_type, data, + inf.file_size, inf.flags, inf.CRC, psw, inf.salt) + + # check crc + if self._crc_check: + crc = rar_crc32(cmt) + if crc != inf.CRC: + return None + + return self._decode_comment(cmt) + + def _decode(self, val): + for c in TRY_ENCODINGS: + try: + return val.decode(c) + except UnicodeError: + pass + return val.decode(self._charset, 'replace') + + def _decode_comment(self, val): + return self._decode(val) + + def process_entry(self, fd, item): + if item.type == RAR_BLOCK_FILE: + # use only first part + if (item.flags & RAR_FILE_SPLIT_BEFORE) == 0: + self._info_map[item.filename] = item + self._info_list.append(item) + elif len(self._info_list) > 0: + # final crc is in last block + old = self._info_list[-1] + old.CRC = item.CRC + old._md_expect = item._md_expect + old.compress_size += item.compress_size + + # parse new-style comment + if item.type == RAR_BLOCK_SUB and item.filename == 'CMT': + if item.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): + pass + elif item.flags & RAR_FILE_SOLID: + # file comment + cmt = self._read_comment_v3(item, self._password) + if len(self._info_list) > 0: + old = self._info_list[-1] + old.comment = cmt + else: + # archive comment + cmt = self._read_comment_v3(item, self._password) + self.comment = cmt + + if item.type == RAR_BLOCK_MAIN: + if item.flags & RAR_MAIN_COMMENT: + self.comment = item.comment + if item.flags & RAR_MAIN_PASSWORD: + self._needs_password = True + + # put file compressed data into temporary .rar archive, and run + # unrar on that, thus avoiding unrar going over whole archive + def _open_hack(self, inf, psw): + # create main header: crc, type, flags, size, res1, res2 + prefix = RAR_ID + S_BLK_HDR.pack(0x90CF, 0x73, 0, 13) + ZERO * (2 + 4) + return self._open_hack_core(inf, psw, prefix, EMPTY) + +# +# RAR5 format +# + +class Rar5Info(RarInfo): + """Shared fields for RAR5 records. + """ + extract_version = 50 + header_crc = None + header_size = None + header_offset = None + data_offset = None + + # type=all + block_type = None + block_flags = None + add_size = 0 + block_extra_size = 0 + + # type=MAIN + volume_number = None + _md_class = None + _md_expect = None + + def _must_disable_hack(self): + return False + + +class Rar5BaseFile(Rar5Info): + """Shared sturct for file & service record. + """ + type = -1 + file_flags = None + file_encryption = (0, 0, 0, EMPTY, EMPTY, EMPTY) + file_compress_flags = None + file_redir = None + file_owner = None + file_version = None + blake2sp_hash = None + + def _must_disable_hack(self): + if self.flags & RAR_FILE_PASSWORD: + return True + if self.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): + return True + if self.file_compress_flags & RAR5_COMPR_SOLID: + return True + if self.file_redir: + return True + return False + + +class Rar5FileInfo(Rar5BaseFile): + """RAR5 file record. + """ + type = RAR_BLOCK_FILE + + +class Rar5ServiceInfo(Rar5BaseFile): + """RAR5 service record. + """ + type = RAR_BLOCK_SUB + + +class Rar5MainInfo(Rar5Info): + """RAR5 archive main record. + """ + type = RAR_BLOCK_MAIN + main_flags = None + main_volume_number = None + + def _must_disable_hack(self): + if self.main_flags & RAR5_MAIN_FLAG_SOLID: + return True + return False + + +class Rar5EncryptionInfo(Rar5Info): + """RAR5 archive header encryption record. + """ + type = RAR5_BLOCK_ENCRYPTION + encryption_algo = None + encryption_flags = None + encryption_kdf_count = None + encryption_salt = None + encryption_check_value = None + + def needs_password(self): + return True + + +class Rar5EndArcInfo(Rar5Info): + """RAR5 end of archive record. + """ + type = RAR_BLOCK_ENDARC + endarc_flags = None + + +class RAR5Parser(CommonParser): + """Parse RAR5 format. + """ + _expect_sig = RAR5_ID + _hdrenc_main = None + + # AES encrypted headers + _last_aes256_key = (-1, None, None) # (kdf_count, salt, key) + + def _gen_key(self, kdf_count, salt): + if self._last_aes256_key[:2] == (kdf_count, salt): + return self._last_aes256_key[2] + if kdf_count > 24: + raise BadRarFile('Too large kdf_count') + psw = self._password + if isinstance(psw, unicode): + psw = psw.encode('utf8') + key = pbkdf2_sha256(psw, salt, 1 << kdf_count) + self._last_aes256_key = (kdf_count, salt, key) + return key + + def _decrypt_header(self, fd): + if not _have_crypto: + raise NoCrypto('Cannot parse encrypted headers - no crypto') + h = self._hdrenc_main + key = self._gen_key(h.encryption_kdf_count, h.encryption_salt) + iv = fd.read(16) + return HeaderDecrypt(fd, key, iv) + + # common header + def _parse_block_header(self, fd): + header_offset = fd.tell() + + preload = 4 + 3 + start_bytes = fd.read(preload) + header_crc, pos = load_le32(start_bytes, 0) + hdrlen, pos = load_vint(start_bytes, pos) + if hdrlen > 2 * 1024 * 1024: + return None + header_size = pos + hdrlen + + # read full header, check for EOF + hdata = start_bytes + fd.read(header_size - len(start_bytes)) + if len(hdata) != header_size: + self._set_error('Unexpected EOF when reading header') + return None + data_offset = fd.tell() + + calc_crc = rar_crc32(memoryview(hdata)[4:]) + if header_crc != calc_crc: + # header parsing failed. + self._set_error('Header CRC error: exp=%x got=%x (xlen = %d)', + header_crc, calc_crc, len(hdata)) + return None + + block_type, pos = load_vint(hdata, pos) + + if block_type == RAR5_BLOCK_MAIN: + h, pos = self._parse_block_common(Rar5MainInfo(), hdata) + h = self._parse_main_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_FILE: + h, pos = self._parse_block_common(Rar5FileInfo(), hdata) + h = self._parse_file_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_SERVICE: + h, pos = self._parse_block_common(Rar5ServiceInfo(), hdata) + h = self._parse_file_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_ENCRYPTION: + h, pos = self._parse_block_common(Rar5EncryptionInfo(), hdata) + h = self._parse_encryption_block(h, hdata, pos) + elif block_type == RAR5_BLOCK_ENDARC: + h, pos = self._parse_block_common(Rar5EndArcInfo(), hdata) + h = self._parse_endarc_block(h, hdata, pos) + else: + h = None + if h: + h.header_offset = header_offset + h.data_offset = data_offset + return h + + def _parse_block_common(self, h, hdata): + h.header_crc, pos = load_le32(hdata, 0) + hdrlen, pos = load_vint(hdata, pos) + h.header_size = hdrlen + pos + h.block_type, pos = load_vint(hdata, pos) + h.block_flags, pos = load_vint(hdata, pos) + + if h.block_flags & RAR5_BLOCK_FLAG_EXTRA_DATA: + h.block_extra_size, pos = load_vint(hdata, pos) + if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: + h.add_size, pos = load_vint(hdata, pos) + + h.compress_size = h.add_size + + if h.block_flags & RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN: + h.flags |= RAR_SKIP_IF_UNKNOWN + if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: + h.flags |= RAR_LONG_BLOCK + return h, pos + + def _parse_main_block(self, h, hdata, pos): + h.main_flags, pos = load_vint(hdata, pos) + if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR: + h.main_volume_number = load_vint(hdata, pos) + + h.flags |= RAR_MAIN_NEWNUMBERING + if h.main_flags & RAR5_MAIN_FLAG_SOLID: + h.flags |= RAR_MAIN_SOLID + if h.main_flags & RAR5_MAIN_FLAG_ISVOL: + h.flags |= RAR_MAIN_VOLUME + if h.main_flags & RAR5_MAIN_FLAG_RECOVERY: + h.flags |= RAR_MAIN_RECOVERY + if self._hdrenc_main: + h.flags |= RAR_MAIN_PASSWORD + if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR == 0: + h.flags |= RAR_MAIN_FIRSTVOLUME + + return h + + def _parse_file_block(self, h, hdata, pos): + h.file_flags, pos = load_vint(hdata, pos) + h.file_size, pos = load_vint(hdata, pos) + h.mode, pos = load_vint(hdata, pos) + + if h.file_flags & RAR5_FILE_FLAG_HAS_MTIME: + h.mtime, pos = load_unixtime(hdata, pos) + h.date_time = h.mtime.timetuple()[:6] + if h.file_flags & RAR5_FILE_FLAG_HAS_CRC32: + h.CRC, pos = load_le32(hdata, pos) + h._md_class = CRC32Context + h._md_expect = h.CRC + + h.file_compress_flags, pos = load_vint(hdata, pos) + h.file_host_os, pos = load_vint(hdata, pos) + h.orig_filename, pos = load_vstr(hdata, pos) + h.filename = h.orig_filename.decode('utf8', 'replace') + + # use compatible values + if h.file_host_os == RAR5_OS_WINDOWS: + h.host_os = RAR_OS_WIN32 + else: + h.host_os = RAR_OS_UNIX + h.compress_type = RAR_M0 + ((h.file_compress_flags >> 7) & 7) + + if h.block_extra_size: + # allow 1 byte of garbage + while pos < len(hdata) - 1: + xsize, pos = load_vint(hdata, pos) + xdata, pos = load_bytes(hdata, xsize, pos) + self._process_file_extra(h, xdata) + + if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE: + h.flags |= RAR_FILE_SPLIT_BEFORE + if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_AFTER: + h.flags |= RAR_FILE_SPLIT_AFTER + if h.file_flags & RAR5_FILE_FLAG_ISDIR: + h.flags |= RAR_FILE_DIRECTORY + if h.file_compress_flags & RAR5_COMPR_SOLID: + h.flags |= RAR_FILE_SOLID + + return h + + def _parse_endarc_block(self, h, hdata, pos): + h.endarc_flags, pos = load_vint(hdata, pos) + if h.endarc_flags & RAR5_ENDARC_FLAG_NEXT_VOL: + h.flags |= RAR_ENDARC_NEXT_VOLUME + return h + + def _parse_encryption_block(self, h, hdata, pos): + h.encryption_algo, pos = load_vint(hdata, pos) + h.encryption_flags, pos = load_vint(hdata, pos) + h.encryption_kdf_count, pos = load_byte(hdata, pos) + h.encryption_salt, pos = load_bytes(hdata, 16, pos) + if h.encryption_flags & RAR5_ENC_FLAG_HAS_CHECKVAL: + h.encryption_check_value = load_bytes(hdata, 12, pos) + if h.encryption_algo != RAR5_XENC_CIPHER_AES256: + raise BadRarFile('Unsupported header encryption cipher') + self._hdrenc_main = h + return h + + # file extra record + def _process_file_extra(self, h, xdata): + xtype, pos = load_vint(xdata, 0) + if xtype == RAR5_XFILE_TIME: + self._parse_file_xtime(h, xdata, pos) + elif xtype == RAR5_XFILE_ENCRYPTION: + self._parse_file_encryption(h, xdata, pos) + elif xtype == RAR5_XFILE_HASH: + self._parse_file_hash(h, xdata, pos) + elif xtype == RAR5_XFILE_VERSION: + self._parse_file_version(h, xdata, pos) + elif xtype == RAR5_XFILE_REDIR: + self._parse_file_redir(h, xdata, pos) + elif xtype == RAR5_XFILE_OWNER: + self._parse_file_owner(h, xdata, pos) + elif xtype == RAR5_XFILE_SERVICE: + pass + else: + pass + + # extra block for file time record + def _parse_file_xtime(self, h, xdata, pos): + tflags, pos = load_vint(xdata, pos) + ldr = load_windowstime + if tflags & RAR5_XTIME_UNIXTIME: + ldr = load_unixtime + if tflags & RAR5_XTIME_HAS_MTIME: + h.mtime, pos = ldr(xdata, pos) + h.date_time = h.mtime.timetuple()[:6] + if tflags & RAR5_XTIME_HAS_CTIME: + h.ctime, pos = ldr(xdata, pos) + if tflags & RAR5_XTIME_HAS_ATIME: + h.atime, pos = ldr(xdata, pos) + + # just remember encryption info + def _parse_file_encryption(self, h, xdata, pos): + algo, pos = load_vint(xdata, pos) + flags, pos = load_vint(xdata, pos) + kdf_count, pos = load_byte(xdata, pos) + salt, pos = load_bytes(xdata, 16, pos) + iv, pos = load_bytes(xdata, 16, pos) + checkval = None + if flags & RAR5_XENC_CHECKVAL: + checkval, pos = load_bytes(xdata, 12, pos) + if flags & RAR5_XENC_TWEAKED: + h._md_expect = None + h._md_class = NoHashContext + + h.file_encryption = (algo, flags, kdf_count, salt, iv, checkval) + h.flags |= RAR_FILE_PASSWORD + + def _parse_file_hash(self, h, xdata, pos): + hash_type, pos = load_vint(xdata, pos) + if hash_type == RAR5_XHASH_BLAKE2SP: + h.blake2sp_hash, pos = load_bytes(xdata, 32, pos) + if _have_blake2 and (h.file_encryption[1] & RAR5_XENC_TWEAKED) == 0: + h._md_class = Blake2SP + h._md_expect = h.blake2sp_hash + + def _parse_file_version(self, h, xdata, pos): + flags, pos = load_vint(xdata, pos) + version, pos = load_vint(xdata, pos) + h.file_version = (flags, version) + + def _parse_file_redir(self, h, xdata, pos): + redir_type, pos = load_vint(xdata, pos) + redir_flags, pos = load_vint(xdata, pos) + redir_name, pos = load_vstr(xdata, pos) + redir_name = redir_name.decode('utf8', 'replace') + h.file_redir = (redir_type, redir_flags, redir_name) + + def _parse_file_owner(self, h, xdata, pos): + user_name = group_name = user_id = group_id = None + + flags, pos = load_vint(xdata, pos) + if flags & RAR5_XOWNER_UNAME: + user_name, pos = load_vstr(xdata, pos) + if flags & RAR5_XOWNER_GNAME: + group_name, pos = load_vstr(xdata, pos) + if flags & RAR5_XOWNER_UID: + user_id, pos = load_vint(xdata, pos) + if flags & RAR5_XOWNER_GID: + group_id, pos = load_vint(xdata, pos) + + h.file_owner = (user_name, group_name, user_id, group_id) + + def process_entry(self, fd, item): + if item.block_type == RAR5_BLOCK_FILE: + # use only first part + if (item.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE) == 0: + self._info_map[item.filename] = item + self._info_list.append(item) + elif len(self._info_list) > 0: + # final crc is in last block + old = self._info_list[-1] + old.CRC = item.CRC + old._md_expect = item._md_expect + old.blake2sp_hash = item.blake2sp_hash + old.compress_size += item.compress_size + elif item.block_type == RAR5_BLOCK_SERVICE: + if item.filename == 'CMT': + self._load_comment(fd, item) + + def _load_comment(self, fd, item): + if item.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): + return None + if item.compress_type != RAR_M0: + return None + + if item.flags & RAR_FILE_PASSWORD: + algo, ___flags, kdf_count, salt, iv, ___checkval = item.file_encryption + if algo != RAR5_XENC_CIPHER_AES256: + return None + key = self._gen_key(kdf_count, salt) + f = HeaderDecrypt(fd, key, iv) + cmt = f.read(item.file_size) + else: + # archive comment + with self._open_clear(item) as cmtstream: + cmt = cmtstream.read() + + # rar bug? - appends zero to comment + cmt = cmt.split(ZERO, 1)[0] + self.comment = cmt.decode('utf8') + + def _open_hack(self, inf, psw): + # len, type, blk_flags, flags + main_hdr = b'\x03\x01\x00\x00' + endarc_hdr = b'\x03\x05\x00\x00' + main_hdr = S_LONG.pack(rar_crc32(main_hdr)) + main_hdr + endarc_hdr = S_LONG.pack(rar_crc32(endarc_hdr)) + endarc_hdr + return self._open_hack_core(inf, psw, RAR5_ID + main_hdr, endarc_hdr) + +## +## Utility classes +## + +class UnicodeFilename(object): + """Handle RAR3 unicode filename decompression. + """ + def __init__(self, name, encdata): + self.std_name = bytearray(name) + self.encdata = bytearray(encdata) + self.pos = self.encpos = 0 + self.buf = bytearray() + self.failed = 0 + + def enc_byte(self): + """Copy encoded byte.""" + try: + c = self.encdata[self.encpos] + self.encpos += 1 + return c + except IndexError: + self.failed = 1 + return 0 + + def std_byte(self): + """Copy byte from 8-bit representation.""" + try: + return self.std_name[self.pos] + except IndexError: + self.failed = 1 + return ord('?') + + def put(self, lo, hi): + """Copy 16-bit value to result.""" + self.buf.append(lo) + self.buf.append(hi) + self.pos += 1 + + def decode(self): + """Decompress compressed UTF16 value.""" + hi = self.enc_byte() + flagbits = 0 + while self.encpos < len(self.encdata): + if flagbits == 0: + flags = self.enc_byte() + flagbits = 8 + flagbits -= 2 + t = (flags >> flagbits) & 3 + if t == 0: + self.put(self.enc_byte(), 0) + elif t == 1: + self.put(self.enc_byte(), hi) + elif t == 2: + self.put(self.enc_byte(), self.enc_byte()) + else: + n = self.enc_byte() + if n & 0x80: + c = self.enc_byte() + for _ in range((n & 0x7f) + 2): + lo = (self.std_byte() + c) & 0xFF + self.put(lo, hi) + else: + for _ in range(n + 2): + self.put(self.std_byte(), 0) + return self.buf.decode("utf-16le", "replace") + + +class RarExtFile(RawIOBase): + """Base class for file-like object that :meth:`RarFile.open` returns. + + Provides public methods and common crc checking. + + Behaviour: + - no short reads - .read() and .readinfo() read as much as requested. + - no internal buffer, use io.BufferedReader for that. + """ + + #: Filename of the archive entry + name = None + + def __init__(self, parser, inf): + super(RarExtFile, self).__init__() + + # standard io.* properties + self.name = inf.filename + self.mode = 'rb' + + self._parser = parser + self._inf = inf + self._fd = None + self._remain = 0 + self._returncode = 0 + + self._md_context = None + + self._open() + + def _open(self): + if self._fd: + self._fd.close() + md_class = self._inf._md_class or NoHashContext + self._md_context = md_class() + self._fd = None + self._remain = self._inf.file_size + + def read(self, cnt=None): + """Read all or specified amount of data from archive entry.""" + + # sanitize cnt + if cnt is None or cnt < 0: + cnt = self._remain + elif cnt > self._remain: + cnt = self._remain + if cnt == 0: + return EMPTY + + # actual read + data = self._read(cnt) + if data: + self._md_context.update(data) + self._remain -= len(data) + if len(data) != cnt: + raise BadRarFile("Failed the read enough data") + + # done? + if not data or self._remain == 0: + # self.close() + self._check() + return data + + def _check(self): + """Check final CRC.""" + final = self._md_context.digest() + exp = self._inf._md_expect + if exp is None: + return + if final is None: + return + if self._returncode: + check_returncode(self, '') + if self._remain != 0: + raise BadRarFile("Failed the read enough data") + if final != exp: + raise BadRarFile("Corrupt file - CRC check failed: %s - exp=%r got=%r" % ( + self._inf.filename, exp, final)) + + def _read(self, cnt): + """Actual read that gets sanitized cnt.""" + + def close(self): + """Close open resources.""" + + super(RarExtFile, self).close() + + if self._fd: + self._fd.close() + self._fd = None + + def __del__(self): + """Hook delete to make sure tempfile is removed.""" + self.close() + + def readinto(self, buf): + """Zero-copy read directly into buffer. + + Returns bytes read. + """ + raise NotImplementedError('readinto') + + def tell(self): + """Return current reading position in uncompressed data.""" + return self._inf.file_size - self._remain + + def seek(self, ofs, whence=0): + """Seek in data. + + On uncompressed files, the seeking works by actual + seeks so it's fast. On compresses files its slow + - forward seeking happends by reading ahead, + backwards by re-opening and decompressing from the start. + """ + + # disable crc check when seeking + self._md_context = NoHashContext() + + fsize = self._inf.file_size + cur_ofs = self.tell() + + if whence == 0: # seek from beginning of file + new_ofs = ofs + elif whence == 1: # seek from current position + new_ofs = cur_ofs + ofs + elif whence == 2: # seek from end of file + new_ofs = fsize + ofs + else: + raise ValueError('Invalid value for whence') + + # sanity check + if new_ofs < 0: + new_ofs = 0 + elif new_ofs > fsize: + new_ofs = fsize + + # do the actual seek + if new_ofs >= cur_ofs: + self._skip(new_ofs - cur_ofs) + else: + # reopen and seek + self._open() + self._skip(new_ofs) + return self.tell() + + def _skip(self, cnt): + """Read and discard data""" + while cnt > 0: + if cnt > 8192: + buf = self.read(8192) + else: + buf = self.read(cnt) + if not buf: + break + cnt -= len(buf) + + def readable(self): + """Returns True""" + return True + + def writable(self): + """Returns False. + + Writing is not supported.""" + return False + + def seekable(self): + """Returns True. + + Seeking is supported, although it's slow on compressed files. + """ + return True + + def readall(self): + """Read all remaining data""" + # avoid RawIOBase default impl + return self.read() + + +class PipeReader(RarExtFile): + """Read data from pipe, handle tempfile cleanup.""" + + def __init__(self, rf, inf, cmd, tempfile=None): + self._cmd = cmd + self._proc = None + self._tempfile = tempfile + super(PipeReader, self).__init__(rf, inf) + + def _close_proc(self): + if not self._proc: + return + if self._proc.stdout: + self._proc.stdout.close() + if self._proc.stdin: + self._proc.stdin.close() + if self._proc.stderr: + self._proc.stderr.close() + self._proc.wait() + self._returncode = self._proc.returncode + self._proc = None + + def _open(self): + super(PipeReader, self)._open() + + # stop old process + self._close_proc() + + # launch new process + self._returncode = 0 + self._proc = custom_popen(self._cmd) + self._fd = self._proc.stdout + + # avoid situation where unrar waits on stdin + if self._proc.stdin: + self._proc.stdin.close() + + def _read(self, cnt): + """Read from pipe.""" + + # normal read is usually enough + data = self._fd.read(cnt) + if len(data) == cnt or not data: + return data + + # short read, try looping + buf = [data] + cnt -= len(data) + while cnt > 0: + data = self._fd.read(cnt) + if not data: + break + cnt -= len(data) + buf.append(data) + return EMPTY.join(buf) + + def close(self): + """Close open resources.""" + + self._close_proc() + super(PipeReader, self).close() + + if self._tempfile: + try: + os.unlink(self._tempfile) + except OSError: + pass + self._tempfile = None + + def readinto(self, buf): + """Zero-copy read directly into buffer.""" + cnt = len(buf) + if cnt > self._remain: + cnt = self._remain + vbuf = memoryview(buf) + res = got = 0 + while got < cnt: + res = self._fd.readinto(vbuf[got : cnt]) + if not res: + break + self._md_context.update(vbuf[got : got + res]) + self._remain -= res + got += res + return got + + +class DirectReader(RarExtFile): + """Read uncompressed data directly from archive. + """ + _cur = None + _cur_avail = None + _volfile = None + + def _open(self): + super(DirectReader, self)._open() + + self._volfile = self._inf.volume_file + self._fd = XFile(self._volfile, 0) + self._fd.seek(self._inf.header_offset, 0) + self._cur = self._parser._parse_header(self._fd) + self._cur_avail = self._cur.add_size + + def _skip(self, cnt): + """RAR Seek, skipping through rar files to get to correct position + """ + + while cnt > 0: + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # fd is in read pos, do the read + if cnt > self._cur_avail: + cnt -= self._cur_avail + self._remain -= self._cur_avail + self._cur_avail = 0 + else: + self._fd.seek(cnt, 1) + self._cur_avail -= cnt + self._remain -= cnt + cnt = 0 + + def _read(self, cnt): + """Read from potentially multi-volume archive.""" + + buf = [] + while cnt > 0: + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # fd is in read pos, do the read + if cnt > self._cur_avail: + data = self._fd.read(self._cur_avail) + else: + data = self._fd.read(cnt) + if not data: + break + + # got some data + cnt -= len(data) + self._cur_avail -= len(data) + buf.append(data) + + if len(buf) == 1: + return buf[0] + return EMPTY.join(buf) + + def _open_next(self): + """Proceed to next volume.""" + + # is the file split over archives? + if (self._cur.flags & RAR_FILE_SPLIT_AFTER) == 0: + return False + + if self._fd: + self._fd.close() + self._fd = None + + # open next part + self._volfile = self._parser._next_volname(self._volfile) + fd = open(self._volfile, "rb", 0) + self._fd = fd + sig = fd.read(len(self._parser._expect_sig)) + if sig != self._parser._expect_sig: + raise BadRarFile("Invalid signature") + + # loop until first file header + while 1: + cur = self._parser._parse_header(fd) + if not cur: + raise BadRarFile("Unexpected EOF") + if cur.type in (RAR_BLOCK_MARK, RAR_BLOCK_MAIN): + if cur.add_size: + fd.seek(cur.add_size, 1) + continue + if cur.orig_filename != self._inf.orig_filename: + raise BadRarFile("Did not found file entry") + self._cur = cur + self._cur_avail = cur.add_size + return True + + def readinto(self, buf): + """Zero-copy read directly into buffer.""" + got = 0 + vbuf = memoryview(buf) + while got < len(buf): + # next vol needed? + if self._cur_avail == 0: + if not self._open_next(): + break + + # length for next read + cnt = len(buf) - got + if cnt > self._cur_avail: + cnt = self._cur_avail + + # read into temp view + res = self._fd.readinto(vbuf[got : got + cnt]) + if not res: + break + self._md_context.update(vbuf[got : got + res]) + self._cur_avail -= res + self._remain -= res + got += res + return got + + +class HeaderDecrypt(object): + """File-like object that decrypts from another file""" + def __init__(self, f, key, iv): + self.f = f + self.ciph = AES_CBC_Decrypt(key, iv) + self.buf = EMPTY + + def tell(self): + """Current file pos - works only on block boundaries.""" + return self.f.tell() + + def read(self, cnt=None): + """Read and decrypt.""" + if cnt > 8 * 1024: + raise BadRarFile('Bad count to header decrypt - wrong password?') + + # consume old data + if cnt <= len(self.buf): + res = self.buf[:cnt] + self.buf = self.buf[cnt:] + return res + res = self.buf + self.buf = EMPTY + cnt -= len(res) + + # decrypt new data + blklen = 16 + while cnt > 0: + enc = self.f.read(blklen) + if len(enc) < blklen: + break + dec = self.ciph.decrypt(enc) + if cnt >= len(dec): + res += dec + cnt -= len(dec) + else: + res += dec[:cnt] + self.buf = dec[cnt:] + cnt = 0 + + return res + + +# handle (filename|filelike) object +class XFile(object): + """Input may be filename or file object. + """ + __slots__ = ('_fd', '_need_close') + + def __init__(self, xfile, bufsize=1024): + if is_filelike(xfile): + self._need_close = False + self._fd = xfile + self._fd.seek(0) + else: + self._need_close = True + self._fd = open(xfile, 'rb', bufsize) + + def read(self, n=None): + """Read from file.""" + return self._fd.read(n) + + def tell(self): + """Return file pos.""" + return self._fd.tell() + + def seek(self, ofs, whence=0): + """Move file pos.""" + return self._fd.seek(ofs, whence) + + def readinto(self, dst): + """Read into buffer.""" + return self._fd.readinto(dst) + + def close(self): + """Close file object.""" + if self._need_close: + self._fd.close() + + def __enter__(self): + return self + + def __exit__(self, typ, val, tb): + self.close() + + +class NoHashContext(object): + """No-op hash function.""" + def __init__(self, data=None): + """Initialize""" + def update(self, data): + """Update data""" + def digest(self): + """Final hash""" + def hexdigest(self): + """Hexadecimal digest.""" + + +class CRC32Context(object): + """Hash context that uses CRC32.""" + __slots__ = ['_crc'] + + def __init__(self, data=None): + self._crc = 0 + if data: + self.update(data) + + def update(self, data): + """Process data.""" + self._crc = rar_crc32(data, self._crc) + + def digest(self): + """Final hash.""" + return self._crc + + def hexdigest(self): + """Hexadecimal digest.""" + return '%08x' % self.digest() + + +class Blake2SP(object): + """Blake2sp hash context. + """ + __slots__ = ['_thread', '_buf', '_cur', '_digest'] + digest_size = 32 + block_size = 64 + parallelism = 8 + + def __init__(self, data=None): + self._buf = b'' + self._cur = 0 + self._digest = None + self._thread = [] + + for i in range(self.parallelism): + ctx = self._blake2s(i, 0, i == (self.parallelism - 1)) + self._thread.append(ctx) + + if data: + self.update(data) + + def _blake2s(self, ofs, depth, is_last): + return blake2s(node_offset=ofs, node_depth=depth, last_node=is_last, + depth=2, inner_size=32, fanout=self.parallelism) + + def _add_block(self, blk): + self._thread[self._cur].update(blk) + self._cur = (self._cur + 1) % self.parallelism + + def update(self, data): + """Hash data. + """ + view = memoryview(data) + bs = self.block_size + if self._buf: + need = bs - len(self._buf) + if len(view) < need: + self._buf += view.tobytes() + return + self._add_block(self._buf + view[:need].tobytes()) + view = view[need:] + while len(view) >= bs: + self._add_block(view[:bs]) + view = view[bs:] + self._buf = view.tobytes() + + def digest(self): + """Return final digest value. + """ + if self._digest is None: + if self._buf: + self._add_block(self._buf) + self._buf = EMPTY + ctx = self._blake2s(0, 1, True) + for t in self._thread: + ctx.update(t.digest()) + self._digest = ctx.digest() + return self._digest + + def hexdigest(self): + """Hexadecimal digest.""" + return tohex(self.digest()) + +## +## Utility functions +## + +S_LONG = Struct(' len(buf): + raise BadRarFile('cannot load byte') + return S_BYTE.unpack_from(buf, pos)[0], end + +def load_le32(buf, pos): + """Load little-endian 32-bit integer""" + end = pos + 4 + if end > len(buf): + raise BadRarFile('cannot load le32') + return S_LONG.unpack_from(buf, pos)[0], pos + 4 + +def load_bytes(buf, num, pos): + """Load sequence of bytes""" + end = pos + num + if end > len(buf): + raise BadRarFile('cannot load bytes') + return buf[pos : end], end + +def load_vstr(buf, pos): + """Load bytes prefixed by vint length""" + slen, pos = load_vint(buf, pos) + return load_bytes(buf, slen, pos) + +def load_dostime(buf, pos): + """Load LE32 dos timestamp""" + stamp, pos = load_le32(buf, pos) + tup = parse_dos_time(stamp) + return to_datetime(tup), pos + +def load_unixtime(buf, pos): + """Load LE32 unix timestamp""" + secs, pos = load_le32(buf, pos) + dt = datetime.fromtimestamp(secs, UTC) + return dt, pos + +def load_windowstime(buf, pos): + """Load LE64 windows timestamp""" + # unix epoch (1970) in seconds from windows epoch (1601) + unix_epoch = 11644473600 + val1, pos = load_le32(buf, pos) + val2, pos = load_le32(buf, pos) + secs, n1secs = divmod((val2 << 32) | val1, 10000000) + dt = datetime.fromtimestamp(secs - unix_epoch, UTC) + dt = dt.replace(microsecond=n1secs // 10) + return dt, pos + +# new-style next volume +def _next_newvol(volfile): + i = len(volfile) - 1 + while i >= 0: + if volfile[i] >= '0' and volfile[i] <= '9': + return _inc_volname(volfile, i) + i -= 1 + raise BadRarName("Cannot construct volume name: " + volfile) + +# old-style next volume +def _next_oldvol(volfile): + # rar -> r00 + if volfile[-4:].lower() == '.rar': + return volfile[:-2] + '00' + return _inc_volname(volfile, len(volfile) - 1) + +# increase digits with carry, otherwise just increment char +def _inc_volname(volfile, i): + fn = list(volfile) + while i >= 0: + if fn[i] != '9': + fn[i] = chr(ord(fn[i]) + 1) + break + fn[i] = '0' + i -= 1 + return ''.join(fn) + +# rar3 extended time fields +def _parse_ext_time(h, data, pos): + # flags and rest of data can be missing + flags = 0 + if pos + 2 <= len(data): + flags = S_SHORT.unpack_from(data, pos)[0] + pos += 2 + + mtime, pos = _parse_xtime(flags >> 3 * 4, data, pos, h.mtime) + h.ctime, pos = _parse_xtime(flags >> 2 * 4, data, pos) + h.atime, pos = _parse_xtime(flags >> 1 * 4, data, pos) + h.arctime, pos = _parse_xtime(flags >> 0 * 4, data, pos) + if mtime: + h.mtime = mtime + h.date_time = mtime.timetuple()[:6] + return pos + +# rar3 one extended time field +def _parse_xtime(flag, data, pos, basetime=None): + res = None + if flag & 8: + if not basetime: + basetime, pos = load_dostime(data, pos) + + # load second fractions + rem = 0 + cnt = flag & 3 + for _ in range(cnt): + b, pos = load_byte(data, pos) + rem = (b << 16) | (rem >> 8) + + # convert 100ns units to microseconds + usec = rem // 10 + if usec > 1000000: + usec = 999999 + + # dostime has room for 30 seconds only, correct if needed + if flag & 4 and basetime.second < 59: + res = basetime.replace(microsecond=usec, second=basetime.second + 1) + else: + res = basetime.replace(microsecond=usec) + return res, pos + +def is_filelike(obj): + """Filename or file object? + """ + if isinstance(obj, str) or isinstance(obj, unicode): + return False + res = True + for a in ('read', 'tell', 'seek'): + res = res and hasattr(obj, a) + if not res: + raise ValueError("Invalid object passed as file") + return True + +def rar3_s2k(psw, salt): + """String-to-key hash for RAR3. + """ + if not isinstance(psw, unicode): + psw = psw.decode('utf8') + seed = psw.encode('utf-16le') + salt + iv = EMPTY + h = sha1() + for i in range(16): + for j in range(0x4000): + cnt = S_LONG.pack(i * 0x4000 + j) + h.update(seed + cnt[:3]) + if j == 0: + iv += h.digest()[19:20] + key_be = h.digest()[:16] + key_le = pack("LLLL", key_be)) + return key_le, iv + +def rar3_decompress(vers, meth, data, declen=0, flags=0, crc=0, psw=None, salt=None): + """Decompress blob of compressed data. + + Used for data with non-standard header - eg. comments. + """ + # already uncompressed? + if meth == RAR_M0 and (flags & RAR_FILE_PASSWORD) == 0: + return data + + # take only necessary flags + flags = flags & (RAR_FILE_PASSWORD | RAR_FILE_SALT | RAR_FILE_DICTMASK) + flags |= RAR_LONG_BLOCK + + # file header + fname = b'data' + date = 0 + mode = 0x20 + fhdr = S_FILE_HDR.pack(len(data), declen, RAR_OS_MSDOS, crc, + date, vers, meth, len(fname), mode) + fhdr += fname + if flags & RAR_FILE_SALT: + if not salt: + return EMPTY + fhdr += salt + + # full header + hlen = S_BLK_HDR.size + len(fhdr) + hdr = S_BLK_HDR.pack(0, RAR_BLOCK_FILE, flags, hlen) + fhdr + hcrc = rar_crc32(hdr[2:]) & 0xFFFF + hdr = S_BLK_HDR.pack(hcrc, RAR_BLOCK_FILE, flags, hlen) + fhdr + + # archive main header + mh = S_BLK_HDR.pack(0x90CF, RAR_BLOCK_MAIN, 0, 13) + ZERO * (2 + 4) + + # decompress via temp rar + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + try: + tmpf.write(RAR_ID + mh + hdr + data) + tmpf.close() + + cmd = [UNRAR_TOOL] + list(OPEN_ARGS) + add_password_arg(cmd, psw, (flags & RAR_FILE_PASSWORD)) + cmd.append(tmpname) + + p = custom_popen(cmd) + return p.communicate()[0] + finally: + tmpf.close() + os.unlink(tmpname) + +def to_datetime(t): + """Convert 6-part time tuple into datetime object. + """ + if t is None: + return None + + # extract values + year, mon, day, h, m, s = t + + # assume the values are valid + try: + return datetime(year, mon, day, h, m, s) + except ValueError: + pass + + # sanitize invalid values + mday = (0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) + if mon < 1: + mon = 1 + if mon > 12: + mon = 12 + if day < 1: + day = 1 + if day > mday[mon]: + day = mday[mon] + if h > 23: + h = 23 + if m > 59: + m = 59 + if s > 59: + s = 59 + if mon == 2 and day == 29: + try: + return datetime(year, mon, day, h, m, s) + except ValueError: + day = 28 + return datetime(year, mon, day, h, m, s) + +def parse_dos_time(stamp): + """Parse standard 32-bit DOS timestamp. + """ + sec, stamp = stamp & 0x1F, stamp >> 5 + mn, stamp = stamp & 0x3F, stamp >> 6 + hr, stamp = stamp & 0x1F, stamp >> 5 + day, stamp = stamp & 0x1F, stamp >> 5 + mon, stamp = stamp & 0x0F, stamp >> 4 + yr = (stamp & 0x7F) + 1980 + return (yr, mon, day, hr, mn, sec * 2) + +def custom_popen(cmd): + """Disconnect cmd from parent fds, read only from stdout. + """ + # needed for py2exe + creationflags = 0 + if sys.platform == 'win32': + creationflags = 0x08000000 # CREATE_NO_WINDOW + + # run command + try: + p = Popen(cmd, bufsize=0, stdout=PIPE, stdin=PIPE, stderr=STDOUT, + creationflags=creationflags) + except OSError as ex: + if ex.errno == errno.ENOENT: + raise RarCannotExec("Unrar not installed? (rarfile.UNRAR_TOOL=%r)" % UNRAR_TOOL) + raise + return p + +def custom_check(cmd, ignore_retcode=False): + """Run command, collect output, raise error if needed. + """ + p = custom_popen(cmd) + out, _ = p.communicate() + if p.returncode and not ignore_retcode: + raise RarExecError("Check-run failed") + return out + +def add_password_arg(cmd, psw, ___required=False): + """Append password switch to commandline. + """ + if UNRAR_TOOL == ALT_TOOL: + return + if psw is not None: + cmd.append('-p' + psw) + else: + cmd.append('-p-') + +def check_returncode(p, out): + """Raise exception according to unrar exit code. + """ + code = p.returncode + if code == 0: + return + + # map return code to exception class, codes from rar.txt + errmap = [None, + RarWarning, RarFatalError, RarCRCError, RarLockedArchiveError, # 1..4 + RarWriteError, RarOpenError, RarUserError, RarMemoryError, # 5..8 + RarCreateError, RarNoFilesError, RarWrongPassword] # 9..11 + if UNRAR_TOOL == ALT_TOOL: + errmap = [None] + if code > 0 and code < len(errmap): + exc = errmap[code] + elif code == 255: + exc = RarUserBreak + elif code < 0: + exc = RarSignalExit + else: + exc = RarUnknownError + + # format message + if out: + msg = "%s [%d]: %s" % (exc.__doc__, p.returncode, out) + else: + msg = "%s [%d]" % (exc.__doc__, p.returncode) + + raise exc(msg) + +def hmac_sha256(key, data): + """HMAC-SHA256""" + return HMAC(key, data, sha256).digest() + +def membuf_tempfile(memfile): + memfile.seek(0, 0) + + tmpfd, tmpname = mkstemp(suffix='.rar') + tmpf = os.fdopen(tmpfd, "wb") + + try: + while True: + buf = memfile.read(BSIZE) + if not buf: + break + tmpf.write(buf) + tmpf.close() + except: + tmpf.close() + os.unlink(tmpname) + raise + return tmpname + +class XTempFile(object): + __slots__ = ('_tmpfile', '_filename') + + def __init__(self, rarfile): + if is_filelike(rarfile): + self._tmpfile = membuf_tempfile(rarfile) + self._filename = self._tmpfile + else: + self._tmpfile = None + self._filename = rarfile + + def __enter__(self): + return self._filename + + def __exit__(self, exc_type, exc_value, tb): + if self._tmpfile: + try: + os.unlink(self._tmpfile) + except OSError: + pass + self._tmpfile = None + +# +# Check if unrar works +# + +ORIG_UNRAR_TOOL = UNRAR_TOOL +ORIG_OPEN_ARGS = OPEN_ARGS +ORIG_EXTRACT_ARGS = EXTRACT_ARGS +ORIG_TEST_ARGS = TEST_ARGS + +def _check_unrar_tool(): + global UNRAR_TOOL, OPEN_ARGS, EXTRACT_ARGS, TEST_ARGS + try: + # does UNRAR_TOOL work? + custom_check([ORIG_UNRAR_TOOL], True) + + UNRAR_TOOL = ORIG_UNRAR_TOOL + OPEN_ARGS = ORIG_OPEN_ARGS + EXTRACT_ARGS = ORIG_EXTRACT_ARGS + TEST_ARGS = ORIG_TEST_ARGS + except RarCannotExec: + try: + # does ALT_TOOL work? + custom_check([ALT_TOOL] + list(ALT_CHECK_ARGS), True) + # replace config + UNRAR_TOOL = ALT_TOOL + OPEN_ARGS = ALT_OPEN_ARGS + EXTRACT_ARGS = ALT_EXTRACT_ARGS + TEST_ARGS = ALT_TEST_ARGS + except RarCannotExec: + # no usable tool, only uncompressed archives work + pass + +_check_unrar_tool() + From d8e8cff0733afb26f82fcdf4b27c45c900b9a6a4 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:19:16 +0000 Subject: [PATCH 051/185] Create DeleteSamples.py --- .../templates/nzgbet/scripts/DeleteSamples.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/DeleteSamples.py diff --git a/apps/templates/nzgbet/scripts/DeleteSamples.py b/apps/templates/nzgbet/scripts/DeleteSamples.py new file mode 100644 index 0000000..395628f --- /dev/null +++ b/apps/templates/nzgbet/scripts/DeleteSamples.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +def is_sample(filePath, inputName, maxSampleSize, SampleIDs): + # 200 MB in bytes + SIZE_CUTOFF = int(maxSampleSize) * 1024 * 1024 + if os.path.getsize(filePath) < SIZE_CUTOFF: + if 'SizeOnly' in SampleIDs: + return True + # Ignore 'sample' in files unless 'sample' in Torrent Name + for ident in SampleIDs: + if ident.lower() in filePath.lower() and not ident.lower() in inputName.lower(): + return True + # Return False if none of these were met. + return False + +if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + +if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + +print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) +status = 0 +if os.environ.has_key('NZBPP_TOTALSTATUS'): + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + +else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." + print "Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." + print "Please check your Par-check/repair settings for future downloads." + +# Check if destination directory exists (important for reprocessing of history items) +if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + +# All checks done, now launching the script. +if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + +mediaContainer = os.environ['NZBPO_MEDIAEXTENSIONS'].split(',') +SampleIDs = os.environ['NZBPO_SAMPLEIDS'].split(',') +for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): + for file in filenames: + filePath = os.path.join(dirpath, file) + fileName, fileExtension = os.path.splitext(file) + if fileExtension in mediaContainer or ".*" in mediaContainer : # If the file is a video file + if is_sample(filePath, os.environ['NZBPP_NZBNAME'], os.environ['NZBPO_MAXSAMPLESIZE'], SampleIDs): # Ignore samples + print "Deleting sample file: ", filePath + try: + os.unlink(filePath) + except: + print "Error: unable to delete file", filePath + sys.exit(NZBGET_POSTPROCESS_ERROR) +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From 17ef17859feb676dd3da81ecb258f91d8d5bdf28 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:19:32 +0000 Subject: [PATCH 052/185] Create flatten.py --- apps/templates/nzgbet/scripts/flatten.py | 106 +++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/flatten.py diff --git a/apps/templates/nzgbet/scripts/flatten.py b/apps/templates/nzgbet/scripts/flatten.py new file mode 100644 index 0000000..73dd0d5 --- /dev/null +++ b/apps/templates/nzgbet/scripts/flatten.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys +import shutil + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + +if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + +print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) +status = 0 +if os.environ.has_key('NZBPP_TOTALSTATUS'): + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + +else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "[ERROR] Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "[ERROR] Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." + print "[ERROR] Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." + print "[WARNING] Please check your Par-check/repair settings for future downloads." + +# Check if destination directory exists (important for reprocessing of history items) +if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "[ERROR] Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + +# All checks done, now launching the script. +if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + +def removeEmptyFolders(path, removeRoot=True): + #Function to remove empty folders + if not os.path.isdir(path): + return + + # remove empty subfolders + print "[INFO] Checking for empty folders in:%s" % path + files = os.listdir(path) + if len(files): + for f in files: + fullpath = os.path.join(path, f) + if os.path.isdir(fullpath): + removeEmptyFolders(fullpath) + + # if folder empty, delete it + files = os.listdir(path) + if len(files) == 0 and removeRoot: + print "[INFO] Removing empty folder:%s" % path + os.rmdir(path) + +directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) +if os.environ['NZBPO_DESTINATIONDIRECTORY'] and os.path.isdir(os.environ['NZBPO_DESTINATIONDIRECTORY']): + destination = os.environ['NZBPO_DESTINATIONDIRECTORY'] + if os.environ['NZBPO_APPENDCATEGORIES'] == 'yes': + destination = os.path.join(destination, os.environ['NZBPP_CATEGORY']) +else: + destination = directory +print "Flattening directory: %s" % (directory) +for dirpath, dirnames, filenames in os.walk(directory): + for fileName in filenames: + outputFile = os.path.join(dirpath, fileName) + if dirpath == directory: + continue + target = os.path.join(destination, fileName) + try: + shutil.move(outputFile, target) + except: + print "[ERROR] Could not flatten %s" % outputFile +removeEmptyFolders(directory) # Cleanup empty directories +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From e4496b31fcc9b8999205acf14dbabb61c32ffd44 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:19:51 +0000 Subject: [PATCH 053/185] Create hash.py --- apps/templates/nzgbet/scripts/hash.py | 167 ++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/hash.py diff --git a/apps/templates/nzgbet/scripts/hash.py b/apps/templates/nzgbet/scripts/hash.py new file mode 100644 index 0000000..f2fbec1 --- /dev/null +++ b/apps/templates/nzgbet/scripts/hash.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Original Author: clinton-hall +# https://github.com/clinton-hall/GetScripts/blob/master/SafeRename.py +# +# Modified By: desimaniac (No Acknowledgement of Source Above) +################################################################################ +import os +import re +import shutil +import sys + +# NZBGet Exit Codes +NZBGET_POSTPROCESS_PARCHECK = 92 +NZBGET_POSTPROCESS_SUCCESS = 93 +NZBGET_POSTPROCESS_ERROR = 94 +NZBGET_POSTPROCESS_NONE = 95 + +############################################################ +# EXTENSION STUFF +############################################################ + +def do_check(): + if not os.environ.has_key('NZBOP_SCRIPTDIR'): + print "This script can only be called from NZBGet (11.0 or later)." + sys.exit(0) + + if os.environ['NZBOP_VERSION'][0:5] < '11.0': + print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) + sys.exit(0) + + print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) + + status = 0 + if 'NZBPP_TOTALSTATUS' in os.environ: + if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': + print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) + status = 1 + else: + # Check par status + if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': + print "[ERROR] Par-repair failed, setting status \"failed\"." + status = 1 + + # Check unpack status + if os.environ['NZBPP_UNPACKSTATUS'] == '1': + print "[ERROR] Unpack failed, setting status \"failed\"." + status = 1 + + if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': + # Unpack was skipped due to nzb-file properties or due to errors during par-check + + if os.environ['NZBPP_HEALTH'] < 1000: + print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. " \ + "Setting status \"failed\"." + print "[ERROR] Please check your Par-check/repair settings for future downloads." + status = 1 + + else: + print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is " \ + "ok so handle as though download successful." + print "[WARNING] Please check your Par-check/repair settings for future downloads." + + # Check if destination directory exists (important for reprocessing of history items) + if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): + print "[ERROR] Nothing to post-process: destination directory", os.environ[ + 'NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." + status = 1 + + # All checks done, now launching the script. + if status == 1: + sys.exit(NZBGET_POSTPROCESS_NONE) + + +def get_file_name(path): + try: + file_name = os.path.basename(path) + extensions = re.findall(r'\.([^.]+)', file_name) + ext = '.'.join(extensions) + name = file_name.replace(".%s" % ext, '') + return name, ext + except Exception: + pass + return None + + +def is_file_hash(file_name): + hash_regexp = [ + r'^[a-fA-F0-9]{40}$', + r'^[a-fA-F0-9]{32}$', + r'^[a-f0-9]{128}$', + r'^[a-zA-Z0-9]{42}$' + ] + for hash in hash_regexp: + if re.match(hash, file_name): + return True + return False + + +def find_files(folder, extension=None, depth=None): + file_list = [] + start_count = folder.count(os.sep) + for path, subdirs, files in os.walk(folder, topdown=True): + for name in files: + if depth and path.count(os.sep) - start_count >= depth: + del subdirs[:] + continue + file = os.path.join(path, name) + if not extension: + file_list.append(file) + else: + if file.lower().endswith(extension.lower()): + file_list.append(file) + + return sorted(file_list, key=lambda x: x.count(os.path.sep), reverse=True) + + +############################################################ +# MAIN +############################################################ + +# do checks +do_check() + +# retrieve required variables +directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) +nzb_name = os.environ['NZBPP_NZBFILENAME'] +if nzb_name is None: + print("[ERROR] Unable to retrieve NZBPP_NZBFILENAME") + sys.exit(NZBGET_POSTPROCESS_ERROR) +nzb_name = nzb_name.replace('.nzb', '') + +print("[INFO] Using \"%s\" for hashed filenames" % nzb_name) +print("[INFO] Scanning \"%s\" for hashed filenames" % directory) + +# scan for files +found_files = find_files(directory) +if not found_files: + print("[INFO] No files were found in \"%s\"" % directory) + sys.exit(NZBGET_POSTPROCESS_NONE) +else: + print("[INFO] Found %d files to check for hashed filenames" % len(found_files)) + # loop files checking for file hash + moved_files = 0 + for found_file_path in found_files: + # set variable + dir_name = os.path.dirname(found_file_path) + file_name, file_ext = get_file_name(found_file_path) + + # is this a file hash + if is_file_hash(file_name): + new_file_path = os.path.join(dir_name, "%s.%s" % (nzb_name, file_ext)) + print("[INFO] Moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) + try: + shutil.move(found_file_path, new_file_path) + moved_files += 1 + except Exception: + print("[ERROR] Failed moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) + + print("[INFO] Finished processing \"%s\", moved %d files" % (directory, moved_files)) + +sys.exit(NZBGET_POSTPROCESS_SUCCESS) From b4834608ba66469e090e4bb1eb69939d79e166f3 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:20:07 +0000 Subject: [PATCH 054/185] Create reverse_name.py --- apps/templates/nzgbet/scripts/reverse_name.py | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/reverse_name.py diff --git a/apps/templates/nzgbet/scripts/reverse_name.py b/apps/templates/nzgbet/scripts/reverse_name.py new file mode 100644 index 0000000..98819d0 --- /dev/null +++ b/apps/templates/nzgbet/scripts/reverse_name.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python +# +# Title: PGBlitz (Reference Title File) +# Maintainer: Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +# +# Additions: clinton-hall - https://github.com/Prinz23 +################################################################################ +import os +import sys +import re +import locale + +reverse_list = [r"\.\d{2}e\d{2}s\.", r"\.p0612\.", r"\.[pi]0801\.", r"\.p027\.", r"\.[pi]675\.", r"\.[pi]084\.", r"\.p063\.", r"\b[45]62[xh]\.", r"\.yarulb\.", r"\.vtd[hp]\.", + r'\.(?:ld[.-]?)?bew\.', r"\.pir.?(shv|dov|bew|dvd|db|rb)\.", r"\brdvd\.", r"\.vts\.", r"\.reneercs\.", r"\.dcv\.", r"\b(pir|mac)dh\b", r"\.reporp\.", r"\.kcaper\.", + r"\.lanretni\.", r"\b3ca\b", r"\bcaa\b", r"\b3pm\b", r"\.cstn\.", r"\.5r\.", r"\brcs\b"] +reverse_pattern = re.compile('|'.join(reverse_list), flags=re.IGNORECASE) +season_pattern = re.compile(r"(.*\.\d{2}e\d{2}s\.)(.*)", flags=re.IGNORECASE) +word_pattern = re.compile(r"([^A-Z0-9]*[A-Z0-9]+)") +char_replace = [[r"(\w)1\.(\w)",r"\1i\2"] +] +garbage_name = re.compile(r"^[a-zA-Z0-9]{2,}$") +media_list = [r"\.s\d{2}e\d{2}\.", r"\.2160p\.", r"\.1080[pi]\.", r"\.720p\.", r"\.576[pi]\.", r"\.480[pi]\.", r"\.360p\.", r"\.[xh]26[45]\b", r"\.bluray\.", r"\.[hp]dtv\.", + r'\.web(?:[.-]?dl)?\.', r"\.(vhs|vod|dvd|web|bd|br).?rip\.", r"\.dvdr\b", r"\.stv\.", r"\.screener\.", r"\.vcd\.", r"\bhd(cam|rip)\b", r"\.proper\.", r"\.repack\.", + r"\.internal\.", r"\bac3\b", r"\baac\b", r"\bmp3\b", r"\.ntsc\.", r"\.pal\.", r"\.secam\.", r"\bdivx\b", r"\bxvid\b", r"\.r5\.", r"\.scr\."] +media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) +media_extentions = [".mkv", ".mp4", ".avi", ".wmv", ".divx", ".xvid"] + +if 'nt' == os.name: + import ctypes + + class WinEnv: + def __init__(self): + pass + + @staticmethod + def get_environment_variable(name): + name = unicode(name) # ensures string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + env_value = None + if n: + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + env_value = buf.value + return env_value + + def __getitem__(self, key): + return self.get_environment_variable(key) + + def get(self, key, default=None): + r = self.get_environment_variable(key) + return r if r is not None else default + + evn = WinEnv() +else: + class LinuxEnv(object): + def __init__(self, environ): + self.environ = environ + + def __getitem__(self, key): + v = self.environ.get(key) + try: + return v.decode(SYS_ENCODING) if isinstance(v, str) else v + except (UnicodeDecodeError, UnicodeEncodeError): + return v + + def get(self, key, default=None): + v = self[key] + return v if v is not None else default + + evn = LinuxEnv(os.environ) + +SYS_ENCODING = None + +try: + locale.setlocale(locale.LC_ALL, '') +except (locale.Error, IOError): + pass +try: + SYS_ENCODING = locale.getpreferredencoding() +except (locale.Error, IOError): + pass + +if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + SYS_ENCODING = 'UTF-8' + + +class ek: + def __init__(self): + pass + + @staticmethod + def fix_string_encoding(x): + if str == type(x): + try: + return x.decode(SYS_ENCODING) + except UnicodeDecodeError: + return None + elif unicode == type(x): + return x + return None + + @staticmethod + def fix_out_encoding(x): + if isinstance(x, basestring): + return ek.fix_string_encoding(x) + return x + + @staticmethod + def fix_list_encoding(x): + if type(x) not in (list, tuple): + return x + return filter(lambda i: None is not i, map(ek.fix_out_encoding, x)) + + @staticmethod + def encode_item(x): + try: + return x.encode(SYS_ENCODING) + except UnicodeEncodeError: + return x.encode(SYS_ENCODING, 'ignore') + + @staticmethod + def win_encode_unicode(x): + if isinstance(x, str): + try: + return x.decode('UTF-8') + except UnicodeDecodeError: + return x + return x + + @staticmethod + def ek(func, *args, **kwargs): + if 'nt' == os.name: + # convert all str parameter values to unicode + args = tuple([x if not isinstance(x, str) else ek.win_encode_unicode(x) for x in args]) + kwargs = {k: x if not isinstance(x, str) else ek.win_encode_unicode(x) for k, x in + kwargs.iteritems()} + func_result = func(*args, **kwargs) + else: + func_result = func(*[ek.encode_item(x) if type(x) == str else x for x in args], **kwargs) + + if type(func_result) in (list, tuple): + return ek.fix_list_encoding(func_result) + elif str == type(func_result): + return ek.fix_string_encoding(func_result) + return func_result + + +class logger: + INFO = 'INFO' + DETAIL = 'DETAIL' + ERROR = 'ERROR' + WARNING = 'WARNING' + + @staticmethod + def log(message, msg_type=INFO): + print('[%s] %s' % (msg_type, message)) + + +def tryInt(s, s_default=0): + try: + return int(s) + except: + return s_default + +# NZBGet V11+ +# Check if the script is called from nzbget 11.0 or later +nzbget_version = evn.get('NZBOP_VERSION', '0.1') +nzbget_version = tryInt(nzbget_version[:nzbget_version.find(".")]) +if nzbget_version >= 11: + logger.log("Script triggered from NZBGet (11.0 or later).") + + # NZBGet argv: all passed as environment variables. + clientAgent = "nzbget" + # Exit codes used by NZBGet + POSTPROCESS_PARCHECK=92 + POSTPROCESS_SUCCESS=93 + POSTPROCESS_ERROR=94 + POSTPROCESS_NONE=95 + + # Check nzbget.conf options + status = 0 + + if evn['NZBOP_UNPACK'] != 'yes': + logger.log("Please enable option \"Unpack\" in nzbget configuration file, exiting") + sys.exit(POSTPROCESS_NONE) + + parstatus = evn['NZBPP_PARSTATUS'] + + # Check par status + if parstatus == '3': + logger.log("Par-check successful, but Par-repair disabled, exiting") + sys.exit(POSTPROCESS_NONE) + + if parstatus == '1': + logger.log("Par-check failed, setting status \"failed\"") + status = 1 + sys.exit(POSTPROCESS_NONE) + + unpackstatus = evn['NZBPP_UNPACKSTATUS'] + + # Check unpack status + if unpackstatus == '1': + logger.log("Unpack failed, setting status \"failed\"") + status = 1 + sys.exit(POSTPROCESS_NONE) + + directory = evn['NZBPP_DIRECTORY'] + + if unpackstatus == '0' and parstatus != '2': + # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check + + for dirpath, dirnames, filenames in ek.ek(os.walk, directory): + for file in filenames: + fileExtension = ek.ek(os.path.splitext, file)[1] + + if fileExtension in ['.par2']: + logger.log("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\"g") + status = 1 + break + + if ek.ek(os.path.isfile, ek.ek(os.path.join, directory, "_brokenlog.txt")) and not status == 1: + logger.log("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting") + status = 1 + + if not status == 1: + logger.log("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful") + + # Check if destination directory exists (important for reprocessing of history items) + if not ek.ek(os.path.isdir, directory): + logger.log("Post-Process: Nothing to post-process: destination directory %s doesn't exist" % directory) + status = 1 + + # All checks done, now launching the script. + + rd = False + videos = 0 + old_name = None + new_name = None + base_name = ek.ek(os.path.basename, directory) + for dirpath, dirnames, filenames in ek.ek(os.walk, directory): + for file in filenames: + + filePath = ek.ek(os.path.join, dirpath, file) + fileName, fileExtension = ek.ek(os.path.splitext, file) + dirname = ek.ek(os.path.dirname, filePath) + + if reverse_pattern.search(fileName) is not None: + na_parts = season_pattern.search(fileName) + if na_parts is not None: + word_p = word_pattern.findall(na_parts.group(2)) + new_words = "" + for wp in word_p: + if wp[0] == ".": + new_words += "." + new_words += re.sub(r"\W","",wp) + for cr in char_replace: + new_words = re.sub(cr[0],cr[1],new_words) + new_filename = new_words[::-1] + na_parts.group(1)[::-1] + else: + new_filename = fileName[::-1] + logger.log("reversing filename from: %s to %s" % (fileName, new_filename)) + try: + ek.ek(os.rename, filePath, ek.ek(os.path.join, dirpath, new_filename + fileExtension)) + rd = True + except Exception,e: + logger.log(e, logger.ERROR) + logger.log("Error: unable to rename file %s" % file, logger.ERROR) + pass + elif (fileExtension.lower() in media_extentions) and (garbage_name.search(fileName) is not None) and (media_pattern.search(base_name) is not None): + videos += 1 + old_name = filePath + new_name = ek.ek(os.path.join, dirname, '%s%s' % (base_name, fileExtension)) + + if not rd and videos == 1 and old_name is not None and new_name is not None: + logger.log("renaming the File %s to the Dirname %s" % (ek.ek(os.path.basename, old_name), base_name)) + try: + ek.ek(os.rename, old_name, new_name) + rd = True + except Exception,e: + logger.log(e, logger.ERROR) + logger.log("Error unable to rename file %s" % old_name, logger.ERROR) + pass + + if rd: + sys.exit(POSTPROCESS_SUCCESS) + else: + sys.exit(POSTPROCESS_NONE) + +else: + logger.log("This script can only be called from NZBGet (11.0 or later).", logger.ERROR) + sys.exit(0) From 85837ddeade65829d65b6228e651ef050cdb5ba8 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 20:20:22 +0000 Subject: [PATCH 055/185] Create unzip.py --- apps/templates/nzgbet/scripts/unzip.py | 294 +++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 apps/templates/nzgbet/scripts/unzip.py diff --git a/apps/templates/nzgbet/scripts/unzip.py b/apps/templates/nzgbet/scripts/unzip.py new file mode 100644 index 0000000..521a50f --- /dev/null +++ b/apps/templates/nzgbet/scripts/unzip.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# +############################################################################## +### NZBGET SCAN SCRIPT ### + +# Unzips zipped nzbs. +# +# NOTE: This script requires Python to be installed on your system. + +############################################################################## +### OPTIONS ### +### NZBGET SCAN SCRIPT ### +############################################################################## + +import os, zipfile, tarfile, gzip, pickle, datetime, re, struct, locale +import rarfile.rarfile as rarfile + +from gzip import FEXTRA, FNAME + +if 'nt' == os.name: + import ctypes + + class WinEnv: + def __init__(self): + pass + + @staticmethod + def get_environment_variable(name): + name = unicode(name) # ensures string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + result = None + if n: + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + result = buf.value + return result + + def __getitem__(self, key): + return self.get_environment_variable(key) + + def get(self, key, default=None): + r = self.get_environment_variable(key) + return r if r is not None else default + + env_var = WinEnv() +else: + class LinuxEnv(object): + def __init__(self, environ): + self.environ = environ + + def __getitem__(self, key): + v = self.environ.get(key) + try: + return v.decode(SYS_ENCODING) if isinstance(v, str) else v + except (UnicodeDecodeError, UnicodeEncodeError): + return v + + def get(self, key, default=None): + v = self[key] + return v if v is not None else default + + env_var = LinuxEnv(os.environ) + + +SYS_ENCODING = None +try: + locale.setlocale(locale.LC_ALL, '') +except (locale.Error, IOError): + pass +try: + SYS_ENCODING = locale.getpreferredencoding() +except (locale.Error, IOError): + pass +if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): + SYS_ENCODING = 'UTF-8' + + +filename = env_var.get('NZBNP_FILENAME') +if re.search(r"\.tar\.gz$", filename, flags=re.I) is None: + ext = os.path.splitext(filename)[1].lower() +else: + ext = '.tar.gz' +cat = env_var.get('NZBNP_CATEGORY') +dir = env_var.get('NZBNP_DIRECTORY') +prio = env_var.get('NZBNP_PRIORITY') +top = env_var.get('NZBNP_TOP') +pause = env_var.get('NZBNP_PAUSED') +if 'NZBNP_DUPEKEY' in os.environ: + dupekey = env_var.get('NZBNP_DUPEKEY') + dupescore = env_var.get('NZBNP_DUPESCORE') + dupemode = env_var.get('NZBNP_DUPEMODE') +else: + dupekey = None + dupescore = None + dupemode = None + +tmp_zipinfo = os.path.join(os.environ.get('NZBOP_TEMPDIR'), r'nzbget\unzip_scan\info') +nzb_list = [] + +def read_gzip_info(gzipfile): + gf = gzipfile.fileobj + pos = gf.tell() + + # Read archive size + gf.seek(-4, 2) + size = struct.unpack('= 11: - logger.log("Script triggered from NZBGet (11.0 or later).") - - # NZBGet argv: all passed as environment variables. - clientAgent = "nzbget" - # Exit codes used by NZBGet - POSTPROCESS_PARCHECK=92 - POSTPROCESS_SUCCESS=93 - POSTPROCESS_ERROR=94 - POSTPROCESS_NONE=95 - - # Check nzbget.conf options - status = 0 - - if evn['NZBOP_UNPACK'] != 'yes': - logger.log("Please enable option \"Unpack\" in nzbget configuration file, exiting") - sys.exit(POSTPROCESS_NONE) - - parstatus = evn['NZBPP_PARSTATUS'] - - # Check par status - if parstatus == '3': - logger.log("Par-check successful, but Par-repair disabled, exiting") - sys.exit(POSTPROCESS_NONE) - - if parstatus == '1': - logger.log("Par-check failed, setting status \"failed\"") - status = 1 - sys.exit(POSTPROCESS_NONE) - - unpackstatus = evn['NZBPP_UNPACKSTATUS'] - - # Check unpack status - if unpackstatus == '1': - logger.log("Unpack failed, setting status \"failed\"") - status = 1 - sys.exit(POSTPROCESS_NONE) - - directory = evn['NZBPP_DIRECTORY'] - - if unpackstatus == '0' and parstatus != '2': - # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check - - for dirpath, dirnames, filenames in ek.ek(os.walk, directory): - for file in filenames: - fileExtension = ek.ek(os.path.splitext, file)[1] - - if fileExtension in ['.par2']: - logger.log("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\"g") - status = 1 - break - - if ek.ek(os.path.isfile, ek.ek(os.path.join, directory, "_brokenlog.txt")) and not status == 1: - logger.log("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting") - status = 1 - - if not status == 1: - logger.log("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful") - - # Check if destination directory exists (important for reprocessing of history items) - if not ek.ek(os.path.isdir, directory): - logger.log("Post-Process: Nothing to post-process: destination directory %s doesn't exist" % directory) - status = 1 - - # All checks done, now launching the script. - - rd = False - videos = 0 - old_name = None - new_name = None - base_name = ek.ek(os.path.basename, directory) - for dirpath, dirnames, filenames in ek.ek(os.walk, directory): - for file in filenames: - - filePath = ek.ek(os.path.join, dirpath, file) - fileName, fileExtension = ek.ek(os.path.splitext, file) - dirname = ek.ek(os.path.dirname, filePath) - - if reverse_pattern.search(fileName) is not None: - na_parts = season_pattern.search(fileName) - if na_parts is not None: - word_p = word_pattern.findall(na_parts.group(2)) - new_words = "" - for wp in word_p: - if wp[0] == ".": - new_words += "." - new_words += re.sub(r"\W","",wp) - for cr in char_replace: - new_words = re.sub(cr[0],cr[1],new_words) - new_filename = new_words[::-1] + na_parts.group(1)[::-1] - else: - new_filename = fileName[::-1] - logger.log("reversing filename from: %s to %s" % (fileName, new_filename)) - try: - ek.ek(os.rename, filePath, ek.ek(os.path.join, dirpath, new_filename + fileExtension)) - rd = True - except Exception,e: - logger.log(e, logger.ERROR) - logger.log("Error: unable to rename file %s" % file, logger.ERROR) - pass - elif (fileExtension.lower() in media_extentions) and (garbage_name.search(fileName) is not None) and (media_pattern.search(base_name) is not None): - videos += 1 - old_name = filePath - new_name = ek.ek(os.path.join, dirname, '%s%s' % (base_name, fileExtension)) - - if not rd and videos == 1 and old_name is not None and new_name is not None: - logger.log("renaming the File %s to the Dirname %s" % (ek.ek(os.path.basename, old_name), base_name)) - try: - ek.ek(os.rename, old_name, new_name) - rd = True - except Exception,e: - logger.log(e, logger.ERROR) - logger.log("Error unable to rename file %s" % old_name, logger.ERROR) - pass - - if rd: - sys.exit(POSTPROCESS_SUCCESS) - else: - sys.exit(POSTPROCESS_NONE) - -else: - logger.log("This script can only be called from NZBGet (11.0 or later).", logger.ERROR) - sys.exit(0) From 5aa46bf941837c2e065df21472cc0986962da901 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:35:41 +0000 Subject: [PATCH 060/185] Delete hash.py --- apps/templates/nzbget-mp4/scripts/hash.py | 167 ---------------------- 1 file changed, 167 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/hash.py diff --git a/apps/templates/nzbget-mp4/scripts/hash.py b/apps/templates/nzbget-mp4/scripts/hash.py deleted file mode 100644 index f2fbec1..0000000 --- a/apps/templates/nzbget-mp4/scripts/hash.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python -# -# Title: PGBlitz (Reference Title File) -# Maintainer: Admin9705 -# URL: https://pgblitz.com - http://github.pgblitz.com -# GNU: General Public License v3.0 -# -# Original Author: clinton-hall -# https://github.com/clinton-hall/GetScripts/blob/master/SafeRename.py -# -# Modified By: desimaniac (No Acknowledgement of Source Above) -################################################################################ -import os -import re -import shutil -import sys - -# NZBGet Exit Codes -NZBGET_POSTPROCESS_PARCHECK = 92 -NZBGET_POSTPROCESS_SUCCESS = 93 -NZBGET_POSTPROCESS_ERROR = 94 -NZBGET_POSTPROCESS_NONE = 95 - -############################################################ -# EXTENSION STUFF -############################################################ - -def do_check(): - if not os.environ.has_key('NZBOP_SCRIPTDIR'): - print "This script can only be called from NZBGet (11.0 or later)." - sys.exit(0) - - if os.environ['NZBOP_VERSION'][0:5] < '11.0': - print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) - sys.exit(0) - - print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) - - status = 0 - if 'NZBPP_TOTALSTATUS' in os.environ: - if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': - print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) - status = 1 - else: - # Check par status - if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': - print "[ERROR] Par-repair failed, setting status \"failed\"." - status = 1 - - # Check unpack status - if os.environ['NZBPP_UNPACKSTATUS'] == '1': - print "[ERROR] Unpack failed, setting status \"failed\"." - status = 1 - - if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': - # Unpack was skipped due to nzb-file properties or due to errors during par-check - - if os.environ['NZBPP_HEALTH'] < 1000: - print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. " \ - "Setting status \"failed\"." - print "[ERROR] Please check your Par-check/repair settings for future downloads." - status = 1 - - else: - print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is " \ - "ok so handle as though download successful." - print "[WARNING] Please check your Par-check/repair settings for future downloads." - - # Check if destination directory exists (important for reprocessing of history items) - if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): - print "[ERROR] Nothing to post-process: destination directory", os.environ[ - 'NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." - status = 1 - - # All checks done, now launching the script. - if status == 1: - sys.exit(NZBGET_POSTPROCESS_NONE) - - -def get_file_name(path): - try: - file_name = os.path.basename(path) - extensions = re.findall(r'\.([^.]+)', file_name) - ext = '.'.join(extensions) - name = file_name.replace(".%s" % ext, '') - return name, ext - except Exception: - pass - return None - - -def is_file_hash(file_name): - hash_regexp = [ - r'^[a-fA-F0-9]{40}$', - r'^[a-fA-F0-9]{32}$', - r'^[a-f0-9]{128}$', - r'^[a-zA-Z0-9]{42}$' - ] - for hash in hash_regexp: - if re.match(hash, file_name): - return True - return False - - -def find_files(folder, extension=None, depth=None): - file_list = [] - start_count = folder.count(os.sep) - for path, subdirs, files in os.walk(folder, topdown=True): - for name in files: - if depth and path.count(os.sep) - start_count >= depth: - del subdirs[:] - continue - file = os.path.join(path, name) - if not extension: - file_list.append(file) - else: - if file.lower().endswith(extension.lower()): - file_list.append(file) - - return sorted(file_list, key=lambda x: x.count(os.path.sep), reverse=True) - - -############################################################ -# MAIN -############################################################ - -# do checks -do_check() - -# retrieve required variables -directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) -nzb_name = os.environ['NZBPP_NZBFILENAME'] -if nzb_name is None: - print("[ERROR] Unable to retrieve NZBPP_NZBFILENAME") - sys.exit(NZBGET_POSTPROCESS_ERROR) -nzb_name = nzb_name.replace('.nzb', '') - -print("[INFO] Using \"%s\" for hashed filenames" % nzb_name) -print("[INFO] Scanning \"%s\" for hashed filenames" % directory) - -# scan for files -found_files = find_files(directory) -if not found_files: - print("[INFO] No files were found in \"%s\"" % directory) - sys.exit(NZBGET_POSTPROCESS_NONE) -else: - print("[INFO] Found %d files to check for hashed filenames" % len(found_files)) - # loop files checking for file hash - moved_files = 0 - for found_file_path in found_files: - # set variable - dir_name = os.path.dirname(found_file_path) - file_name, file_ext = get_file_name(found_file_path) - - # is this a file hash - if is_file_hash(file_name): - new_file_path = os.path.join(dir_name, "%s.%s" % (nzb_name, file_ext)) - print("[INFO] Moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) - try: - shutil.move(found_file_path, new_file_path) - moved_files += 1 - except Exception: - print("[ERROR] Failed moving \"%s\" to \"%s\"" % (found_file_path, new_file_path)) - - print("[INFO] Finished processing \"%s\", moved %d files" % (directory, moved_files)) - -sys.exit(NZBGET_POSTPROCESS_SUCCESS) From b74965ad0469263daa401fa15c37308b6d61bb8c Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:35:51 +0000 Subject: [PATCH 061/185] Delete flatten.py --- apps/templates/nzbget-mp4/scripts/flatten.py | 106 ------------------- 1 file changed, 106 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/flatten.py diff --git a/apps/templates/nzbget-mp4/scripts/flatten.py b/apps/templates/nzbget-mp4/scripts/flatten.py deleted file mode 100644 index 73dd0d5..0000000 --- a/apps/templates/nzbget-mp4/scripts/flatten.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python -# -# Title: PGBlitz (Reference Title File) -# Maintainer: Admin9705 -# URL: https://pgblitz.com - http://github.pgblitz.com -# GNU: General Public License v3.0 -# -# Additions: clinton-hall - https://github.com/Prinz23 -################################################################################ -import os -import sys -import shutil - -# NZBGet Exit Codes -NZBGET_POSTPROCESS_PARCHECK = 92 -NZBGET_POSTPROCESS_SUCCESS = 93 -NZBGET_POSTPROCESS_ERROR = 94 -NZBGET_POSTPROCESS_NONE = 95 - -if not os.environ.has_key('NZBOP_SCRIPTDIR'): - print "This script can only be called from NZBGet (11.0 or later)." - sys.exit(0) - -if os.environ['NZBOP_VERSION'][0:5] < '11.0': - print "[ERROR] NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) - sys.exit(0) - -print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) -status = 0 -if os.environ.has_key('NZBPP_TOTALSTATUS'): - if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': - print "[ERROR] Download failed with status %s." % (os.environ['NZBPP_STATUS']) - status = 1 - -else: - # Check par status - if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': - print "[ERROR] Par-repair failed, setting status \"failed\"." - status = 1 - - # Check unpack status - if os.environ['NZBPP_UNPACKSTATUS'] == '1': - print "[ERROR] Unpack failed, setting status \"failed\"." - status = 1 - - if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': - # Unpack was skipped due to nzb-file properties or due to errors during par-check - - if os.environ['NZBPP_HEALTH'] < 1000: - print "[ERROR] Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." - print "[ERROR] Please check your Par-check/repair settings for future downloads." - status = 1 - - else: - print "[ERROR] Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." - print "[WARNING] Please check your Par-check/repair settings for future downloads." - -# Check if destination directory exists (important for reprocessing of history items) -if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): - print "[ERROR] Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." - status = 1 - -# All checks done, now launching the script. -if status == 1: - sys.exit(NZBGET_POSTPROCESS_NONE) - -def removeEmptyFolders(path, removeRoot=True): - #Function to remove empty folders - if not os.path.isdir(path): - return - - # remove empty subfolders - print "[INFO] Checking for empty folders in:%s" % path - files = os.listdir(path) - if len(files): - for f in files: - fullpath = os.path.join(path, f) - if os.path.isdir(fullpath): - removeEmptyFolders(fullpath) - - # if folder empty, delete it - files = os.listdir(path) - if len(files) == 0 and removeRoot: - print "[INFO] Removing empty folder:%s" % path - os.rmdir(path) - -directory = os.path.normpath(os.environ['NZBPP_DIRECTORY']) -if os.environ['NZBPO_DESTINATIONDIRECTORY'] and os.path.isdir(os.environ['NZBPO_DESTINATIONDIRECTORY']): - destination = os.environ['NZBPO_DESTINATIONDIRECTORY'] - if os.environ['NZBPO_APPENDCATEGORIES'] == 'yes': - destination = os.path.join(destination, os.environ['NZBPP_CATEGORY']) -else: - destination = directory -print "Flattening directory: %s" % (directory) -for dirpath, dirnames, filenames in os.walk(directory): - for fileName in filenames: - outputFile = os.path.join(dirpath, fileName) - if dirpath == directory: - continue - target = os.path.join(destination, fileName) - try: - shutil.move(outputFile, target) - except: - print "[ERROR] Could not flatten %s" % outputFile -removeEmptyFolders(directory) # Cleanup empty directories -sys.exit(NZBGET_POSTPROCESS_SUCCESS) From 56e3b16ab377ffa1442eadb2733044db8769ab4f Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:36:03 +0000 Subject: [PATCH 062/185] Delete DeleteSamples.py --- .../nzbget-mp4/scripts/DeleteSamples.py | 93 ------------------- 1 file changed, 93 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/DeleteSamples.py diff --git a/apps/templates/nzbget-mp4/scripts/DeleteSamples.py b/apps/templates/nzbget-mp4/scripts/DeleteSamples.py deleted file mode 100644 index 395628f..0000000 --- a/apps/templates/nzbget-mp4/scripts/DeleteSamples.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -# Title: PGBlitz (Reference Title File) -# Maintainer: Admin9705 -# URL: https://pgblitz.com - http://github.pgblitz.com -# GNU: General Public License v3.0 -# -# Additions: clinton-hall - https://github.com/Prinz23 -################################################################################ -import os -import sys - -# NZBGet Exit Codes -NZBGET_POSTPROCESS_PARCHECK = 92 -NZBGET_POSTPROCESS_SUCCESS = 93 -NZBGET_POSTPROCESS_ERROR = 94 -NZBGET_POSTPROCESS_NONE = 95 - -def is_sample(filePath, inputName, maxSampleSize, SampleIDs): - # 200 MB in bytes - SIZE_CUTOFF = int(maxSampleSize) * 1024 * 1024 - if os.path.getsize(filePath) < SIZE_CUTOFF: - if 'SizeOnly' in SampleIDs: - return True - # Ignore 'sample' in files unless 'sample' in Torrent Name - for ident in SampleIDs: - if ident.lower() in filePath.lower() and not ident.lower() in inputName.lower(): - return True - # Return False if none of these were met. - return False - -if not os.environ.has_key('NZBOP_SCRIPTDIR'): - print "This script can only be called from NZBGet (11.0 or later)." - sys.exit(0) - -if os.environ['NZBOP_VERSION'][0:5] < '11.0': - print "NZBGet Version %s is not supported. Please update NZBGet." % (str(os.environ['NZBOP_VERSION'])) - sys.exit(0) - -print "Script triggered from NZBGet Version %s." % (str(os.environ['NZBOP_VERSION'])) -status = 0 -if os.environ.has_key('NZBPP_TOTALSTATUS'): - if not os.environ['NZBPP_TOTALSTATUS'] == 'SUCCESS': - print "Download failed with status %s." % (os.environ['NZBPP_STATUS']) - status = 1 - -else: - # Check par status - if os.environ['NZBPP_PARSTATUS'] == '1' or os.environ['NZBPP_PARSTATUS'] == '4': - print "Par-repair failed, setting status \"failed\"." - status = 1 - - # Check unpack status - if os.environ['NZBPP_UNPACKSTATUS'] == '1': - print "Unpack failed, setting status \"failed\"." - status = 1 - - if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] == '0': - # Unpack was skipped due to nzb-file properties or due to errors during par-check - - if os.environ['NZBPP_HEALTH'] < 1000: - print "Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \"failed\"." - print "Please check your Par-check/repair settings for future downloads." - status = 1 - - else: - print "Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful." - print "Please check your Par-check/repair settings for future downloads." - -# Check if destination directory exists (important for reprocessing of history items) -if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): - print "Nothing to post-process: destination directory", os.environ['NZBPP_DIRECTORY'], "doesn't exist. Setting status \"failed\"." - status = 1 - -# All checks done, now launching the script. -if status == 1: - sys.exit(NZBGET_POSTPROCESS_NONE) - -mediaContainer = os.environ['NZBPO_MEDIAEXTENSIONS'].split(',') -SampleIDs = os.environ['NZBPO_SAMPLEIDS'].split(',') -for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): - for file in filenames: - filePath = os.path.join(dirpath, file) - fileName, fileExtension = os.path.splitext(file) - if fileExtension in mediaContainer or ".*" in mediaContainer : # If the file is a video file - if is_sample(filePath, os.environ['NZBPP_NZBNAME'], os.environ['NZBPO_MAXSAMPLESIZE'], SampleIDs): # Ignore samples - print "Deleting sample file: ", filePath - try: - os.unlink(filePath) - except: - print "Error: unable to delete file", filePath - sys.exit(NZBGET_POSTPROCESS_ERROR) -sys.exit(NZBGET_POSTPROCESS_SUCCESS) From f0ba686d0e6a97e27fce220dbd6bfca970b80e18 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:36:20 +0000 Subject: [PATCH 063/185] Delete rarfile.py --- .../nzbget-mp4/scripts/rarfile/rarfile.py | 2931 ----------------- 1 file changed, 2931 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py diff --git a/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py b/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py deleted file mode 100644 index 6a3a647..0000000 --- a/apps/templates/nzbget-mp4/scripts/rarfile/rarfile.py +++ /dev/null @@ -1,2931 +0,0 @@ -# rarfile.py -# -# Copyright (c) 2005-2016 Marko Kreen -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -r"""RAR archive reader. - -This is Python module for Rar archive reading. The interface -is made as :mod:`zipfile`-like as possible. - -Basic logic: - - Parse archive structure with Python. - - Extract non-compressed files with Python - - Extract compressed files with unrar. - - Optionally write compressed data to temp file to speed up unrar, - otherwise it needs to scan whole archive on each execution. - -Example:: - - import rarfile - - rf = rarfile.RarFile('myarchive.rar') - for f in rf.infolist(): - print f.filename, f.file_size - if f.filename == 'README': - print(rf.read(f)) - -Archive files can also be accessed via file-like object returned -by :meth:`RarFile.open`:: - - import rarfile - - with rarfile.RarFile('archive.rar') as rf: - with rf.open('README') as f: - for ln in f: - print(ln.strip()) - -There are few module-level parameters to tune behaviour, -here they are with defaults, and reason to change it:: - - import rarfile - - # Set to full path of unrar.exe if it is not in PATH - rarfile.UNRAR_TOOL = "unrar" - - # Set to '\\' to be more compatible with old rarfile - rarfile.PATH_SEP = '/' - -For more details, refer to source. - -""" - -from __future__ import division, print_function - -## -## Imports and compat - support both Python 2.x and 3.x -## - -import sys -import os -import errno -import struct - -from struct import pack, unpack, Struct -from binascii import crc32, hexlify -from tempfile import mkstemp -from subprocess import Popen, PIPE, STDOUT -from io import RawIOBase -from hashlib import sha1, sha256 -from hmac import HMAC -from datetime import datetime, timedelta, tzinfo - -# fixed offset timezone, for UTC -try: - from datetime import timezone -except ImportError: - class timezone(tzinfo): - """Compat timezone.""" - __slots__ = ('_ofs', '_name') - _DST = timedelta(0) - - def __init__(self, offset, name): - super(timezone, self).__init__() - self._ofs, self._name = offset, name - - def utcoffset(self, dt): - return self._ofs - - def tzname(self, dt): - return self._name - - def dst(self, dt): - return self._DST - -# only needed for encryped headers -try: - try: - from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.kdf import pbkdf2 - - class AES_CBC_Decrypt(object): - """Decrypt API""" - def __init__(self, key, iv): - ciph = Cipher(algorithms.AES(key), modes.CBC(iv), default_backend()) - self.decrypt = ciph.decryptor().update - - def pbkdf2_sha256(password, salt, iters): - """PBKDF2 with HMAC-SHA256""" - ctx = pbkdf2.PBKDF2HMAC(hashes.SHA256(), 32, salt, iters, default_backend()) - return ctx.derive(password) - - except ImportError: - from Crypto.Cipher import AES - from Crypto.Protocol import KDF - - class AES_CBC_Decrypt(object): - """Decrypt API""" - def __init__(self, key, iv): - self.decrypt = AES.new(key, AES.MODE_CBC, iv).decrypt - - def pbkdf2_sha256(password, salt, iters): - """PBKDF2 with HMAC-SHA256""" - return KDF.PBKDF2(password, salt, 32, iters, hmac_sha256) - - _have_crypto = 1 -except ImportError: - _have_crypto = 0 - -try: - from pyblake2 import blake2s - _have_blake2 = True -except ImportError: - _have_blake2 = False - -# compat with 2.x -if sys.hexversion < 0x3000000: - def rar_crc32(data, prev=0): - """CRC32 with unsigned values. - """ - if (prev > 0) and (prev & 0x80000000): - prev -= (1 << 32) - res = crc32(data, prev) - if res < 0: - res += (1 << 32) - return res - tohex = hexlify - _byte_code = ord -else: # pragma: no cover - def tohex(data): - """Return hex string.""" - return hexlify(data).decode('ascii') - rar_crc32 = crc32 - unicode = str - _byte_code = int # noqa - - -__version__ = '3.0' - -# export only interesting items -__all__ = ['is_rarfile', 'RarInfo', 'RarFile', 'RarExtFile'] - -## -## Module configuration. Can be tuned after importing. -## - -#: default fallback charset -DEFAULT_CHARSET = "windows-1252" - -#: list of encodings to try, with fallback to DEFAULT_CHARSET if none succeed -TRY_ENCODINGS = ('utf8', 'utf-16le') - -#: 'unrar', 'rar' or full path to either one -UNRAR_TOOL = "unrar" - -#: Command line args to use for opening file for reading. -OPEN_ARGS = ('p', '-inul') - -#: Command line args to use for extracting file to disk. -EXTRACT_ARGS = ('x', '-y', '-idq') - -#: args for testrar() -TEST_ARGS = ('t', '-idq') - -# -# Allow use of tool that is not compatible with unrar. -# -# By default use 'bsdtar' which is 'tar' program that -# sits on top of libarchive. -# -# Problems with libarchive RAR backend: -# - Does not support solid archives. -# - Does not support password-protected archives. -# - -ALT_TOOL = 'bsdtar' -ALT_OPEN_ARGS = ('-x', '--to-stdout', '-f') -ALT_EXTRACT_ARGS = ('-x', '-f') -ALT_TEST_ARGS = ('-t', '-f') -ALT_CHECK_ARGS = ('--help',) - -#: whether to speed up decompression by using tmp archive -USE_EXTRACT_HACK = 1 - -#: limit the filesize for tmp archive usage -HACK_SIZE_LIMIT = 20 * 1024 * 1024 - -#: Separator for path name components. RAR internally uses '\\'. -#: Use '/' to be similar with zipfile. -PATH_SEP = '/' - -## -## rar constants -## - -# block types -RAR_BLOCK_MARK = 0x72 # r -RAR_BLOCK_MAIN = 0x73 # s -RAR_BLOCK_FILE = 0x74 # t -RAR_BLOCK_OLD_COMMENT = 0x75 # u -RAR_BLOCK_OLD_EXTRA = 0x76 # v -RAR_BLOCK_OLD_SUB = 0x77 # w -RAR_BLOCK_OLD_RECOVERY = 0x78 # x -RAR_BLOCK_OLD_AUTH = 0x79 # y -RAR_BLOCK_SUB = 0x7a # z -RAR_BLOCK_ENDARC = 0x7b # { - -# flags for RAR_BLOCK_MAIN -RAR_MAIN_VOLUME = 0x0001 -RAR_MAIN_COMMENT = 0x0002 -RAR_MAIN_LOCK = 0x0004 -RAR_MAIN_SOLID = 0x0008 -RAR_MAIN_NEWNUMBERING = 0x0010 -RAR_MAIN_AUTH = 0x0020 -RAR_MAIN_RECOVERY = 0x0040 -RAR_MAIN_PASSWORD = 0x0080 -RAR_MAIN_FIRSTVOLUME = 0x0100 -RAR_MAIN_ENCRYPTVER = 0x0200 - -# flags for RAR_BLOCK_FILE -RAR_FILE_SPLIT_BEFORE = 0x0001 -RAR_FILE_SPLIT_AFTER = 0x0002 -RAR_FILE_PASSWORD = 0x0004 -RAR_FILE_COMMENT = 0x0008 -RAR_FILE_SOLID = 0x0010 -RAR_FILE_DICTMASK = 0x00e0 -RAR_FILE_DICT64 = 0x0000 -RAR_FILE_DICT128 = 0x0020 -RAR_FILE_DICT256 = 0x0040 -RAR_FILE_DICT512 = 0x0060 -RAR_FILE_DICT1024 = 0x0080 -RAR_FILE_DICT2048 = 0x00a0 -RAR_FILE_DICT4096 = 0x00c0 -RAR_FILE_DIRECTORY = 0x00e0 -RAR_FILE_LARGE = 0x0100 -RAR_FILE_UNICODE = 0x0200 -RAR_FILE_SALT = 0x0400 -RAR_FILE_VERSION = 0x0800 -RAR_FILE_EXTTIME = 0x1000 -RAR_FILE_EXTFLAGS = 0x2000 - -# flags for RAR_BLOCK_ENDARC -RAR_ENDARC_NEXT_VOLUME = 0x0001 -RAR_ENDARC_DATACRC = 0x0002 -RAR_ENDARC_REVSPACE = 0x0004 -RAR_ENDARC_VOLNR = 0x0008 - -# flags common to all blocks -RAR_SKIP_IF_UNKNOWN = 0x4000 -RAR_LONG_BLOCK = 0x8000 - -# Host OS types -RAR_OS_MSDOS = 0 -RAR_OS_OS2 = 1 -RAR_OS_WIN32 = 2 -RAR_OS_UNIX = 3 -RAR_OS_MACOS = 4 -RAR_OS_BEOS = 5 - -# Compression methods - '0'..'5' -RAR_M0 = 0x30 -RAR_M1 = 0x31 -RAR_M2 = 0x32 -RAR_M3 = 0x33 -RAR_M4 = 0x34 -RAR_M5 = 0x35 - -# -# RAR5 constants -# - -RAR5_BLOCK_MAIN = 1 -RAR5_BLOCK_FILE = 2 -RAR5_BLOCK_SERVICE = 3 -RAR5_BLOCK_ENCRYPTION = 4 -RAR5_BLOCK_ENDARC = 5 - -RAR5_BLOCK_FLAG_EXTRA_DATA = 0x01 -RAR5_BLOCK_FLAG_DATA_AREA = 0x02 -RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN = 0x04 -RAR5_BLOCK_FLAG_SPLIT_BEFORE = 0x08 -RAR5_BLOCK_FLAG_SPLIT_AFTER = 0x10 -RAR5_BLOCK_FLAG_DEPENDS_PREV = 0x20 -RAR5_BLOCK_FLAG_KEEP_WITH_PARENT = 0x40 - -RAR5_MAIN_FLAG_ISVOL = 0x01 -RAR5_MAIN_FLAG_HAS_VOLNR = 0x02 -RAR5_MAIN_FLAG_SOLID = 0x04 -RAR5_MAIN_FLAG_RECOVERY = 0x08 -RAR5_MAIN_FLAG_LOCKED = 0x10 - -RAR5_FILE_FLAG_ISDIR = 0x01 -RAR5_FILE_FLAG_HAS_MTIME = 0x02 -RAR5_FILE_FLAG_HAS_CRC32 = 0x04 -RAR5_FILE_FLAG_UNKNOWN_SIZE = 0x08 - -RAR5_COMPR_SOLID = 0x40 - -RAR5_ENC_FLAG_HAS_CHECKVAL = 0x01 - -RAR5_ENDARC_FLAG_NEXT_VOL = 0x01 - -RAR5_XFILE_ENCRYPTION = 1 -RAR5_XFILE_HASH = 2 -RAR5_XFILE_TIME = 3 -RAR5_XFILE_VERSION = 4 -RAR5_XFILE_REDIR = 5 -RAR5_XFILE_OWNER = 6 -RAR5_XFILE_SERVICE = 7 - -RAR5_XTIME_UNIXTIME = 0x01 -RAR5_XTIME_HAS_MTIME = 0x02 -RAR5_XTIME_HAS_CTIME = 0x04 -RAR5_XTIME_HAS_ATIME = 0x08 - -RAR5_XENC_CIPHER_AES256 = 0 - -RAR5_XENC_CHECKVAL = 0x01 -RAR5_XENC_TWEAKED = 0x02 - -RAR5_XHASH_BLAKE2SP = 0 - -RAR5_XREDIR_UNIX_SYMLINK = 1 -RAR5_XREDIR_WINDOWS_SYMLINK = 2 -RAR5_XREDIR_WINDOWS_JUNCTION = 3 -RAR5_XREDIR_HARD_LINK = 4 -RAR5_XREDIR_FILE_COPY = 5 - -RAR5_XREDIR_ISDIR = 0x01 - -RAR5_XOWNER_UNAME = 0x01 -RAR5_XOWNER_GNAME = 0x02 -RAR5_XOWNER_UID = 0x04 -RAR5_XOWNER_GID = 0x08 - -RAR5_OS_WINDOWS = 0 -RAR5_OS_UNIX = 1 - -## -## internal constants -## - -RAR_ID = b"Rar!\x1a\x07\x00" -RAR5_ID = b"Rar!\x1a\x07\x01\x00" -ZERO = b'\0' -EMPTY = b'' -UTC = timezone(timedelta(0), 'UTC') -BSIZE = 32 * 1024 - -def _get_rar_version(xfile): - '''Check quickly whether file is rar archive. - ''' - with XFile(xfile) as fd: - buf = fd.read(len(RAR5_ID)) - if buf.startswith(RAR_ID): - return 3 - elif buf.startswith(RAR5_ID): - return 5 - return 0 - -## -## Public interface -## - -def is_rarfile(xfile): - '''Check quickly whether file is rar archive. - ''' - return _get_rar_version(xfile) > 0 - -class Error(Exception): - """Base class for rarfile errors.""" - -class BadRarFile(Error): - """Incorrect data in archive.""" - -class NotRarFile(Error): - """The file is not RAR archive.""" - -class BadRarName(Error): - """Cannot guess multipart name components.""" - -class NoRarEntry(Error): - """File not found in RAR""" - -class PasswordRequired(Error): - """File requires password""" - -class NeedFirstVolume(Error): - """Need to start from first volume.""" - -class NoCrypto(Error): - """Cannot parse encrypted headers - no crypto available.""" - -class RarExecError(Error): - """Problem reported by unrar/rar.""" - -class RarWarning(RarExecError): - """Non-fatal error""" - -class RarFatalError(RarExecError): - """Fatal error""" - -class RarCRCError(RarExecError): - """CRC error during unpacking""" - -class RarLockedArchiveError(RarExecError): - """Must not modify locked archive""" - -class RarWriteError(RarExecError): - """Write error""" - -class RarOpenError(RarExecError): - """Open error""" - -class RarUserError(RarExecError): - """User error""" - -class RarMemoryError(RarExecError): - """Memory error""" - -class RarCreateError(RarExecError): - """Create error""" - -class RarNoFilesError(RarExecError): - """No files that match pattern were found""" - -class RarUserBreak(RarExecError): - """User stop""" - -class RarWrongPassword(RarExecError): - """Incorrect password""" - -class RarUnknownError(RarExecError): - """Unknown exit code""" - -class RarSignalExit(RarExecError): - """Unrar exited with signal""" - -class RarCannotExec(RarExecError): - """Executable not found.""" - - -class RarInfo(object): - r'''An entry in rar archive. - - RAR3 extended timestamps are :class:`datetime.datetime` objects without timezone. - RAR5 extended timestamps are :class:`datetime.datetime` objects with UTC timezone. - - Attributes: - - filename - File name with relative path. - Path separator is '/'. Always unicode string. - - date_time - File modification timestamp. As tuple of (year, month, day, hour, minute, second). - RAR5 allows archives where it is missing, it's None then. - - file_size - Uncompressed size. - - compress_size - Compressed size. - - compress_type - Compression method: one of :data:`RAR_M0` .. :data:`RAR_M5` constants. - - extract_version - Minimal Rar version needed for decompressing. As (major*10 + minor), - so 2.9 is 29. - - RAR3: 10, 20, 29 - - RAR5 does not have such field in archive, it's simply set to 50. - - host_os - Host OS type, one of RAR_OS_* constants. - - RAR3: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`, :data:`RAR_OS_MSDOS`, - :data:`RAR_OS_OS2`, :data:`RAR_OS_BEOS`. - - RAR5: :data:`RAR_OS_WIN32`, :data:`RAR_OS_UNIX`. - - mode - File attributes. May be either dos-style or unix-style, depending on host_os. - - mtime - File modification time. Same value as :attr:`date_time` - but as :class:`datetime.datetime` object with extended precision. - - ctime - Optional time field: creation time. As :class:`datetime.datetime` object. - - atime - Optional time field: last access time. As :class:`datetime.datetime` object. - - arctime - Optional time field: archival time. As :class:`datetime.datetime` object. - (RAR3-only) - - CRC - CRC-32 of uncompressed file, unsigned int. - - RAR5: may be None. - - blake2sp_hash - Blake2SP hash over decompressed data. (RAR5-only) - - comment - Optional file comment field. Unicode string. (RAR3-only) - - file_redir - If not None, file is link of some sort. Contains tuple of (type, flags, target). - (RAR5-only) - - Type is one of constants: - - :data:`RAR5_XREDIR_UNIX_SYMLINK` - unix symlink to target. - :data:`RAR5_XREDIR_WINDOWS_SYMLINK` - windows symlink to target. - :data:`RAR5_XREDIR_WINDOWS_JUNCTION` - windows junction. - :data:`RAR5_XREDIR_HARD_LINK` - hard link to target. - :data:`RAR5_XREDIR_FILE_COPY` - current file is copy of another archive entry. - - Flags may contain :data:`RAR5_XREDIR_ISDIR` bit. - - volume - Volume nr, starting from 0. - - volume_file - Volume file name, where file starts. - - ''' - - # zipfile-compatible fields - filename = None - file_size = None - compress_size = None - date_time = None - comment = None - CRC = None - volume = None - orig_filename = None - - # optional extended time fields, datetime() objects. - mtime = None - ctime = None - atime = None - - extract_version = None - mode = None - host_os = None - compress_type = None - - # rar3-only fields - comment = None - arctime = None - - # rar5-only fields - blake2sp_hash = None - file_redir = None - - # internal fields - flags = 0 - type = None - - def isdir(self): - """Returns True if entry is a directory. - """ - if self.type == RAR_BLOCK_FILE: - return (self.flags & RAR_FILE_DIRECTORY) == RAR_FILE_DIRECTORY - return False - - def needs_password(self): - """Returns True if data is stored password-protected. - """ - if self.type == RAR_BLOCK_FILE: - return (self.flags & RAR_FILE_PASSWORD) > 0 - return False - - -class RarFile(object): - '''Parse RAR structure, provide access to files in archive. - ''' - - #: Archive comment. Unicode string or None. - comment = None - - def __init__(self, rarfile, mode="r", charset=None, info_callback=None, - crc_check=True, errors="stop"): - """Open and parse a RAR archive. - - Parameters: - - rarfile - archive file name - mode - only 'r' is supported. - charset - fallback charset to use, if filenames are not already Unicode-enabled. - info_callback - debug callback, gets to see all archive entries. - crc_check - set to False to disable CRC checks - errors - Either "stop" to quietly stop parsing on errors, - or "strict" to raise errors. Default is "stop". - """ - self._rarfile = rarfile - self._charset = charset or DEFAULT_CHARSET - self._info_callback = info_callback - self._crc_check = crc_check - self._password = None - self._file_parser = None - - if errors == "stop": - self._strict = False - elif errors == "strict": - self._strict = True - else: - raise ValueError("Invalid value for 'errors' parameter.") - - if mode != "r": - raise NotImplementedError("RarFile supports only mode=r") - - self._parse() - - def __enter__(self): - return self - - def __exit__(self, typ, value, traceback): - self.close() - - def setpassword(self, password): - '''Sets the password to use when extracting.''' - self._password = password - if self._file_parser: - if self._file_parser.has_header_encryption(): - self._file_parser = None - if not self._file_parser: - self._parse() - else: - self._file_parser.setpassword(self._password) - - def needs_password(self): - '''Returns True if any archive entries require password for extraction.''' - return self._file_parser.needs_password() - - def namelist(self): - '''Return list of filenames in archive.''' - return [f.filename for f in self.infolist()] - - def infolist(self): - '''Return RarInfo objects for all files/directories in archive.''' - return self._file_parser.infolist() - - def volumelist(self): - '''Returns filenames of archive volumes. - - In case of single-volume archive, the list contains - just the name of main archive file. - ''' - return self._file_parser.volumelist() - - def getinfo(self, fname): - '''Return RarInfo for file. - ''' - return self._file_parser.getinfo(fname) - - def open(self, fname, mode='r', psw=None): - '''Returns file-like object (:class:`RarExtFile`), - from where the data can be read. - - The object implements :class:`io.RawIOBase` interface, so it can - be further wrapped with :class:`io.BufferedReader` - and :class:`io.TextIOWrapper`. - - On older Python where io module is not available, it implements - only .read(), .seek(), .tell() and .close() methods. - - The object is seekable, although the seeking is fast only on - uncompressed files, on compressed files the seeking is implemented - by reading ahead and/or restarting the decompression. - - Parameters: - - fname - file name or RarInfo instance. - mode - must be 'r' - psw - password to use for extracting. - ''' - - if mode != 'r': - raise NotImplementedError("RarFile.open() supports only mode=r") - - # entry lookup - inf = self.getinfo(fname) - if inf.isdir(): - raise TypeError("Directory does not have any data: " + inf.filename) - - # check password - if inf.needs_password(): - psw = psw or self._password - if psw is None: - raise PasswordRequired("File %s requires password" % inf.filename) - else: - psw = None - - return self._file_parser.open(inf, psw) - - def read(self, fname, psw=None): - """Return uncompressed data for archive entry. - - For longer files using :meth:`RarFile.open` may be better idea. - - Parameters: - - fname - filename or RarInfo instance - psw - password to use for extracting. - """ - - with self.open(fname, 'r', psw) as f: - return f.read() - - def close(self): - """Release open resources.""" - pass - - def printdir(self): - """Print archive file list to stdout.""" - for f in self.infolist(): - print(f.filename) - - def extract(self, member, path=None, pwd=None): - """Extract single file into current directory. - - Parameters: - - member - filename or :class:`RarInfo` instance - path - optional destination path - pwd - optional password to use - """ - if isinstance(member, RarInfo): - fname = member.filename - else: - fname = member - self._extract([fname], path, pwd) - - def extractall(self, path=None, members=None, pwd=None): - """Extract all files into current directory. - - Parameters: - - path - optional destination path - members - optional filename or :class:`RarInfo` instance list to extract - pwd - optional password to use - """ - fnlist = [] - if members is not None: - for m in members: - if isinstance(m, RarInfo): - fnlist.append(m.filename) - else: - fnlist.append(m) - self._extract(fnlist, path, pwd) - - def testrar(self): - """Let 'unrar' test the archive. - """ - cmd = [UNRAR_TOOL] + list(TEST_ARGS) - add_password_arg(cmd, self._password) - cmd.append('--') - with XTempFile(self._rarfile) as rarfile: - cmd.append(rarfile) - p = custom_popen(cmd) - output = p.communicate()[0] - check_returncode(p, output) - - def strerror(self): - """Return error string if parsing failed, - or None if no problems. - """ - if not self._file_parser: - return "Not a RAR file" - return self._file_parser.strerror() - - ## - ## private methods - ## - - def _parse(self): - ver = _get_rar_version(self._rarfile) - if ver == 3: - p3 = RAR3Parser(self._rarfile, self._password, self._crc_check, - self._charset, self._strict, self._info_callback) - self._file_parser = p3 # noqa - elif ver == 5: - p5 = RAR5Parser(self._rarfile, self._password, self._crc_check, - self._charset, self._strict, self._info_callback) - self._file_parser = p5 # noqa - else: - raise BadRarFile("Not a RAR file") - - self._file_parser.parse() - self.comment = self._file_parser.comment - - # call unrar to extract a file - def _extract(self, fnlist, path=None, psw=None): - cmd = [UNRAR_TOOL] + list(EXTRACT_ARGS) - - # pasoword - psw = psw or self._password - add_password_arg(cmd, psw) - cmd.append('--') - - # rar file - with XTempFile(self._rarfile) as rarfn: - cmd.append(rarfn) - - # file list - for fn in fnlist: - if os.sep != PATH_SEP: - fn = fn.replace(PATH_SEP, os.sep) - cmd.append(fn) - - # destination path - if path is not None: - cmd.append(path + os.sep) - - # call - p = custom_popen(cmd) - output = p.communicate()[0] - check_returncode(p, output) - -# -# File format parsing -# - -class CommonParser(object): - """Shared parser parts.""" - _main = None - _hdrenc_main = None - _needs_password = False - _fd = None - _expect_sig = None - _parse_error = None - _password = None - comment = None - - def __init__(self, rarfile, password, crc_check, charset, strict, info_cb): - self._rarfile = rarfile - self._password = password - self._crc_check = crc_check - self._charset = charset - self._strict = strict - self._info_callback = info_cb - self._info_list = [] - self._info_map = {} - self._vol_list = [] - - def has_header_encryption(self): - """Returns True if headers are encrypted - """ - if self._hdrenc_main: - return True - if self._main: - if self._main.flags & RAR_MAIN_PASSWORD: - return True - return False - - def setpassword(self, psw): - """Set cached password.""" - self._password = psw - - def volumelist(self): - """Volume files""" - return self._vol_list - - def needs_password(self): - """Is password required""" - return self._needs_password - - def strerror(self): - """Last error""" - return self._parse_error - - def infolist(self): - """List of RarInfo records. - """ - return self._info_list - - def getinfo(self, fname): - """Return RarInfo for filename - """ - # accept both ways here - if PATH_SEP == '/': - fname2 = fname.replace("\\", "/") - else: - fname2 = fname.replace("/", "\\") - - try: - return self._info_map[fname] - except KeyError: - try: - return self._info_map[fname2] - except KeyError: - raise NoRarEntry("No such file: %s" % fname) - - # read rar - def parse(self): - """Process file.""" - self._fd = None - try: - self._parse_real() - finally: - if self._fd: - self._fd.close() - self._fd = None - - def _parse_real(self): - fd = XFile(self._rarfile) - self._fd = fd - sig = fd.read(len(self._expect_sig)) - if sig != self._expect_sig: - if isinstance(self._rarfile, (str, unicode)): - raise NotRarFile("Not a Rar archive: {}".format(self._rarfile)) - raise NotRarFile("Not a Rar archive") - - volume = 0 # first vol (.rar) is 0 - more_vols = False - endarc = False - volfile = self._rarfile - self._vol_list = [self._rarfile] - while 1: - if endarc: - h = None # don't read past ENDARC - else: - h = self._parse_header(fd) - if not h: - if more_vols: - volume += 1 - fd.close() - try: - volfile = self._next_volname(volfile) - fd = XFile(volfile) - except IOError: - self._set_error("Cannot open next volume: %s", volfile) - break - self._fd = fd - sig = fd.read(len(self._expect_sig)) - if sig != self._expect_sig: - self._set_error("Invalid volume sig: %s", volfile) - break - more_vols = False - endarc = False - self._vol_list.append(volfile) - continue - break - h.volume = volume - h.volume_file = volfile - - if h.type == RAR_BLOCK_MAIN and not self._main: - self._main = h - if h.flags & RAR_MAIN_NEWNUMBERING: - # RAR 2.x does not set FIRSTVOLUME, - # so check it only if NEWNUMBERING is used - if (h.flags & RAR_MAIN_FIRSTVOLUME) == 0: - raise NeedFirstVolume("Need to start from first volume") - if h.flags & RAR_MAIN_PASSWORD: - self._needs_password = True - if not self._password: - break - elif h.type == RAR_BLOCK_ENDARC: - more_vols = (h.flags & RAR_ENDARC_NEXT_VOLUME) > 0 - endarc = True - elif h.type == RAR_BLOCK_FILE: - # RAR 2.x does not write RAR_BLOCK_ENDARC - if h.flags & RAR_FILE_SPLIT_AFTER: - more_vols = True - # RAR 2.x does not set RAR_MAIN_FIRSTVOLUME - if volume == 0 and h.flags & RAR_FILE_SPLIT_BEFORE: - raise NeedFirstVolume("Need to start from first volume") - - if h.needs_password(): - self._needs_password = True - - # store it - self.process_entry(fd, h) - - if self._info_callback: - self._info_callback(h) - - # go to next header - if h.add_size > 0: - fd.seek(h.data_offset + h.add_size, 0) - - def process_entry(self, fd, item): - """Examine item, add into lookup cache.""" - raise NotImplementedError() - - def _decrypt_header(self, fd): - raise NotImplementedError('_decrypt_header') - - def _parse_block_header(self, fd): - raise NotImplementedError('_parse_block_header') - - def _open_hack(self, inf, psw): - raise NotImplementedError('_open_hack') - - # read single header - def _parse_header(self, fd): - try: - # handle encrypted headers - if (self._main and self._main.flags & RAR_MAIN_PASSWORD) or self._hdrenc_main: - if not self._password: - return - fd = self._decrypt_header(fd) - - # now read actual header - return self._parse_block_header(fd) - except struct.error: - self._set_error('Broken header in RAR file') - return None - - # given current vol name, construct next one - def _next_volname(self, volfile): - if is_filelike(volfile): - raise IOError("Working on single FD") - if self._main.flags & RAR_MAIN_NEWNUMBERING: - return _next_newvol(volfile) - return _next_oldvol(volfile) - - def _set_error(self, msg, *args): - if args: - msg = msg % args - self._parse_error = msg - if self._strict: - raise BadRarFile(msg) - - def open(self, inf, psw): - """Return stream object for file data.""" - - if inf.file_redir: - # cannot leave to unrar as it expects copied file to exist - if inf.file_redir[0] in (RAR5_XREDIR_FILE_COPY, RAR5_XREDIR_HARD_LINK): - inf = self.getinfo(inf.file_redir[2]) - if not inf: - raise BadRarFile('cannot find copied file') - - if inf.flags & RAR_FILE_SPLIT_BEFORE: - raise NeedFirstVolume("Partial file, please start from first volume: " + inf.filename) - - # is temp write usable? - use_hack = 1 - if not self._main: - use_hack = 0 - elif self._main._must_disable_hack(): - use_hack = 0 - elif inf._must_disable_hack(): - use_hack = 0 - elif is_filelike(self._rarfile): - pass - elif inf.file_size > HACK_SIZE_LIMIT: - use_hack = 0 - elif not USE_EXTRACT_HACK: - use_hack = 0 - - # now extract - if inf.compress_type == RAR_M0 and (inf.flags & RAR_FILE_PASSWORD) == 0 and inf.file_redir is None: - return self._open_clear(inf) - elif use_hack: - return self._open_hack(inf, psw) - elif is_filelike(self._rarfile): - return self._open_unrar_membuf(self._rarfile, inf, psw) - else: - return self._open_unrar(self._rarfile, inf, psw) - - def _open_clear(self, inf): - return DirectReader(self, inf) - - def _open_hack_core(self, inf, psw, prefix, suffix): - - size = inf.compress_size + inf.header_size - rf = XFile(inf.volume_file, 0) - rf.seek(inf.header_offset) - - tmpfd, tmpname = mkstemp(suffix='.rar') - tmpf = os.fdopen(tmpfd, "wb") - - try: - tmpf.write(prefix) - while size > 0: - if size > BSIZE: - buf = rf.read(BSIZE) - else: - buf = rf.read(size) - if not buf: - raise BadRarFile('read failed: ' + inf.filename) - tmpf.write(buf) - size -= len(buf) - tmpf.write(suffix) - tmpf.close() - rf.close() - except: - rf.close() - tmpf.close() - os.unlink(tmpname) - raise - - return self._open_unrar(tmpname, inf, psw, tmpname) - - # write in-memory archive to temp file - needed for solid archives - def _open_unrar_membuf(self, memfile, inf, psw): - tmpname = membuf_tempfile(memfile) - return self._open_unrar(tmpname, inf, psw, tmpname, force_file=True) - - # extract using unrar - def _open_unrar(self, rarfile, inf, psw=None, tmpfile=None, force_file=False): - cmd = [UNRAR_TOOL] + list(OPEN_ARGS) - add_password_arg(cmd, psw) - cmd.append("--") - cmd.append(rarfile) - - # not giving filename avoids encoding related problems - if not tmpfile or force_file: - fn = inf.filename - if PATH_SEP != os.sep: - fn = fn.replace(PATH_SEP, os.sep) - cmd.append(fn) - - # read from unrar pipe - return PipeReader(self, inf, cmd, tmpfile) - -# -# RAR3 format -# - -class Rar3Info(RarInfo): - """RAR3 specific fields.""" - extract_version = 15 - salt = None - add_size = 0 - header_crc = None - header_size = None - header_offset = None - data_offset = None - _md_class = None - _md_expect = None - - # make sure some rar5 fields are always present - file_redir = None - blake2sp_hash = None - - def _must_disable_hack(self): - if self.type == RAR_BLOCK_FILE: - if self.flags & RAR_FILE_PASSWORD: - return True - elif self.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): - return True - elif self.type == RAR_BLOCK_MAIN: - if self.flags & (RAR_MAIN_SOLID | RAR_MAIN_PASSWORD): - return True - return False - - -class RAR3Parser(CommonParser): - """Parse RAR3 file format. - """ - _expect_sig = RAR_ID - _last_aes_key = (None, None, None) # (salt, key, iv) - - def _decrypt_header(self, fd): - if not _have_crypto: - raise NoCrypto('Cannot parse encrypted headers - no crypto') - salt = fd.read(8) - if self._last_aes_key[0] == salt: - key, iv = self._last_aes_key[1:] - else: - key, iv = rar3_s2k(self._password, salt) - self._last_aes_key = (salt, key, iv) - return HeaderDecrypt(fd, key, iv) - - # common header - def _parse_block_header(self, fd): - h = Rar3Info() - h.header_offset = fd.tell() - - # read and parse base header - buf = fd.read(S_BLK_HDR.size) - if not buf: - return None - t = S_BLK_HDR.unpack_from(buf) - h.header_crc, h.type, h.flags, h.header_size = t - - # read full header - if h.header_size > S_BLK_HDR.size: - hdata = buf + fd.read(h.header_size - S_BLK_HDR.size) - else: - hdata = buf - h.data_offset = fd.tell() - - # unexpected EOF? - if len(hdata) != h.header_size: - self._set_error('Unexpected EOF when reading header') - return None - - pos = S_BLK_HDR.size - - # block has data assiciated with it? - if h.flags & RAR_LONG_BLOCK: - h.add_size, pos = load_le32(hdata, pos) - else: - h.add_size = 0 - - # parse interesting ones, decide header boundaries for crc - if h.type == RAR_BLOCK_MARK: - return h - elif h.type == RAR_BLOCK_MAIN: - pos += 6 - if h.flags & RAR_MAIN_ENCRYPTVER: - pos += 1 - crc_pos = pos - if h.flags & RAR_MAIN_COMMENT: - self._parse_subblocks(h, hdata, pos) - elif h.type == RAR_BLOCK_FILE: - pos = self._parse_file_header(h, hdata, pos - 4) - crc_pos = pos - if h.flags & RAR_FILE_COMMENT: - pos = self._parse_subblocks(h, hdata, pos) - elif h.type == RAR_BLOCK_SUB: - pos = self._parse_file_header(h, hdata, pos - 4) - crc_pos = h.header_size - elif h.type == RAR_BLOCK_OLD_AUTH: - pos += 8 - crc_pos = pos - elif h.type == RAR_BLOCK_OLD_EXTRA: - pos += 7 - crc_pos = pos - else: - crc_pos = h.header_size - - # check crc - if h.type == RAR_BLOCK_OLD_SUB: - crcdat = hdata[2:] + fd.read(h.add_size) - else: - crcdat = hdata[2:crc_pos] - - calc_crc = rar_crc32(crcdat) & 0xFFFF - - # return good header - if h.header_crc == calc_crc: - return h - - # header parsing failed. - self._set_error('Header CRC error (%02x): exp=%x got=%x (xlen = %d)', - h.type, h.header_crc, calc_crc, len(crcdat)) - - # instead panicing, send eof - return None - - # read file-specific header - def _parse_file_header(self, h, hdata, pos): - fld = S_FILE_HDR.unpack_from(hdata, pos) - pos += S_FILE_HDR.size - - h.compress_size = fld[0] - h.file_size = fld[1] - h.host_os = fld[2] - h.CRC = fld[3] - h.date_time = parse_dos_time(fld[4]) - h.mtime = to_datetime(h.date_time) - h.extract_version = fld[5] - h.compress_type = fld[6] - name_size = fld[7] - h.mode = fld[8] - - h._md_class = CRC32Context - h._md_expect = h.CRC - - if h.flags & RAR_FILE_LARGE: - h1, pos = load_le32(hdata, pos) - h2, pos = load_le32(hdata, pos) - h.compress_size |= h1 << 32 - h.file_size |= h2 << 32 - h.add_size = h.compress_size - - name, pos = load_bytes(hdata, name_size, pos) - if h.flags & RAR_FILE_UNICODE: - nul = name.find(ZERO) - h.orig_filename = name[:nul] - u = UnicodeFilename(h.orig_filename, name[nul + 1:]) - h.filename = u.decode() - - # if parsing failed fall back to simple name - if u.failed: - h.filename = self._decode(h.orig_filename) - else: - h.orig_filename = name - h.filename = self._decode(name) - - # change separator, if requested - if PATH_SEP != '\\': - h.filename = h.filename.replace('\\', PATH_SEP) - - if h.flags & RAR_FILE_SALT: - h.salt, pos = load_bytes(hdata, 8, pos) - else: - h.salt = None - - # optional extended time stamps - if h.flags & RAR_FILE_EXTTIME: - pos = _parse_ext_time(h, hdata, pos) - else: - h.mtime = h.atime = h.ctime = h.arctime = None - - return pos - - # find old-style comment subblock - def _parse_subblocks(self, h, hdata, pos): - while pos < len(hdata): - # ordinary block header - t = S_BLK_HDR.unpack_from(hdata, pos) - ___scrc, stype, sflags, slen = t - pos_next = pos + slen - pos += S_BLK_HDR.size - - # corrupt header - if pos_next < pos: - break - - # followed by block-specific header - if stype == RAR_BLOCK_OLD_COMMENT and pos + S_COMMENT_HDR.size <= pos_next: - declen, ver, meth, crc = S_COMMENT_HDR.unpack_from(hdata, pos) - pos += S_COMMENT_HDR.size - data = hdata[pos : pos_next] - cmt = rar3_decompress(ver, meth, data, declen, sflags, - crc, self._password) - if not self._crc_check: - h.comment = self._decode_comment(cmt) - elif rar_crc32(cmt) & 0xFFFF == crc: - h.comment = self._decode_comment(cmt) - - pos = pos_next - return pos - - def _read_comment_v3(self, inf, psw=None): - - # read data - with XFile(inf.volume_file) as rf: - rf.seek(inf.data_offset) - data = rf.read(inf.compress_size) - - # decompress - cmt = rar3_decompress(inf.extract_version, inf.compress_type, data, - inf.file_size, inf.flags, inf.CRC, psw, inf.salt) - - # check crc - if self._crc_check: - crc = rar_crc32(cmt) - if crc != inf.CRC: - return None - - return self._decode_comment(cmt) - - def _decode(self, val): - for c in TRY_ENCODINGS: - try: - return val.decode(c) - except UnicodeError: - pass - return val.decode(self._charset, 'replace') - - def _decode_comment(self, val): - return self._decode(val) - - def process_entry(self, fd, item): - if item.type == RAR_BLOCK_FILE: - # use only first part - if (item.flags & RAR_FILE_SPLIT_BEFORE) == 0: - self._info_map[item.filename] = item - self._info_list.append(item) - elif len(self._info_list) > 0: - # final crc is in last block - old = self._info_list[-1] - old.CRC = item.CRC - old._md_expect = item._md_expect - old.compress_size += item.compress_size - - # parse new-style comment - if item.type == RAR_BLOCK_SUB and item.filename == 'CMT': - if item.flags & (RAR_FILE_SPLIT_BEFORE | RAR_FILE_SPLIT_AFTER): - pass - elif item.flags & RAR_FILE_SOLID: - # file comment - cmt = self._read_comment_v3(item, self._password) - if len(self._info_list) > 0: - old = self._info_list[-1] - old.comment = cmt - else: - # archive comment - cmt = self._read_comment_v3(item, self._password) - self.comment = cmt - - if item.type == RAR_BLOCK_MAIN: - if item.flags & RAR_MAIN_COMMENT: - self.comment = item.comment - if item.flags & RAR_MAIN_PASSWORD: - self._needs_password = True - - # put file compressed data into temporary .rar archive, and run - # unrar on that, thus avoiding unrar going over whole archive - def _open_hack(self, inf, psw): - # create main header: crc, type, flags, size, res1, res2 - prefix = RAR_ID + S_BLK_HDR.pack(0x90CF, 0x73, 0, 13) + ZERO * (2 + 4) - return self._open_hack_core(inf, psw, prefix, EMPTY) - -# -# RAR5 format -# - -class Rar5Info(RarInfo): - """Shared fields for RAR5 records. - """ - extract_version = 50 - header_crc = None - header_size = None - header_offset = None - data_offset = None - - # type=all - block_type = None - block_flags = None - add_size = 0 - block_extra_size = 0 - - # type=MAIN - volume_number = None - _md_class = None - _md_expect = None - - def _must_disable_hack(self): - return False - - -class Rar5BaseFile(Rar5Info): - """Shared sturct for file & service record. - """ - type = -1 - file_flags = None - file_encryption = (0, 0, 0, EMPTY, EMPTY, EMPTY) - file_compress_flags = None - file_redir = None - file_owner = None - file_version = None - blake2sp_hash = None - - def _must_disable_hack(self): - if self.flags & RAR_FILE_PASSWORD: - return True - if self.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): - return True - if self.file_compress_flags & RAR5_COMPR_SOLID: - return True - if self.file_redir: - return True - return False - - -class Rar5FileInfo(Rar5BaseFile): - """RAR5 file record. - """ - type = RAR_BLOCK_FILE - - -class Rar5ServiceInfo(Rar5BaseFile): - """RAR5 service record. - """ - type = RAR_BLOCK_SUB - - -class Rar5MainInfo(Rar5Info): - """RAR5 archive main record. - """ - type = RAR_BLOCK_MAIN - main_flags = None - main_volume_number = None - - def _must_disable_hack(self): - if self.main_flags & RAR5_MAIN_FLAG_SOLID: - return True - return False - - -class Rar5EncryptionInfo(Rar5Info): - """RAR5 archive header encryption record. - """ - type = RAR5_BLOCK_ENCRYPTION - encryption_algo = None - encryption_flags = None - encryption_kdf_count = None - encryption_salt = None - encryption_check_value = None - - def needs_password(self): - return True - - -class Rar5EndArcInfo(Rar5Info): - """RAR5 end of archive record. - """ - type = RAR_BLOCK_ENDARC - endarc_flags = None - - -class RAR5Parser(CommonParser): - """Parse RAR5 format. - """ - _expect_sig = RAR5_ID - _hdrenc_main = None - - # AES encrypted headers - _last_aes256_key = (-1, None, None) # (kdf_count, salt, key) - - def _gen_key(self, kdf_count, salt): - if self._last_aes256_key[:2] == (kdf_count, salt): - return self._last_aes256_key[2] - if kdf_count > 24: - raise BadRarFile('Too large kdf_count') - psw = self._password - if isinstance(psw, unicode): - psw = psw.encode('utf8') - key = pbkdf2_sha256(psw, salt, 1 << kdf_count) - self._last_aes256_key = (kdf_count, salt, key) - return key - - def _decrypt_header(self, fd): - if not _have_crypto: - raise NoCrypto('Cannot parse encrypted headers - no crypto') - h = self._hdrenc_main - key = self._gen_key(h.encryption_kdf_count, h.encryption_salt) - iv = fd.read(16) - return HeaderDecrypt(fd, key, iv) - - # common header - def _parse_block_header(self, fd): - header_offset = fd.tell() - - preload = 4 + 3 - start_bytes = fd.read(preload) - header_crc, pos = load_le32(start_bytes, 0) - hdrlen, pos = load_vint(start_bytes, pos) - if hdrlen > 2 * 1024 * 1024: - return None - header_size = pos + hdrlen - - # read full header, check for EOF - hdata = start_bytes + fd.read(header_size - len(start_bytes)) - if len(hdata) != header_size: - self._set_error('Unexpected EOF when reading header') - return None - data_offset = fd.tell() - - calc_crc = rar_crc32(memoryview(hdata)[4:]) - if header_crc != calc_crc: - # header parsing failed. - self._set_error('Header CRC error: exp=%x got=%x (xlen = %d)', - header_crc, calc_crc, len(hdata)) - return None - - block_type, pos = load_vint(hdata, pos) - - if block_type == RAR5_BLOCK_MAIN: - h, pos = self._parse_block_common(Rar5MainInfo(), hdata) - h = self._parse_main_block(h, hdata, pos) - elif block_type == RAR5_BLOCK_FILE: - h, pos = self._parse_block_common(Rar5FileInfo(), hdata) - h = self._parse_file_block(h, hdata, pos) - elif block_type == RAR5_BLOCK_SERVICE: - h, pos = self._parse_block_common(Rar5ServiceInfo(), hdata) - h = self._parse_file_block(h, hdata, pos) - elif block_type == RAR5_BLOCK_ENCRYPTION: - h, pos = self._parse_block_common(Rar5EncryptionInfo(), hdata) - h = self._parse_encryption_block(h, hdata, pos) - elif block_type == RAR5_BLOCK_ENDARC: - h, pos = self._parse_block_common(Rar5EndArcInfo(), hdata) - h = self._parse_endarc_block(h, hdata, pos) - else: - h = None - if h: - h.header_offset = header_offset - h.data_offset = data_offset - return h - - def _parse_block_common(self, h, hdata): - h.header_crc, pos = load_le32(hdata, 0) - hdrlen, pos = load_vint(hdata, pos) - h.header_size = hdrlen + pos - h.block_type, pos = load_vint(hdata, pos) - h.block_flags, pos = load_vint(hdata, pos) - - if h.block_flags & RAR5_BLOCK_FLAG_EXTRA_DATA: - h.block_extra_size, pos = load_vint(hdata, pos) - if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: - h.add_size, pos = load_vint(hdata, pos) - - h.compress_size = h.add_size - - if h.block_flags & RAR5_BLOCK_FLAG_SKIP_IF_UNKNOWN: - h.flags |= RAR_SKIP_IF_UNKNOWN - if h.block_flags & RAR5_BLOCK_FLAG_DATA_AREA: - h.flags |= RAR_LONG_BLOCK - return h, pos - - def _parse_main_block(self, h, hdata, pos): - h.main_flags, pos = load_vint(hdata, pos) - if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR: - h.main_volume_number = load_vint(hdata, pos) - - h.flags |= RAR_MAIN_NEWNUMBERING - if h.main_flags & RAR5_MAIN_FLAG_SOLID: - h.flags |= RAR_MAIN_SOLID - if h.main_flags & RAR5_MAIN_FLAG_ISVOL: - h.flags |= RAR_MAIN_VOLUME - if h.main_flags & RAR5_MAIN_FLAG_RECOVERY: - h.flags |= RAR_MAIN_RECOVERY - if self._hdrenc_main: - h.flags |= RAR_MAIN_PASSWORD - if h.main_flags & RAR5_MAIN_FLAG_HAS_VOLNR == 0: - h.flags |= RAR_MAIN_FIRSTVOLUME - - return h - - def _parse_file_block(self, h, hdata, pos): - h.file_flags, pos = load_vint(hdata, pos) - h.file_size, pos = load_vint(hdata, pos) - h.mode, pos = load_vint(hdata, pos) - - if h.file_flags & RAR5_FILE_FLAG_HAS_MTIME: - h.mtime, pos = load_unixtime(hdata, pos) - h.date_time = h.mtime.timetuple()[:6] - if h.file_flags & RAR5_FILE_FLAG_HAS_CRC32: - h.CRC, pos = load_le32(hdata, pos) - h._md_class = CRC32Context - h._md_expect = h.CRC - - h.file_compress_flags, pos = load_vint(hdata, pos) - h.file_host_os, pos = load_vint(hdata, pos) - h.orig_filename, pos = load_vstr(hdata, pos) - h.filename = h.orig_filename.decode('utf8', 'replace') - - # use compatible values - if h.file_host_os == RAR5_OS_WINDOWS: - h.host_os = RAR_OS_WIN32 - else: - h.host_os = RAR_OS_UNIX - h.compress_type = RAR_M0 + ((h.file_compress_flags >> 7) & 7) - - if h.block_extra_size: - # allow 1 byte of garbage - while pos < len(hdata) - 1: - xsize, pos = load_vint(hdata, pos) - xdata, pos = load_bytes(hdata, xsize, pos) - self._process_file_extra(h, xdata) - - if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE: - h.flags |= RAR_FILE_SPLIT_BEFORE - if h.block_flags & RAR5_BLOCK_FLAG_SPLIT_AFTER: - h.flags |= RAR_FILE_SPLIT_AFTER - if h.file_flags & RAR5_FILE_FLAG_ISDIR: - h.flags |= RAR_FILE_DIRECTORY - if h.file_compress_flags & RAR5_COMPR_SOLID: - h.flags |= RAR_FILE_SOLID - - return h - - def _parse_endarc_block(self, h, hdata, pos): - h.endarc_flags, pos = load_vint(hdata, pos) - if h.endarc_flags & RAR5_ENDARC_FLAG_NEXT_VOL: - h.flags |= RAR_ENDARC_NEXT_VOLUME - return h - - def _parse_encryption_block(self, h, hdata, pos): - h.encryption_algo, pos = load_vint(hdata, pos) - h.encryption_flags, pos = load_vint(hdata, pos) - h.encryption_kdf_count, pos = load_byte(hdata, pos) - h.encryption_salt, pos = load_bytes(hdata, 16, pos) - if h.encryption_flags & RAR5_ENC_FLAG_HAS_CHECKVAL: - h.encryption_check_value = load_bytes(hdata, 12, pos) - if h.encryption_algo != RAR5_XENC_CIPHER_AES256: - raise BadRarFile('Unsupported header encryption cipher') - self._hdrenc_main = h - return h - - # file extra record - def _process_file_extra(self, h, xdata): - xtype, pos = load_vint(xdata, 0) - if xtype == RAR5_XFILE_TIME: - self._parse_file_xtime(h, xdata, pos) - elif xtype == RAR5_XFILE_ENCRYPTION: - self._parse_file_encryption(h, xdata, pos) - elif xtype == RAR5_XFILE_HASH: - self._parse_file_hash(h, xdata, pos) - elif xtype == RAR5_XFILE_VERSION: - self._parse_file_version(h, xdata, pos) - elif xtype == RAR5_XFILE_REDIR: - self._parse_file_redir(h, xdata, pos) - elif xtype == RAR5_XFILE_OWNER: - self._parse_file_owner(h, xdata, pos) - elif xtype == RAR5_XFILE_SERVICE: - pass - else: - pass - - # extra block for file time record - def _parse_file_xtime(self, h, xdata, pos): - tflags, pos = load_vint(xdata, pos) - ldr = load_windowstime - if tflags & RAR5_XTIME_UNIXTIME: - ldr = load_unixtime - if tflags & RAR5_XTIME_HAS_MTIME: - h.mtime, pos = ldr(xdata, pos) - h.date_time = h.mtime.timetuple()[:6] - if tflags & RAR5_XTIME_HAS_CTIME: - h.ctime, pos = ldr(xdata, pos) - if tflags & RAR5_XTIME_HAS_ATIME: - h.atime, pos = ldr(xdata, pos) - - # just remember encryption info - def _parse_file_encryption(self, h, xdata, pos): - algo, pos = load_vint(xdata, pos) - flags, pos = load_vint(xdata, pos) - kdf_count, pos = load_byte(xdata, pos) - salt, pos = load_bytes(xdata, 16, pos) - iv, pos = load_bytes(xdata, 16, pos) - checkval = None - if flags & RAR5_XENC_CHECKVAL: - checkval, pos = load_bytes(xdata, 12, pos) - if flags & RAR5_XENC_TWEAKED: - h._md_expect = None - h._md_class = NoHashContext - - h.file_encryption = (algo, flags, kdf_count, salt, iv, checkval) - h.flags |= RAR_FILE_PASSWORD - - def _parse_file_hash(self, h, xdata, pos): - hash_type, pos = load_vint(xdata, pos) - if hash_type == RAR5_XHASH_BLAKE2SP: - h.blake2sp_hash, pos = load_bytes(xdata, 32, pos) - if _have_blake2 and (h.file_encryption[1] & RAR5_XENC_TWEAKED) == 0: - h._md_class = Blake2SP - h._md_expect = h.blake2sp_hash - - def _parse_file_version(self, h, xdata, pos): - flags, pos = load_vint(xdata, pos) - version, pos = load_vint(xdata, pos) - h.file_version = (flags, version) - - def _parse_file_redir(self, h, xdata, pos): - redir_type, pos = load_vint(xdata, pos) - redir_flags, pos = load_vint(xdata, pos) - redir_name, pos = load_vstr(xdata, pos) - redir_name = redir_name.decode('utf8', 'replace') - h.file_redir = (redir_type, redir_flags, redir_name) - - def _parse_file_owner(self, h, xdata, pos): - user_name = group_name = user_id = group_id = None - - flags, pos = load_vint(xdata, pos) - if flags & RAR5_XOWNER_UNAME: - user_name, pos = load_vstr(xdata, pos) - if flags & RAR5_XOWNER_GNAME: - group_name, pos = load_vstr(xdata, pos) - if flags & RAR5_XOWNER_UID: - user_id, pos = load_vint(xdata, pos) - if flags & RAR5_XOWNER_GID: - group_id, pos = load_vint(xdata, pos) - - h.file_owner = (user_name, group_name, user_id, group_id) - - def process_entry(self, fd, item): - if item.block_type == RAR5_BLOCK_FILE: - # use only first part - if (item.block_flags & RAR5_BLOCK_FLAG_SPLIT_BEFORE) == 0: - self._info_map[item.filename] = item - self._info_list.append(item) - elif len(self._info_list) > 0: - # final crc is in last block - old = self._info_list[-1] - old.CRC = item.CRC - old._md_expect = item._md_expect - old.blake2sp_hash = item.blake2sp_hash - old.compress_size += item.compress_size - elif item.block_type == RAR5_BLOCK_SERVICE: - if item.filename == 'CMT': - self._load_comment(fd, item) - - def _load_comment(self, fd, item): - if item.block_flags & (RAR5_BLOCK_FLAG_SPLIT_BEFORE | RAR5_BLOCK_FLAG_SPLIT_AFTER): - return None - if item.compress_type != RAR_M0: - return None - - if item.flags & RAR_FILE_PASSWORD: - algo, ___flags, kdf_count, salt, iv, ___checkval = item.file_encryption - if algo != RAR5_XENC_CIPHER_AES256: - return None - key = self._gen_key(kdf_count, salt) - f = HeaderDecrypt(fd, key, iv) - cmt = f.read(item.file_size) - else: - # archive comment - with self._open_clear(item) as cmtstream: - cmt = cmtstream.read() - - # rar bug? - appends zero to comment - cmt = cmt.split(ZERO, 1)[0] - self.comment = cmt.decode('utf8') - - def _open_hack(self, inf, psw): - # len, type, blk_flags, flags - main_hdr = b'\x03\x01\x00\x00' - endarc_hdr = b'\x03\x05\x00\x00' - main_hdr = S_LONG.pack(rar_crc32(main_hdr)) + main_hdr - endarc_hdr = S_LONG.pack(rar_crc32(endarc_hdr)) + endarc_hdr - return self._open_hack_core(inf, psw, RAR5_ID + main_hdr, endarc_hdr) - -## -## Utility classes -## - -class UnicodeFilename(object): - """Handle RAR3 unicode filename decompression. - """ - def __init__(self, name, encdata): - self.std_name = bytearray(name) - self.encdata = bytearray(encdata) - self.pos = self.encpos = 0 - self.buf = bytearray() - self.failed = 0 - - def enc_byte(self): - """Copy encoded byte.""" - try: - c = self.encdata[self.encpos] - self.encpos += 1 - return c - except IndexError: - self.failed = 1 - return 0 - - def std_byte(self): - """Copy byte from 8-bit representation.""" - try: - return self.std_name[self.pos] - except IndexError: - self.failed = 1 - return ord('?') - - def put(self, lo, hi): - """Copy 16-bit value to result.""" - self.buf.append(lo) - self.buf.append(hi) - self.pos += 1 - - def decode(self): - """Decompress compressed UTF16 value.""" - hi = self.enc_byte() - flagbits = 0 - while self.encpos < len(self.encdata): - if flagbits == 0: - flags = self.enc_byte() - flagbits = 8 - flagbits -= 2 - t = (flags >> flagbits) & 3 - if t == 0: - self.put(self.enc_byte(), 0) - elif t == 1: - self.put(self.enc_byte(), hi) - elif t == 2: - self.put(self.enc_byte(), self.enc_byte()) - else: - n = self.enc_byte() - if n & 0x80: - c = self.enc_byte() - for _ in range((n & 0x7f) + 2): - lo = (self.std_byte() + c) & 0xFF - self.put(lo, hi) - else: - for _ in range(n + 2): - self.put(self.std_byte(), 0) - return self.buf.decode("utf-16le", "replace") - - -class RarExtFile(RawIOBase): - """Base class for file-like object that :meth:`RarFile.open` returns. - - Provides public methods and common crc checking. - - Behaviour: - - no short reads - .read() and .readinfo() read as much as requested. - - no internal buffer, use io.BufferedReader for that. - """ - - #: Filename of the archive entry - name = None - - def __init__(self, parser, inf): - super(RarExtFile, self).__init__() - - # standard io.* properties - self.name = inf.filename - self.mode = 'rb' - - self._parser = parser - self._inf = inf - self._fd = None - self._remain = 0 - self._returncode = 0 - - self._md_context = None - - self._open() - - def _open(self): - if self._fd: - self._fd.close() - md_class = self._inf._md_class or NoHashContext - self._md_context = md_class() - self._fd = None - self._remain = self._inf.file_size - - def read(self, cnt=None): - """Read all or specified amount of data from archive entry.""" - - # sanitize cnt - if cnt is None or cnt < 0: - cnt = self._remain - elif cnt > self._remain: - cnt = self._remain - if cnt == 0: - return EMPTY - - # actual read - data = self._read(cnt) - if data: - self._md_context.update(data) - self._remain -= len(data) - if len(data) != cnt: - raise BadRarFile("Failed the read enough data") - - # done? - if not data or self._remain == 0: - # self.close() - self._check() - return data - - def _check(self): - """Check final CRC.""" - final = self._md_context.digest() - exp = self._inf._md_expect - if exp is None: - return - if final is None: - return - if self._returncode: - check_returncode(self, '') - if self._remain != 0: - raise BadRarFile("Failed the read enough data") - if final != exp: - raise BadRarFile("Corrupt file - CRC check failed: %s - exp=%r got=%r" % ( - self._inf.filename, exp, final)) - - def _read(self, cnt): - """Actual read that gets sanitized cnt.""" - - def close(self): - """Close open resources.""" - - super(RarExtFile, self).close() - - if self._fd: - self._fd.close() - self._fd = None - - def __del__(self): - """Hook delete to make sure tempfile is removed.""" - self.close() - - def readinto(self, buf): - """Zero-copy read directly into buffer. - - Returns bytes read. - """ - raise NotImplementedError('readinto') - - def tell(self): - """Return current reading position in uncompressed data.""" - return self._inf.file_size - self._remain - - def seek(self, ofs, whence=0): - """Seek in data. - - On uncompressed files, the seeking works by actual - seeks so it's fast. On compresses files its slow - - forward seeking happends by reading ahead, - backwards by re-opening and decompressing from the start. - """ - - # disable crc check when seeking - self._md_context = NoHashContext() - - fsize = self._inf.file_size - cur_ofs = self.tell() - - if whence == 0: # seek from beginning of file - new_ofs = ofs - elif whence == 1: # seek from current position - new_ofs = cur_ofs + ofs - elif whence == 2: # seek from end of file - new_ofs = fsize + ofs - else: - raise ValueError('Invalid value for whence') - - # sanity check - if new_ofs < 0: - new_ofs = 0 - elif new_ofs > fsize: - new_ofs = fsize - - # do the actual seek - if new_ofs >= cur_ofs: - self._skip(new_ofs - cur_ofs) - else: - # reopen and seek - self._open() - self._skip(new_ofs) - return self.tell() - - def _skip(self, cnt): - """Read and discard data""" - while cnt > 0: - if cnt > 8192: - buf = self.read(8192) - else: - buf = self.read(cnt) - if not buf: - break - cnt -= len(buf) - - def readable(self): - """Returns True""" - return True - - def writable(self): - """Returns False. - - Writing is not supported.""" - return False - - def seekable(self): - """Returns True. - - Seeking is supported, although it's slow on compressed files. - """ - return True - - def readall(self): - """Read all remaining data""" - # avoid RawIOBase default impl - return self.read() - - -class PipeReader(RarExtFile): - """Read data from pipe, handle tempfile cleanup.""" - - def __init__(self, rf, inf, cmd, tempfile=None): - self._cmd = cmd - self._proc = None - self._tempfile = tempfile - super(PipeReader, self).__init__(rf, inf) - - def _close_proc(self): - if not self._proc: - return - if self._proc.stdout: - self._proc.stdout.close() - if self._proc.stdin: - self._proc.stdin.close() - if self._proc.stderr: - self._proc.stderr.close() - self._proc.wait() - self._returncode = self._proc.returncode - self._proc = None - - def _open(self): - super(PipeReader, self)._open() - - # stop old process - self._close_proc() - - # launch new process - self._returncode = 0 - self._proc = custom_popen(self._cmd) - self._fd = self._proc.stdout - - # avoid situation where unrar waits on stdin - if self._proc.stdin: - self._proc.stdin.close() - - def _read(self, cnt): - """Read from pipe.""" - - # normal read is usually enough - data = self._fd.read(cnt) - if len(data) == cnt or not data: - return data - - # short read, try looping - buf = [data] - cnt -= len(data) - while cnt > 0: - data = self._fd.read(cnt) - if not data: - break - cnt -= len(data) - buf.append(data) - return EMPTY.join(buf) - - def close(self): - """Close open resources.""" - - self._close_proc() - super(PipeReader, self).close() - - if self._tempfile: - try: - os.unlink(self._tempfile) - except OSError: - pass - self._tempfile = None - - def readinto(self, buf): - """Zero-copy read directly into buffer.""" - cnt = len(buf) - if cnt > self._remain: - cnt = self._remain - vbuf = memoryview(buf) - res = got = 0 - while got < cnt: - res = self._fd.readinto(vbuf[got : cnt]) - if not res: - break - self._md_context.update(vbuf[got : got + res]) - self._remain -= res - got += res - return got - - -class DirectReader(RarExtFile): - """Read uncompressed data directly from archive. - """ - _cur = None - _cur_avail = None - _volfile = None - - def _open(self): - super(DirectReader, self)._open() - - self._volfile = self._inf.volume_file - self._fd = XFile(self._volfile, 0) - self._fd.seek(self._inf.header_offset, 0) - self._cur = self._parser._parse_header(self._fd) - self._cur_avail = self._cur.add_size - - def _skip(self, cnt): - """RAR Seek, skipping through rar files to get to correct position - """ - - while cnt > 0: - # next vol needed? - if self._cur_avail == 0: - if not self._open_next(): - break - - # fd is in read pos, do the read - if cnt > self._cur_avail: - cnt -= self._cur_avail - self._remain -= self._cur_avail - self._cur_avail = 0 - else: - self._fd.seek(cnt, 1) - self._cur_avail -= cnt - self._remain -= cnt - cnt = 0 - - def _read(self, cnt): - """Read from potentially multi-volume archive.""" - - buf = [] - while cnt > 0: - # next vol needed? - if self._cur_avail == 0: - if not self._open_next(): - break - - # fd is in read pos, do the read - if cnt > self._cur_avail: - data = self._fd.read(self._cur_avail) - else: - data = self._fd.read(cnt) - if not data: - break - - # got some data - cnt -= len(data) - self._cur_avail -= len(data) - buf.append(data) - - if len(buf) == 1: - return buf[0] - return EMPTY.join(buf) - - def _open_next(self): - """Proceed to next volume.""" - - # is the file split over archives? - if (self._cur.flags & RAR_FILE_SPLIT_AFTER) == 0: - return False - - if self._fd: - self._fd.close() - self._fd = None - - # open next part - self._volfile = self._parser._next_volname(self._volfile) - fd = open(self._volfile, "rb", 0) - self._fd = fd - sig = fd.read(len(self._parser._expect_sig)) - if sig != self._parser._expect_sig: - raise BadRarFile("Invalid signature") - - # loop until first file header - while 1: - cur = self._parser._parse_header(fd) - if not cur: - raise BadRarFile("Unexpected EOF") - if cur.type in (RAR_BLOCK_MARK, RAR_BLOCK_MAIN): - if cur.add_size: - fd.seek(cur.add_size, 1) - continue - if cur.orig_filename != self._inf.orig_filename: - raise BadRarFile("Did not found file entry") - self._cur = cur - self._cur_avail = cur.add_size - return True - - def readinto(self, buf): - """Zero-copy read directly into buffer.""" - got = 0 - vbuf = memoryview(buf) - while got < len(buf): - # next vol needed? - if self._cur_avail == 0: - if not self._open_next(): - break - - # length for next read - cnt = len(buf) - got - if cnt > self._cur_avail: - cnt = self._cur_avail - - # read into temp view - res = self._fd.readinto(vbuf[got : got + cnt]) - if not res: - break - self._md_context.update(vbuf[got : got + res]) - self._cur_avail -= res - self._remain -= res - got += res - return got - - -class HeaderDecrypt(object): - """File-like object that decrypts from another file""" - def __init__(self, f, key, iv): - self.f = f - self.ciph = AES_CBC_Decrypt(key, iv) - self.buf = EMPTY - - def tell(self): - """Current file pos - works only on block boundaries.""" - return self.f.tell() - - def read(self, cnt=None): - """Read and decrypt.""" - if cnt > 8 * 1024: - raise BadRarFile('Bad count to header decrypt - wrong password?') - - # consume old data - if cnt <= len(self.buf): - res = self.buf[:cnt] - self.buf = self.buf[cnt:] - return res - res = self.buf - self.buf = EMPTY - cnt -= len(res) - - # decrypt new data - blklen = 16 - while cnt > 0: - enc = self.f.read(blklen) - if len(enc) < blklen: - break - dec = self.ciph.decrypt(enc) - if cnt >= len(dec): - res += dec - cnt -= len(dec) - else: - res += dec[:cnt] - self.buf = dec[cnt:] - cnt = 0 - - return res - - -# handle (filename|filelike) object -class XFile(object): - """Input may be filename or file object. - """ - __slots__ = ('_fd', '_need_close') - - def __init__(self, xfile, bufsize=1024): - if is_filelike(xfile): - self._need_close = False - self._fd = xfile - self._fd.seek(0) - else: - self._need_close = True - self._fd = open(xfile, 'rb', bufsize) - - def read(self, n=None): - """Read from file.""" - return self._fd.read(n) - - def tell(self): - """Return file pos.""" - return self._fd.tell() - - def seek(self, ofs, whence=0): - """Move file pos.""" - return self._fd.seek(ofs, whence) - - def readinto(self, dst): - """Read into buffer.""" - return self._fd.readinto(dst) - - def close(self): - """Close file object.""" - if self._need_close: - self._fd.close() - - def __enter__(self): - return self - - def __exit__(self, typ, val, tb): - self.close() - - -class NoHashContext(object): - """No-op hash function.""" - def __init__(self, data=None): - """Initialize""" - def update(self, data): - """Update data""" - def digest(self): - """Final hash""" - def hexdigest(self): - """Hexadecimal digest.""" - - -class CRC32Context(object): - """Hash context that uses CRC32.""" - __slots__ = ['_crc'] - - def __init__(self, data=None): - self._crc = 0 - if data: - self.update(data) - - def update(self, data): - """Process data.""" - self._crc = rar_crc32(data, self._crc) - - def digest(self): - """Final hash.""" - return self._crc - - def hexdigest(self): - """Hexadecimal digest.""" - return '%08x' % self.digest() - - -class Blake2SP(object): - """Blake2sp hash context. - """ - __slots__ = ['_thread', '_buf', '_cur', '_digest'] - digest_size = 32 - block_size = 64 - parallelism = 8 - - def __init__(self, data=None): - self._buf = b'' - self._cur = 0 - self._digest = None - self._thread = [] - - for i in range(self.parallelism): - ctx = self._blake2s(i, 0, i == (self.parallelism - 1)) - self._thread.append(ctx) - - if data: - self.update(data) - - def _blake2s(self, ofs, depth, is_last): - return blake2s(node_offset=ofs, node_depth=depth, last_node=is_last, - depth=2, inner_size=32, fanout=self.parallelism) - - def _add_block(self, blk): - self._thread[self._cur].update(blk) - self._cur = (self._cur + 1) % self.parallelism - - def update(self, data): - """Hash data. - """ - view = memoryview(data) - bs = self.block_size - if self._buf: - need = bs - len(self._buf) - if len(view) < need: - self._buf += view.tobytes() - return - self._add_block(self._buf + view[:need].tobytes()) - view = view[need:] - while len(view) >= bs: - self._add_block(view[:bs]) - view = view[bs:] - self._buf = view.tobytes() - - def digest(self): - """Return final digest value. - """ - if self._digest is None: - if self._buf: - self._add_block(self._buf) - self._buf = EMPTY - ctx = self._blake2s(0, 1, True) - for t in self._thread: - ctx.update(t.digest()) - self._digest = ctx.digest() - return self._digest - - def hexdigest(self): - """Hexadecimal digest.""" - return tohex(self.digest()) - -## -## Utility functions -## - -S_LONG = Struct(' len(buf): - raise BadRarFile('cannot load byte') - return S_BYTE.unpack_from(buf, pos)[0], end - -def load_le32(buf, pos): - """Load little-endian 32-bit integer""" - end = pos + 4 - if end > len(buf): - raise BadRarFile('cannot load le32') - return S_LONG.unpack_from(buf, pos)[0], pos + 4 - -def load_bytes(buf, num, pos): - """Load sequence of bytes""" - end = pos + num - if end > len(buf): - raise BadRarFile('cannot load bytes') - return buf[pos : end], end - -def load_vstr(buf, pos): - """Load bytes prefixed by vint length""" - slen, pos = load_vint(buf, pos) - return load_bytes(buf, slen, pos) - -def load_dostime(buf, pos): - """Load LE32 dos timestamp""" - stamp, pos = load_le32(buf, pos) - tup = parse_dos_time(stamp) - return to_datetime(tup), pos - -def load_unixtime(buf, pos): - """Load LE32 unix timestamp""" - secs, pos = load_le32(buf, pos) - dt = datetime.fromtimestamp(secs, UTC) - return dt, pos - -def load_windowstime(buf, pos): - """Load LE64 windows timestamp""" - # unix epoch (1970) in seconds from windows epoch (1601) - unix_epoch = 11644473600 - val1, pos = load_le32(buf, pos) - val2, pos = load_le32(buf, pos) - secs, n1secs = divmod((val2 << 32) | val1, 10000000) - dt = datetime.fromtimestamp(secs - unix_epoch, UTC) - dt = dt.replace(microsecond=n1secs // 10) - return dt, pos - -# new-style next volume -def _next_newvol(volfile): - i = len(volfile) - 1 - while i >= 0: - if volfile[i] >= '0' and volfile[i] <= '9': - return _inc_volname(volfile, i) - i -= 1 - raise BadRarName("Cannot construct volume name: " + volfile) - -# old-style next volume -def _next_oldvol(volfile): - # rar -> r00 - if volfile[-4:].lower() == '.rar': - return volfile[:-2] + '00' - return _inc_volname(volfile, len(volfile) - 1) - -# increase digits with carry, otherwise just increment char -def _inc_volname(volfile, i): - fn = list(volfile) - while i >= 0: - if fn[i] != '9': - fn[i] = chr(ord(fn[i]) + 1) - break - fn[i] = '0' - i -= 1 - return ''.join(fn) - -# rar3 extended time fields -def _parse_ext_time(h, data, pos): - # flags and rest of data can be missing - flags = 0 - if pos + 2 <= len(data): - flags = S_SHORT.unpack_from(data, pos)[0] - pos += 2 - - mtime, pos = _parse_xtime(flags >> 3 * 4, data, pos, h.mtime) - h.ctime, pos = _parse_xtime(flags >> 2 * 4, data, pos) - h.atime, pos = _parse_xtime(flags >> 1 * 4, data, pos) - h.arctime, pos = _parse_xtime(flags >> 0 * 4, data, pos) - if mtime: - h.mtime = mtime - h.date_time = mtime.timetuple()[:6] - return pos - -# rar3 one extended time field -def _parse_xtime(flag, data, pos, basetime=None): - res = None - if flag & 8: - if not basetime: - basetime, pos = load_dostime(data, pos) - - # load second fractions - rem = 0 - cnt = flag & 3 - for _ in range(cnt): - b, pos = load_byte(data, pos) - rem = (b << 16) | (rem >> 8) - - # convert 100ns units to microseconds - usec = rem // 10 - if usec > 1000000: - usec = 999999 - - # dostime has room for 30 seconds only, correct if needed - if flag & 4 and basetime.second < 59: - res = basetime.replace(microsecond=usec, second=basetime.second + 1) - else: - res = basetime.replace(microsecond=usec) - return res, pos - -def is_filelike(obj): - """Filename or file object? - """ - if isinstance(obj, str) or isinstance(obj, unicode): - return False - res = True - for a in ('read', 'tell', 'seek'): - res = res and hasattr(obj, a) - if not res: - raise ValueError("Invalid object passed as file") - return True - -def rar3_s2k(psw, salt): - """String-to-key hash for RAR3. - """ - if not isinstance(psw, unicode): - psw = psw.decode('utf8') - seed = psw.encode('utf-16le') + salt - iv = EMPTY - h = sha1() - for i in range(16): - for j in range(0x4000): - cnt = S_LONG.pack(i * 0x4000 + j) - h.update(seed + cnt[:3]) - if j == 0: - iv += h.digest()[19:20] - key_be = h.digest()[:16] - key_le = pack("LLLL", key_be)) - return key_le, iv - -def rar3_decompress(vers, meth, data, declen=0, flags=0, crc=0, psw=None, salt=None): - """Decompress blob of compressed data. - - Used for data with non-standard header - eg. comments. - """ - # already uncompressed? - if meth == RAR_M0 and (flags & RAR_FILE_PASSWORD) == 0: - return data - - # take only necessary flags - flags = flags & (RAR_FILE_PASSWORD | RAR_FILE_SALT | RAR_FILE_DICTMASK) - flags |= RAR_LONG_BLOCK - - # file header - fname = b'data' - date = 0 - mode = 0x20 - fhdr = S_FILE_HDR.pack(len(data), declen, RAR_OS_MSDOS, crc, - date, vers, meth, len(fname), mode) - fhdr += fname - if flags & RAR_FILE_SALT: - if not salt: - return EMPTY - fhdr += salt - - # full header - hlen = S_BLK_HDR.size + len(fhdr) - hdr = S_BLK_HDR.pack(0, RAR_BLOCK_FILE, flags, hlen) + fhdr - hcrc = rar_crc32(hdr[2:]) & 0xFFFF - hdr = S_BLK_HDR.pack(hcrc, RAR_BLOCK_FILE, flags, hlen) + fhdr - - # archive main header - mh = S_BLK_HDR.pack(0x90CF, RAR_BLOCK_MAIN, 0, 13) + ZERO * (2 + 4) - - # decompress via temp rar - tmpfd, tmpname = mkstemp(suffix='.rar') - tmpf = os.fdopen(tmpfd, "wb") - try: - tmpf.write(RAR_ID + mh + hdr + data) - tmpf.close() - - cmd = [UNRAR_TOOL] + list(OPEN_ARGS) - add_password_arg(cmd, psw, (flags & RAR_FILE_PASSWORD)) - cmd.append(tmpname) - - p = custom_popen(cmd) - return p.communicate()[0] - finally: - tmpf.close() - os.unlink(tmpname) - -def to_datetime(t): - """Convert 6-part time tuple into datetime object. - """ - if t is None: - return None - - # extract values - year, mon, day, h, m, s = t - - # assume the values are valid - try: - return datetime(year, mon, day, h, m, s) - except ValueError: - pass - - # sanitize invalid values - mday = (0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) - if mon < 1: - mon = 1 - if mon > 12: - mon = 12 - if day < 1: - day = 1 - if day > mday[mon]: - day = mday[mon] - if h > 23: - h = 23 - if m > 59: - m = 59 - if s > 59: - s = 59 - if mon == 2 and day == 29: - try: - return datetime(year, mon, day, h, m, s) - except ValueError: - day = 28 - return datetime(year, mon, day, h, m, s) - -def parse_dos_time(stamp): - """Parse standard 32-bit DOS timestamp. - """ - sec, stamp = stamp & 0x1F, stamp >> 5 - mn, stamp = stamp & 0x3F, stamp >> 6 - hr, stamp = stamp & 0x1F, stamp >> 5 - day, stamp = stamp & 0x1F, stamp >> 5 - mon, stamp = stamp & 0x0F, stamp >> 4 - yr = (stamp & 0x7F) + 1980 - return (yr, mon, day, hr, mn, sec * 2) - -def custom_popen(cmd): - """Disconnect cmd from parent fds, read only from stdout. - """ - # needed for py2exe - creationflags = 0 - if sys.platform == 'win32': - creationflags = 0x08000000 # CREATE_NO_WINDOW - - # run command - try: - p = Popen(cmd, bufsize=0, stdout=PIPE, stdin=PIPE, stderr=STDOUT, - creationflags=creationflags) - except OSError as ex: - if ex.errno == errno.ENOENT: - raise RarCannotExec("Unrar not installed? (rarfile.UNRAR_TOOL=%r)" % UNRAR_TOOL) - raise - return p - -def custom_check(cmd, ignore_retcode=False): - """Run command, collect output, raise error if needed. - """ - p = custom_popen(cmd) - out, _ = p.communicate() - if p.returncode and not ignore_retcode: - raise RarExecError("Check-run failed") - return out - -def add_password_arg(cmd, psw, ___required=False): - """Append password switch to commandline. - """ - if UNRAR_TOOL == ALT_TOOL: - return - if psw is not None: - cmd.append('-p' + psw) - else: - cmd.append('-p-') - -def check_returncode(p, out): - """Raise exception according to unrar exit code. - """ - code = p.returncode - if code == 0: - return - - # map return code to exception class, codes from rar.txt - errmap = [None, - RarWarning, RarFatalError, RarCRCError, RarLockedArchiveError, # 1..4 - RarWriteError, RarOpenError, RarUserError, RarMemoryError, # 5..8 - RarCreateError, RarNoFilesError, RarWrongPassword] # 9..11 - if UNRAR_TOOL == ALT_TOOL: - errmap = [None] - if code > 0 and code < len(errmap): - exc = errmap[code] - elif code == 255: - exc = RarUserBreak - elif code < 0: - exc = RarSignalExit - else: - exc = RarUnknownError - - # format message - if out: - msg = "%s [%d]: %s" % (exc.__doc__, p.returncode, out) - else: - msg = "%s [%d]" % (exc.__doc__, p.returncode) - - raise exc(msg) - -def hmac_sha256(key, data): - """HMAC-SHA256""" - return HMAC(key, data, sha256).digest() - -def membuf_tempfile(memfile): - memfile.seek(0, 0) - - tmpfd, tmpname = mkstemp(suffix='.rar') - tmpf = os.fdopen(tmpfd, "wb") - - try: - while True: - buf = memfile.read(BSIZE) - if not buf: - break - tmpf.write(buf) - tmpf.close() - except: - tmpf.close() - os.unlink(tmpname) - raise - return tmpname - -class XTempFile(object): - __slots__ = ('_tmpfile', '_filename') - - def __init__(self, rarfile): - if is_filelike(rarfile): - self._tmpfile = membuf_tempfile(rarfile) - self._filename = self._tmpfile - else: - self._tmpfile = None - self._filename = rarfile - - def __enter__(self): - return self._filename - - def __exit__(self, exc_type, exc_value, tb): - if self._tmpfile: - try: - os.unlink(self._tmpfile) - except OSError: - pass - self._tmpfile = None - -# -# Check if unrar works -# - -ORIG_UNRAR_TOOL = UNRAR_TOOL -ORIG_OPEN_ARGS = OPEN_ARGS -ORIG_EXTRACT_ARGS = EXTRACT_ARGS -ORIG_TEST_ARGS = TEST_ARGS - -def _check_unrar_tool(): - global UNRAR_TOOL, OPEN_ARGS, EXTRACT_ARGS, TEST_ARGS - try: - # does UNRAR_TOOL work? - custom_check([ORIG_UNRAR_TOOL], True) - - UNRAR_TOOL = ORIG_UNRAR_TOOL - OPEN_ARGS = ORIG_OPEN_ARGS - EXTRACT_ARGS = ORIG_EXTRACT_ARGS - TEST_ARGS = ORIG_TEST_ARGS - except RarCannotExec: - try: - # does ALT_TOOL work? - custom_check([ALT_TOOL] + list(ALT_CHECK_ARGS), True) - # replace config - UNRAR_TOOL = ALT_TOOL - OPEN_ARGS = ALT_OPEN_ARGS - EXTRACT_ARGS = ALT_EXTRACT_ARGS - TEST_ARGS = ALT_TEST_ARGS - except RarCannotExec: - # no usable tool, only uncompressed archives work - pass - -_check_unrar_tool() From a4dbf11dca5bc5456ca2473afbb428ca1bbc5db9 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:36:31 +0000 Subject: [PATCH 064/185] Delete __init__.py --- apps/templates/nzbget-mp4/scripts/rarfile/__init__.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 apps/templates/nzbget-mp4/scripts/rarfile/__init__.py diff --git a/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py b/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/apps/templates/nzbget-mp4/scripts/rarfile/__init__.py +++ /dev/null @@ -1 +0,0 @@ - From a52ded07be36cdf100065fbaee85847fdc0450ff Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:36:49 +0000 Subject: [PATCH 065/185] Delete installer.sh --- .../nzbget-mp4/installer/installer.sh | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/installer/installer.sh diff --git a/apps/templates/nzbget-mp4/installer/installer.sh b/apps/templates/nzbget-mp4/installer/installer.sh deleted file mode 100644 index 8c26cf9..0000000 --- a/apps/templates/nzbget-mp4/installer/installer.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -apk update -apk upgrade -apk add --no-cache git -apk add build-base gcc wget diffutils perl -apk add curl -git clone https://github.com/mdhiggins/sickbeard_mp4_automator.git /config/scripts/MP4_Automator/tmp -cp -r /config/scripts/MP4_Automator/tmp/* /config/scripts/MP4_Automator -rm -rf /config/scripts/MP4_Automator/tmp -git unstage -apk add --no-cache py-setuptools py-pip python-dev libffi-dev gcc musl-dev openssl-dev -pip install --upgrade PIP -pip install requests -pip install requests[security] -pip install requests-cache -pip install babelfish -pip install "guessit<2" -pip install "subliminal<2" -pip install qtfaststart -# As per https://github.com/mdhiggins/sickbeard_mp4_automator/issues/643 -pip uninstall -y stevedore -pip install stevedore==1.19.1 -#Remove default NZBGetPostProcess script settings, and replace with our own -rm /config/scripts/MP4_Automator/NZBGetPostProcess.py -cp /config/TEMPLATEPPScript /config/scripts/MP4_Automator/NZBGetPostProcess.py -#Build ffmpeg -cd /config -. /config/ffmpeg-build/web-install.sh -#Set script file permissions -chmod 777 -R /config/scripts From b8fe69ab0d4144970d2484aaf17b108169b01ff5 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:02 +0000 Subject: [PATCH 066/185] Delete web-install.sh --- .../nzbget-mp4/ffmpeg-build/web-install.sh | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh b/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh deleted file mode 100644 index ffe4158..0000000 --- a/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Helper script to download and run the build-ffmpeg script. - -make_dir () { - if [ ! -d $1 ]; then - if ! mkdir $1; then - printf "\n Failed to create dir %s" "$1"; - exit 1 - fi - fi -} - -command_exists() { - if ! [[ -x $(command -v "$1") ]]; then - return 1 - fi - - return 0 -} - -TARGET='ffmpeg-build' - -if ! command_exists "curl"; then - echo "curl not installed."; - exit 1 -fi - -echo "ffmpeg-build-script-downloader v0.1" -echo "=========================================" -echo "" - -echo "First we create the ffmpeg build directory $TARGET" -make_dir $TARGET -cd $TARGET - -echo "Now we download and execute the build script" -echo "" - -bash build-ffmpeg --build - From 6706b384debad37457ec96b55522eabc4c70126d Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:14 +0000 Subject: [PATCH 067/185] Delete build-ffmpeg --- .../nzbget-mp4/ffmpeg-build/build-ffmpeg | 400 ------------------ 1 file changed, 400 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg diff --git a/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg b/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg deleted file mode 100644 index 9aad5c4..0000000 --- a/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg +++ /dev/null @@ -1,400 +0,0 @@ -#!/bin/bash - -# https://github.com/markus-perl/ffmpeg-build-script - -VERSION=1.1 -CWD=$(pwd) -PACKAGES="$CWD/packages" -WORKSPACE="$CWD/workspace" -CC=clang -LDFLAGS="-L${WORKSPACE}/lib -lm" -CFLAGS="-I${WORKSPACE}/include" -PKG_CONFIG_PATH="${WORKSPACE}/lib/pkgconfig" -ADDITIONAL_CONFIGURE_OPTIONS="" - -# Speed up the process -# Env Var NUMJOBS overrides automatic detection -if [[ -n $NUMJOBS ]]; then - MJOBS=$NUMJOBS -elif [[ -f /proc/cpuinfo ]]; then - MJOBS=$(grep -c processor /proc/cpuinfo) -elif [[ "$OSTYPE" == "darwin"* ]]; then - MJOBS=$(sysctl -n machdep.cpu.thread_count) - ADDITIONAL_CONFIGURE_OPTIONS="--enable-videotoolbox" -else - MJOBS=4 -fi - -make_dir () { - if [ ! -d $1 ]; then - if ! mkdir $1; then - printf "\n Failed to create dir %s" "$1"; - exit 1 - fi - fi -} - -remove_dir () { - if [ -d $1 ]; then - rm -r "$1" - fi -} - -download () { - - DOWNLOAD_PATH=$PACKAGES; - - if [ ! -z "$3" ]; then - mkdir -p $PACKAGES/$3 - DOWNLOAD_PATH=$PACKAGES/$3 - fi; - - if [ ! -f "$DOWNLOAD_PATH/$2" ]; then - - echo "Downloading $1" - curl -L --silent -o "$DOWNLOAD_PATH/$2" "$1" - - EXITCODE=$? - if [ $EXITCODE -ne 0 ]; then - echo "" - echo "Failed to download $1. Exitcode $EXITCODE. Retrying in 10 seconds"; - sleep 10 - curl -L --silent -o "$DOWNLOAD_PATH/$2" "$1" - fi - - EXITCODE=$? - if [ $EXITCODE -ne 0 ]; then - echo "" - echo "Failed to download $1. Exitcode $EXITCODE"; - exit 1 - fi - - echo "... Done" - - if ! tar -xvf "$DOWNLOAD_PATH/$2" -C "$DOWNLOAD_PATH" 2>/dev/null >/dev/null; then - echo "Failed to extract $2"; - exit 1 - fi - - fi -} - -execute () { - echo "$ $*" - - OUTPUT=$($@ 2>&1) - - if [ $? -ne 0 ]; then - echo "$OUTPUT" - echo "" - echo "Failed to Execute $*" >&2 - exit 1 - fi -} - -build () { - echo "" - echo "building $1" - echo "=======================" - - if [ -f "$PACKAGES/$1.done" ]; then - echo "$1 already built. Remove $PACKAGES/$1.done lockfile to rebuild it." - return 1 - fi - - return 0 -} - -command_exists() { - if ! [[ -x $(command -v "$1") ]]; then - return 1 - fi - - return 0 -} - - -build_done () { - touch "$PACKAGES/$1.done" -} - -echo "ffmpeg-build-script v$VERSION" -echo "=========================" -echo "" - -case "$1" in -"--cleanup") - remove_dir $PACKAGES - remove_dir $WORKSPACE - echo "Cleanup done." - echo "" - exit 0 - ;; -"--build") - - ;; -*) - echo "Usage: $0" - echo " --build: start building process" - echo " --cleanup: remove all working dirs" - echo " --help: show this help" - echo "" - exit 0 - ;; -esac - -echo "Using $MJOBS make jobs simultaneously." - -make_dir $PACKAGES -make_dir $WORKSPACE - -export PATH=${WORKSPACE}/bin:$PATH - -if ! command_exists "make"; then - echo "make not installed."; - exit 1 -fi - -if ! command_exists "g++"; then - echo "g++ not installed."; - exit 1 -fi - -if ! command_exists "curl"; then - echo "curl not installed."; - exit 1 -fi - -if build "yasm"; then - download "http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz" "yasm-1.3.0.tar.gz" - cd $PACKAGES/yasm-1.3.0 || exit - execute ./configure --prefix=${WORKSPACE} - execute make -j $MJOBS - execute make install - build_done "yasm" -fi - -if build "nasm"; then - download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.13.03.tar.gz" "nasm.tar.gz" - cd $PACKAGES/nasm-2.13.03 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "nasm" -fi - -if build "opencore"; then - download "http://downloads.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-0.1.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Fopencore-amr%2F&ts=1442256558&use_mirror=netassist" "opencore-amr-0.1.5.tar.gz" - cd $PACKAGES/opencore-amr-0.1.5 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "opencore" -fi - -if build "libvpx"; then - download "https://github.com/webmproject/libvpx/archive/v1.7.0.tar.gz" "libvpx-1.7.0.tar.gz" - cd $PACKAGES/libvpx-*0 || exit - - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "Applying Darwin patch" - sed "s/,--version-script//g" build/make/Makefile > build/make/Makefile.patched - sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched > build/make/Makefile - fi - - execute ./configure --prefix=${WORKSPACE} --disable-unit-tests --disable-shared - execute make -j $MJOBS - execute make install - build_done "libvpx" -fi - -if build "lame"; then - download "http://kent.dl.sourceforge.net/project/lame/lame/3.100/lame-3.100.tar.gz" "lame-3.100.tar.gz" - cd $PACKAGES/lame-3.100 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "lame" -fi - -if build "xvidcore"; then - download "http://downloads.xvid.org/downloads/xvidcore-1.3.4.tar.gz" "xvidcore-1.3.4.tar.gz" - cd $PACKAGES/xvidcore || exit - cd build/generic || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - - if [[ -f ${WORKSPACE}/lib/libxvidcore.4.dylib ]]; then - execute rm "${WORKSPACE}/lib/libxvidcore.4.dylib" - fi - - build_done "xvidcore" -fi - -if build "x264"; then - download "http://ftp.videolan.org/pub/x264/snapshots/x264-snapshot-20190204-2245-stable.tar.bz2" "last_x264.tar.bz2" - cd $PACKAGES/x264-snapshot-* || exit - - if [[ "$OSTYPE" == "linux-gnu" ]]; then - execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic CXXFLAGS="-fPIC" - else - execute ./configure --prefix=${WORKSPACE} --enable-static --enable-pic - fi - - execute make -j $MJOBS - execute make install - execute make install-lib-static - build_done "x264" -fi - -if build "libogg"; then - download "http://downloads.xiph.org/releases/ogg/libogg-1.3.3.tar.gz" "libogg-1.3.3.tar.gz" - cd $PACKAGES/libogg-1.3.3 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "libogg" -fi - -if build "libvorbis"; then - download "http://downloads.xiph.org/releases/vorbis/libvorbis-1.3.6.tar.gz" "libvorbis-1.3.6.tar.gz" - cd $PACKAGES/libvorbis-1.3.6 || exit - execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest - execute make -j $MJOBS - execute make install - build_done "libvorbis" -fi - -if build "libtheora"; then - download "http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.gz" "libtheora-1.1.1.tar.bz" - cd $PACKAGES/libtheora-1.1.1 || exit - sed "s/-fforce-addr//g" configure > configure.patched - chmod +x configure.patched - mv configure.patched configure - execute ./configure --prefix=${WORKSPACE} --with-ogg-libraries=${WORKSPACE}/lib --with-ogg-includes=${WORKSPACE}/include/ --with-vorbis-libraries=${WORKSPACE}/lib --with-vorbis-includes=${WORKSPACE}/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm - execute make -j $MJOBS - execute make install - build_done "libtheora" -fi - -if build "pkg-config"; then - download "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" "pkg-config-0.29.2.tar.gz" - cd $PACKAGES/pkg-config-0.29.2 || exit - execute ./configure --silent --prefix=${WORKSPACE} --with-pc-path=${WORKSPACE}/lib/pkgconfig --with-internal-glib - execute make -j $MJOBS - execute make install - build_done "pkg-config" -fi - -if build "cmake"; then - download "https://cmake.org/files/v3.11/cmake-3.11.3.tar.gz" "cmake-3.11.3.tar.gz" - cd $PACKAGES/cmake-3.11.3 || exit - rm Modules/FindJava.cmake - perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt - perl -p -i -e "s/get_filename_component.JNIPATH/#get_filename_component(JNIPATH/g" Tests/CMakeLists.txt - execute ./configure --prefix=${WORKSPACE} - execute make -j $MJOBS - execute make install - build_done "cmake" -fi - -if build "vid_stab"; then - download "https://codeload.github.com/georgmartius/vid.stab/legacy.tar.gz/release-0.98b" "vid.stab-0.98b-transcode-1.1-binary-x86_64.tgz" - cd $PACKAGES/georgmartius-vid* || exit - perl -p -i -e "s/vidstab SHARED/vidstab STATIC/" CMakeLists.txt - execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} . - execute make install - build_done "vid_stab" -fi - -if build "x265"; then - download "https://bitbucket.org/multicoreware/x265/downloads/x265_3.0.tar.gz" "x265-3.0.tar.gz" - cd $PACKAGES/x265_* || exit - cd source || exit - execute cmake -DCMAKE_INSTALL_PREFIX:PATH=${WORKSPACE} -DENABLE_SHARED:bool=off . - execute make -j $MJOBS - execute make install - sed "s/-lx265/-lx265 -lstdc++/g" "$WORKSPACE/lib/pkgconfig/x265.pc" > "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" - mv "$WORKSPACE/lib/pkgconfig/x265.pc.tmp" "$WORKSPACE/lib/pkgconfig/x265.pc" - build_done "x265" -fi - -if build "fdk_aac"; then - download "http://downloads.sourceforge.net/project/opencore-amr/fdk-aac/fdk-aac-0.1.6.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fopencore-amr%2Ffiles%2Ffdk-aac%2F&ts=1457561564&use_mirror=kent" "fdk-aac-0.1.6.tar.gz" - cd $PACKAGES/fdk-aac-0.1.6 || exit - execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "fdk_aac" -fi - - -build "ffmpeg" -download "http://ffmpeg.org/releases/ffmpeg-4.1.tar.bz2" "ffmpeg-snapshot.tar.bz2" -cd $PACKAGES/ffmpeg-4.1 || exit -./configure $ADDITIONAL_CONFIGURE_OPTIONS \ - --pkgconfigdir="$WORKSPACE/lib/pkgconfig" \ - --prefix=${WORKSPACE} \ - --pkg-config-flags="--static" \ - --extra-cflags="-I$WORKSPACE/include" \ - --extra-ldflags="-L$WORKSPACE/lib" \ - --extra-libs="-lpthread -lm" \ - --enable-static \ - --disable-debug \ - --disable-shared \ - --disable-ffplay \ - --disable-doc \ - --enable-gpl \ - --enable-version3 \ - --enable-nonfree \ - --enable-pthreads \ - --enable-libvpx \ - --enable-libmp3lame \ - --enable-libtheora \ - --enable-libvorbis \ - --enable-libx264 \ - --enable-libx265 \ - --enable-runtime-cpudetect \ - --enable-libfdk-aac \ - --enable-avfilter \ - --enable-libopencore_amrwb \ - --enable-libopencore_amrnb \ - --enable-filters \ - --enable-libvidstab - -execute make -j $MJOBS -execute make install - -INSTALL_FOLDER="/usr/bin" -if [[ "$OSTYPE" == "darwin"* ]]; then -INSTALL_FOLDER="/usr/local/bin" -fi - -echo "" -echo "Building done. The binary can be found here: $WORKSPACE/bin/ffmpeg" -echo "" - - -if [[ $AUTOINSTALL == "yes" ]]; then - if command_exists "sudo"; then - sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" - sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" - echo "Done. ffmpeg is now installed to your system" - fi -elif [[ ! $SKIPINSTALL == "yes" ]]; then - if command_exists "sudo"; then - - read -r -p "Install the binary to your $INSTALL_FOLDER folder? [Y/n] " response - - case $response in - [yY][eE][sS]|[yY]) - sudo cp "$WORKSPACE/bin/ffmpeg" "$INSTALL_FOLDER/ffmpeg" - sudo cp "$WORKSPACE/bin/ffprobe" "$INSTALL_FOLDER/ffprobe" - echo "Done. ffmpeg is now installed to your system" - ;; - esac - fi -fi - -exit 0 From 9bd40af054169b5faa404119127a2042c6d9ec0f Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:26 +0000 Subject: [PATCH 068/185] Delete 30-config --- .../nzbget-mp4/cont-init.d/30-config | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/cont-init.d/30-config diff --git a/apps/templates/nzbget-mp4/cont-init.d/30-config b/apps/templates/nzbget-mp4/cont-init.d/30-config deleted file mode 100644 index deaaacd..0000000 --- a/apps/templates/nzbget-mp4/cont-init.d/30-config +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/with-contenv bash - -# delete lock file if found -[[ -f /downloads/nzbget.lock ]] && \ - rm /downloads/nzbget.lock - -# check if config file exists in /config -[[ ! -f /config/nzbget.conf ]] && \ - cp /defaults/nzbget.conf /config/nzbget.conf - -# permissions -chown 1000:1000 \ - /downloads -chown 1000:1000 -R \ - /app/nzbget \ - /config -chmod u+rw \ - /config/nzbget.conf - -chmod 777 -R \ - /config -chmod 777 -R \ - /app/nzbget -chmod 777 -R \ - /downloads - -exec /config/installer/installer.sh From 2916e91fc9d76f37f9f1634608cbe16ddb48b23b Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:37 +0000 Subject: [PATCH 069/185] Delete autoProcess.ini --- .../nzbget-mp4/MP4_Automator/autoProcess.ini | 143 ------------------ 1 file changed, 143 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini diff --git a/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini b/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini deleted file mode 100644 index 83b18aa..0000000 --- a/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini +++ /dev/null @@ -1,143 +0,0 @@ -[SickBeard] -host = sickbeard -port = 8081 -username = -password = -web_root = -ssl = False -api_key = - -[Sonarr] -host = sonarr -port = 8989 -web_root = -ssl = False -apikey = - -[Radarr] -host = radarr -port = 7878 -web_root = -ssl = False -apikey = - -[MP4] -ffmpeg = /config/ffmpeg-build/workspace/bin/ffmpeg -ffprobe = /config/ffmpeg-build/workspace/bin/ffprobe -threads = 1 -output_directory = -copy_to = -move_to = -output_extension = mp4 -output_format = mp4 -delete_original = True -relocate_moov = True -video-codec = h264,x264 -video-bitrate = -video-crf = 18 -video-max-width = -h264-max-level = 4.1 -use-qsv-decoder-with-encoder = True -ios-audio = libfdk_aac -ios-first-track-only = False -ios-audio-filter = dynaudnorm -max-audio-channels = -audio-codec = ac3,mp3,dts,dca,aac,libfdk_aac -audio-language = eng -audio-default-language = eng -audio-channel-bitrate = 256 -audio-filter = -subtitle-codec = srt -subtitle-language = eng -subtitle-default-language = -subtitle-encoding = -fullpathguess = True -convert-mp4 = True -tagfile = True -tag-language = en -download-artwork = Poster -download-subs = True -embed-subs = False -sub-providers = addic7ed,podnapisi,thesubdb,opensubtitles -permissions = 0777 -post-process = False -pix-fmt = -aac_adtstoasc = False -postopts = -preset,slower -preopts = -audio-copy-original = False -enable_dxva2_gpu_decode = False -ios-move-last = False -use-hevc-qsv-decoder = False -embed-only-internal-subs = False -audio-first-track-of-language = False -video-profile = - -[CouchPotato] -host = couchpotato -port = 5050 -username = -password = -web_root = -ssl = False -apikey = -delay = 65 -method = renamer -delete_failed = False - -[uTorrent] -convert = -couchpotato-label = couchpotato -sickbeard-label = sickbeard -sonarr-label = sonarr -bypass-label = bypass -sickrage-label = sickrage -webui = False -action_before = stop -action_after = removedata -host = http://utorrent:8080/ -username = -password = -output_directory = -radarr-label = radarr - -[Deluge] -host = deluge -username = -convert = True -password = -sonarr-label = sonarr -radarr-label = radarr -bypass-label = bypass -sickbeard-label = sickbeard -port = 12569 -sickrage-label = sickrage -couchpotato-label = couchpotato -output_directory = -remove = true - -[SABNZBD] -convert = True -sickrage-category = sickrage -sonarr-category = sonarr -radarr-category = radarr -bypass-category = bypass -couchpotato-category = couchpotato -sickbeard-category = sickbeard -output_directory = - -[Sickrage] -host = sickrage -port = 8081 -username = -password = -web_root = -ssl = False -api_key = - -[Plex] -host = plex -port = 32400 -refresh = False -token = - From 43d7dbdb86711f0bbebbf198a27bf30e57627b39 Mon Sep 17 00:00:00 2001 From: DaRealestUK Date: Wed, 13 Mar 2019 22:37:47 +0000 Subject: [PATCH 070/185] Delete TEMPLATEPPScript --- .../nzbget-mp4/MP4_Automator/TEMPLATEPPScript | 244 ------------------ 1 file changed, 244 deletions(-) delete mode 100644 apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript diff --git a/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript b/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript deleted file mode 100644 index 6eb7055..0000000 --- a/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# -############################################################################## -### NZBGET POST-PROCESSING SCRIPT ### - -# Modified to enable multiple bypass categories, -# as per: https://github.com/mdhiggins/sickbeard_mp4_automator/issues/509 -# -# Converts files and passes them to Sonarr for further processing. -# -# NOTE: This script requires Python to be installed on your system. - -############################################################################## -### OPTIONS ### - -# Change to full path to MP4 Automator folder. No quotes and a trailing / -#MP4_FOLDER=~/sickbeard_mp4_automator/ - -# Convert file before passing to destination (True, False) -#SHOULDCONVERT=False - -# Category for Couchpotato -#CP_CAT=Couchpotato - -# Category for Sonarr -#SONARR_CAT=Sonarr - -# Category for Radarr -#RADARR_CAT=Radarr - -# Category for Sickbeard -#SICKBEARD_CAT=Sickbeard - -# Category for Sickrage -#SICKRAGE_CAT=Sickrage - -# Category list (comma seperated) for bypassing any further processing but still converting -#BYPASS_CAT=tv,movies - -# Custom output_directory setting -#OUTPUT_DIR= - -### NZBGET POST-PROCESSING SCRIPT ### -############################################################################## - -import os -import sys -import re -import json -import traceback - -# Sanity checks for path string -MP4folder = os.environ['NZBPO_MP4_FOLDER'].strip() -MP4folder = MP4folder.replace('"', '') -MP4folder = MP4folder.replace("'", "") -MP4folder = MP4folder.replace("\\", "/") -if not(MP4folder.endswith("/")): - MP4folder += "/" -#DEBUG#print MP4folder+" the original is "+os.environ['NZBPO_MP4_FOLDER'] - -output_dir = None -if 'NZBPO_OUTPUT_DIR' in os.environ: - output_dir = os.environ['NZBPO_OUTPUT_DIR'].strip() - if len(output_dir) > 0: - output_dir = output_dir.replace('"', '') - output_dir = output_dir.replace("'", "") - output_dir = output_dir.replace("\\", "/") - if not(output_dir.endswith("/")): - output_dir += "/" - #DEBUG#print Overriding output directory - -sys.path.append(MP4folder) -try: - from readSettings import ReadSettings - from mkvtomp4 import MkvtoMp4 - from autoprocess import autoProcessMovie, autoProcessTV, autoProcessTVSR, sonarr, radarr - import logging - from logging.config import fileConfig -except ImportError: - print("[ERROR] Wrong path to sickbeard_mp4_automator: " + os.environ['NZBPO_MP4_FOLDER']) - print("[ERROR] %s" % traceback.print_exc()) - sys.exit(0) - -# Setup Logging -logpath = '/var/log/sickbeard_mp4_automator' -if os.name == 'nt': - logpath = MP4folder -elif not os.path.isdir(logpath): - try: - os.mkdir(logpath) - except: - logpath = MP4folder -configPath = os.path.abspath(os.path.join(MP4folder, 'logging.ini')).replace("\\", "\\\\") -logPath = os.path.abspath(os.path.join(logpath, 'index.log')).replace("\\", "\\\\") -fileConfig(configPath, defaults={'logfilename': logPath}) -log = logging.getLogger("NZBGetPostProcess") - -# Determine if conversion will take place -shouldConvert = (os.environ['NZBPO_SHOULDCONVERT'].lower() in ("yes", "true", "t", "1")) - -if 'NZBOP_SCRIPTDIR' in os.environ and not os.environ['NZBOP_VERSION'][0:5] < '11.0': - log.info("Script triggered from NZBGet (11.0 or later).") - - path = os.environ['NZBPP_DIRECTORY'] # Path to NZB directory - nzb = os.environ['NZBPP_NZBFILENAME'] # Original NZB name - category = os.environ['NZBPP_CATEGORY'] # NZB Category to determine destination - #DEBUG#print "Category is %s." % category - - couchcat = os.environ['NZBPO_CP_CAT'].lower() - sonarrcat = os.environ['NZBPO_SONARR_CAT'].lower() - radarrcat = os.environ['NZBPO_RADARR_CAT'].lower() - sickbeardcat = os.environ['NZBPO_SICKBEARD_CAT'].lower() - sickragecat = os.environ['NZBPO_SICKRAGE_CAT'].lower() - bypass = os.environ['NZBPO_BYPASS_CAT'].lower().replace(' ','').split(',') - - categories = [sickbeardcat, couchcat, sonarrcat, radarrcat, sickragecat] - - log.debug("Path: %s" % path) - log.debug("NZB: %s" % nzb) - log.debug("Category: %s" % category) - log.debug("Categories: %s" % categories) - - # NZBGet argv: all passed as environment variables. - clientAgent = "nzbget" - # Exit codes used by NZBGet - POSTPROCESS_PARCHECK = 92 - POSTPROCESS_SUCCESS = 93 - POSTPROCESS_ERROR = 94 - POSTPROCESS_NONE = 95 - - # Check nzbget.conf options - status = 0 - - if os.environ['NZBOP_UNPACK'] != 'yes': - log.error("Please enable option \"Unpack\" in nzbget configuration file, exiting.") - sys.exit(POSTPROCESS_NONE) - - # Check par status - if os.environ['NZBPP_PARSTATUS'] == '3': - log.error("Par-check successful, but Par-repair disabled, exiting") - sys.exit(POSTPROCESS_NONE) - - if os.environ['NZBPP_PARSTATUS'] == '1': - log.error("Par-check failed, setting status \"failed\".") - status = 1 - sys.exit(POSTPROCESS_NONE) - - # Check unpack status - if os.environ['NZBPP_UNPACKSTATUS'] == '1': - log.error("Unpack failed, setting status \"failed\".") - status = 1 - sys.exit(POSTPROCESS_NONE) - - if os.environ['NZBPP_UNPACKSTATUS'] == '0' and os.environ['NZBPP_PARSTATUS'] != '2': - # Unpack is disabled or was skipped due to nzb-file properties or due to errors during par-check - - for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']): - for file in filenames: - fileExtension = os.path.splitext(file)[1] - - if fileExtension in ['.par2']: - log.error("Post-Process: Unpack skipped and par-check skipped (although par2-files exist), setting status \"failed\".") - status = 1 - break - - if os.path.isfile(os.path.join(os.environ['NZBPP_DIRECTORY'], "_brokenlog.txt")) and not status == 1: - log.error("Post-Process: _brokenlog.txt exists, download is probably damaged, exiting.") - status = 1 - - if not status == 1: - log.error("Neither par2-files found, _brokenlog.txt doesn't exist, considering download successful.") - - # Check if destination directory exists (important for reprocessing of history items) - if not os.path.isdir(os.environ['NZBPP_DIRECTORY']): - log.error("Post-Process: Nothing to post-process: destination directory ", os.environ['NZBPP_DIRECTORY'], "doesn't exist.") - status = 1 - sys.exit(POSTPROCESS_NONE) - - # Make sure one of the appropriate categories is set - if category.lower() not in categories and category.lower() not in bypass: - log.error("Post-Process: No valid category detected. Category was %s." % (category)) - status = 1 - sys.exit(POSTPROCESS_NONE) - - # Make sure there are no duplicate categories - if len(categories) != len(set(categories)): - log.error("Duplicate category detected. Category names must be unique.") - status = 1 - sys.exit(POSTPROCESS_NONE) - - # All checks done, now launching the script. - settings = ReadSettings(MP4folder, "autoProcess.ini") - - if shouldConvert: - if output_dir: - settings.output_dir = output_dir - converter = MkvtoMp4(settings, logger=log) - for r, d, f in os.walk(path): - for files in f: - inputfile = os.path.join(r, files) - #DEBUG#print inputfile - #Ignores files under 50MB - if os.path.getsize(inputfile) > 50000000: - if MkvtoMp4(settings, logger=log).validSource(inputfile): - try: - output = converter.process(inputfile) - log.info("Successfully processed %s." % inputfile) - except: - log.exception("File processing failed.") - if converter.output_dir: - path = converter.output_dir - if (category.lower() == categories[0]): - #DEBUG#print "Sickbeard Processing Activated" - autoProcessTV.processEpisode(path, settings, nzb) - sys.exit(POSTPROCESS_SUCCESS) - elif (category.lower() == categories[1]): - #DEBUG#print "CouchPotato Processing Activated" - autoProcessMovie.process(path, settings, nzb, status) - sys.exit(POSTPROCESS_SUCCESS) - elif (category.lower() == categories[2]): - #DEBUG#print "Sonarr Processing Activated" - success = sonarr.processEpisode(path, settings, True) - if success: - sys.exit(POSTPROCESS_SUCCESS) - else: - sys.exit(POSTPROCESS_ERROR) - elif (category.lower() == categories[3]): - #DEBUG#print "Radarr Processing Activated" - success = radarr.processMovie(path, settings, True) - if success: - sys.exit(POSTPROCESS_SUCCESS) - else: - sys.exit(POSTPROCESS_ERROR) - elif (category.lower() == categories[4]): - #DEBUG#print "Sickrage Processing Activated" - autoProcessTVSR.processEpisode(path, settings, nzb) - sys.exit(POSTPROCESS_SUCCESS) - elif (category.lower() in bypass): - #DEBUG#print "Bypass Further Processing" - sys.exit(POSTPROCESS_NONE) - -else: - log.error("This script can only be called from NZBGet (11.0 or later).") - sys.exit(0) From e88b2b23a7bbf16a780e4d1cc30bc6c2249065b5 Mon Sep 17 00:00:00 2001 From: h1f0x Date: Thu, 14 Mar 2019 08:33:01 +0100 Subject: [PATCH 071/185] Trakto.or Created a simple application with the free API of trakt.tv to track the progress of your tv-show libs in plex. https://github.com/h1f0x/docker-trackt-plex-tracker Cheers :-) H1f0x --- apps/traktor.yml | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 apps/traktor.yml diff --git a/apps/traktor.yml b/apps/traktor.yml new file mode 100644 index 0000000..f1bf601 --- /dev/null +++ b/apps/traktor.yml @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Title: Traktor docker container - track you Plex TV-Show progress +# Author(s): h1f0x) +# URL: https://github.com/h1f0x/docker-trackt-plex-tracker +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'traktor' + intport: '80' + extport: '5892' + image: 'h1f0x/docker-traktor' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + - name: 'Including plugins' + include_tasks: '/opt/communityapps/apps/_plugins.yml' + + - name: 'Including folders' + include_tasks: '/opt/communityapps/apps/_downloaders.yml' + + - name: 'Ini Check' + stat: + path: /opt/appdata/{{pgrole}}/core.conf + register: inicheck + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/plex/database/Library/Application Support/Plex Media Server/Plug-in Support/Databases:/plex:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + UID: 1000 + GID: 1000 + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + privileged: yes + devices: + - '/dev/net/tun' + state: started + labels: '{{pg_labels}}' From a71d18df763c42408e3c173fbef261c11b4417c3 Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 14 Mar 2019 17:48:02 -0400 Subject: [PATCH 072/185] Initialize Flexget --- apps/flexget.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/apps/flexget.yml b/apps/flexget.yml index cadd05c..25faf86 100644 --- a/apps/flexget.yml +++ b/apps/flexget.yml @@ -35,7 +35,6 @@ pg_volumes: - '/etc/localtime:/etc/localtime:ro' - '/opt/appdata/{{pgrole}}:/config' - - '/mnt/downloads/{{pgrole}}:/downloads' - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' @@ -64,19 +63,3 @@ - '{{pgrole}}' state: started labels: '{{pg_labels}}' - - # CONFIGURATION ############################################################# - #- name: Checking for existing app data - # stat: - # path: "/opt/appdata/{{pgrole}}/config.yml" - # register: cfcheck - # - #- name: "Configuring {{pgrole}} for first time use" - # block: - # - name: Creating config file - # shell: 'touch /opt/appdata/{{pgrole}}/config.yml' - # - name: Populate example config - # shell: 'echo -e ''web_server:\n bind: 0.0.0.0\n port: {{intport}}\n\ntasks:\n example-task:\n rss: hxxp://example.com/feed.xml\n download: /download\n series:\n - some series 1\n - some series 2'' > /opt/appdata/{{pgrole}}/config.yml' - # - name: Fix permissions - # shell: 'chown {{PUID}}:{{PGID}} /opt/appdata/{{pgrole}}/config.yml' - # when: not cfcheck.stat.exists \ No newline at end of file From 093776b8108481f66d8f448d0d75d82ddc151068 Mon Sep 17 00:00:00 2001 From: timekills Date: Mon, 18 Mar 2019 01:07:08 +0430 Subject: [PATCH 073/185] Deconflict external port with Synclounge Both used 8088 --- apps/mcmyadmin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/mcmyadmin.yml b/apps/mcmyadmin.yml index 8e55ebd..ff4957e 100644 --- a/apps/mcmyadmin.yml +++ b/apps/mcmyadmin.yml @@ -15,7 +15,7 @@ set_fact: pgrole: 'mcmyadmin' intport: '8080' - extport: '8088' + extport: '8087' image: 'linuxserver/mcmyadmin2' - name: 'Including cron job' From 370b497b6e02fcc0cf6fe30906f979763718ea70 Mon Sep 17 00:00:00 2001 From: timekills Date: Mon, 18 Mar 2019 01:13:03 +0430 Subject: [PATCH 074/185] No longer needed --- apps/handbrake2.yml | 95 --------------------------------------------- 1 file changed, 95 deletions(-) delete mode 100644 apps/handbrake2.yml diff --git a/apps/handbrake2.yml b/apps/handbrake2.yml deleted file mode 100644 index 6e3e0cb..0000000 --- a/apps/handbrake2.yml +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -# -# Title: Handbrake for PGBlitz Community (user/password) -# Author(s): timekills -# URL: https://pgblitz.com - https://github.com/timekills -# GNU: General Public License v3.0 -################################################################################ ---- -- hosts: localhost - gather_facts: false - tasks: - # FACTS ####################################################################### - - - name: 'Set Known Facts' - set_fact: - pgrole: 'handbrake' - intport: '5800' - extport: '5800' - image: 'jlesage/handbrake:dev-latest' - - # CORE (MANDATORY) ############################################################ - - name: 'Including cron job' - include_tasks: '/opt/communityapps/apps/_core.yml' - - # LABELS ###################################################################### - - name: 'Adding Traefik' - set_fact: - pg_labels: - traefik.enable: 'true' - traefik.port: '{{intport}}' - traefik.frontend.auth.basic.users: "plex:$apr1$tosnCNtX$XKXnDaIiW7f0y1nwmd.KL0" - traefik.frontend.rule: 'Host:handbrake.{{domain.stdout}},{{tldset}}' - - - name: 'Setting PG Volumes' - set_fact: - pg_volumes: - - '/opt/appdata/{{pgrole}}/config:/config:rw' - - '/:/storage:ro' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt:/mnt' - - '{{path.stdout}}/{{pgrole}}/watch/Very_Fast_1080p30:/watch:rw' - - '{{path.stdout}}/{{pgrole}}/watch/HQ_1080p30_Surround:/watch2:rw' - - '{{path.stdout}}/{{pgrole}}/watch/1080mkv:/watch3:rw' - - '{{path.stdout}}/{{pgrole}}/complete/:/output:rw' - - '/dev/dri:/dev/dri:rw' - - '/etc/localtime:/etc/localtime:ro' - - - name: 'Setting PG ENV' - set_fact: - pg_env: - USER_ID: '1000' - GROUP_ID: '1000' - AUTOMATED_CONVERSION_PRESET: "Very Fast 1080p30" - AUTOMATED_CONVERSION_FORMAT: "mp4" - AUTOMATED_CONVERSION_PRESET_2: "HQ 1080p30 Surround" - AUTOMATED_CONVERSION_FORMAT_2: "mp4" - AUTOMATED_CONVERSION_PRESET_3: "H.264 MKV 1080p30" - AUTOMATED_CONVERSION_FORMAT_3: "mkv" -# SECURE_CONNECTION: 1 - - # MAIN DEPLOYMENT ############################################################# - - name: 'Create watch directory for {{pgrole}}' - file: - path: '{{path.stdout}}/handbrake/watch' - state: directory - owner: '1000' - group: '1000' - mode: 0755 - recurse: yes - - - name: 'Create complete directory for {{pgrole}}' - file: - path: '{{path.stdout}}/handbrake/complete' - state: directory - owner: '1000' - group: '1000' - mode: 0755 - recurse: yes - - - name: 'Deploying {{pgrole}}' - docker_container: - name: '{{pgrole}}' - image: '{{image}}' - pull: yes - published_ports: - - '{{ports.stdout}}{{extport}}:{{intport}}' - volumes: '{{pg_volumes}}' - env: '{{pg_env}}' - restart_policy: unless-stopped - networks: - - name: plexguide - aliases: - - '{{pgrole}}' - state: started - labels: '{{pg_labels}}' From e86ed7381bac06c5f16fcbdac903e7472f27f4e3 Mon Sep 17 00:00:00 2001 From: Robert Baker Date: Sun, 17 Mar 2019 20:54:02 -0700 Subject: [PATCH 075/185] fix --- apps/_plugins.yml | 2 +- apps/handbrake.yml | 4 ++-- apps/logarr.yml | 2 +- apps/nzbget-mp4.yml | 14 +++++++------- apps/templates/broken/nzbthrottle.yml | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/_plugins.yml b/apps/_plugins.yml index 7e6a747..7a557be 100644 --- a/apps/_plugins.yml +++ b/apps/_plugins.yml @@ -17,5 +17,5 @@ force: yes owner: '1000' group: '1000' - mode: 0755 + mode: 0775 when: copycheck.stat.exists diff --git a/apps/handbrake.yml b/apps/handbrake.yml index 991120f..d1013f4 100644 --- a/apps/handbrake.yml +++ b/apps/handbrake.yml @@ -65,7 +65,7 @@ state: directory owner: '1000' group: '1000' - mode: 0755 + mode: 0775 recurse: yes - name: 'Create complete directory for {{pgrole}}' @@ -74,7 +74,7 @@ state: directory owner: '1000' group: '1000' - mode: 0755 + mode: 0775 recurse: yes - name: 'Deploying {{pgrole}}' diff --git a/apps/logarr.yml b/apps/logarr.yml index e98dc4e..9e59173 100644 --- a/apps/logarr.yml +++ b/apps/logarr.yml @@ -14,7 +14,7 @@ set_fact: pgrole: 'logarr' intport: '80' - extport: '7555' + extport: '7755' image: 'monitorr/logarr:develop' # CORE (MANDATORY) ############################################################ diff --git a/apps/nzbget-mp4.yml b/apps/nzbget-mp4.yml index 32404cd..df25703 100644 --- a/apps/nzbget-mp4.yml +++ b/apps/nzbget-mp4.yml @@ -63,7 +63,7 @@ force: yes owner: 1000 group: 1000 - mode: 0755 + mode: 0775 - name: 'Copy custom install scripts into directory for {{pgrole}}' copy: @@ -73,7 +73,7 @@ force: yes owner: 1000 group: 1000 - mode: 0755 + mode: 0775 - name: 'Copy custom mp4 config into directory for {{pgrole}}' copy: @@ -83,7 +83,7 @@ force: yes owner: 1000 group: 1000 - mode: 0755 + mode: 0775 - name: 'Copy custom NZBGetPostProcess script config into directory for {{pgrole}}' copy: @@ -93,7 +93,7 @@ force: yes owner: 1000 group: 1000 - mode: 0755 + mode: 0775 - name: 'Copy ffmpeg build script into directory for {{pgrole}}' copy: @@ -103,7 +103,7 @@ force: yes owner: 1000 group: 1000 - mode: 0755 + mode: 0775 - name: 'Copy ffmpeg build script into directory for {{pgrole}}' copy: @@ -113,7 +113,7 @@ force: yes owner: 1000 group: 1000 - mode: 0755 + mode: 0775 - name: 'Copy custom init scripts into directory for {{pgrole}}' copy: @@ -123,7 +123,7 @@ force: yes owner: 1000 group: 1000 - mode: 0755 + mode: 0775 # LABELS ###################################################################### - name: 'Adding Traefik' diff --git a/apps/templates/broken/nzbthrottle.yml b/apps/templates/broken/nzbthrottle.yml index 27f3a7d..ac56db7 100644 --- a/apps/templates/broken/nzbthrottle.yml +++ b/apps/templates/broken/nzbthrottle.yml @@ -34,7 +34,7 @@ force: yes owner: '1000' group: '1000' - mode: 0755 + mode: 0775 when: not jsoncheck.stat.exists # LABELS ###################################################################### From fd0b180960993adb67e2a42f3385b32092d1d9c0 Mon Sep 17 00:00:00 2001 From: dhumphrey7 <47798651+dhumphrey7@users.noreply.github.com> Date: Wed, 20 Mar 2019 09:00:22 -0500 Subject: [PATCH 076/185] Changing templates folder to correct path --- apps/nzbget-mp4.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/nzbget-mp4.yml b/apps/nzbget-mp4.yml index 32404cd..a3b65f1 100644 --- a/apps/nzbget-mp4.yml +++ b/apps/nzbget-mp4.yml @@ -57,7 +57,7 @@ - name: 'Copy custom init scripts into directory for {{pgrole}}' copy: - src: /opt/communityapps/apps/templates/nzbget-mp4/cont-init.d/30-config + src: /opt/communityapps/apps/templates/{{pgrole}}/cont-init.d/30-config dest: /opt/appdata/{{pgrole}}/cont-init.d directory_mode: yes force: yes @@ -67,7 +67,7 @@ - name: 'Copy custom install scripts into directory for {{pgrole}}' copy: - src: /opt/communityapps/apps/templates/nzbget-mp4/installer/installer.sh + src: /opt/communityapps/apps/templates/{{pgrole}}/installer/installer.sh dest: /opt/appdata/{{pgrole}}/installer directory_mode: yes force: yes @@ -77,7 +77,7 @@ - name: 'Copy custom mp4 config into directory for {{pgrole}}' copy: - src: /opt/communityapps/apps/templates/nzbget-mp4/MP4_Automator/autoProcess.ini + src: /opt/communityapps/apps/templates/{{pgrole}}/MP4_Automator/autoProcess.ini dest: /opt/appdata/{{pgrole}}/scripts/MP4_Automator directory_mode: yes force: yes @@ -87,7 +87,7 @@ - name: 'Copy custom NZBGetPostProcess script config into directory for {{pgrole}}' copy: - src: /opt/communityapps/apps/templates/nzbget-mp4/MP4_Automator/TEMPLATEPPScript + src: /opt/communityapps/apps/templates/{{pgrole}}/MP4_Automator/TEMPLATEPPScript dest: /opt/appdata/{{pgrole}}/ directory_mode: yes force: yes @@ -97,7 +97,7 @@ - name: 'Copy ffmpeg build script into directory for {{pgrole}}' copy: - src: /opt/communityapps/apps/templates/nzbget-mp4/ffmpeg-build/web-install.sh + src: /opt/communityapps/apps/templates/{{pgrole}}/ffmpeg-build/web-install.sh dest: /opt/appdata/{{pgrole}}/ffmpeg-build directory_mode: yes force: yes @@ -107,7 +107,7 @@ - name: 'Copy ffmpeg build script into directory for {{pgrole}}' copy: - src: /opt/communityapps/apps/templates/nzbget-mp4/ffmpeg-build/build-ffmpeg + src: /opt/communityapps/apps/templates/{{pgrole}}/ffmpeg-build/build-ffmpeg dest: /opt/appdata/{{pgrole}}/ffmpeg-build directory_mode: yes force: yes @@ -117,7 +117,7 @@ - name: 'Copy custom init scripts into directory for {{pgrole}}' copy: - src: /opt/communityapps/apps/templates/nzbget-mp4/services.d/run + src: /opt/communityapps/apps/templates/{{pgrole}}/services.d/run dest: /opt/appdata/{{pgrole}}/services.d directory_mode: yes force: yes From 1457880eac35190d50129e139128171d62364d1b Mon Sep 17 00:00:00 2001 From: dhumphrey7 <47798651+dhumphrey7@users.noreply.github.com> Date: Wed, 20 Mar 2019 09:07:56 -0500 Subject: [PATCH 077/185] Updates nasm to 2.14.02 2.13 has issues with gcc 8 --- apps/templates/nzgbet/ffmpeg-build/build-ffmpeg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/templates/nzgbet/ffmpeg-build/build-ffmpeg b/apps/templates/nzgbet/ffmpeg-build/build-ffmpeg index 9aad5c4..54bd986 100644 --- a/apps/templates/nzgbet/ffmpeg-build/build-ffmpeg +++ b/apps/templates/nzgbet/ffmpeg-build/build-ffmpeg @@ -175,8 +175,8 @@ if build "yasm"; then fi if build "nasm"; then - download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.13.03.tar.gz" "nasm.tar.gz" - cd $PACKAGES/nasm-2.13.03 || exit + download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.14.02.tar.gz" "nasm.tar.gz" + cd $PACKAGES/nasm-2.14.02 || exit execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static execute make -j $MJOBS execute make install From 64aad0e9d18f9b2ae1d417df8ddb74e145992d27 Mon Sep 17 00:00:00 2001 From: dhumphrey7 <47798651+dhumphrey7@users.noreply.github.com> Date: Wed, 20 Mar 2019 09:14:44 -0500 Subject: [PATCH 078/185] Update autoProcess.ini --- apps/templates/nzgbet/mp4_automator/autoProcess.ini | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/templates/nzgbet/mp4_automator/autoProcess.ini b/apps/templates/nzgbet/mp4_automator/autoProcess.ini index 83b18aa..d0427a6 100644 --- a/apps/templates/nzgbet/mp4_automator/autoProcess.ini +++ b/apps/templates/nzgbet/mp4_automator/autoProcess.ini @@ -24,7 +24,7 @@ apikey = [MP4] ffmpeg = /config/ffmpeg-build/workspace/bin/ffmpeg ffprobe = /config/ffmpeg-build/workspace/bin/ffprobe -threads = 1 +threads = 0 output_directory = copy_to = move_to = @@ -38,7 +38,8 @@ video-crf = 18 video-max-width = h264-max-level = 4.1 use-qsv-decoder-with-encoder = True -ios-audio = libfdk_aac +use-hevc-qsv-decoder = False +ios-audio = True ios-first-track-only = False ios-audio-filter = dynaudnorm max-audio-channels = @@ -47,17 +48,17 @@ audio-language = eng audio-default-language = eng audio-channel-bitrate = 256 audio-filter = -subtitle-codec = srt +subtitle-codec = mov_text subtitle-language = eng subtitle-default-language = subtitle-encoding = fullpathguess = True -convert-mp4 = True +convert-mp4 = False tagfile = True tag-language = en download-artwork = Poster download-subs = True -embed-subs = False +embed-subs = True sub-providers = addic7ed,podnapisi,thesubdb,opensubtitles permissions = 0777 post-process = False From da0317efb492c9b5cd5a295cdee7ef6faece2d7b Mon Sep 17 00:00:00 2001 From: root Date: Wed, 20 Mar 2019 12:25:32 -0500 Subject: [PATCH 079/185] fixed directory names --- .../mp4_automator => nzbget/MP4_Automator}/TEMPLATEPPScript | 0 .../mp4_automator => nzbget/MP4_Automator}/autoProcess.ini | 0 apps/templates/{nzgbet => nzbget}/cont-init.d/30-config | 0 apps/templates/{nzgbet => nzbget}/ffmpeg-build/build-ffmpeg | 0 apps/templates/{nzgbet => nzbget}/ffmpeg-build/web-install.sh | 0 apps/templates/{nzgbet => nzbget}/installer/installer.sh | 0 apps/templates/{nzgbet => nzbget}/scripts/DeleteSamples.py | 0 apps/templates/{nzgbet => nzbget}/scripts/flatten.py | 0 apps/templates/{nzgbet => nzbget}/scripts/hash.py | 0 apps/templates/{nzgbet => nzbget}/scripts/rarfile/__init__.py | 0 apps/templates/{nzgbet => nzbget}/scripts/rarfile/rarfile.py | 0 apps/templates/{nzgbet => nzbget}/scripts/reverse_name.py | 0 apps/templates/{nzgbet => nzbget}/scripts/unzip.py | 0 apps/templates/{nzgbet => nzbget}/services.d/run | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename apps/templates/{nzgbet/mp4_automator => nzbget/MP4_Automator}/TEMPLATEPPScript (100%) rename apps/templates/{nzgbet/mp4_automator => nzbget/MP4_Automator}/autoProcess.ini (100%) rename apps/templates/{nzgbet => nzbget}/cont-init.d/30-config (100%) rename apps/templates/{nzgbet => nzbget}/ffmpeg-build/build-ffmpeg (100%) rename apps/templates/{nzgbet => nzbget}/ffmpeg-build/web-install.sh (100%) rename apps/templates/{nzgbet => nzbget}/installer/installer.sh (100%) rename apps/templates/{nzgbet => nzbget}/scripts/DeleteSamples.py (100%) rename apps/templates/{nzgbet => nzbget}/scripts/flatten.py (100%) rename apps/templates/{nzgbet => nzbget}/scripts/hash.py (100%) rename apps/templates/{nzgbet => nzbget}/scripts/rarfile/__init__.py (100%) rename apps/templates/{nzgbet => nzbget}/scripts/rarfile/rarfile.py (100%) rename apps/templates/{nzgbet => nzbget}/scripts/reverse_name.py (100%) rename apps/templates/{nzgbet => nzbget}/scripts/unzip.py (100%) rename apps/templates/{nzgbet => nzbget}/services.d/run (100%) diff --git a/apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript b/apps/templates/nzbget/MP4_Automator/TEMPLATEPPScript similarity index 100% rename from apps/templates/nzgbet/mp4_automator/TEMPLATEPPScript rename to apps/templates/nzbget/MP4_Automator/TEMPLATEPPScript diff --git a/apps/templates/nzgbet/mp4_automator/autoProcess.ini b/apps/templates/nzbget/MP4_Automator/autoProcess.ini similarity index 100% rename from apps/templates/nzgbet/mp4_automator/autoProcess.ini rename to apps/templates/nzbget/MP4_Automator/autoProcess.ini diff --git a/apps/templates/nzgbet/cont-init.d/30-config b/apps/templates/nzbget/cont-init.d/30-config similarity index 100% rename from apps/templates/nzgbet/cont-init.d/30-config rename to apps/templates/nzbget/cont-init.d/30-config diff --git a/apps/templates/nzgbet/ffmpeg-build/build-ffmpeg b/apps/templates/nzbget/ffmpeg-build/build-ffmpeg similarity index 100% rename from apps/templates/nzgbet/ffmpeg-build/build-ffmpeg rename to apps/templates/nzbget/ffmpeg-build/build-ffmpeg diff --git a/apps/templates/nzgbet/ffmpeg-build/web-install.sh b/apps/templates/nzbget/ffmpeg-build/web-install.sh similarity index 100% rename from apps/templates/nzgbet/ffmpeg-build/web-install.sh rename to apps/templates/nzbget/ffmpeg-build/web-install.sh diff --git a/apps/templates/nzgbet/installer/installer.sh b/apps/templates/nzbget/installer/installer.sh similarity index 100% rename from apps/templates/nzgbet/installer/installer.sh rename to apps/templates/nzbget/installer/installer.sh diff --git a/apps/templates/nzgbet/scripts/DeleteSamples.py b/apps/templates/nzbget/scripts/DeleteSamples.py similarity index 100% rename from apps/templates/nzgbet/scripts/DeleteSamples.py rename to apps/templates/nzbget/scripts/DeleteSamples.py diff --git a/apps/templates/nzgbet/scripts/flatten.py b/apps/templates/nzbget/scripts/flatten.py similarity index 100% rename from apps/templates/nzgbet/scripts/flatten.py rename to apps/templates/nzbget/scripts/flatten.py diff --git a/apps/templates/nzgbet/scripts/hash.py b/apps/templates/nzbget/scripts/hash.py similarity index 100% rename from apps/templates/nzgbet/scripts/hash.py rename to apps/templates/nzbget/scripts/hash.py diff --git a/apps/templates/nzgbet/scripts/rarfile/__init__.py b/apps/templates/nzbget/scripts/rarfile/__init__.py similarity index 100% rename from apps/templates/nzgbet/scripts/rarfile/__init__.py rename to apps/templates/nzbget/scripts/rarfile/__init__.py diff --git a/apps/templates/nzgbet/scripts/rarfile/rarfile.py b/apps/templates/nzbget/scripts/rarfile/rarfile.py similarity index 100% rename from apps/templates/nzgbet/scripts/rarfile/rarfile.py rename to apps/templates/nzbget/scripts/rarfile/rarfile.py diff --git a/apps/templates/nzgbet/scripts/reverse_name.py b/apps/templates/nzbget/scripts/reverse_name.py similarity index 100% rename from apps/templates/nzgbet/scripts/reverse_name.py rename to apps/templates/nzbget/scripts/reverse_name.py diff --git a/apps/templates/nzgbet/scripts/unzip.py b/apps/templates/nzbget/scripts/unzip.py similarity index 100% rename from apps/templates/nzgbet/scripts/unzip.py rename to apps/templates/nzbget/scripts/unzip.py diff --git a/apps/templates/nzgbet/services.d/run b/apps/templates/nzbget/services.d/run similarity index 100% rename from apps/templates/nzgbet/services.d/run rename to apps/templates/nzbget/services.d/run From 06e438352e4ae05509594eca6b8ecaaa75032d2e Mon Sep 17 00:00:00 2001 From: h1f0x Date: Wed, 20 Mar 2019 21:55:31 +0100 Subject: [PATCH 080/185] rTorrent + FloodUI + OpenVPN Hi all, Created the first of a series of torrent clients with openvpn integration: Github: https://github.com/h1f0x/rtorrent-flood-openvpn Docker Hub: https://hub.docker.com/r/h1f0x/rtorrent-flood-openvpn I will create forum post, for feedback! :-) If it works, i will create other UI's too! Cheers H1f0x --- apps/rflood-openvpn.yml | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 apps/rflood-openvpn.yml diff --git a/apps/rflood-openvpn.yml b/apps/rflood-openvpn.yml new file mode 100644 index 0000000..552de75 --- /dev/null +++ b/apps/rflood-openvpn.yml @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Title: rTorrent + FloodUI integrated OpenVPN +# Author(s): h1f0x +# URL: https://github.com/h1f0x/rtorrent-flood-openvpn +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'rflood-openvpn' + intport: '80' + extport: '5896' + image: 'h1f0x/rtorrent-flood-openvpn' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + - name: 'Including plugins' + include_tasks: '/opt/communityapps/apps/_plugins.yml' + + - name: 'Including folders' + include_tasks: '/opt/communityapps/apps/_downloaders.yml' + + - name: 'Ini Check' + stat: + path: /opt/appdata/{{pgrole}}/core.conf + register: inicheck + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/unionfs' + - '/mnt/downloads/{{pgrole}}:/output/complete' + - '/mnt/incomplete/{{pgrole}}:/output/incomplete' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + UID: 1000 + GID: 1000 + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + privileged: yes + state: started + labels: '{{pg_labels}}' From 31ff3d760f418a1e132d57c94460a1426a2bdf3b Mon Sep 17 00:00:00 2001 From: Admin9705 Date: Thu, 21 Mar 2019 11:58:12 -0400 Subject: [PATCH 081/185] update --- apps/zammad.yml | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 apps/zammad.yml diff --git a/apps/zammad.yml b/apps/zammad.yml new file mode 100644 index 0000000..3481879 --- /dev/null +++ b/apps/zammad.yml @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Title: PGBlitz (Reference Title File) +# Author(s): Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: +# CORE (MANDATORY) DO NOT CHANGE ########################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: "zammad" + intport: "80" + extport: "8777" + image: "zammad/zammad" + + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + +# EXTRA FUNCTIONS REQUIRED BY THE ROLE ##################################### + +##### NOTHING REQUIRED + +# LABELS #### KEEPS BOTTOM CLEAN ########################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + + # MAIN SCRIPT ############################################################## + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From b7f6b95a1b0945931194f1bbdbee91533f2f7711 Mon Sep 17 00:00:00 2001 From: John Gibson Date: Fri, 22 Mar 2019 00:33:50 -0400 Subject: [PATCH 082/185] Fix missing quotations on braces expression --- apps/synclounge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/synclounge.yml b/apps/synclounge.yml index 9523a6d..80eb4fa 100644 --- a/apps/synclounge.yml +++ b/apps/synclounge.yml @@ -48,7 +48,7 @@ pg_env: UID: '1000' GID: '1000' - DOMAIN: {{pgrole}}.{{domain.stdout}} + DOMAIN: '{{pgrole}}.{{domain.stdout}}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' From 99fc259857d487b75f26209d97453b1cd86196d0 Mon Sep 17 00:00:00 2001 From: h1f0x Date: Sat, 23 Mar 2019 10:02:53 +0100 Subject: [PATCH 083/185] Updated yml for Ansible 2.6 --- apps/traktor.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/apps/traktor.yml b/apps/traktor.yml index f1bf601..0b46d33 100644 --- a/apps/traktor.yml +++ b/apps/traktor.yml @@ -1,7 +1,7 @@ #!/bin/bash # # Title: Traktor docker container - track you Plex TV-Show progress -# Author(s): h1f0x) +# Author(s): h1f0x # URL: https://github.com/h1f0x/docker-trackt-plex-tracker # GNU: General Public License v3.0 ################################################################################ @@ -22,17 +22,6 @@ - name: 'Including cron job' include_tasks: '/opt/communityapps/apps/_core.yml' - - name: 'Including plugins' - include_tasks: '/opt/communityapps/apps/_plugins.yml' - - - name: 'Including folders' - include_tasks: '/opt/communityapps/apps/_downloaders.yml' - - - name: 'Ini Check' - stat: - path: /opt/appdata/{{pgrole}}/core.conf - register: inicheck - # LABELS ###################################################################### - name: 'Adding Traefik' set_fact: @@ -53,8 +42,8 @@ - name: 'Setting PG ENV' set_fact: pg_env: - UID: 1000 - GID: 1000 + UID: '1000' + GID: '1000' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' From 6d9bc83cbb7d7316edbd741fdf4943934b71624a Mon Sep 17 00:00:00 2001 From: h1f0x Date: Sat, 23 Mar 2019 10:03:24 +0100 Subject: [PATCH 084/185] Removed missconfigured tun interface --- apps/traktor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/traktor.yml b/apps/traktor.yml index 0b46d33..4452565 100644 --- a/apps/traktor.yml +++ b/apps/traktor.yml @@ -61,7 +61,5 @@ aliases: - '{{pgrole}}' privileged: yes - devices: - - '/dev/net/tun' state: started labels: '{{pg_labels}}' From e45211e31efac000c58e4c5e5e3ef1152727f953 Mon Sep 17 00:00:00 2001 From: h1f0x Date: Sat, 23 Mar 2019 10:04:22 +0100 Subject: [PATCH 085/185] Prepared Ansible 2.6 und cleanup --- apps/traktor.yml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/apps/traktor.yml b/apps/traktor.yml index f1bf601..4452565 100644 --- a/apps/traktor.yml +++ b/apps/traktor.yml @@ -1,7 +1,7 @@ #!/bin/bash # # Title: Traktor docker container - track you Plex TV-Show progress -# Author(s): h1f0x) +# Author(s): h1f0x # URL: https://github.com/h1f0x/docker-trackt-plex-tracker # GNU: General Public License v3.0 ################################################################################ @@ -22,17 +22,6 @@ - name: 'Including cron job' include_tasks: '/opt/communityapps/apps/_core.yml' - - name: 'Including plugins' - include_tasks: '/opt/communityapps/apps/_plugins.yml' - - - name: 'Including folders' - include_tasks: '/opt/communityapps/apps/_downloaders.yml' - - - name: 'Ini Check' - stat: - path: /opt/appdata/{{pgrole}}/core.conf - register: inicheck - # LABELS ###################################################################### - name: 'Adding Traefik' set_fact: @@ -53,8 +42,8 @@ - name: 'Setting PG ENV' set_fact: pg_env: - UID: 1000 - GID: 1000 + UID: '1000' + GID: '1000' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' @@ -72,7 +61,5 @@ aliases: - '{{pgrole}}' privileged: yes - devices: - - '/dev/net/tun' state: started labels: '{{pg_labels}}' From 4202795f06aa0353a1797925908fe1ccc7590191 Mon Sep 17 00:00:00 2001 From: h1f0x Date: Sun, 24 Mar 2019 16:21:52 +0100 Subject: [PATCH 086/185] rTorrent + ruTorrent integrated OpenVPN Hi all, As promised i created a another openvpn container, this time: - Docker container for "rTorrent (with ruTorrent) OpenVPN integrated" As the name may says it already this App offers the following features: Starting and maintaining active OpenVPN connection of your choice (capable of User/Pass/Cert authentication) Starts rTorrent Service + ruTorrent with autodl-irssi All configuration files can be edited an accessed from outside the docker container on the usual /opt/appdata/rflood-openvpn/ path. A detailed Guide and instruction can be found at: Github: https://github.com/h1f0x/rtorrent-rutorrent-openvpn Docker Hub: https://hub.docker.com/r/h1f0x/rtorrent-rutorrent-openvpn The app is already in the repository for testing. You can install it with the normal procedures. - choose: rutorrent-openvpn Short Usage: Install the app modify the VPN configuration to your needs wait about 1-2 min till the service is restarted, you can verify by the "my-external-ip.txt" in /opt/appdata/rutorrent-openvpn/ access your page and login with rtorrent : rtorrent Enjoy All config changes and from https://pgblitz.com/threads/rtorrent-floodui-openvpn-integrated.4156/ are built in already: - Added Sonarr support. Modification at the nginx.conf was needed. You can configure Sonarr like this: Name: rutorrent-openvpn Enable: Yes Host: rutorrent-openvpn Port: 8080 Username & Password: empty - Downloads will be copied to /mnt/downloads/rflood-openvpn, the original files will remain at /mnt/incomplete/rflood-openvpn to proceed seeding. (Info: Once the downloads hit /mn/downloads/rflood-openvpn/ they get moved to /mnt/move/rflood-openvpn -> /mnt/unionfs/rflood-openvpn - Sonarr needs to monior this path.) - force a re-hashing after the files get copied - without it bugged sometimes. - added `log-append /config/vpn/openvpn.log` to the client.conf for logging of vpn - Downloads which are tagged for example with "tv" will be copied once their completed to /mnt/downloads/rflood-openvpn/tv - Downloads which have no tag will be copied on default to /mnt/downloads/rflood-openvpn/unsorted Feedback is welcome and needed! Thats it for me regarding app-development. I will try to maintain my 4 released apps for now :) Cheers and Enjoy H1f0x --- apps/rutorrent-openvpn.yml | 78 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 apps/rutorrent-openvpn.yml diff --git a/apps/rutorrent-openvpn.yml b/apps/rutorrent-openvpn.yml new file mode 100644 index 0000000..9fd1766 --- /dev/null +++ b/apps/rutorrent-openvpn.yml @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Title: rTorrent + ruTorrent integrated OpenVPN +# Author(s): h1f0x +# URL: https://github.com/h1f0x/rtorrent-flood-openvpn +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'rutorrent-openvpn' + intport: '80' + extport: '5897' + image: 'h1f0x/rtorrent-rutorrent-openvpn' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + - name: 'Including plugins' + include_tasks: '/opt/communityapps/apps/_plugins.yml' + + - name: 'Including folders' + include_tasks: '/opt/communityapps/apps/_downloaders.yml' + + - name: 'Ini Check' + stat: + path: /opt/appdata/{{pgrole}}/core.conf + register: inicheck + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/unionfs' + - '/mnt/downloads/{{pgrole}}:/output/complete' + - '/mnt/incomplete/{{pgrole}}:/output/incomplete' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + UID: '1000' + GID: '1000' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + privileged: yes + state: started + labels: '{{pg_labels}}' From 50a75a5780d867476a7287f161d24d44ce2dbe0b Mon Sep 17 00:00:00 2001 From: Zoink5 Date: Tue, 26 Mar 2019 11:31:31 +0100 Subject: [PATCH 087/185] Removed OAUTH (breaks bitwarden-apps) Using oauth will cause Browser Extensions and Applications to be unable to load the server url properly. Removing this line fixes the problem. Login page is secured by accounts already. --- apps/bitwarden.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/bitwarden.yml b/apps/bitwarden.yml index 7870044..f886a82 100644 --- a/apps/bitwarden.yml +++ b/apps/bitwarden.yml @@ -28,7 +28,6 @@ traefik.enable: 'true' traefik.backend: "{{pgrole}}" traefik.port: '80' - traefik.frontend.auth.forward.address: '{{gauth}}' traefik.frontend.rule: 'Host:bit.{{domain.stdout}},{{pgrole}}.{{domain.stdout}},{{tldset}}' - name: 'Setting PG Volumes' From b0343f8476e58c03c762229a76fcb228ad1bd766 Mon Sep 17 00:00:00 2001 From: dhumphrey7 <47798651+dhumphrey7@users.noreply.github.com> Date: Thu, 28 Mar 2019 16:36:21 -0500 Subject: [PATCH 088/185] Fixed a typo to Nasm package URL was bad for 2.14.02 --- apps/templates/nzbget/ffmpeg-build/build-ffmpeg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/templates/nzbget/ffmpeg-build/build-ffmpeg b/apps/templates/nzbget/ffmpeg-build/build-ffmpeg index 54bd986..fc9fee9 100644 --- a/apps/templates/nzbget/ffmpeg-build/build-ffmpeg +++ b/apps/templates/nzbget/ffmpeg-build/build-ffmpeg @@ -175,7 +175,7 @@ if build "yasm"; then fi if build "nasm"; then - download "http://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.14.02.tar.gz" "nasm.tar.gz" + download "http://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz" "nasm.tar.gz" cd $PACKAGES/nasm-2.14.02 || exit execute ./configure --prefix=${WORKSPACE} --disable-shared --enable-static execute make -j $MJOBS From f20f6135f10726914b7cbef813d4b386590018c1 Mon Sep 17 00:00:00 2001 From: dhumphrey7 <47798651+dhumphrey7@users.noreply.github.com> Date: Thu, 28 Mar 2019 19:06:32 -0500 Subject: [PATCH 089/185] Update autoProcess.ini --- apps/templates/nzbget/MP4_Automator/autoProcess.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/templates/nzbget/MP4_Automator/autoProcess.ini b/apps/templates/nzbget/MP4_Automator/autoProcess.ini index d0427a6..8122743 100644 --- a/apps/templates/nzbget/MP4_Automator/autoProcess.ini +++ b/apps/templates/nzbget/MP4_Automator/autoProcess.ini @@ -39,7 +39,7 @@ video-max-width = h264-max-level = 4.1 use-qsv-decoder-with-encoder = True use-hevc-qsv-decoder = False -ios-audio = True +ios-audio = libfdk_aac ios-first-track-only = False ios-audio-filter = dynaudnorm max-audio-channels = @@ -48,7 +48,7 @@ audio-language = eng audio-default-language = eng audio-channel-bitrate = 256 audio-filter = -subtitle-codec = mov_text +subtitle-codec = srt subtitle-language = eng subtitle-default-language = subtitle-encoding = @@ -58,7 +58,7 @@ tagfile = True tag-language = en download-artwork = Poster download-subs = True -embed-subs = True +embed-subs = False sub-providers = addic7ed,podnapisi,thesubdb,opensubtitles permissions = 0777 post-process = False From a143e8965e4ff97fc853b6b42af1647be23ecd8b Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sun, 31 Mar 2019 21:58:44 +0200 Subject: [PATCH 090/185] Repo change Old repo dont work anymore --- apps/deezloaderremix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/deezloaderremix.yml b/apps/deezloaderremix.yml index 6399ca9..c0a1354 100644 --- a/apps/deezloaderremix.yml +++ b/apps/deezloaderremix.yml @@ -15,7 +15,7 @@ pgrole: 'deezloaderremix' intport: '1730' extport: '1730' - image: 'zachawii/deezloaderremix' + image: 'bocki/deezloaderrmx' # CORE (MANDATORY) ############################################################ - name: 'Always have a backup plan' From fd1b313099781dbecd46ca66745a1a853ef6eb7c Mon Sep 17 00:00:00 2001 From: Admin9705 <24727006+Admin9705@users.noreply.github.com> Date: Tue, 2 Apr 2019 00:19:21 -0400 Subject: [PATCH 091/185] Update README.md --- README.md | 64 ++++++++++++++++++------------------------------------- 1 file changed, 21 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index cbab32b..64834bd 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,23 @@ -📂 [**Click Here**](https://goo.gl/7NR3Da) - Sign up for Google's Suite for Business - Unlimited Space - -📂 [**Click Here**](https://controlpanel.newshosting.com/signup/index.php?promo=partners&a_aid=5a65169240efd&a_bid=5ecfe99b) - NZB's with from NewsHost - PG Members Receive a 58% Discount - +##### WANT TO HELP? CLICK THE ★ (STAR LOGO) in the Upper-Right!

- - - - -

- -_**Table of Contents**_ - -1. [General Intro](#1-general-intro) -2. [YouTube Video - Intro](#2-youtube-video---intro) -3. [Summary](#3-summary) - ----- -# 1. General Intro - -**PG Box Community:** PG Community is a PG repo that collects and stores accepted user container writeups for community usage install. User can add programs such as Radarr4k or other unique programs that others may want to utlize. - -

-

Manage PGBlitz - AnyTime, Anywhere!

- -Forum Node for Community Box: [ CLICK HERE ](https://pgblitz.com/forums/pg-app-community-box.191/) - -**WARNING: Wiki Under Construction** - -(BETTER EXAMPLES will be provided), but Radarr4k and Sonarr4k serve as general containers until posted! - -In order for apps show up under the Community App Store, users must select or push apps from within the app folder of this repo. Do not worry, the apps automatically show up under the community app store and are segerated from the official containers that are built and supported by PG. - -# 2. YouTube Video - Intro -NOT READY - -# 3. Summary - -PG Box Community Edition is dead simple to use! Have a container in mind? Push containers that benifit the community today! - -📂 [**Click Here**](http://usenetserver.com/partners/?a_aid=5a65169240efd&a_bid=5725b6ed) - NZB's from USENET Server - PG Members Receive a 60% Discount - -

- + + + +

+ +* 📂 [**[Click Here]**](https://goo.gl/7NR3Da) - Google G-Suite (Unlimited Hard Drive Space & Storage) +* 📂 [**[Click Here]**](https://controlpanel.newshosting.com/signup/index.php?promo=partners&a_aid=5a65169240efd&a_bid=5ecfe99b) - Top Performance NewsHost! - Blitz Members Receive a 58% Discount +---- +### **Reference Shortcut -** http://wiki.pgblitz.com | Discord ( !wiki ) +---- + +## 1. PG YouTube + +

+

PGBlitz Introduction Video

+ +

+

PGBlitz Installation Video

+ +[**[Click Here]**](https://pgblitz.com/threads/plexguide-install-instructions.243/) for installation instructions to start the process From 944b9e4f9c65408f6356548489ee7033284ba0bd Mon Sep 17 00:00:00 2001 From: Kieran McEniff Date: Wed, 3 Apr 2019 19:09:52 +0100 Subject: [PATCH 092/185] Updating embystats internal port to 5432 instead of 80. Tested working via portainer --- apps/embystats.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/embystats.yml b/apps/embystats.yml index 41651f9..9d67b7a 100644 --- a/apps/embystats.yml +++ b/apps/embystats.yml @@ -14,7 +14,7 @@ - name: 'Set Known Facts' set_fact: pgrole: "embystats" - intport: "80" + intport: "5432" extport: "9049" image: "uping/embystat:beta-linux" From 9e01ca0d31be7f0b919fc6d2b183780e21e68cf7 Mon Sep 17 00:00:00 2001 From: Robert Baker Date: Sun, 17 Mar 2019 21:56:12 -0700 Subject: [PATCH 093/185] ansible warn fix --- apps/deluge.yml | 12 ++++++------ apps/delugevpn.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/deluge.yml b/apps/deluge.yml index 1450fe4..7918f6e 100644 --- a/apps/deluge.yml +++ b/apps/deluge.yml @@ -134,14 +134,14 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"move_completed".*' - line: '"move_completed": true,' + line: '"move_completed": True,' state: present - name: Setting extractor folder lineinfile: path: '/opt/appdata/{{pgrole}}/plugins/extractor.conf' regexp: '"use_name_folder".*' - line: '"use_name_folder": true,' + line: '"use_name_folder": True,' state: present - name: Setting extractor @@ -165,7 +165,7 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"compact_allocation".*' - line: '"compact_allocation": true,' + line: '"compact_allocation": True,' state: present - name: set stop_seed_ratio @@ -186,14 +186,14 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"remove_seed_at_ratio".*' - line: '"remove_seed_at_ratio": true,' + line: '"remove_seed_at_ratio": True,' state: present - name: set enc_prefer_rc4 lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"enc_prefer_rc4".*' - line: '"enc_prefer_rc4": true,' + line: '"enc_prefer_rc4": True,' state: present - name: set enc_level @@ -228,7 +228,7 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"dont_count_slow_torrents".*' - line: '"dont_count_slow_torrents": true,' + line: '"dont_count_slow_torrents": True,' state: present - name: set max_active_seeding diff --git a/apps/delugevpn.yml b/apps/delugevpn.yml index 55976a6..9872189 100644 --- a/apps/delugevpn.yml +++ b/apps/delugevpn.yml @@ -141,14 +141,14 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"move_completed".*' - line: '"move_completed": true,' + line: '"move_completed": True,' state: present - name: Setting extractor folder lineinfile: path: '/opt/appdata/{{pgrole}}/plugins/extractor.conf' regexp: '"use_name_folder".*' - line: '"use_name_folder": true,' + line: '"use_name_folder": True,' state: present - name: Setting extractor @@ -172,7 +172,7 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"compact_allocation".*' - line: '"compact_allocation": true,' + line: '"compact_allocation": True,' state: present - name: set stop_seed_ratio @@ -193,14 +193,14 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"remove_seed_at_ratio".*' - line: '"remove_seed_at_ratio": true,' + line: '"remove_seed_at_ratio": True,' state: present - name: set enc_prefer_rc4 lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"enc_prefer_rc4".*' - line: '"enc_prefer_rc4": true,' + line: '"enc_prefer_rc4": True,' state: present - name: set enc_level @@ -235,7 +235,7 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"dont_count_slow_torrents".*' - line: '"dont_count_slow_torrents": true,' + line: '"dont_count_slow_torrents": True,' state: present - name: set max_active_seeding From 4616dcb025ebe10343d08c7f3a0b008a316ec7cd Mon Sep 17 00:00:00 2001 From: h1f0x Date: Wed, 24 Apr 2019 09:24:38 +0200 Subject: [PATCH 094/185] Varken Docker container Hi Team, Created a single container for Varken! Check it out: https://github.com/h1f0x/centos-varken Cheers --- apps/varken.yml | 66 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 apps/varken.yml diff --git a/apps/varken.yml b/apps/varken.yml new file mode 100644 index 0000000..2011338 --- /dev/null +++ b/apps/varken.yml @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Title: CentOS 7 Varken +# Author(s): h1f0x +# URL: https://github.com/h1f0x/centos-varken +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'varken' + intport: '80' + extport: '5985' + image: 'h1f0x/centos-varken' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + - name: 'Including plugins' + include_tasks: '/opt/communityapps/apps/_plugins.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + UID: '1000' + GID: '1000' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + privileged: yes + state: started + labels: '{{pg_labels}}' From 3f8f55a9a8c584106859ad59f488a4127f8e03cc Mon Sep 17 00:00:00 2001 From: CasperNielsen <27451397+CasperNielsen@users.noreply.github.com> Date: Wed, 1 May 2019 22:45:53 +0200 Subject: [PATCH 095/185] Filezilla Webgui --- apps/filezilla.yml | 79 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 apps/filezilla.yml diff --git a/apps/filezilla.yml b/apps/filezilla.yml new file mode 100644 index 0000000..4eb5f88 --- /dev/null +++ b/apps/filezilla.yml @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Title: Filezilla for PGBlitz (OAuth security) +# Author(s): CasperVN +# URL: https://pgblitz.com - https://github.com/jlesage/docker-filezilla +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'filezilla' + intport: '5800' + extport: '5803' + image: 'jlesage/filezilla' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + - name: 'Including plugins' + include_tasks: '/opt/communityapps/apps/_plugins.yml' + + - name: 'Including folders' + include_tasks: '/opt/communityapps/apps/_downloaders.yml' + + - name: 'Ini Check' + stat: + path: /opt/appdata/{{pgrole}}/core.conf + register: inicheck + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/downloads:/storage:rw' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + UID: '1000' + GID: '1000' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' + +##PG-Community + +##PG-Community From d72aaef950ea33ee81bbf40c4d01f54f697f5aa6 Mon Sep 17 00:00:00 2001 From: Cody Date: Mon, 6 May 2019 11:25:14 -0400 Subject: [PATCH 096/185] Update and rename transmission.yml to transmission-vpn.yml Adding '-vpn' to role to avoid confusion. Added DNS servers. Setting OpenVPN config to 'default'. --- apps/{transmission.yml => transmission-vpn.yml} | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) rename apps/{transmission.yml => transmission-vpn.yml} (96%) diff --git a/apps/transmission.yml b/apps/transmission-vpn.yml similarity index 96% rename from apps/transmission.yml rename to apps/transmission-vpn.yml index 67c1086..0ff2e20 100644 --- a/apps/transmission.yml +++ b/apps/transmission-vpn.yml @@ -12,7 +12,7 @@ # FACTS ####################################################################### - name: 'Set Known Facts' set_fact: - pgrole: 'transmission' + pgrole: 'transmission-vpn' pgrole2: 'transmission-rss' intport: '9091' extport: '9091' @@ -63,7 +63,7 @@ PUID: '1000' PGID: '1000' OPENVPN_PROVIDER: 'see available configs at https://git.io/fpCSF' - OPENVPN_CONFIG: 'see available configs at https://git.io/fpCSF' + OPENVPN_CONFIG: 'default' OPENVPN_USERNAME: 'vpnuser' OPENVPN_PASSWORD: 'vpnpass' OPENVPN_OPTS: '--inactive 3600 --ping 10 --ping-exit 60' @@ -103,6 +103,11 @@ restart_policy: unless-stopped devices: - '/dev/net/tun:/dev/net/tun:rwm' + dns_servers: + - 1.1.1.1 + - 84.200.69.80 + - 45.33.97.5 + - 208.67.222.222 capabilities: - NET_ADMIN networks: From 23c2875275c3e782246fd9e394f8467c326da77b Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 15:20:40 +0200 Subject: [PATCH 097/185] Create domoticz.yml --- apps/domoticz.yml | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 apps/domoticz.yml diff --git a/apps/domoticz.yml b/apps/domoticz.yml new file mode 100644 index 0000000..4054ac5 --- /dev/null +++ b/apps/domoticz.yml @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Title: PGBlitz (linuxserver/domoticz) +# Author(s): Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'domoticz' + intport: '8080/tcp' + extport: '8777' + intport2: '6144/udp' + extport2: '6144' + intport3: '1443/tcp' + extport3: '1443' + + image: 'linuxserver/domoticz:stable' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' + - '{{ports.stdout}}{{extport3}}:{{intport3}}' + - '{{ports.stdout}}{{extport4}}:{{intport4}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 03aba632e0849c676bbda0c8e73b46cc2902b02b Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 15:21:35 +0200 Subject: [PATCH 098/185] Create unifi.yml --- apps/unifi.yml | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 apps/unifi.yml diff --git a/apps/unifi.yml b/apps/unifi.yml new file mode 100644 index 0000000..399c1cb --- /dev/null +++ b/apps/unifi.yml @@ -0,0 +1,89 @@ +#!/bin/bash +# +# Title: PGBlitz (Reference Title File) +# Author(s): Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'unifi' + intport: '3478/udp' + extport: '3478' + intport2: '10001/udp' + extport2: '10001' + intport3: '8080/tcp' + extport3: '8080' + intport4: '8081/tcp' + extport4: '8081' + intport5: '8443/tcp' + extport5: '8443' + intport6: '8880/tcp' + extport6: '8880' + image: 'linuxserver/unifi:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.tags: 'frontend' + traefik.frontend.passHostHeader: 'true' + traefik.backend: '{{pgrole}}' + traefik.admin.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + traefik.admin.port: '{{intport5}}' + traefik.admin.protocol: 'https' + traefik.frontend.headers.SSLRedirect: 'true' + traefik.frontend.headers.STSSeconds: '315360000' + traefik.frontend.headers.browserXSSFilter: 'true' + traefik.frontend.headers.contentTypeNosniff: 'true' + traefik.frontend.headers.forceSTSHeader: 'true' + traefik.frontend.headers.SSLHost: '{{domain.stdout}}' + traefik.frontend.headers.STSIncludeSubdomains: 'true' + traefik.frontend.headers.STSPreload: 'true' + traefik.frontend.headers.frameDeny: 'true' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' + - '{{ports.stdout}}{{extport3}}:{{intport3}}' + - '{{ports.stdout}}{{extport4}}:{{intport4}}' + - '{{ports.stdout}}{{extport5}}:{{intport5}}' + - '{{ports.stdout}}{{extport6}}:{{intport6}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: always + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From ddb3b05091c4c51fb1bdf2ce3ba58f530961cd0c Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 19:40:55 +0200 Subject: [PATCH 099/185] Create flextv.yml --- apps/flextv.yml | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 apps/flextv.yml diff --git a/apps/flextv.yml b/apps/flextv.yml new file mode 100644 index 0000000..4c69656 --- /dev/null +++ b/apps/flextv.yml @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Title: PGBlitz (digitalhigh/FlexTV) +# Author(s): Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'flextv' + intport: '80/tcp' + extport: '8987' + image: 'digitalhigh/flextv:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From fd7467591325d129b754ecf66334475dfa100aea Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 20:46:22 +0200 Subject: [PATCH 100/185] Create filebot.yml --- apps/filebot.yml | 65 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 apps/filebot.yml diff --git a/apps/filebot.yml b/apps/filebot.yml new file mode 100644 index 0000000..d030da2 --- /dev/null +++ b/apps/filebot.yml @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Title: PGBlitz (jlesage/filebot) +# Author(s): MrDoob +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'filebot' + intport: '5900' + extport: '5903' + image: 'jlesage/filebot:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/unionfs:rw' + - '/mnt:/mnt:rw' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 52907c79dfb3cffd4f55b022643928814c727b97 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 20:55:41 +0200 Subject: [PATCH 101/185] Create mediainfo.yml --- apps/mediainfo.yml | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 apps/mediainfo.yml diff --git a/apps/mediainfo.yml b/apps/mediainfo.yml new file mode 100644 index 0000000..f3847f1 --- /dev/null +++ b/apps/mediainfo.yml @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Title: PGBlitz (jlesage/mediainfo) +# Author(s): MrDoob +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'mediainfo' + intport: '5800' + extport: '5806' + intport2: '5900' + extport2: '5907'' + image: 'jlesage/mediainfo:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/unionfs:rw' + - '/mnt:/mnt:rw' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 69c0bc176117c3fba5368ef4b093bc1ca059a511 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 21:34:24 +0200 Subject: [PATCH 102/185] Update mediainfo.yml --- apps/mediainfo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/mediainfo.yml b/apps/mediainfo.yml index f3847f1..3779f7e 100644 --- a/apps/mediainfo.yml +++ b/apps/mediainfo.yml @@ -16,7 +16,7 @@ intport: '5800' extport: '5806' intport2: '5900' - extport2: '5907'' + extport2: '5907' image: 'jlesage/mediainfo:latest' # CORE (MANDATORY) ############################################################ From 1239751d0264dd04862612da2192e4154c60f264 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 21:47:46 +0200 Subject: [PATCH 103/185] Update filebot.yml --- apps/filebot.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/filebot.yml b/apps/filebot.yml index d030da2..bcb6e91 100644 --- a/apps/filebot.yml +++ b/apps/filebot.yml @@ -35,8 +35,7 @@ pg_volumes: - '/opt/appdata/{{pgrole}}:/config' - '{{path.stdout}}:{{path.stdout}}' - - '/mnt/unionfs:/unionfs:rw' - - '/mnt:/mnt:rw' + - '/mnt:/storage:rw' - '/etc/localtime:/etc/localtime:ro' - name: 'Setting PG ENV' From 1a96a82f22b0d458eb06f88b80888f392d2fc5e3 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 21:48:21 +0200 Subject: [PATCH 104/185] Update filebot.yml --- apps/filebot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/filebot.yml b/apps/filebot.yml index bcb6e91..747ae89 100644 --- a/apps/filebot.yml +++ b/apps/filebot.yml @@ -15,7 +15,7 @@ pgrole: 'filebot' intport: '5900' extport: '5903' - image: 'jlesage/filebot:latest' + image: 'jlesage/filebot' # CORE (MANDATORY) ############################################################ - name: 'Including cron job' From 3e1f07b71075218864ec9f2479936e60f064a0b9 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 22:05:53 +0200 Subject: [PATCH 105/185] Update filebot.yml --- apps/filebot.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/filebot.yml b/apps/filebot.yml index 747ae89..e2f0f7c 100644 --- a/apps/filebot.yml +++ b/apps/filebot.yml @@ -13,8 +13,10 @@ - name: 'Set Known Facts' set_fact: pgrole: 'filebot' - intport: '5900' - extport: '5903' + intport: '5800' + extport: '5803' + intport2: '5900' + extport2: '5903' image: 'jlesage/filebot' # CORE (MANDATORY) ############################################################ @@ -53,6 +55,7 @@ pull: yes published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: unless-stopped From 6d34432f292b93a42cc06e1af0ad0df277e86856 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 22:06:18 +0200 Subject: [PATCH 106/185] Update mediainfo.yml --- apps/mediainfo.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/mediainfo.yml b/apps/mediainfo.yml index 3779f7e..a989491 100644 --- a/apps/mediainfo.yml +++ b/apps/mediainfo.yml @@ -56,6 +56,7 @@ pull: yes published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: unless-stopped From a6991f162e5223c0143dbcc0800e1f33d22d0ed5 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 10 May 2019 22:41:29 +0200 Subject: [PATCH 107/185] Create rclonebrowser.yml --- apps/rclonebrowser.yml | 67 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 apps/rclonebrowser.yml diff --git a/apps/rclonebrowser.yml b/apps/rclonebrowser.yml new file mode 100644 index 0000000..876bff1 --- /dev/null +++ b/apps/rclonebrowser.yml @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Title: PGBlitz (rclone browser) +# Author(s): MrDoob +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'rclonebrowser' + intport: '5800' + extport: '5808' + intport2: '5900' + extport2: '5909' + image: 'romancin/rclonebrowser:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/unionfs:rw' + - '/mnt:/mnt:rw' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 7d6affb0f74c1d6821dd874d59f65f39e6e78e10 Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 01:32:53 -0600 Subject: [PATCH 108/185] Add teamspeak3 alpine image --- apps/teamspeak3.yml | 75 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 apps/teamspeak3.yml diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml new file mode 100644 index 0000000..d36f3c9 --- /dev/null +++ b/apps/teamspeak3.yml @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Title: Teamspeak 3 Alpine +# Author(s): N1ghtshade +# URL: https://plexguide.com - http://github.plexguide.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'teamspeak3' + intport: '9987/udp' + extport: '9987' + intport2: '10011/tcp' + extport2: '10011' + intport3: '30033/tcp' + extport3: '30033' + image: 'qmcgaw/teamspeak3-alpine:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # EXTRA FOR TS3 ######################################################### + - name: 'Create {{pgrole}} data and log directories' + file: 'path={{item}} state=directory mode=0700 owner=1000 group=1000 recurse=yes' + with_items: + - '/opt/appdata/{{pgrole}}' + - '/opt/appdata/{{pgrole}}/data' + - '/opt/appdata/{{pgrole}}/logs' + + # LABELS ###################################################################### + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}/data:/data' + - '/opt/appdata/{{pgrole}}/logs:/logs' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + + # MAIN DEPLOYMENT ############################################################# + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' + - '{{ports.stdout}}{{extport3}}:{{intport3}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' + + # ENDING TASK FOR TS3 #################################################### + - name: 'Configuring {{pgrole}} for first time use' + block: + - name: 'Wait 30 Seconds' + wait_for: + timeout: 30 From 079355f4bde762b1870156679034e578042a9557 Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 09:44:17 -0600 Subject: [PATCH 109/185] fix pg_labels --- apps/teamspeak3.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index d36f3c9..7d3add8 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -34,7 +34,15 @@ - '/opt/appdata/{{pgrole}}/logs' # LABELS ###################################################################### - - name: 'Setting PG Volumes' +###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'false' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + - name: 'Setting PG Volumes' set_fact: pg_volumes: - '/opt/appdata/{{pgrole}}/data:/data' From 4bada1a014da62e1702bd9c473e9a11e7149457b Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 09:48:04 -0600 Subject: [PATCH 110/185] Fix syntax error --- apps/teamspeak3.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index 7d3add8..a9bd7c7 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -34,7 +34,7 @@ - '/opt/appdata/{{pgrole}}/logs' # LABELS ###################################################################### -###################################################################### + - name: 'Adding Traefik' set_fact: pg_labels: @@ -42,6 +42,8 @@ traefik.port: '{{intport}}' traefik.frontend.auth.forward.address: '{{gauth}}' traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + # VOLUMES ######### + - name: 'Setting PG Volumes' set_fact: pg_volumes: From 12c229caec8cbbbb85410a1a51894a35a3c06474 Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 09:55:40 -0600 Subject: [PATCH 111/185] Fix syntax error --- apps/teamspeak3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index a9bd7c7..3ed04e3 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -44,7 +44,7 @@ traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' # VOLUMES ######### - - name: 'Setting PG Volumes' + - name: 'Setting PG Volumes' set_fact: pg_volumes: - '/opt/appdata/{{pgrole}}/data:/data' From 6aa38d0f1558f3287210207bffe9ae7839bed7c2 Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 10:07:22 -0600 Subject: [PATCH 112/185] Add license_accept=1 --- apps/teamspeak3.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index 3ed04e3..f96adec 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -55,6 +55,8 @@ pg_env: PUID: '1000' PGID: '1000' + pg_cmd: + license_accepted: '1' # MAIN DEPLOYMENT ############################################################# @@ -69,6 +71,7 @@ - '{{ports.stdout}}{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' + cmd: '{{pg_cmd}}' restart_policy: unless-stopped networks: - name: plexguide From 5037bfcebdbc1d11911ffb7780a38953843ed67c Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 10:11:03 -0600 Subject: [PATCH 113/185] Update command label --- apps/teamspeak3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index f96adec..5550fa3 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -71,7 +71,7 @@ - '{{ports.stdout}}{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' - cmd: '{{pg_cmd}}' + command: '{{pg_cmd}}' restart_policy: unless-stopped networks: - name: plexguide From 3075d8e2b3498db26587032d98ebed1363175403 Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 10:16:47 -0600 Subject: [PATCH 114/185] Update teamspeak3.yml --- apps/teamspeak3.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index 5550fa3..3ed04e3 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -55,8 +55,6 @@ pg_env: PUID: '1000' PGID: '1000' - pg_cmd: - license_accepted: '1' # MAIN DEPLOYMENT ############################################################# @@ -71,7 +69,6 @@ - '{{ports.stdout}}{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' - command: '{{pg_cmd}}' restart_policy: unless-stopped networks: - name: plexguide From 04c506c99deab81dad05d2df736d449bf6ac7b72 Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 10:18:17 -0600 Subject: [PATCH 115/185] Accept license --- apps/teamspeak3.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index 3ed04e3..48acc35 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -55,6 +55,7 @@ pg_env: PUID: '1000' PGID: '1000' + command: 'license_accepted=1' # MAIN DEPLOYMENT ############################################################# From 2c92331f6e93f7b5157b629299a7fe806d5977d9 Mon Sep 17 00:00:00 2001 From: Nightshade <38169527+n1ghtshade@users.noreply.github.com> Date: Sun, 12 May 2019 10:25:06 -0600 Subject: [PATCH 116/185] Fix Accept License --- apps/teamspeak3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index 48acc35..30d72ff 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -55,7 +55,6 @@ pg_env: PUID: '1000' PGID: '1000' - command: 'license_accepted=1' # MAIN DEPLOYMENT ############################################################# @@ -70,6 +69,7 @@ - '{{ports.stdout}}{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' + command: license_accepted=1 restart_policy: unless-stopped networks: - name: plexguide From b1bcd033d8ed16720c6905f9b2f43debc49bd885 Mon Sep 17 00:00:00 2001 From: CasperNielsen <27451397+CasperNielsen@users.noreply.github.com> Date: Sun, 12 May 2019 21:01:05 +0200 Subject: [PATCH 117/185] Update filezilla.yml Added higher screen resolution. --- apps/filezilla.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/filezilla.yml b/apps/filezilla.yml index 4eb5f88..ac8c81d 100644 --- a/apps/filezilla.yml +++ b/apps/filezilla.yml @@ -55,6 +55,8 @@ pg_env: UID: '1000' GID: '1000' + DISPLAY_WIDTH: '1920' + DISPLAY_HEIGHT: '1080' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' From e4c4162783ca47084d82a97c9296a3dfe4896a0c Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 17 May 2019 16:36:15 +0200 Subject: [PATCH 118/185] Create vnc-xfce.yml --- apps/vnc-xfce.yml | 65 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 apps/vnc-xfce.yml diff --git a/apps/vnc-xfce.yml b/apps/vnc-xfce.yml new file mode 100644 index 0000000..7ee217d --- /dev/null +++ b/apps/vnc-xfce.yml @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Title: PGBlitz (consol/ubuntu-xfce-vnc) +# Author(s): MrDoob +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'vnc-xfce' + intport: '6901' + extport: '6910' + image: 'consol/ubuntu-xfce-vnc' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/unionfs:rw' + - '/mnt:/mnt:rw' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From ea45234ef62679d0f64e36b04bf5c36443b70c0a Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 17 May 2019 16:39:37 +0200 Subject: [PATCH 119/185] Create calibre-web.yml created by Sejrup --- apps/calibre-web.yml | 66 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 apps/calibre-web.yml diff --git a/apps/calibre-web.yml b/apps/calibre-web.yml new file mode 100644 index 0000000..b2e93ac --- /dev/null +++ b/apps/calibre-web.yml @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Title: PGBlitz (calibre-web) +# Author(s): Sejrup +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'calibre-web' + intport: '8083' + extport: '8083' + image: 'technosoft2000/calibre-web' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/books' + - '/opt/appdata/{{pgrole}}/config:/calibre-web/config' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + USE_CONFIG_DIR: 'true' + SET_CONTAINER_TIMEZONE: 'true' + CONTAINER_TIMEZONE: 'Europe/Copenhagen' + PUID: '1000' + PGID: '1000' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: always + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From eb4ddf981a8a0456f86b13d76ace39cf973e893a Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 17 May 2019 21:31:44 +0200 Subject: [PATCH 120/185] Update unifi.yml --- apps/unifi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/unifi.yml b/apps/unifi.yml index 399c1cb..857a9ac 100644 --- a/apps/unifi.yml +++ b/apps/unifi.yml @@ -25,7 +25,7 @@ extport5: '8443' intport6: '8880/tcp' extport6: '8880' - image: 'linuxserver/unifi:latest' + image: 'linuxserver/unifi-controller:LTS' # CORE (MANDATORY) ############################################################ - name: 'Including cron job' From e473383bd4a4e2fe3edec30dba1d108430e95157 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 17 May 2019 21:42:37 +0200 Subject: [PATCH 121/185] ports exposed this ports must be exposed for communitcation --- apps/unifi.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/unifi.yml b/apps/unifi.yml index 857a9ac..2daf68e 100644 --- a/apps/unifi.yml +++ b/apps/unifi.yml @@ -72,12 +72,12 @@ image: '{{image}}' pull: yes published_ports: - - '{{ports.stdout}}{{extport}}:{{intport}}' - - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{ports.stdout}}{{extport3}}:{{intport3}}' - - '{{ports.stdout}}{{extport4}}:{{intport4}}' - - '{{ports.stdout}}{{extport5}}:{{intport5}}' - - '{{ports.stdout}}{{extport6}}:{{intport6}}' + - '{{extport}}:{{intport}}' + - '{{extport2}}:{{intport2}}' + - '{{extport3}}:{{intport3}}' + - '{{extport4}}:{{intport4}}' + - '{{extport5}}:{{intport5}}' + - '{{extport6}}:{{intport6}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: always From fc6f2c97fcbca0b8827132e4ce3a91dde416804b Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 17 May 2019 21:45:09 +0200 Subject: [PATCH 122/185] Update unifi.yml --- apps/unifi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/unifi.yml b/apps/unifi.yml index 2daf68e..eec7e2d 100644 --- a/apps/unifi.yml +++ b/apps/unifi.yml @@ -18,7 +18,7 @@ intport2: '10001/udp' extport2: '10001' intport3: '8080/tcp' - extport3: '8080' + extport3: '8088' intport4: '8081/tcp' extport4: '8081' intport5: '8443/tcp' From aba3ae95988937648112b7d37e7cfd632ddd4185 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sat, 18 May 2019 11:30:55 +0200 Subject: [PATCH 123/185] update port conflict with filezilla --- apps/filebot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/filebot.yml b/apps/filebot.yml index e2f0f7c..5fd6f77 100644 --- a/apps/filebot.yml +++ b/apps/filebot.yml @@ -14,7 +14,7 @@ set_fact: pgrole: 'filebot' intport: '5800' - extport: '5803' + extport: '5804' intport2: '5900' extport2: '5903' image: 'jlesage/filebot' From aa8165975590e59644fcbf8542450ff265859b40 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sat, 18 May 2019 11:40:45 +0200 Subject: [PATCH 124/185] Create rdp-calibre.yml --- apps/rdp-calibre.yml | 71 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 apps/rdp-calibre.yml diff --git a/apps/rdp-calibre.yml b/apps/rdp-calibre.yml new file mode 100644 index 0000000..8b5576d --- /dev/null +++ b/apps/rdp-calibre.yml @@ -0,0 +1,71 @@ +#!/bin/bash +# +# Title: PGBlitz (rdp-calibre) +# Author(s): MrDoob +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'rdp-calibre' + intport: '8080' + extport: '8089' + intport2: '8081' + extport2: '8090' + image: 'aptalca/docker-rdp-calibre:latest' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config:rw' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/unionfs:rw' + - '/mnt:/mnt:rw' + - '/etc/localtime:/etc/localtime:ro' + - '/mnt/unionfs/libary:/library:rw' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + WIDTH: '1280' + HEIGHT: '720' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 9a4e3f8bf31e25bd26436519dc6e7e54e958a8d5 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sat, 18 May 2019 21:08:54 +0200 Subject: [PATCH 125/185] port exposed --- apps/teamspeak3.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/teamspeak3.yml b/apps/teamspeak3.yml index 30d72ff..cf1b51e 100644 --- a/apps/teamspeak3.yml +++ b/apps/teamspeak3.yml @@ -64,9 +64,9 @@ image: '{{image}}' pull: yes published_ports: - - '{{ports.stdout}}{{extport}}:{{intport}}' - - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{ports.stdout}}{{extport3}}:{{intport3}}' + - '{{extport}}:{{intport}}' + - '{{extport2}}:{{intport2}}' + - '{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' command: license_accepted=1 From 47c68732e28176d72a1269b336e381e2dde20fc7 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sun, 19 May 2019 14:46:08 +0200 Subject: [PATCH 126/185] Update sonarrhdr.yml --- apps/sonarrhdr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sonarrhdr.yml b/apps/sonarrhdr.yml index 0541208..3a80027 100644 --- a/apps/sonarrhdr.yml +++ b/apps/sonarrhdr.yml @@ -16,7 +16,7 @@ pgrole: 'sonarrhdr' intport: '8989' extport: '8987' - image: 'lsiodev/sonarr-preview' + image: 'linuxserver/sonarr:preview' # CORE (MANDATORY) ############################################################ - name: 'Including cron job' From 15639d8aa75446d0fc29dfd329b4a234cf09b176 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 17:32:50 +0200 Subject: [PATCH 127/185] Update unifi.yml --- apps/unifi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/unifi.yml b/apps/unifi.yml index eec7e2d..ed6cef6 100644 --- a/apps/unifi.yml +++ b/apps/unifi.yml @@ -25,7 +25,7 @@ extport5: '8443' intport6: '8880/tcp' extport6: '8880' - image: 'linuxserver/unifi-controller:LTS' + image: 'linuxserver/unifi-controller:latest' # CORE (MANDATORY) ############################################################ - name: 'Including cron job' From 71b9860b5a78d4107bae0c16159d107b5730ebb8 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 17:37:14 +0200 Subject: [PATCH 128/185] Create unifi-controller --- apps/image/unifi-controller | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 apps/image/unifi-controller diff --git a/apps/image/unifi-controller b/apps/image/unifi-controller new file mode 100644 index 0000000..371ec56 --- /dev/null +++ b/apps/image/unifi-controller @@ -0,0 +1,2 @@ +linuxserver/unifi-controller:latest +linuxserver/unifi-controller:LTS From 2f40d8e002e7968275e91c7c7eebebade79df621 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 17:46:00 +0200 Subject: [PATCH 129/185] Create kodi-headless.yml --- apps/kodi-headless.yml | 74 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 apps/kodi-headless.yml diff --git a/apps/kodi-headless.yml b/apps/kodi-headless.yml new file mode 100644 index 0000000..86816f2 --- /dev/null +++ b/apps/kodi-headless.yml @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Title: PGBlitz (kodi-headless) +# Author(s): MrDoob +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'kodi-headless' + intport: '8080' + extport: '8282' + intport2: '9090' + extport2: '9292' + intport3: '9777' + extport3: '9777/udp' + image: 'linuxserver/kodi-headless:latest' + + # CORE (MANDATORY) ############################################################ + + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config/.kodi:rw' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt:rw' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' + - '{{extport3}}:{{intport3}}' + + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 11965be81a3bbac2fa3e8aa9bfd9c524445bbd52 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 18:11:07 +0200 Subject: [PATCH 130/185] Update kodi-headless.yml --- apps/kodi-headless.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/apps/kodi-headless.yml b/apps/kodi-headless.yml index 86816f2..132cc86 100644 --- a/apps/kodi-headless.yml +++ b/apps/kodi-headless.yml @@ -1,6 +1,6 @@ #!/bin/bash # -# Title: PGBlitz (kodi-headless) +# Title: PGBlitz (linuxserver/kodi-headless) # Author(s): MrDoob # URL: https://pgblitz.com - http://github.pgblitz.com # GNU: General Public License v3.0 @@ -10,27 +10,25 @@ gather_facts: false tasks: # FACTS ####################################################################### - - name: 'Set Known Facts' set_fact: - pgrole: 'kodi-headless' + pgrole: 'linuxserver/kodi-headless' intport: '8080' - extport: '8282' + extport: '8088' intport2: '9090' - extport2: '9292' + extport2: '9099' intport3: '9777' - extport3: '9777/udp' + extport3: '9777/udp' image: 'linuxserver/kodi-headless:latest' # CORE (MANDATORY) ############################################################ - - name: 'Including cron job' include_tasks: '/opt/communityapps/apps/_core.yml' - - # LABELS ###################################################################### + # LABELS ###################################################################### - name: 'Adding Traefik' set_fact: + pg_labels: pg_labels: traefik.enable: 'true' traefik.port: '{{intport}}' @@ -40,8 +38,7 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config/.kodi:rw' - - '{{path.stdout}}:{{path.stdout}}' + - '/opt/appdata/{{pgrole}}/config:/config/.kodi:rw' - '/mnt:/mnt:rw' - '/etc/localtime:/etc/localtime:ro' @@ -61,11 +58,10 @@ published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{extport3}}:{{intport3}}' - + - '{{ports.stdout}}{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' - restart_policy: unless-stopped + restart_policy: always networks: - name: plexguide aliases: From 2333b477281effdb757aa5b8404b7263ae7ce783 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 18:20:07 +0200 Subject: [PATCH 131/185] Update kodi-headless.yml --- apps/kodi-headless.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/kodi-headless.yml b/apps/kodi-headless.yml index 132cc86..e4e756c 100644 --- a/apps/kodi-headless.yml +++ b/apps/kodi-headless.yml @@ -28,7 +28,6 @@ # LABELS ###################################################################### - name: 'Adding Traefik' set_fact: - pg_labels: pg_labels: traefik.enable: 'true' traefik.port: '{{intport}}' From b41a216139718180254cef238d2818af863def1e Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 18:32:39 +0200 Subject: [PATCH 132/185] Update kodi-headless.yml --- apps/kodi-headless.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/kodi-headless.yml b/apps/kodi-headless.yml index e4e756c..20e0745 100644 --- a/apps/kodi-headless.yml +++ b/apps/kodi-headless.yml @@ -18,7 +18,7 @@ intport2: '9090' extport2: '9099' intport3: '9777' - extport3: '9777/udp' + extport3: '9777/udp' image: 'linuxserver/kodi-headless:latest' # CORE (MANDATORY) ############################################################ From ca26550001787ffa9288f37af1a4a6af0bc45f6f Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 18:43:24 +0200 Subject: [PATCH 133/185] Update kodi-headless.yml --- apps/kodi-headless.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/kodi-headless.yml b/apps/kodi-headless.yml index 20e0745..69dcd05 100644 --- a/apps/kodi-headless.yml +++ b/apps/kodi-headless.yml @@ -57,7 +57,7 @@ published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{ports.stdout}}{{extport3}}:{{intport3}}' + - '{{ports.stdout}}{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: always From 860be554a6d9eef1705ad100fee89d9bdbba82a6 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 18:56:36 +0200 Subject: [PATCH 134/185] Rename apps/kodi-headless.yml to apps/templates/broken/kodi-headless.yml --- apps/{ => templates/broken}/kodi-headless.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/{ => templates/broken}/kodi-headless.yml (100%) diff --git a/apps/kodi-headless.yml b/apps/templates/broken/kodi-headless.yml similarity index 100% rename from apps/kodi-headless.yml rename to apps/templates/broken/kodi-headless.yml From 047713ae7621353ac32a0f94059e693953c56057 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 19:06:48 +0200 Subject: [PATCH 135/185] Update kodi-headless.yml --- apps/templates/broken/kodi-headless.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/templates/broken/kodi-headless.yml b/apps/templates/broken/kodi-headless.yml index 69dcd05..88ccc68 100644 --- a/apps/templates/broken/kodi-headless.yml +++ b/apps/templates/broken/kodi-headless.yml @@ -12,7 +12,7 @@ # FACTS ####################################################################### - name: 'Set Known Facts' set_fact: - pgrole: 'linuxserver/kodi-headless' + pgrole: 'kodi-headless' intport: '8080' extport: '8088' intport2: '9090' @@ -55,9 +55,9 @@ image: '{{image}}' pull: yes published_ports: - - '{{ports.stdout}}{{extport}}:{{intport}}' - - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{ports.stdout}}{{extport3}}:{{intport3}}' + - '{{extport}}:{{intport}}' + - '{{extport2}}:{{intport2}}' + - '{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: always From 2aa9e939b41e5820741925d789137dff853b1666 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 19:07:02 +0200 Subject: [PATCH 136/185] Rename apps/templates/broken/kodi-headless.yml to apps/kodi-headless.yml --- apps/{templates/broken => }/kodi-headless.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/{templates/broken => }/kodi-headless.yml (100%) diff --git a/apps/templates/broken/kodi-headless.yml b/apps/kodi-headless.yml similarity index 100% rename from apps/templates/broken/kodi-headless.yml rename to apps/kodi-headless.yml From 4cbfe25c739c07f4562dde4c34918258f382ad08 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 19:46:51 +0200 Subject: [PATCH 137/185] Create users.auth.php --- apps/templates/dokuwiki/users.auth.php | 1 + 1 file changed, 1 insertion(+) create mode 100644 apps/templates/dokuwiki/users.auth.php diff --git a/apps/templates/dokuwiki/users.auth.php b/apps/templates/dokuwiki/users.auth.php new file mode 100644 index 0000000..052a42b --- /dev/null +++ b/apps/templates/dokuwiki/users.auth.php @@ -0,0 +1 @@ +admin:$1$4fd0ad31$.cId7p1uxI4a.RcrH81On0:DokuWiki Administrator:mail@host.com:admin,user From 1c91bdf9cec788186695a20f5d720ec80e99d497 Mon Sep 17 00:00:00 2001 From: Cody Date: Thu, 23 May 2019 15:16:47 -0400 Subject: [PATCH 138/185] Initial Dozzle commit --- apps/dozzle.yml | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 apps/dozzle.yml diff --git a/apps/dozzle.yml b/apps/dozzle.yml new file mode 100644 index 0000000..f7f87a9 --- /dev/null +++ b/apps/dozzle.yml @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Title: Dozzle +# Author(s): Amir Raminfar (amir20) +# URL: https://dozzle.dev/ - https://github.com/amir20/dozzle +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # CORE (MANDATORY) DO NOT CHANGE ########################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'dozzle' + intport: '8080' + extport: '8383' + image: 'amir20/dozzle' + + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS #### KEEPS BOTTOM CLEAN ########################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/var/run/docker.sock:/var/run/docker.sock' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + DOZZLE_ADDR: ':8080' + DOZZLE_BASE: '/' + DOZZLE_LEVEL: 'info' + DOZZLE_TAILSIZE: '300' + + # MAIN SCRIPT ############################################################## + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From deccf1de9168edb096ccacdb14b6f26a65edf961 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Thu, 23 May 2019 21:37:45 +0200 Subject: [PATCH 139/185] Rename apps/kodi-headless.yml to apps/templates/broken/kodi-headless.yml --- apps/{ => templates/broken}/kodi-headless.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/{ => templates/broken}/kodi-headless.yml (100%) diff --git a/apps/kodi-headless.yml b/apps/templates/broken/kodi-headless.yml similarity index 100% rename from apps/kodi-headless.yml rename to apps/templates/broken/kodi-headless.yml From 4e0e586e77c5d92aa5c10e72411aa011736e6aa6 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 24 May 2019 22:28:08 +0200 Subject: [PATCH 140/185] Update flexget.yml line fix empty deleted --- apps/flexget.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/flexget.yml b/apps/flexget.yml index 25faf86..7b375a6 100644 --- a/apps/flexget.yml +++ b/apps/flexget.yml @@ -44,8 +44,6 @@ PUID: '1000' PGID: '1000' - - # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' docker_container: From 6bbd15fe55f9aab70b06591801faeff26895969e Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Fri, 24 May 2019 23:50:16 +0200 Subject: [PATCH 141/185] Add files via upload --- apps/airsonic.yml | 129 +++++++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 65 deletions(-) diff --git a/apps/airsonic.yml b/apps/airsonic.yml index 25f9cf7..4b38a71 100644 --- a/apps/airsonic.yml +++ b/apps/airsonic.yml @@ -1,65 +1,64 @@ -#!/bin/bash -# -# Title: PGBlitz (Reference Title File) -# Author(s): Admin9705 -# URL: https://pgblitz.com - http://github.pgblitz.com -# GNU: General Public License v3.0 -################################################################################ ---- -- hosts: localhost - gather_facts: false - tasks: - # FACTS ####################################################################### - - name: 'Set Known Facts' - set_fact: - pgrole: 'airsonic' - intport: '4040' - extport: '4040' - image: 'airsonic/airsonic' - - # CORE (MANDATORY) ############################################################ - - name: 'Including cron job' - include_tasks: '/opt/communityapps/apps/_core.yml' - - # LABELS ###################################################################### - - name: 'Adding Traefik' - set_fact: - pg_labels: - traefik.enable: 'true' - traefik.port: '{{intport}}' - traefik.frontend.auth.forward.address: '{{gauth}}' - traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' - - - name: 'Setting PG Volumes' - set_fact: - pg_volumes: - - '/etc/localtime:/etc/localtime:ro' - - '/opt/appdata/{{pgrole}}:/airsonic/data' - - '/opt/appdata/{{pgrole}}/playlists:/airsonic/playlists' - - '/opt/appdata/{{pgrole}}/podcasts:/airsonic/podcasts' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt:/mnt' - - - name: 'Setting PG ENV' - set_fact: - pg_env: - PUID: '1000' - PGID: '1000' - - # MAIN DEPLOYMENT ############################################################# - - name: 'Deploying {{pgrole}}' - docker_container: - name: '{{pgrole}}' - image: '{{image}}' - pull: yes - published_ports: - - '{{ports.stdout}}{{extport}}:{{intport}}' - volumes: '{{pg_volumes}}' - env: '{{pg_env}}' - restart_policy: unless-stopped - networks: - - name: plexguide - aliases: - - '{{pgrole}}' - state: started - labels: '{{pg_labels}}' +#!/bin/bash +# +# Title: PGBlitz (linuxserver/airsonic) +# Author(s): MrDoob +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'airsonic' + intport: '4040' + extport: '4040' + image: 'linuxserver/airsonic' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}/config:/config:rw' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/media:ro' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: <<< Date: Fri, 24 May 2019 23:55:51 +0200 Subject: [PATCH 142/185] Update airsonic.yml --- apps/airsonic.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/airsonic.yml b/apps/airsonic.yml index 4b38a71..06e8f5d 100644 --- a/apps/airsonic.yml +++ b/apps/airsonic.yml @@ -44,14 +44,14 @@ PUID: '1000' PGID: '1000' TZ: '${TZ}' - + # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' docker_container: name: '{{pgrole}}' image: '{{image}}' pull: yes - published_ports: <<< Date: Sat, 25 May 2019 00:09:40 +0200 Subject: [PATCH 143/185] Add files via upload --- apps/airsonic.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/airsonic.yml b/apps/airsonic.yml index 06e8f5d..4b38a71 100644 --- a/apps/airsonic.yml +++ b/apps/airsonic.yml @@ -44,14 +44,14 @@ PUID: '1000' PGID: '1000' TZ: '${TZ}' - + # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' docker_container: name: '{{pgrole}}' image: '{{image}}' pull: yes - published_ports: + published_ports: <<< Date: Sat, 25 May 2019 00:29:42 +0200 Subject: [PATCH 144/185] Add files via upload --- apps/subsonic.yml | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 apps/subsonic.yml diff --git a/apps/subsonic.yml b/apps/subsonic.yml new file mode 100644 index 0000000..bb68be9 --- /dev/null +++ b/apps/subsonic.yml @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Title: PGBlitz (danisla/subsonic) +# Author(s): MrDoob +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + - name: 'Set Known Facts' + set_fact: + pgrole: 'subsonic' + intport: '4040' + extport: '4040' + image: 'danisla/subsonic' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}/state:/opt/app/state:rw' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt/unionfs:/mnt/music:ro' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + TZ: '${TZ}' + + # MAIN DEPLOYMENT ############################################################# + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' \ No newline at end of file From 0fedc45c4b6cbbc927608df600a3dd32a4d7b295 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sat, 25 May 2019 00:31:18 +0200 Subject: [PATCH 145/185] Update airsonic.yml --- apps/airsonic.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/airsonic.yml b/apps/airsonic.yml index 4b38a71..369bc0e 100644 --- a/apps/airsonic.yml +++ b/apps/airsonic.yml @@ -33,10 +33,12 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}/config:/config:rw' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt/unionfs:/media:ro' - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/airsonic/data' + - '/opt/appdata/{{pgrole}}/playlists:/airsonic/playlists' + - '/opt/appdata/{{pgrole}}/podcasts:/airsonic/podcasts' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' - name: 'Setting PG ENV' set_fact: @@ -51,7 +53,7 @@ name: '{{pgrole}}' image: '{{image}}' pull: yes - published_ports: <<< Date: Mon, 27 May 2019 06:47:35 -0700 Subject: [PATCH 146/185] AVIdemux Avidemux is a free video editor designed for simple cutting, filtering and encoding tasks. It supports many file types, including AVI, DVD compatible MPEG files, MP4 and ASF, using a variety of codecs. Tasks can be automated using projects, job queue and powerful scripting capabilities. --- apps/avidemux.ym; | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 apps/avidemux.ym; diff --git a/apps/avidemux.ym; b/apps/avidemux.ym; new file mode 100644 index 0000000..8b78e56 --- /dev/null +++ b/apps/avidemux.ym; @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Title: avidemux +# Author(s): timekills +# URL: https://plexguide.com - http://github.plexguide.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # CORE (MANDATORY) DO NOT CHANGE ########################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'avidemux' + intport: '5800' + extport: '5806' +# intport2: '25565' +# extport2: '25565' + image: 'jlesage/avidemux' + + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # EXTRA FUNCTIONS REQUIRED BY THE ROLE ##################################### + + - name: 'Create scripts directory for {{pgrole}}' + file: + path: /opt/appdata/{{pgrole}}/scripts + state: directory + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Create plugins directory for {{pgrole}}' + file: + path: /opt/appdata/{{pgrole}}/plugins + state: directory + owner: 1000 + group: 1000 + mode: 0755 + + # LABELS #### KEEPS BOTTOM CLEAN ########################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config:rw' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: 1000 + PGID: 1000 + # Default width is 1280 If you comment out the next line it will use 1280. + DISPLAY_WIDTH: 1600 + # Default height is 768. If you comment out the next line it will use 768. + DISPLAY_HEIGHT: 960 + + # MAIN SCRIPT ############################################################## + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 8678f3b6e17523daecb292092ce5c9a41d7734ec Mon Sep 17 00:00:00 2001 From: timekills Date: Mon, 27 May 2019 06:47:56 -0700 Subject: [PATCH 147/185] Delete avidemux.ym; --- apps/avidemux.ym; | 88 ----------------------------------------------- 1 file changed, 88 deletions(-) delete mode 100644 apps/avidemux.ym; diff --git a/apps/avidemux.ym; b/apps/avidemux.ym; deleted file mode 100644 index 8b78e56..0000000 --- a/apps/avidemux.ym; +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# -# Title: avidemux -# Author(s): timekills -# URL: https://plexguide.com - http://github.plexguide.com -# GNU: General Public License v3.0 -################################################################################ ---- -- hosts: localhost - gather_facts: false - tasks: - # CORE (MANDATORY) DO NOT CHANGE ########################################### - - - name: 'Set Known Facts' - set_fact: - pgrole: 'avidemux' - intport: '5800' - extport: '5806' -# intport2: '25565' -# extport2: '25565' - image: 'jlesage/avidemux' - - - name: 'Including cron job' - include_tasks: '/opt/communityapps/apps/_core.yml' - - # EXTRA FUNCTIONS REQUIRED BY THE ROLE ##################################### - - - name: 'Create scripts directory for {{pgrole}}' - file: - path: /opt/appdata/{{pgrole}}/scripts - state: directory - owner: 1000 - group: 1000 - mode: 0755 - - - name: 'Create plugins directory for {{pgrole}}' - file: - path: /opt/appdata/{{pgrole}}/plugins - state: directory - owner: 1000 - group: 1000 - mode: 0755 - - # LABELS #### KEEPS BOTTOM CLEAN ########################################### - - name: 'Adding Traefik' - set_fact: - pg_labels: - traefik.frontend.auth.forward.address: '{{gauth}}' - traefik.enable: 'true' - traefik.port: '{{intport}}' - traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' - - - name: 'Setting PG Volumes' - set_fact: - pg_volumes: - - '/opt/appdata/{{pgrole}}:/config:rw' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt:/mnt' - - '/etc/localtime:/etc/localtime:ro' - - - name: 'Setting PG ENV' - set_fact: - pg_env: - PUID: 1000 - PGID: 1000 - # Default width is 1280 If you comment out the next line it will use 1280. - DISPLAY_WIDTH: 1600 - # Default height is 768. If you comment out the next line it will use 768. - DISPLAY_HEIGHT: 960 - - # MAIN SCRIPT ############################################################## - - - name: 'Deploying {{pgrole}}' - docker_container: - name: '{{pgrole}}' - image: '{{image}}' - pull: yes - published_ports: - - '{{ports.stdout}}{{extport}}:{{intport}}' - volumes: '{{pg_volumes}}' - env: '{{pg_env}}' - restart_policy: unless-stopped - networks: - - name: plexguide - aliases: - - '{{pgrole}}' - state: started - labels: '{{pg_labels}}' From ed6f0bbf2c9f354c3bae244cf663dc3820f4fe82 Mon Sep 17 00:00:00 2001 From: timekills Date: Mon, 27 May 2019 06:48:29 -0700 Subject: [PATCH 148/185] AVIDemux Avidemux is a free video editor designed for simple cutting, filtering and encoding tasks. It supports many file types, including AVI, DVD compatible MPEG files, MP4 and ASF, using a variety of codecs. Tasks can be automated using projects, job queue and powerful scripting capabilities. --- apps/avidemux.yml | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 apps/avidemux.yml diff --git a/apps/avidemux.yml b/apps/avidemux.yml new file mode 100644 index 0000000..8b78e56 --- /dev/null +++ b/apps/avidemux.yml @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Title: avidemux +# Author(s): timekills +# URL: https://plexguide.com - http://github.plexguide.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # CORE (MANDATORY) DO NOT CHANGE ########################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'avidemux' + intport: '5800' + extport: '5806' +# intport2: '25565' +# extport2: '25565' + image: 'jlesage/avidemux' + + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # EXTRA FUNCTIONS REQUIRED BY THE ROLE ##################################### + + - name: 'Create scripts directory for {{pgrole}}' + file: + path: /opt/appdata/{{pgrole}}/scripts + state: directory + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Create plugins directory for {{pgrole}}' + file: + path: /opt/appdata/{{pgrole}}/plugins + state: directory + owner: 1000 + group: 1000 + mode: 0755 + + # LABELS #### KEEPS BOTTOM CLEAN ########################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config:rw' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/etc/localtime:/etc/localtime:ro' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: 1000 + PGID: 1000 + # Default width is 1280 If you comment out the next line it will use 1280. + DISPLAY_WIDTH: 1600 + # Default height is 768. If you comment out the next line it will use 768. + DISPLAY_HEIGHT: 960 + + # MAIN SCRIPT ############################################################## + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 107a21900550e2aed341192ad3b5c78920d1e380 Mon Sep 17 00:00:00 2001 From: timekills Date: Thu, 6 Jun 2019 11:48:26 +0430 Subject: [PATCH 149/185] Changed default width and height of GUI Lines 65 and 66. Default width = 1280 Default height =768 You can comment out those two lines to go back to default. --- apps/jd2-openvpn.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/jd2-openvpn.yml b/apps/jd2-openvpn.yml index 292946c..9a0b600 100644 --- a/apps/jd2-openvpn.yml +++ b/apps/jd2-openvpn.yml @@ -62,6 +62,8 @@ pg_env: UID: '1000' GID: '1000' + DISPLAY_WIDTH: '1600' + DISPLAY_HEIGHT: '960' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' From dc2dbf63053da88f366f54a2bfd95eb3e8f6702f Mon Sep 17 00:00:00 2001 From: timekills Date: Thu, 6 Jun 2019 12:18:42 +0430 Subject: [PATCH 150/185] Create firefox Can change width and height of GUI by changing values in the variables under pg-env. Commenting out "DISPLAY_WIDTH" and "DISPLAY HEIGHT" sets the width to 1280 and height to 768 (default values.) --- apps/firefox | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 apps/firefox diff --git a/apps/firefox b/apps/firefox new file mode 100644 index 0000000..b273802 --- /dev/null +++ b/apps/firefox @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Title: firefox +# Author(s): timekills +# URL: https://plexguide.com - http://github.plexguide.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # CORE (MANDATORY) DO NOT CHANGE ########################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'avidemux' + intport: '5800' + extport: '5810' +# intport2: '25565' +# extport2: '25565' + image: 'jlesage/docker-firefox' + + - name: 'Including cron job' + include_tasks: '/opt/communityapps/apps/_core.yml' + + # EXTRA FUNCTIONS REQUIRED BY THE ROLE ##################################### + + - name: 'Create scripts directory for {{pgrole}}' + file: + path: /opt/appdata/{{pgrole}}/scripts + state: directory + owner: 1000 + group: 1000 + mode: 0755 + + - name: 'Create plugins directory for {{pgrole}}' + file: + path: /opt/appdata/{{pgrole}}/plugins + state: directory + owner: 1000 + group: 1000 + mode: 0755 + + # LABELS #### KEEPS BOTTOM CLEAN ########################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config:rw' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/etc/localtime:/etc/localtime:ro' + - '/dev/shm:/dev/shm:rw' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: 1000 + PGID: 1000 + DISPLAY_WIDTH: 1600 + DISPLAY_HEIGHT: 960 + + # MAIN SCRIPT ############################################################## + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From d24321b22bf65d02cc4277c21b0ad8191f5d71ee Mon Sep 17 00:00:00 2001 From: timekills Date: Thu, 6 Jun 2019 12:21:28 +0430 Subject: [PATCH 151/185] Updated GUI size for Firefox Can change width and height of GUI by changing values in the variables under pg-env. Commenting out "DISPLAY_WIDTH" and "DISPLAY HEIGHT" sets the width to 1280 and height to 768 (default values.) --- apps/{firefox => firefox.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/{firefox => firefox.yml} (100%) diff --git a/apps/firefox b/apps/firefox.yml similarity index 100% rename from apps/firefox rename to apps/firefox.yml From 6288cff5c293b9ec157760c1a12630f0050cd285 Mon Sep 17 00:00:00 2001 From: timekills Date: Thu, 6 Jun 2019 12:22:44 +0430 Subject: [PATCH 152/185] Updated GUI size and pgrole for firefox Can change width and height of GUI by changing values in the variables under pg-env. Commenting out "DISPLAY_WIDTH" and "DISPLAY HEIGHT" sets the width to 1280 and height to 768 (default values.) --- apps/firefox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/firefox.yml b/apps/firefox.yml index b273802..4e52dc6 100644 --- a/apps/firefox.yml +++ b/apps/firefox.yml @@ -13,7 +13,7 @@ - name: 'Set Known Facts' set_fact: - pgrole: 'avidemux' + pgrole: 'firefox' intport: '5800' extport: '5810' # intport2: '25565' From 7e1aed27eef41fe1fec7f8b1f514e0368417840a Mon Sep 17 00:00:00 2001 From: timekills Date: Thu, 6 Jun 2019 12:25:13 +0430 Subject: [PATCH 153/185] Update Firefox image to more recent version --- apps/firefox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/firefox.yml b/apps/firefox.yml index 4e52dc6..87c4bab 100644 --- a/apps/firefox.yml +++ b/apps/firefox.yml @@ -18,7 +18,7 @@ extport: '5810' # intport2: '25565' # extport2: '25565' - image: 'jlesage/docker-firefox' + image: 'jlesage/firefox' - name: 'Including cron job' include_tasks: '/opt/communityapps/apps/_core.yml' From 570bca9bf5691276238efed109fafc1d691ecd52 Mon Sep 17 00:00:00 2001 From: timekills Date: Thu, 6 Jun 2019 13:08:47 +0430 Subject: [PATCH 154/185] Create ombi4k.yml By request --- ombi4k.yml | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ombi4k.yml diff --git a/ombi4k.yml b/ombi4k.yml new file mode 100644 index 0000000..8cd097e --- /dev/null +++ b/ombi4k.yml @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Title: PGBlitz (Reference Title File) +# Author(s): Admin9705 +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'ombi4k' + intport: '3579' + extport: '3580' + image: 'linuxserver/ombi' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + # PRETASKS #################################################################### + - name: Check JSON exists + stat: + path: '/opt/appdata/{{pgrole}}/appsettings.json' + register: jsonfile + + - name: 'Download {{pgrole}} appsettings.json config file' + get_url: + url: https://raw.githubusercontent.com/tidusjar/Ombi/master/src/Ombi/appsettings.json + dest: /opt/appdata/{{pgrole}}/appsettings.json + owner: '1000' + group: '1000' + force: no + ignore_errors: True + when: jsonfile.stat.exists == False + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}/appsettings.json:/opt/{{pgrole}}/appsettings.json' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + + # MAIN DEPLOYMENT ############################################################# + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 9cd9d90bb3abf92113351754360161eaf95829c1 Mon Sep 17 00:00:00 2001 From: timekills Date: Thu, 6 Jun 2019 13:13:10 +0430 Subject: [PATCH 155/185] Delete ombi4k.yml --- ombi4k.yml | 80 ------------------------------------------------------ 1 file changed, 80 deletions(-) delete mode 100644 ombi4k.yml diff --git a/ombi4k.yml b/ombi4k.yml deleted file mode 100644 index 8cd097e..0000000 --- a/ombi4k.yml +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# -# Title: PGBlitz (Reference Title File) -# Author(s): Admin9705 -# URL: https://pgblitz.com - http://github.pgblitz.com -# GNU: General Public License v3.0 -################################################################################ ---- -- hosts: localhost - gather_facts: false - tasks: - # FACTS ####################################################################### - - - name: 'Set Known Facts' - set_fact: - pgrole: 'ombi4k' - intport: '3579' - extport: '3580' - image: 'linuxserver/ombi' - - # CORE (MANDATORY) ############################################################ - - name: 'Including cron job' - include_tasks: '/opt/coreapps/apps/_core.yml' - - # PRETASKS #################################################################### - - name: Check JSON exists - stat: - path: '/opt/appdata/{{pgrole}}/appsettings.json' - register: jsonfile - - - name: 'Download {{pgrole}} appsettings.json config file' - get_url: - url: https://raw.githubusercontent.com/tidusjar/Ombi/master/src/Ombi/appsettings.json - dest: /opt/appdata/{{pgrole}}/appsettings.json - owner: '1000' - group: '1000' - force: no - ignore_errors: True - when: jsonfile.stat.exists == False - - # LABELS ###################################################################### - - name: 'Adding Traefik' - set_fact: - pg_labels: - traefik.enable: 'true' - traefik.port: '{{intport}}' - traefik.frontend.auth.forward.address: '{{gauth}}' - traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' - - - name: 'Setting PG Volumes' - set_fact: - pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - - '/etc/localtime:/etc/localtime:ro' - - '/opt/appdata/{{pgrole}}/appsettings.json:/opt/{{pgrole}}/appsettings.json' - - - name: 'Setting PG ENV' - set_fact: - pg_env: - PUID: '1000' - PGID: '1000' - - # MAIN DEPLOYMENT ############################################################# - - - name: 'Deploying {{pgrole}}' - docker_container: - name: '{{pgrole}}' - image: '{{image}}' - pull: yes - published_ports: - - '{{ports.stdout}}{{extport}}:{{intport}}' - volumes: '{{pg_volumes}}' - env: '{{pg_env}}' - restart_policy: unless-stopped - networks: - - name: plexguide - aliases: - - '{{pgrole}}' - state: started - labels: '{{pg_labels}}' From 84b870b373ea279fda91aa658f532d525180189f Mon Sep 17 00:00:00 2001 From: timekills Date: Thu, 6 Jun 2019 13:14:11 +0430 Subject: [PATCH 156/185] Create ombi4k.yml By request --- apps/ombi4k.yml | 80 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 apps/ombi4k.yml diff --git a/apps/ombi4k.yml b/apps/ombi4k.yml new file mode 100644 index 0000000..7674cc2 --- /dev/null +++ b/apps/ombi4k.yml @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Title: Ombi4k (Reference Title File) +# Author(s): Admin9705; timekills mod +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'ombi4k' + intport: '3579' + extport: '3580' + image: 'linuxserver/ombi' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + # PRETASKS #################################################################### + - name: Check JSON exists + stat: + path: '/opt/appdata/{{pgrole}}/appsettings.json' + register: jsonfile + + - name: 'Download {{pgrole}} appsettings.json config file' + get_url: + url: https://raw.githubusercontent.com/tidusjar/Ombi/master/src/Ombi/appsettings.json + dest: /opt/appdata/{{pgrole}}/appsettings.json + owner: '1000' + group: '1000' + force: no + ignore_errors: True + when: jsonfile.stat.exists == False + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}/appsettings.json:/opt/{{pgrole}}/appsettings.json' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + + # MAIN DEPLOYMENT ############################################################# + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From 5e1fc0ebc2761b4def7ab3dd2ad3e3386d4a0ce4 Mon Sep 17 00:00:00 2001 From: Artiume Date: Fri, 7 Jun 2019 20:24:19 -0400 Subject: [PATCH 157/185] Update firefox.yml Adds HSTS Compliance --- apps/firefox.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apps/firefox.yml b/apps/firefox.yml index 87c4bab..c1bd84c 100644 --- a/apps/firefox.yml +++ b/apps/firefox.yml @@ -49,6 +49,16 @@ traefik.enable: 'true' traefik.port: '{{intport}}' traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + traefik.frontend.headers.SSLHost: '{{domain.stdout}}' + traefik.frontend.headers.SSLRedirect: 'true' + traefik.frontend.headers.STSIncludeSubdomains: 'true' + traefik.frontend.headers.STSPreload: 'true' + traefik.frontend.headers.STSSeconds: '315360000' + traefik.frontend.headers.browserXSSFilter: 'true' + traefik.frontend.headers.contentTypeNosniff: 'true' + traefik.frontend.headers.customResponseHeaders: 'X-Robots-Tag:noindex,nofollow,nosnippet,noarchive,notranslate,noimageindex' + traefik.frontend.headers.forceSTSHeader: 'true' + #traefik.frontend.headers.frameDeny: 'true' - name: 'Setting PG Volumes' set_fact: From 962a15e1e1ee6942377c8520bb96f58031abe536 Mon Sep 17 00:00:00 2001 From: Artiume Date: Tue, 11 Jun 2019 21:13:06 -0400 Subject: [PATCH 158/185] Update Organizr for HSTS Compliance Adds update for HSTS Compliance. frameDeny enabled will prevent Organizr from being iFrame'd. --- apps/organizr.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apps/organizr.yml b/apps/organizr.yml index 70530a1..2a89008 100644 --- a/apps/organizr.yml +++ b/apps/organizr.yml @@ -29,6 +29,16 @@ traefik.enable: 'true' traefik.port: '{{intport}}' traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + traefik.frontend.headers.SSLHost: '{{domain.stdout}}' + traefik.frontend.headers.SSLRedirect: 'true' + traefik.frontend.headers.STSIncludeSubdomains: 'true' + traefik.frontend.headers.STSPreload: 'true' + traefik.frontend.headers.STSSeconds: '315360000' + traefik.frontend.headers.browserXSSFilter: 'true' + traefik.frontend.headers.contentTypeNosniff: 'true' + traefik.frontend.headers.customResponseHeaders: 'X-Robots-Tag:noindex,nofollow,nosnippet,noarchive,notranslate,noimageindex' + traefik.frontend.headers.forceSTSHeader: 'true' + traefik.frontend.headers.frameDeny: 'true' - name: 'Setting PG Volumes' set_fact: From 022e47a0974520dce20e4a248a03189d0c7a561a Mon Sep 17 00:00:00 2001 From: Cody Date: Sat, 15 Jun 2019 05:58:44 -0400 Subject: [PATCH 159/185] Update and rename rutorrent-openvpn.yml to rutorrent-vpn.yml File/Role rename for simplicity/conformity. --- apps/{rutorrent-openvpn.yml => rutorrent-vpn.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename apps/{rutorrent-openvpn.yml => rutorrent-vpn.yml} (98%) diff --git a/apps/rutorrent-openvpn.yml b/apps/rutorrent-vpn.yml similarity index 98% rename from apps/rutorrent-openvpn.yml rename to apps/rutorrent-vpn.yml index 9fd1766..4824609 100644 --- a/apps/rutorrent-openvpn.yml +++ b/apps/rutorrent-vpn.yml @@ -13,7 +13,7 @@ - name: 'Set Known Facts' set_fact: - pgrole: 'rutorrent-openvpn' + pgrole: 'rutorrent-vpn' intport: '80' extport: '5897' image: 'h1f0x/rtorrent-rutorrent-openvpn' From 5db185c793657da450ffd74d3cf8a23d18ee4c75 Mon Sep 17 00:00:00 2001 From: Cody Date: Sat, 15 Jun 2019 06:00:05 -0400 Subject: [PATCH 160/185] Update and rename rflood-openvpn.yml to rflood-vpn.yml File/Role rename for simplicity/conformity. --- apps/{rflood-openvpn.yml => rflood-vpn.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename apps/{rflood-openvpn.yml => rflood-vpn.yml} (98%) diff --git a/apps/rflood-openvpn.yml b/apps/rflood-vpn.yml similarity index 98% rename from apps/rflood-openvpn.yml rename to apps/rflood-vpn.yml index 552de75..331cbe3 100644 --- a/apps/rflood-openvpn.yml +++ b/apps/rflood-vpn.yml @@ -13,7 +13,7 @@ - name: 'Set Known Facts' set_fact: - pgrole: 'rflood-openvpn' + pgrole: 'rflood' intport: '80' extport: '5896' image: 'h1f0x/rtorrent-flood-openvpn' From ec48789f83cb93046ef4bb33b54cd05bfaed71f2 Mon Sep 17 00:00:00 2001 From: Cody Date: Sat, 15 Jun 2019 06:00:59 -0400 Subject: [PATCH 161/185] Rename delugevpn.yml to deluge-vpn.yml File rename for simplicity/conformity. --- apps/{delugevpn.yml => deluge-vpn.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/{delugevpn.yml => deluge-vpn.yml} (100%) diff --git a/apps/delugevpn.yml b/apps/deluge-vpn.yml similarity index 100% rename from apps/delugevpn.yml rename to apps/deluge-vpn.yml From 453fa4c16c36952bdb6507338ded276be454ba1d Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 13:00:59 -0400 Subject: [PATCH 162/185] Initial qbittorrent-vpn commit --- apps/qbittorrent-vpn.yml | 292 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 apps/qbittorrent-vpn.yml diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml new file mode 100644 index 0000000..edb0d85 --- /dev/null +++ b/apps/qbittorrent-vpn.yml @@ -0,0 +1,292 @@ +#!/bin/bash +# +# Title: qBittorrent (with OpenVPN and Privoxy) +# Author(s): Mike Tzou (Chocobo1), Vladimir Golovnev (glassez), Diego Heras (ngosang), binhex +# URL: https://www.qbittorrent.org/ - https://github.com/binhex/arch-qbittorrentvpn +# GNU: General Public License v2.0 (w/ exception) +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'qbittorrent' + intport: '8998' + extport: '8998' + intport1: '8118' + extport1: '8118' + intport2: '6881' + extport2: '6881' + dnsserver1: '1.1.1.1' + dnsserver2: '84.200.69.80' + dnsserver3: '37.235.1.174' + dnsserver4: '185.121.177.177' + image: 'binhex/arch-qbittorrentvpn:latest' + + # CORE (MANDATORY) ############################################################# + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + - name: 'Including folders' + include_tasks: '/opt/coreapps/apps/_downloaders.yml' + + - name: 'Including plugins' + include_tasks: '/opt/coreapps/apps/_plugins.yml' + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + UMASK: '000' + VPN_ENABLED: 'no' + VPN_USER: 'username' + VPN_PASS: 'password' + VPN_PROV: 'custom' + VPN_OPTIONS: '--inactive 3600 --ping 10 --ping-exit 60' + STRICT_PORT_FORWARD: 'no' + ENABLE_PRIVOXY: 'no' + LAN_NETWORK: '127.20.0.0/16' + NAME_SERVERS: '{{dnsserver1}},{{dnsserver2}},{{dnsserver3}},{{dnsserver4}}' + DEBUG: 'false' + WEBUI_PORT: '{{intport}}' + + + + + # MAIN DEPLOYMENT ############################################################# + - name: 'Checking for existing app data' + stat: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + register: confcheck + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + devices: + - '/dev/net/tun:/dev/net/tun:rwm' + dns_servers: + - '{{dnsserver1}}' + - '{{dnsserver2}}' + - '{{dnsserver3}}' + - '{{dnsserver4}}' + capabilities: + - NET_ADMIN + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' + + # CONFIGURATION #################################################### + - name: 'Waiting for {{pgrole}} to initialize' + wait_for: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + state: present + timeout: 15 + delay: 5 + + - name: 'Stopping {{pgrole}}' + docker_container: + name: '{{pgrole}}' + state: stopped + + - name: Set SavePath + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Downloads\SavePath + value: '{{path.stdout}}/downloads/{{pgrole}}' + state: present + + + - name: Set TempPathEnabled + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Downloads\TempPathEnabled + value: True + state: present + + # FIRST TIME CONFIGURATION #################################################### + - name: 'Configuring {{pgrole}} for first time use' + block: + - name: Enable AutoRun + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: AutoRun + option: enabled + value: True + state: present + + - name: Enable Auto UnRar + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: AutoRun + option: program + value: '/usr/bin/unrar x -r \"%F/.\" \"%F/\"' + state: present + + - name: Disable DHT + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Bittorrent\DHT + value: false + state: present + + - name: Disable PeX + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Bittorrent\PeX + value: false + state: present + + - name: Disable LSD + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Bittorrent\LSD + value: false + state: present + + - name: Enable Encryption + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Bittorrent\Encryption + value: 1 + state: present + + - name: Don't use incomplete extension + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Downloads\UseIncompleteExtension + value: false + state: present + - name: Ignore slow torrents + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Queueing\IgnoreSlowTorrents + value: True + state: present + - name: Set MaxActiveDownloads + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Queueing\MaxActiveDownloads + value: 10 + state: present + - name: Set MaxActiveTorrents + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Queueing\MaxActiveTorrents + value: 100 + state: present + - name: Set MaxActiveUploads + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Queueing\MaxActiveUploads + value: 100 + state: present + - name: Set GlobalMaxSeedingMinutes + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: BitTorrent + option: Session\GlobalMaxSeedingMinutes + value: 20160 + state: present + - name: Set MaxRatio + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Bittorrent\MaxRatio + value: 1 + state: present + - name: pause on MaxRatioAction + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: Bittorrent\MaxRatioAction + value: 1 + state: present + - name: Disable csrf + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: WebUI\CSRFProtection + value: false + force: yes + state: present + - name: Disable HostHeaderValidation + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: WebUI\HostHeaderValidation + value: false + force: yes + state: present + - name: Disable ClickjackingProtection + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: WebUI\ClickjackingProtection + value: false + force: yes + state: present + + - name: Set ServerDomains + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: WebUI\ServerDomains + value: '*' + force: yes + state: present + - name: Set Address + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + section: Preferences + option: WebUI\Address + value: '*' + force: yes + state: present + + when: not confcheck.stat.exists + - name: Restart {{pgrole}} + docker_container: + name: '{{pgrole}}' + state: started \ No newline at end of file From 1b0cc0aa176d27cf685cd3f83bf4dd03eb593c69 Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 13:21:55 -0400 Subject: [PATCH 163/185] Update config path --- apps/qbittorrent-vpn.yml | 46 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index edb0d85..b84a794 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -77,7 +77,7 @@ # MAIN DEPLOYMENT ############################################################# - name: 'Checking for existing app data' stat: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' register: confcheck - name: 'Deploying {{pgrole}}' @@ -109,7 +109,7 @@ # CONFIGURATION #################################################### - name: 'Waiting for {{pgrole}} to initialize' wait_for: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' state: present timeout: 15 delay: 5 @@ -121,7 +121,7 @@ - name: Set SavePath ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Downloads\SavePath value: '{{path.stdout}}/downloads/{{pgrole}}' @@ -130,7 +130,7 @@ - name: Set TempPathEnabled ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Downloads\TempPathEnabled value: True @@ -141,7 +141,7 @@ block: - name: Enable AutoRun ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: AutoRun option: enabled value: True @@ -149,7 +149,7 @@ - name: Enable Auto UnRar ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: AutoRun option: program value: '/usr/bin/unrar x -r \"%F/.\" \"%F/\"' @@ -157,7 +157,7 @@ - name: Disable DHT ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Bittorrent\DHT value: false @@ -165,7 +165,7 @@ - name: Disable PeX ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Bittorrent\PeX value: false @@ -173,7 +173,7 @@ - name: Disable LSD ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Bittorrent\LSD value: false @@ -181,7 +181,7 @@ - name: Enable Encryption ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Bittorrent\Encryption value: 1 @@ -189,63 +189,63 @@ - name: Don't use incomplete extension ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Downloads\UseIncompleteExtension value: false state: present - name: Ignore slow torrents ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Queueing\IgnoreSlowTorrents value: True state: present - name: Set MaxActiveDownloads ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Queueing\MaxActiveDownloads value: 10 state: present - name: Set MaxActiveTorrents ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Queueing\MaxActiveTorrents value: 100 state: present - name: Set MaxActiveUploads ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Queueing\MaxActiveUploads value: 100 state: present - name: Set GlobalMaxSeedingMinutes ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: BitTorrent option: Session\GlobalMaxSeedingMinutes value: 20160 state: present - name: Set MaxRatio ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Bittorrent\MaxRatio value: 1 state: present - name: pause on MaxRatioAction ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: Bittorrent\MaxRatioAction value: 1 state: present - name: Disable csrf ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: WebUI\CSRFProtection value: false @@ -253,7 +253,7 @@ state: present - name: Disable HostHeaderValidation ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: WebUI\HostHeaderValidation value: false @@ -261,7 +261,7 @@ state: present - name: Disable ClickjackingProtection ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: WebUI\ClickjackingProtection value: false @@ -270,7 +270,7 @@ - name: Set ServerDomains ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: WebUI\ServerDomains value: '*' @@ -278,7 +278,7 @@ state: present - name: Set Address ini_file: - path: '/opt/appdata/{{pgrole}}/qBittorrent/qBittorrent.conf' + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' section: Preferences option: WebUI\Address value: '*' From e871e79fcd4693cfd8a7658e7c1d1a7ac5067c30 Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 14:40:32 -0400 Subject: [PATCH 164/185] Additional WebUI, blocklist config --- apps/qbittorrent-vpn.yml | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index b84a794..46f70d5 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -87,6 +87,8 @@ pull: yes published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' + - '{{ports.stdout}}{{extport1}}:{{intport1}}' + - '{{ports.stdout}}{{extport2}}:{{intport2}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: unless-stopped @@ -136,6 +138,10 @@ value: True state: present + - name: Get latest blocklist + shell: /usr/bin/curl -L http://john.bitsurge.net/public/biglist.p2p.gz | /usr/bin/gzip -cdf > /config/qBittorrent/config/biglist.p2p + + # FIRST TIME CONFIGURATION #################################################### - name: 'Configuring {{pgrole}} for first time use' block: @@ -284,6 +290,51 @@ value: '*' force: yes state: present + + - name: Set Port + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' + section: Preferences + option: Connection\PortRangeMin + value: '{{intport2}}' + force: yes + state: present + + - name: Set torrent Watch folder + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' + section: Preferences + option: Downloads\ScanDirsV2 + value: '@Variant(\0\0\0\x1c\0\0\0\x1\0\0\0$\0/\0m\0n\0t\0/\0t\0o\0r\0r\0\x65\0n\0t\0/\0w\0\x61\0t\0\x63\0h\0\0\0\x2\0\0\0\0)' + force: yes + state: present + + - name: Set Blocklist enabled + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' + section: Preferences + option: IPFilter\Enabled + value: 'true' + force: yes + state: present + + - name: Set Blocklist location + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' + section: Preferences + option: IPFilter\File + value: '/config/qBittorrent/config/biglist.p2p' + force: yes + state: present + + - name: Set default login + ini_file: + path: '/opt/appdata/{{pgrole}}/qBittorrent/config/qBittorrent.conf' + section: Preferences + option: WebUI\Password_ha1 + value: '@ByteArray(22f616dbc8cdb4aa96105b1c8f36ea63)' + force: yes + state: present when: not confcheck.stat.exists - name: Restart {{pgrole}} From cb039c47f35f186a754b8119ff4263b13d3ae05e Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 14:42:34 -0400 Subject: [PATCH 165/185] Blocklist path update --- apps/qbittorrent-vpn.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index 46f70d5..8a7a93d 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -139,7 +139,7 @@ state: present - name: Get latest blocklist - shell: /usr/bin/curl -L http://john.bitsurge.net/public/biglist.p2p.gz | /usr/bin/gzip -cdf > /config/qBittorrent/config/biglist.p2p + shell: /usr/bin/curl -L http://john.bitsurge.net/public/biglist.p2p.gz | /usr/bin/gzip -cdf > /opt/appdata/{{pgrole}}/qBittorrent/config/biglist.p2p # FIRST TIME CONFIGURATION #################################################### From 23aec4d022b311af26906d3599c252a050ab8772 Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 14:46:42 -0400 Subject: [PATCH 166/185] Removed hardcoded exe location --- apps/qbittorrent-vpn.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index 8a7a93d..b160be6 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -139,7 +139,7 @@ state: present - name: Get latest blocklist - shell: /usr/bin/curl -L http://john.bitsurge.net/public/biglist.p2p.gz | /usr/bin/gzip -cdf > /opt/appdata/{{pgrole}}/qBittorrent/config/biglist.p2p + shell: curl -L http://john.bitsurge.net/public/biglist.p2p.gz | gzip -cdf > /opt/appdata/{{pgrole}}/qBittorrent/config/biglist.p2p # FIRST TIME CONFIGURATION #################################################### From 8e1f47c95e7dde9bc1f2c50f2985f0e21f1f6672 Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 15:08:04 -0400 Subject: [PATCH 167/185] Initialize ovpn folder --- apps/qbittorrent-vpn.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index b160be6..5fad3fd 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -140,6 +140,17 @@ - name: Get latest blocklist shell: curl -L http://john.bitsurge.net/public/biglist.p2p.gz | gzip -cdf > /opt/appdata/{{pgrole}}/qBittorrent/config/biglist.p2p + + - name: 'Checking for existing openvpn folder' + stat: + path: '/opt/appdata/{{pgrole}}/openvpn' + register: ovpncheck + + - name: Initialize opvn + block: + - name: Create opvn folder + shell: mkdir /opt/appdata/{{pgrole}}/openvpn && touch mkdir /opt/appdata/{{pgrole}}/openvpn/OPVN\ files\ go\ here > && chown -R {{PUID}}:{{PGID}} /opt/appdata/{{pgrole}}/openvpn + when: not opvncheck.stat.exists # FIRST TIME CONFIGURATION #################################################### From 0d82d0ae93523fb109ee4efd3b5896497ef34d41 Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 15:11:23 -0400 Subject: [PATCH 168/185] Fix typo --- apps/qbittorrent-vpn.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index 5fad3fd..9aa131f 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -150,7 +150,7 @@ block: - name: Create opvn folder shell: mkdir /opt/appdata/{{pgrole}}/openvpn && touch mkdir /opt/appdata/{{pgrole}}/openvpn/OPVN\ files\ go\ here > && chown -R {{PUID}}:{{PGID}} /opt/appdata/{{pgrole}}/openvpn - when: not opvncheck.stat.exists + when: not ovpncheck.stat.exists # FIRST TIME CONFIGURATION #################################################### From fd1aa0d61e407009d82c5fa19222616a4ac94d1e Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 15:16:24 -0400 Subject: [PATCH 169/185] Set puid,pgid as global attrib --- apps/qbittorrent-vpn.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index 9aa131f..38f132d 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -24,6 +24,8 @@ dnsserver2: '84.200.69.80' dnsserver3: '37.235.1.174' dnsserver4: '185.121.177.177' + puid: '1000' + pgid: '1000' image: 'binhex/arch-qbittorrentvpn:latest' # CORE (MANDATORY) ############################################################# @@ -56,8 +58,8 @@ - name: 'Setting PG ENV' set_fact: pg_env: - PUID: '1000' - PGID: '1000' + PUID: '{{puid}}' + PGID: '{{pgid}}' UMASK: '000' VPN_ENABLED: 'no' VPN_USER: 'username' @@ -149,7 +151,7 @@ - name: Initialize opvn block: - name: Create opvn folder - shell: mkdir /opt/appdata/{{pgrole}}/openvpn && touch mkdir /opt/appdata/{{pgrole}}/openvpn/OPVN\ files\ go\ here > && chown -R {{PUID}}:{{PGID}} /opt/appdata/{{pgrole}}/openvpn + shell: mkdir /opt/appdata/{{pgrole}}/openvpn && touch mkdir /opt/appdata/{{pgrole}}/openvpn/OPVN\ files\ go\ here > && chown -R {{puid}}:{{pgid}} /opt/appdata/{{pgrole}}/openvpn when: not ovpncheck.stat.exists From 8fa8512c20e86861e9206d2b3187ed7269124e17 Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 15:19:08 -0400 Subject: [PATCH 170/185] Fix typo --- apps/qbittorrent-vpn.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index 38f132d..bec102d 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -151,7 +151,7 @@ - name: Initialize opvn block: - name: Create opvn folder - shell: mkdir /opt/appdata/{{pgrole}}/openvpn && touch mkdir /opt/appdata/{{pgrole}}/openvpn/OPVN\ files\ go\ here > && chown -R {{puid}}:{{pgid}} /opt/appdata/{{pgrole}}/openvpn + shell: mkdir /opt/appdata/{{pgrole}}/openvpn && touch mkdir /opt/appdata/{{pgrole}}/openvpn/OPVN\ files\ go\ here && chown -R {{puid}}:{{pgid}} /opt/appdata/{{pgrole}}/openvpn when: not ovpncheck.stat.exists From fe9df6d4e23037195905379825f75161303cea76 Mon Sep 17 00:00:00 2001 From: ChaosZero112 Date: Sat, 15 Jun 2019 17:46:13 -0400 Subject: [PATCH 171/185] Removed VPN Options as they could conflict with some configurations --- apps/qbittorrent-vpn.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index bec102d..5c4e955 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -65,7 +65,7 @@ VPN_USER: 'username' VPN_PASS: 'password' VPN_PROV: 'custom' - VPN_OPTIONS: '--inactive 3600 --ping 10 --ping-exit 60' + VPN_OPTIONS: '' STRICT_PORT_FORWARD: 'no' ENABLE_PRIVOXY: 'no' LAN_NETWORK: '127.20.0.0/16' @@ -151,7 +151,7 @@ - name: Initialize opvn block: - name: Create opvn folder - shell: mkdir /opt/appdata/{{pgrole}}/openvpn && touch mkdir /opt/appdata/{{pgrole}}/openvpn/OPVN\ files\ go\ here && chown -R {{puid}}:{{pgid}} /opt/appdata/{{pgrole}}/openvpn + shell: mkdir /opt/appdata/{{pgrole}}/openvpn && touch mkdir /opt/appdata/{{pgrole}}/openvpn/OVPN\ files\ go\ here && chown -R {{puid}}:{{pgid}} /opt/appdata/{{pgrole}}/openvpn when: not ovpncheck.stat.exists From ea04517bbe77e49c3231102ed0b51b7934bd10d0 Mon Sep 17 00:00:00 2001 From: timekills Date: Fri, 21 Jun 2019 01:45:22 +0430 Subject: [PATCH 172/185] Ported host (PGBlitz) rclone.conf to Docker container In RcloneBrowser's File:Preferences where it asks for the rclone.conf file, the host (PGBlitz's) directory containing rclone.conf is ported to the Docker container as Read Only. -Update the rclone.conf file through the File then Preferences setting in the GUI.Choose the rclone.conf file in the "host_rcloneconf_folder" directory. --- apps/rclonebrowser.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/rclonebrowser.yml b/apps/rclonebrowser.yml index 876bff1..c9bb3aa 100644 --- a/apps/rclonebrowser.yml +++ b/apps/rclonebrowser.yml @@ -39,6 +39,7 @@ - '{{path.stdout}}:{{path.stdout}}' - '/mnt/unionfs:/unionfs:rw' - '/mnt:/mnt:rw' + - '/opt/appdata/plexguide:/host_rcloneconf_folder:ro' - '/etc/localtime:/etc/localtime:ro' - name: 'Setting PG ENV' From d4d6b9b338c8970e6a185b3646a3f605dbf1b679 Mon Sep 17 00:00:00 2001 From: timekills Date: Sat, 22 Jun 2019 17:54:44 +0430 Subject: [PATCH 173/185] Create ombiHDR.yml --- apps/ombiHDR.yml | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 apps/ombiHDR.yml diff --git a/apps/ombiHDR.yml b/apps/ombiHDR.yml new file mode 100644 index 0000000..e9f87f5 --- /dev/null +++ b/apps/ombiHDR.yml @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Title: OmbiHDR +# Author(s): Admin9705; timekills mod +# URL: https://pgblitz.com - http://github.pgblitz.com +# GNU: General Public License v3.0 +################################################################################ +--- +- hosts: localhost + gather_facts: false + tasks: + # FACTS ####################################################################### + + - name: 'Set Known Facts' + set_fact: + pgrole: 'ombiHDR' + intport: '3579' + extport: '3581' + image: 'linuxserver/ombi' + + # CORE (MANDATORY) ############################################################ + - name: 'Including cron job' + include_tasks: '/opt/coreapps/apps/_core.yml' + + # PRETASKS #################################################################### + - name: Check JSON exists + stat: + path: '/opt/appdata/{{pgrole}}/appsettings.json' + register: jsonfile + + - name: 'Download {{pgrole}} appsettings.json config file' + get_url: + url: https://raw.githubusercontent.com/tidusjar/Ombi/master/src/Ombi/appsettings.json + dest: /opt/appdata/{{pgrole}}/appsettings.json + owner: '1000' + group: '1000' + force: no + ignore_errors: True + when: jsonfile.stat.exists == False + + # LABELS ###################################################################### + - name: 'Adding Traefik' + set_fact: + pg_labels: + traefik.enable: 'true' + traefik.port: '{{intport}}' + traefik.frontend.auth.forward.address: '{{gauth}}' + traefik.frontend.rule: 'Host:{{pgrole}}.{{domain.stdout}},{{tldset}}' + + - name: 'Setting PG Volumes' + set_fact: + pg_volumes: + - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}/appsettings.json:/opt/{{pgrole}}/appsettings.json' + + - name: 'Setting PG ENV' + set_fact: + pg_env: + PUID: '1000' + PGID: '1000' + + # MAIN DEPLOYMENT ############################################################# + + - name: 'Deploying {{pgrole}}' + docker_container: + name: '{{pgrole}}' + image: '{{image}}' + pull: yes + published_ports: + - '{{ports.stdout}}{{extport}}:{{intport}}' + volumes: '{{pg_volumes}}' + env: '{{pg_env}}' + restart_policy: unless-stopped + networks: + - name: plexguide + aliases: + - '{{pgrole}}' + state: started + labels: '{{pg_labels}}' From e468a44e042262766ec573a398519ed27e24851b Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sat, 22 Jun 2019 18:14:21 +0200 Subject: [PATCH 174/185] Update domoticz.yml --- apps/domoticz.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/domoticz.yml b/apps/domoticz.yml index 4054ac5..ee465ea 100644 --- a/apps/domoticz.yml +++ b/apps/domoticz.yml @@ -19,7 +19,6 @@ extport2: '6144' intport3: '1443/tcp' extport3: '1443' - image: 'linuxserver/domoticz:stable' # CORE (MANDATORY) ############################################################ @@ -47,7 +46,7 @@ PUID: '1000' PGID: '1000' TZ: '${TZ}' - + # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' docker_container: From 4d9823cdfe6b9fcb4745cfd9b5d30c76dc66a214 Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sat, 22 Jun 2019 19:05:44 +0200 Subject: [PATCH 175/185] Update domoticz.yml Puah --- apps/domoticz.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/domoticz.yml b/apps/domoticz.yml index ee465ea..425176a 100644 --- a/apps/domoticz.yml +++ b/apps/domoticz.yml @@ -56,8 +56,7 @@ published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{ports.stdout}}{{extport3}}:{{intport3}}' - - '{{ports.stdout}}{{extport4}}:{{intport4}}' + - '{{ports.stdout}}{{extport3}}:{{intport3}}'}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: unless-stopped From f139de00bbe466a2bab1b95488e253f514d17e1b Mon Sep 17 00:00:00 2001 From: MrDoobPG <46342172+MrDoobPG@users.noreply.github.com> Date: Sat, 22 Jun 2019 19:55:22 +0200 Subject: [PATCH 176/185] Update domoticz.yml Push v3 --- apps/domoticz.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/domoticz.yml b/apps/domoticz.yml index 425176a..63ed8d3 100644 --- a/apps/domoticz.yml +++ b/apps/domoticz.yml @@ -56,7 +56,7 @@ published_ports: - '{{ports.stdout}}{{extport}}:{{intport}}' - '{{ports.stdout}}{{extport2}}:{{intport2}}' - - '{{ports.stdout}}{{extport3}}:{{intport3}}'}}' + - '{{ports.stdout}}{{extport3}}:{{intport3}}' volumes: '{{pg_volumes}}' env: '{{pg_env}}' restart_policy: unless-stopped From 56db76fd5de7ce6da4efaa58a75f1bf351f433d5 Mon Sep 17 00:00:00 2001 From: LooseSeal2 Date: Sun, 23 Jun 2019 12:00:16 -0700 Subject: [PATCH 177/185] fix --- apps/_appsgen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/_appsgen.sh b/apps/_appsgen.sh index aa87c16..3f4754d 100644 --- a/apps/_appsgen.sh +++ b/apps/_appsgen.sh @@ -7,7 +7,7 @@ ################################################################################ # Generates App List -ls -la /opt/coreapps/apps/ | sed -e 's/.yml//g' \ +ls -la /opt/communityapps/apps/ | sed -e 's/.yml//g' \ | awk '{print $9}' | tail -n +4 > /var/plexguide/app.list ls -la /opt/mycontainers/ | sed -e 's/.yml//g' \ From 886ff9f1012a501ee03ab58105004edd0f1218d9 Mon Sep 17 00:00:00 2001 From: LooseSeal2 Date: Tue, 25 Jun 2019 19:04:03 -0700 Subject: [PATCH 178/185] fix --- apps/templates/broken/nzbthrottle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/templates/broken/nzbthrottle.yml b/apps/templates/broken/nzbthrottle.yml index ac56db7..550ecaf 100644 --- a/apps/templates/broken/nzbthrottle.yml +++ b/apps/templates/broken/nzbthrottle.yml @@ -21,7 +21,7 @@ - name: 'Including cron job' include_tasks: '/opt/communityapps/apps/_core.yml' - - name: 'Checking {{pgrole}}'s json existance' + - name: 'Checking {{pgrole}} for existing app data' stat: path: '/opt/communityapps/apps/templates/{{pgrole}}/config.json' register: jsoncheck From 06d3121aab49cebe965d19b8f01433dd2fd5c77e Mon Sep 17 00:00:00 2001 From: CasperNielsen <27451397+CasperNielsen@users.noreply.github.com> Date: Fri, 28 Jun 2019 11:10:38 +0200 Subject: [PATCH 179/185] Update deluge.yml Edited True values to true, Deluge 2.0 is case sensitive. --- apps/deluge.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/deluge.yml b/apps/deluge.yml index 7918f6e..1450fe4 100644 --- a/apps/deluge.yml +++ b/apps/deluge.yml @@ -134,14 +134,14 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"move_completed".*' - line: '"move_completed": True,' + line: '"move_completed": true,' state: present - name: Setting extractor folder lineinfile: path: '/opt/appdata/{{pgrole}}/plugins/extractor.conf' regexp: '"use_name_folder".*' - line: '"use_name_folder": True,' + line: '"use_name_folder": true,' state: present - name: Setting extractor @@ -165,7 +165,7 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"compact_allocation".*' - line: '"compact_allocation": True,' + line: '"compact_allocation": true,' state: present - name: set stop_seed_ratio @@ -186,14 +186,14 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"remove_seed_at_ratio".*' - line: '"remove_seed_at_ratio": True,' + line: '"remove_seed_at_ratio": true,' state: present - name: set enc_prefer_rc4 lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"enc_prefer_rc4".*' - line: '"enc_prefer_rc4": True,' + line: '"enc_prefer_rc4": true,' state: present - name: set enc_level @@ -228,7 +228,7 @@ lineinfile: path: '/opt/appdata/{{pgrole}}/core.conf' regexp: '"dont_count_slow_torrents".*' - line: '"dont_count_slow_torrents": True,' + line: '"dont_count_slow_torrents": true,' state: present - name: set max_active_seeding From be3f7eb3e275acd885f26745e7a26f7dbfbbe809 Mon Sep 17 00:00:00 2001 From: LooseSeal2 Date: Tue, 9 Jul 2019 23:17:52 -0700 Subject: [PATCH 180/185] shield fix --- apps/_appsgen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/_appsgen.sh b/apps/_appsgen.sh index 3f4754d..a0e5ccd 100644 --- a/apps/_appsgen.sh +++ b/apps/_appsgen.sh @@ -8,7 +8,7 @@ # Generates App List ls -la /opt/communityapps/apps/ | sed -e 's/.yml//g' \ -| awk '{print $9}' | tail -n +4 > /var/plexguide/app.list +| awk '{print $9}' | tail -n +4 >> /var/plexguide/app.list ls -la /opt/mycontainers/ | sed -e 's/.yml//g' \ | awk '{print $9}' | tail -n +4 >> /var/plexguide/app.list From 1eb59dc5ebfed5d0a01845ab3e44310701a7358c Mon Sep 17 00:00:00 2001 From: LooseSeal2 Date: Wed, 10 Jul 2019 12:26:42 -0700 Subject: [PATCH 181/185] fix --- apps/_cron.list | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/_cron.list b/apps/_cron.list index 51857fe..d7eff0f 100644 --- a/apps/_cron.list +++ b/apps/_cron.list @@ -4,3 +4,4 @@ netdata alltube dockergc blitzui +watchtower From 142817d166177f2821c5d8b578eb032bba32353c Mon Sep 17 00:00:00 2001 From: Ryan Fisher Date: Thu, 11 Jul 2019 09:44:32 -0500 Subject: [PATCH 182/185] Don't let the community appsgen.sh clobber the core, just append --- apps/_appsgen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/_appsgen.sh b/apps/_appsgen.sh index 3f4754d..a0e5ccd 100644 --- a/apps/_appsgen.sh +++ b/apps/_appsgen.sh @@ -8,7 +8,7 @@ # Generates App List ls -la /opt/communityapps/apps/ | sed -e 's/.yml//g' \ -| awk '{print $9}' | tail -n +4 > /var/plexguide/app.list +| awk '{print $9}' | tail -n +4 >> /var/plexguide/app.list ls -la /opt/mycontainers/ | sed -e 's/.yml//g' \ | awk '{print $9}' | tail -n +4 >> /var/plexguide/app.list From b392f73a1b5c90ef591bd871eab7d7cffc7ee73a Mon Sep 17 00:00:00 2001 From: LooseSeal2 Date: Thu, 11 Jul 2019 16:06:19 -0700 Subject: [PATCH 183/185] vpn volume fixes --- apps/qbittorrent-vpn.yml | 4 ++-- apps/rflood-vpn.yml | 8 +++----- apps/rutorrent-vpn.yml | 9 ++++----- apps/transmission-vpn.yml | 2 +- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/apps/qbittorrent-vpn.yml b/apps/qbittorrent-vpn.yml index 5c4e955..74d5c73 100644 --- a/apps/qbittorrent-vpn.yml +++ b/apps/qbittorrent-vpn.yml @@ -51,9 +51,9 @@ set_fact: pg_volumes: - '/etc/localtime:/etc/localtime:ro' - - '/opt/appdata/{{pgrole}}:/config' - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' - name: 'Setting PG ENV' set_fact: @@ -353,4 +353,4 @@ - name: Restart {{pgrole}} docker_container: name: '{{pgrole}}' - state: started \ No newline at end of file + state: started diff --git a/apps/rflood-vpn.yml b/apps/rflood-vpn.yml index 331cbe3..c5cd676 100644 --- a/apps/rflood-vpn.yml +++ b/apps/rflood-vpn.yml @@ -45,12 +45,10 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt/unionfs:/unionfs' - - '/mnt/downloads/{{pgrole}}:/output/complete' - - '/mnt/incomplete/{{pgrole}}:/output/incomplete' - '/etc/localtime:/etc/localtime:ro' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' - name: 'Setting PG ENV' set_fact: diff --git a/apps/rutorrent-vpn.yml b/apps/rutorrent-vpn.yml index 4824609..efc5406 100644 --- a/apps/rutorrent-vpn.yml +++ b/apps/rutorrent-vpn.yml @@ -45,12 +45,11 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt/unionfs:/unionfs' - - '/mnt/downloads/{{pgrole}}:/output/complete' - - '/mnt/incomplete/{{pgrole}}:/output/incomplete' - '/etc/localtime:/etc/localtime:ro' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' + - '/opt/appdata/{{pgrole}}/sock:/run/php' - name: 'Setting PG ENV' set_fact: diff --git a/apps/transmission-vpn.yml b/apps/transmission-vpn.yml index 0ff2e20..254e2c8 100644 --- a/apps/transmission-vpn.yml +++ b/apps/transmission-vpn.yml @@ -48,9 +48,9 @@ set_fact: pg_volumes: - '/etc/localtime:/etc/localtime:ro' - - '/opt/appdata/{{pgrole}}:/config' - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' - name: 'Setting {{pgrole2}} Volumes' set_fact: From 9e4c6bfd826af13f4559cbd104eff092740af02e Mon Sep 17 00:00:00 2001 From: LooseSeal2 Date: Thu, 11 Jul 2019 16:10:25 -0700 Subject: [PATCH 184/185] fixes --- apps/jd2-openvpn.yml | 9 ++++----- apps/jdownloader2.yml | 9 ++++----- apps/makemkv.yml | 8 ++++---- apps/nzbget-mp4.yml | 2 +- apps/ombi4k.yml | 2 +- apps/ombiHDR.yml | 2 +- apps/radarr4k.yml | 4 ++-- apps/radarrhdr.yml | 4 ++-- apps/sonarr4k.yml | 4 ++-- apps/sonarrhdr.yml | 4 ++-- apps/templates/broken/kodi-headless.yml | 5 +++-- apps/traktor.yml | 5 +++-- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/apps/jd2-openvpn.yml b/apps/jd2-openvpn.yml index 9a0b600..26cf971 100644 --- a/apps/jd2-openvpn.yml +++ b/apps/jd2-openvpn.yml @@ -49,12 +49,11 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt/unionfs:/unionfs' - - '/mnt:/mnt' - - '{{path.stdout}}/downloads/{{pgrole}}/:/output:rw' - '/etc/localtime:/etc/localtime:ro' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}/downloads/{{pgrole}}/:/output:rw' - '/opt/appdata/{{pgrole}}/vpn:/vpn:rw' - name: 'Setting PG ENV' diff --git a/apps/jdownloader2.yml b/apps/jdownloader2.yml index b3e56b8..addee0d 100644 --- a/apps/jdownloader2.yml +++ b/apps/jdownloader2.yml @@ -49,12 +49,11 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt/unionfs:/unionfs' - - '/mnt:/mnt' - - '{{path.stdout}}/downloads/{{pgrole}}/:/output:rw' - '/etc/localtime:/etc/localtime:ro' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}/downloads/{{pgrole}}/:/output:rw' - name: 'Setting PG ENV' set_fact: diff --git a/apps/makemkv.yml b/apps/makemkv.yml index 70d4c8e..5171a58 100644 --- a/apps/makemkv.yml +++ b/apps/makemkv.yml @@ -36,11 +36,11 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - - '{{path.stdout}}:{{path.stdout}}' - - '/mnt/unionfs:/unionfs' - - '{{path.stdout}}/downloads/{{pgrole}}/:/output:rw' - '/etc/localtime:/etc/localtime:ro' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' + - '{{path.stdout}}/downloads/{{pgrole}}/:/output:rw' - name: 'Setting PG ENV' set_fact: diff --git a/apps/nzbget-mp4.yml b/apps/nzbget-mp4.yml index 33a33ad..3303ba7 100644 --- a/apps/nzbget-mp4.yml +++ b/apps/nzbget-mp4.yml @@ -138,9 +138,9 @@ set_fact: pg_volumes: - '/etc/localtime:/etc/localtime:ro' - - '/opt/appdata/{{pgrole}}:/config' - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' - '/tmp:/tmp' - '/opt/appdata/{{pgrole}}/cont-init.d:/etc/cont-init.d' - '/opt/appdata/{{pgrole}}/services.d:/etc/services.d/nzbget' diff --git a/apps/ombi4k.yml b/apps/ombi4k.yml index 7674cc2..fcfd8f1 100644 --- a/apps/ombi4k.yml +++ b/apps/ombi4k.yml @@ -50,8 +50,8 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' - '/opt/appdata/{{pgrole}}/appsettings.json:/opt/{{pgrole}}/appsettings.json' - name: 'Setting PG ENV' diff --git a/apps/ombiHDR.yml b/apps/ombiHDR.yml index e9f87f5..61d0a23 100644 --- a/apps/ombiHDR.yml +++ b/apps/ombiHDR.yml @@ -50,8 +50,8 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' - '/opt/appdata/{{pgrole}}/appsettings.json:/opt/{{pgrole}}/appsettings.json' - name: 'Setting PG ENV' diff --git a/apps/radarr4k.yml b/apps/radarr4k.yml index 7c3a2cd..4437ba2 100644 --- a/apps/radarr4k.yml +++ b/apps/radarr4k.yml @@ -33,10 +33,10 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' - - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' - name: 'Setting PG ENV' set_fact: diff --git a/apps/radarrhdr.yml b/apps/radarrhdr.yml index 8c468d7..b9674b2 100644 --- a/apps/radarrhdr.yml +++ b/apps/radarrhdr.yml @@ -33,10 +33,10 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' - - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' - name: 'Setting PG ENV' set_fact: diff --git a/apps/sonarr4k.yml b/apps/sonarr4k.yml index fe4fbaf..f3aa995 100644 --- a/apps/sonarr4k.yml +++ b/apps/sonarr4k.yml @@ -34,10 +34,10 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' - - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' - name: 'Setting PG ENV' set_fact: diff --git a/apps/sonarrhdr.yml b/apps/sonarrhdr.yml index 3a80027..22699eb 100644 --- a/apps/sonarrhdr.yml +++ b/apps/sonarrhdr.yml @@ -34,10 +34,10 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' + - '/etc/localtime:/etc/localtime:ro' - '{{path.stdout}}:{{path.stdout}}' - '/mnt:/mnt' - - '/etc/localtime:/etc/localtime:ro' + - '/opt/appdata/{{pgrole}}:/config' - name: 'Setting PG ENV' set_fact: diff --git a/apps/templates/broken/kodi-headless.yml b/apps/templates/broken/kodi-headless.yml index 88ccc68..06e07b7 100644 --- a/apps/templates/broken/kodi-headless.yml +++ b/apps/templates/broken/kodi-headless.yml @@ -37,9 +37,10 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}/config:/config/.kodi:rw' - - '/mnt:/mnt:rw' - '/etc/localtime:/etc/localtime:ro' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}/config:/config/.kodi:rw' - name: 'Setting PG ENV' set_fact: diff --git a/apps/traktor.yml b/apps/traktor.yml index 4452565..56e0d67 100644 --- a/apps/traktor.yml +++ b/apps/traktor.yml @@ -34,9 +34,10 @@ - name: 'Setting PG Volumes' set_fact: pg_volumes: - - '/opt/appdata/{{pgrole}}:/config' - - '{{path.stdout}}:{{path.stdout}}' - '/etc/localtime:/etc/localtime:ro' + - '{{path.stdout}}:{{path.stdout}}' + - '/mnt:/mnt' + - '/opt/appdata/{{pgrole}}:/config' - '/opt/appdata/plex/database/Library/Application Support/Plex Media Server/Plug-in Support/Databases:/plex:ro' - name: 'Setting PG ENV' From 067ef0515b14c8d8a119a13d23c48c43a762b265 Mon Sep 17 00:00:00 2001 From: LooseSeal2 Date: Sat, 13 Jul 2019 19:15:40 -0700 Subject: [PATCH 185/185] not needed env --- apps/airsonic.yml | 1 - apps/domoticz.yml | 1 - apps/filebot.yml | 1 - apps/flextv.yml | 1 - apps/mediainfo.yml | 1 - apps/rclonebrowser.yml | 1 - apps/rdp-calibre.yml | 1 - apps/subsonic.yml | 3 +-- apps/templates/broken/kodi-headless.yml | 1 - apps/unifi.yml | 1 - apps/vnc-xfce.yml | 1 - 11 files changed, 1 insertion(+), 12 deletions(-) diff --git a/apps/airsonic.yml b/apps/airsonic.yml index 369bc0e..733afac 100644 --- a/apps/airsonic.yml +++ b/apps/airsonic.yml @@ -45,7 +45,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' diff --git a/apps/domoticz.yml b/apps/domoticz.yml index 63ed8d3..dc35552 100644 --- a/apps/domoticz.yml +++ b/apps/domoticz.yml @@ -45,7 +45,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' diff --git a/apps/filebot.yml b/apps/filebot.yml index 5fd6f77..7ba2310 100644 --- a/apps/filebot.yml +++ b/apps/filebot.yml @@ -45,7 +45,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' diff --git a/apps/flextv.yml b/apps/flextv.yml index 4c69656..ee73fbf 100644 --- a/apps/flextv.yml +++ b/apps/flextv.yml @@ -41,7 +41,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' diff --git a/apps/mediainfo.yml b/apps/mediainfo.yml index a989491..026f6b5 100644 --- a/apps/mediainfo.yml +++ b/apps/mediainfo.yml @@ -46,7 +46,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' diff --git a/apps/rclonebrowser.yml b/apps/rclonebrowser.yml index c9bb3aa..d8ad471 100644 --- a/apps/rclonebrowser.yml +++ b/apps/rclonebrowser.yml @@ -47,7 +47,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' diff --git a/apps/rdp-calibre.yml b/apps/rdp-calibre.yml index 8b5576d..41f6d98 100644 --- a/apps/rdp-calibre.yml +++ b/apps/rdp-calibre.yml @@ -47,7 +47,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' WIDTH: '1280' HEIGHT: '720' diff --git a/apps/subsonic.yml b/apps/subsonic.yml index bb68be9..b81db75 100644 --- a/apps/subsonic.yml +++ b/apps/subsonic.yml @@ -43,7 +43,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' @@ -61,4 +60,4 @@ aliases: - '{{pgrole}}' state: started - labels: '{{pg_labels}}' \ No newline at end of file + labels: '{{pg_labels}}' diff --git a/apps/templates/broken/kodi-headless.yml b/apps/templates/broken/kodi-headless.yml index 06e07b7..ff51a2e 100644 --- a/apps/templates/broken/kodi-headless.yml +++ b/apps/templates/broken/kodi-headless.yml @@ -47,7 +47,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' diff --git a/apps/unifi.yml b/apps/unifi.yml index ed6cef6..403107d 100644 --- a/apps/unifi.yml +++ b/apps/unifi.yml @@ -63,7 +63,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}' diff --git a/apps/vnc-xfce.yml b/apps/vnc-xfce.yml index 7ee217d..54b8739 100644 --- a/apps/vnc-xfce.yml +++ b/apps/vnc-xfce.yml @@ -44,7 +44,6 @@ pg_env: PUID: '1000' PGID: '1000' - TZ: '${TZ}' # MAIN DEPLOYMENT ############################################################# - name: 'Deploying {{pgrole}}'