From 2d0ecd79b0bc606253c479d9468a09ddc7bb2fe2 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 25 Apr 2025 22:41:18 +0530 Subject: [PATCH 01/13] change for installing custom rclone version --- script/get-rclone/customize.py | 3 + script/get-rclone/install-system-macos.sh | 8 +- script/get-rclone/install-system.sh | 2 +- script/get-rclone/rclone-custom-install.sh | 240 +++++++++++++++++++++ 4 files changed, 251 insertions(+), 2 deletions(-) create mode 100755 script/get-rclone/rclone-custom-install.sh diff --git a/script/get-rclone/customize.py b/script/get-rclone/customize.py index 0b3a05065..9ae7eeb6e 100644 --- a/script/get-rclone/customize.py +++ b/script/get-rclone/customize.py @@ -19,6 +19,9 @@ def preprocess(i): automation = i['automation'] need_version = env.get('MLC_VERSION', '') + if need_version != '': + env['MLC_RCLONE_CUSTOM_VERSION'] = need_version + env['MLC_RCLONE_CUSTOM_VERSION_FLAG'] = '--version' host_os_machine = '' if os_info['platform'] != 'windows': diff --git a/script/get-rclone/install-system-macos.sh b/script/get-rclone/install-system-macos.sh index 97f8f41ee..c8d24e980 100644 --- a/script/get-rclone/install-system-macos.sh +++ b/script/get-rclone/install-system-macos.sh @@ -1,3 +1,9 @@ #!/bin/bash -brew install rclone + +if [ -n "$MLC_RCLONE_CUSTOM_VERSION" ]; then + brew install rclone@$MLC_RCLONE_CUSTOM_VERSION +else + brew install rclone +fi + test $? -eq 0 || exit 1 diff --git a/script/get-rclone/install-system.sh b/script/get-rclone/install-system.sh index a08dd54fb..04c826f28 100644 --- a/script/get-rclone/install-system.sh +++ b/script/get-rclone/install-system.sh @@ -1,3 +1,3 @@ #!/bin/bash -sudo -v ; curl -k https://rclone.org/install.sh | sudo bash +sudo -v ; chmod +x "${MLC_TMP_CURRENT_SCRIPT_PATH}/rclone-custom-install.sh" ; sudo "${MLC_TMP_CURRENT_SCRIPT_PATH}/rclone-custom-install.sh" ${MLC_RCLONE_CUSTOM_VERSION_FLAG} ${MLC_RCLONE_CUSTOM_VERSION} --force test $? -eq 0 || exit 1 diff --git a/script/get-rclone/rclone-custom-install.sh b/script/get-rclone/rclone-custom-install.sh new file mode 100755 index 000000000..7eaf96f9c --- /dev/null +++ b/script/get-rclone/rclone-custom-install.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash + +# error codes +# 0 - exited without problems +# 1 - parameters not supported were used or some unexpected error occurred +# 2 - OS not supported by this script +# 3 - installed version of rclone is up to date +# 4 - supported unzip tools are not available + +set -e + +# When adding a tool to the list, make sure to also add its corresponding command further in the script +unzip_tools_list=('unzip' '7z' 'busybox') + +usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta] [--version VERSION] [--force]" 1>&2; exit 1; } + +# Check for flags +install_beta="" +custom_version="" +force_flag="" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --beta) + install_beta="beta " + shift + ;; + --version) + custom_version="$2" + shift 2 + ;; + --force) + force_flag="--force" + shift + ;; + *) + usage + ;; + esac +done + +# Create temp directory and move to it with macOS compatibility fallback +tmp_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'rclone-install.XXXXXXXXXX') +cd "$tmp_dir" + +# Make sure unzip tool is available and choose one to work with +set +e +for tool in ${unzip_tools_list[*]}; do + trash=$(hash "$tool" 2>>errors) + if [ "$?" -eq 0 ]; then + unzip_tool="$tool" + break + fi +done +set -e + +# Exit if no unzip tools available +if [ -z "$unzip_tool" ]; then + printf "\nNone of the supported tools for extracting zip archives (${unzip_tools_list[*]}) were found. " + printf "Please install one of them and try again.\n\n" + exit 4 +fi + +# Make sure we don't create a root owned .config/rclone directory #2127 +export XDG_CONFIG_HOME=config + +# === Determine Current Version === +if command -v rclone &>/dev/null; then + current_installed=$(rclone --version | head -n1 | awk '{print $2}') +else + current_installed="" +fi + +# Check installed version of rclone to determine if update is necessary +version=$(rclone --version 2>>errors | head -n 1) +if [ -z "$custom_version" ]; then + if [ -z "$install_beta" ]; then + current_version=$(curl -fsS https://downloads.rclone.org/version.txt) + else + current_version=$(curl -fsS https://beta.rclone.org/version.txt) + current_version=$(echo "$current_version" | grep -o 'v[0-9]\+\.[0-9]\+\.[0-9]\+-[a-zA-Z0-9.-]\+') + fi +else + current_version="$custom_version" +fi + +# Check if the version is already up to date +if [ "$version" -eq "$current_version" ] && [ -z "$force_flag" ]; then + printf "\nThe ${install_beta}version of rclone ${version} is already installed.\n\n" + exit 3 +fi + +if [ -n "$current_installed" ] && [ -z "$force_flag" ]; then + echo "Rclone version ${current_installed} is installed. Replace with version ${version}? (yes/no)" + read -r answer + if [ "$answer" != "yes" ]; then + echo "Aborted." + exit 0 + fi +fi + +# Detect the platform +OS="$(uname)" +case $OS in + Linux) + OS='linux' + ;; + FreeBSD) + OS='freebsd' + ;; + NetBSD) + OS='netbsd' + ;; + OpenBSD) + OS='openbsd' + ;; + Darwin) + OS='osx' + binTgtDir=/usr/local/bin + man1TgtDir=/usr/local/share/man/man1 + ;; + SunOS) + OS='solaris' + echo 'OS not supported' + exit 2 + ;; + *) + echo 'OS not supported' + exit 2 + ;; +esac + +OS_type="$(uname -m)" +case "$OS_type" in + x86_64|amd64) + OS_type='amd64' + ;; + i?86|x86) + OS_type='386' + ;; + aarch64|arm64) + OS_type='arm64' + ;; + armv7*) + OS_type='arm-v7' + ;; + armv6*) + OS_type='arm-v6' + ;; + arm*) + OS_type='arm' + ;; + *) + echo 'OS type not supported' + exit 2 + ;; +esac + +# Download and unzip +if [ -z "$install_beta" ]; then + download_link="https://downloads.rclone.org/v${current_version}/rclone-v${current_version}-${OS}-${OS_type}.zip" + rclone_zip="rclone-v${current_version}-${OS}-${OS_type}.zip" +else + download_link="https://beta.rclone.org/${current_version}/rclone-${current_version}-${OS}-${OS_type}.zip" + rclone_zip="rclone-${current_version}-${OS}-${OS_type}.zip" +fi + +echo "$download_link" +curl -O "$download_link" +unzip_dir="tmp_unzip_dir_for_rclone" + +# Unzip with the selected tool +case "$unzip_tool" in + 'unzip') + unzip -a "$rclone_zip" -d "$unzip_dir" + ;; + '7z') + 7z x "$rclone_zip" "-o$unzip_dir" + ;; + 'busybox') + mkdir -p "$unzip_dir" + busybox unzip "$rclone_zip" -d "$unzip_dir" + ;; +esac + +cd $unzip_dir/* + +# Mounting rclone to environment +case "$OS" in + 'linux') + # Binary + cp rclone /usr/bin/rclone.new + chmod 755 /usr/bin/rclone.new + chown root:root /usr/bin/rclone.new + mv /usr/bin/rclone.new /usr/bin/rclone + # Manual + if ! [ -x "$(command -v mandb)" ]; then + echo 'mandb not found. The rclone man docs will not be installed.' + else + mkdir -p /usr/local/share/man/man1 + cp rclone.1 /usr/local/share/man/man1/ + mandb + fi + ;; + 'freebsd'|'openbsd'|'netbsd') + # Binary + cp rclone /usr/bin/rclone.new + chown root:wheel /usr/bin/rclone.new + mv /usr/bin/rclone.new /usr/bin/rclone + # Manual + mkdir -p /usr/local/man/man1 + cp rclone.1 /usr/local/man/man1/ + makewhatis + ;; + 'osx') + # Binary + mkdir -m 0555 -p ${binTgtDir} + cp rclone ${binTgtDir}/rclone.new + mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone + chmod a=x ${binTgtDir}/rclone + # Manual + mkdir -m 0555 -p ${man1TgtDir} + cp rclone.1 ${man1TgtDir} + chmod a=r ${man1TgtDir}/rclone.1 + ;; + *) + echo 'OS not supported' + exit 2 +esac + +# Update version variable post-install +version=$(rclone --version 2>>errors | head -n 1) + +# Cleanup +rm -rf "$tmp_dir" + +printf "\n${version} has successfully installed." +printf '\nNow run "rclone config" for setup. Check https://rclone.org/docs/ for more details.\n\n' +exit 0 \ No newline at end of file From d4f39e552b340b114923cfb567ed0f7539851c76 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 5 May 2025 13:12:44 +0530 Subject: [PATCH 02/13] add --multi-thread-streams=0 for rclone version >= 1.60.0 --- script/download-file/customize.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 6200cc223..38b8c4b26 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -234,6 +234,15 @@ def preprocess(i): pre_clean = False if not verify_ssl: extra_download_options += " --no-check-certificate " + # check rclojne version to add --multi-thread-streams=0 + tmp_rclone_version = env.get('MLC_RCLONE_VERSION', '') + if tmp_rclone_version == '': + logger.warning("MLC_RCLONE_VERSION not set, was get-rclone called as dependency?") + else: + ref_version = list(map(int, "1.60.0")) + rclone_version = list(map(int, tmp_rclone_version)) + if rclone_version >= ref_version: + extra_download_options += " --multi-thread-streams=0 " if env["MLC_HOST_OS_TYPE"] == "windows": # have to modify the variable from url to temp_url if it is # going to be used anywhere after this point From bf3515cac65b7160e10cfb0ff25a332832b803c0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 May 2025 07:42:57 +0000 Subject: [PATCH 03/13] [Automated Commit] Format Codebase [skip ci] --- script/download-file/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 38b8c4b26..edf2ea128 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -237,7 +237,8 @@ def preprocess(i): # check rclojne version to add --multi-thread-streams=0 tmp_rclone_version = env.get('MLC_RCLONE_VERSION', '') if tmp_rclone_version == '': - logger.warning("MLC_RCLONE_VERSION not set, was get-rclone called as dependency?") + logger.warning( + "MLC_RCLONE_VERSION not set, was get-rclone called as dependency?") else: ref_version = list(map(int, "1.60.0")) rclone_version = list(map(int, tmp_rclone_version)) From 8b8341cb3cfe233123608a04fa76eb1810457b21 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 5 May 2025 13:13:28 +0530 Subject: [PATCH 04/13] Update customize.py --- script/download-file/customize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index edf2ea128..6479a95cc 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -3,7 +3,6 @@ import subprocess from utils import * - def escape_special_chars(text, tool=None): special_chars = [ '&', '|', '(', ')' From 2e91eb868f173f4a4dc481c9ebb08ffd86fad051 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 May 2025 07:43:38 +0000 Subject: [PATCH 05/13] [Automated Commit] Format Codebase [skip ci] --- script/download-file/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 6479a95cc..edf2ea128 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -3,6 +3,7 @@ import subprocess from utils import * + def escape_special_chars(text, tool=None): special_chars = [ '&', '|', '(', ')' From f9215304a75a8b81e769a1d226f9fc87c29bd232 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 5 May 2025 13:14:37 +0530 Subject: [PATCH 06/13] Update customize.py --- script/download-file/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index edf2ea128..ea0e772b0 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -234,7 +234,7 @@ def preprocess(i): pre_clean = False if not verify_ssl: extra_download_options += " --no-check-certificate " - # check rclojne version to add --multi-thread-streams=0 + # check rclone version to add --multi-thread-streams=0 tmp_rclone_version = env.get('MLC_RCLONE_VERSION', '') if tmp_rclone_version == '': logger.warning( From 035d0e0ca0e9ab1c7396b535cd6900d94a50f50f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 5 May 2025 13:17:37 +0530 Subject: [PATCH 07/13] Update customize.py --- script/download-file/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index ea0e772b0..8804a9e8f 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -240,8 +240,8 @@ def preprocess(i): logger.warning( "MLC_RCLONE_VERSION not set, was get-rclone called as dependency?") else: - ref_version = list(map(int, "1.60.0")) - rclone_version = list(map(int, tmp_rclone_version)) + ref_version = list(map(int, "1.60.0".split('.')) + rclone_version = list(map(int, tmp_rclone_version.split('.')) if rclone_version >= ref_version: extra_download_options += " --multi-thread-streams=0 " if env["MLC_HOST_OS_TYPE"] == "windows": From f5b66ddc9ca1f3477a2807f8232db0906caa328e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 May 2025 07:47:48 +0000 Subject: [PATCH 08/13] [Automated Commit] Format Codebase [skip ci] --- script/download-file/customize.py | 59 ++++++++++++++++--------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 8804a9e8f..c7b601709 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -159,7 +159,8 @@ def preprocess(i): "return": 1, "error": f"Permission denied to delete file {env['MLC_DOWNLOAD_FILENAME']}."} mlcutil_require_download = 1 elif "no such file" in checksum_result.stderr.lower(): - # print(f"No file {env['MLC_DOWNLOAD_FILENAME']}. Downloading through mlcutil.") + # print(f"No file {env['MLC_DOWNLOAD_FILENAME']}. + # Downloading through mlcutil.") mlcutil_require_download = 1 elif checksum_result.returncode > 0: return { @@ -241,37 +242,37 @@ def preprocess(i): "MLC_RCLONE_VERSION not set, was get-rclone called as dependency?") else: ref_version = list(map(int, "1.60.0".split('.')) - rclone_version = list(map(int, tmp_rclone_version.split('.')) + rclone_version=list(map(int, tmp_rclone_version.split('.')) if rclone_version >= ref_version: extra_download_options += " --multi-thread-streams=0 " if env["MLC_HOST_OS_TYPE"] == "windows": # have to modify the variable from url to temp_url if it is # going to be used anywhere after this point - url = url.replace("%", "%%") - temp_download_file = env['MLC_DOWNLOAD_FILENAME'].replace( + url=url.replace("%", "%%") + temp_download_file=env['MLC_DOWNLOAD_FILENAME'].replace( "%", "%%") - env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer {extra_download_options}" + env['MLC_DOWNLOAD_CMD']=f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer {extra_download_options}" else: - env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['MLC_DOWNLOAD_FILENAME'])}{q} -P --error-on-no-transfer {extra_download_options}" + env['MLC_DOWNLOAD_CMD']=f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['MLC_DOWNLOAD_FILENAME'])}{q} -P --error-on-no-transfer {extra_download_options}" - filename = env['MLC_DOWNLOAD_FILENAME'] - env['MLC_DOWNLOAD_DOWNLOADED_FILENAME'] = filename + filename=env['MLC_DOWNLOAD_FILENAME'] + env['MLC_DOWNLOAD_DOWNLOADED_FILENAME']=filename - filename = os.path.basename(env['MLC_DOWNLOAD_FILENAME']) - filepath = os.path.join(os.getcwd(), filename) + filename=os.path.basename(env['MLC_DOWNLOAD_FILENAME']) + filepath=os.path.join(os.getcwd(), filename) - env['MLC_DOWNLOAD_DOWNLOADED_PATH'] = filepath + env['MLC_DOWNLOAD_DOWNLOADED_PATH']=filepath # verify checksum if file already present if env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '': - env['MLC_DOWNLOAD_CHECKSUM_CMD'] = f"cd {q}{filepath}{q} {xsep} md5sum -c {x_c} {x}{q}{env['MLC_DOWNLOAD_CHECKSUM_FILE']}{q}" + env['MLC_DOWNLOAD_CHECKSUM_CMD']=f"cd {q}{filepath}{q} {xsep} md5sum -c {x_c} {x}{q}{env['MLC_DOWNLOAD_CHECKSUM_FILE']}{q}" elif env.get('MLC_DOWNLOAD_CHECKSUM', '') != '': if os_info['platform'] == 'windows': - env['MLC_DOWNLOAD_CHECKSUM_CMD'] = "echo {} {}{} | md5sum {} -c -".format( + env['MLC_DOWNLOAD_CHECKSUM_CMD']="echo {} {}{} | md5sum {} -c -".format( env.get('MLC_DOWNLOAD_CHECKSUM'), x, escape_special_chars( env['MLC_DOWNLOAD_FILENAME']), x_c) else: - env['MLC_DOWNLOAD_CHECKSUM_CMD'] = "echo {} {}{}{}{} | md5sum {} -c -".format( + env['MLC_DOWNLOAD_CHECKSUM_CMD']="echo {} {}{}{}{} | md5sum {} -c -".format( env.get('MLC_DOWNLOAD_CHECKSUM'), x, q, env['MLC_DOWNLOAD_FILENAME'], q, x_c) for i in range(1, 5): if env.get('MLC_DOWNLOAD_CHECKSUM' + str(i), '') == '': @@ -299,22 +300,22 @@ def preprocess(i): x_c) # print(env['MLC_DOWNLOAD_CHECKSUM_CMD']) else: - env['MLC_DOWNLOAD_CHECKSUM_CMD'] = "" + env['MLC_DOWNLOAD_CHECKSUM_CMD']="" if not pre_clean: - env['MLC_PRE_DOWNLOAD_CMD'] = '' + env['MLC_PRE_DOWNLOAD_CMD']='' if os_info['platform'] == 'windows' and env.get( 'MLC_DOWNLOAD_CMD', '') != '': - env['MLC_DOWNLOAD_CMD'] = escape_special_chars( + env['MLC_DOWNLOAD_CMD']=escape_special_chars( env['MLC_DOWNLOAD_CMD'], tool) if pre_clean: - env['MLC_PRE_DOWNLOAD_CLEAN_CMD'] = "del /Q %MLC_DOWNLOAD_FILENAME%" + env['MLC_PRE_DOWNLOAD_CLEAN_CMD']="del /Q %MLC_DOWNLOAD_FILENAME%" # Check that if empty CMD, should add "" for x in ['MLC_DOWNLOAD_CMD', 'MLC_DOWNLOAD_CHECKSUM_CMD']: - env[x + '_USED'] = 'YES' if env.get(x, '') != '' else 'NO' + env[x + '_USED']='YES' if env.get(x, '') != '' else 'NO' else: - env['MLC_PRE_DOWNLOAD_CLEAN_CMD'] = "rm -f {}".format( + env['MLC_PRE_DOWNLOAD_CLEAN_CMD']="rm -f {}".format( env['MLC_DOWNLOAD_FILENAME']) return {'return': 0} @@ -322,30 +323,30 @@ def preprocess(i): def postprocess(i): - automation = i['automation'] + automation=i['automation'] - env = i['env'] + env=i['env'] if env.get('MLC_DOWNLOAD_MODE') == "dry": return {'return': 0} - filepath = env['MLC_DOWNLOAD_DOWNLOADED_PATH'] + filepath=env['MLC_DOWNLOAD_DOWNLOADED_PATH'] if not os.path.exists(filepath): return { 'return': 1, 'error': 'Downloaded path {} does not exist. Probably MLC_DOWNLOAD_FILENAME is not set and MLC_DOWNLOAD_URL given is not pointing to a file'.format(filepath)} if env.get('MLC_DOWNLOAD_RENAME_FILE', '') != '': - file_dir = os.path.dirname(filepath) - new_file_name = env['MLC_DOWNLOAD_RENAME_FILE'] - new_file_path = os.path.join(file_dir, new_file_name) + file_dir=os.path.dirname(filepath) + new_file_name=env['MLC_DOWNLOAD_RENAME_FILE'] + new_file_path=os.path.join(file_dir, new_file_name) os.rename(filepath, new_file_path) - filepath = new_file_path + filepath=new_file_path if env.get('MLC_DOWNLOAD_FINAL_ENV_NAME', '') != '': - env[env['MLC_DOWNLOAD_FINAL_ENV_NAME']] = filepath + env[env['MLC_DOWNLOAD_FINAL_ENV_NAME']]=filepath - env['MLC_GET_DEPENDENT_CACHED_PATH'] = filepath + env['MLC_GET_DEPENDENT_CACHED_PATH']=filepath # Since may change directory, check if need to clean some temporal files automation.clean_some_tmp_files({'env': env}) From e0f7cc7d6e7ad4e99afb0efa79959eb9b631aa0d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 5 May 2025 13:19:36 +0530 Subject: [PATCH 09/13] Update customize.py --- script/download-file/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index c7b601709..7cb305731 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -235,7 +235,7 @@ def preprocess(i): pre_clean = False if not verify_ssl: extra_download_options += " --no-check-certificate " - # check rclone version to add --multi-thread-streams=0 + # rlone fix: check rclone version to add --multi-thread-streams=0 tmp_rclone_version = env.get('MLC_RCLONE_VERSION', '') if tmp_rclone_version == '': logger.warning( From a46995c2d9bdfd5b6384f4859db4781e48495363 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 5 May 2025 13:23:03 +0530 Subject: [PATCH 10/13] clean code --- script/download-file/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 7cb305731..bc034fc9d 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -241,8 +241,8 @@ def preprocess(i): logger.warning( "MLC_RCLONE_VERSION not set, was get-rclone called as dependency?") else: - ref_version = list(map(int, "1.60.0".split('.')) - rclone_version=list(map(int, tmp_rclone_version.split('.')) + ref_version = list(map(int, "1.60.0".split('.'))) + rclone_version=list(map(int, tmp_rclone_version.split('.'))) if rclone_version >= ref_version: extra_download_options += " --multi-thread-streams=0 " if env["MLC_HOST_OS_TYPE"] == "windows": From 4ce8dd715cf5a627d1b1a721d785ef097849ff3b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 May 2025 07:53:14 +0000 Subject: [PATCH 11/13] [Automated Commit] Format Codebase [skip ci] --- script/download-file/customize.py | 56 +++++++++++++++---------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index bc034fc9d..4548fe187 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -242,37 +242,37 @@ def preprocess(i): "MLC_RCLONE_VERSION not set, was get-rclone called as dependency?") else: ref_version = list(map(int, "1.60.0".split('.'))) - rclone_version=list(map(int, tmp_rclone_version.split('.'))) + rclone_version = list(map(int, tmp_rclone_version.split('.'))) if rclone_version >= ref_version: extra_download_options += " --multi-thread-streams=0 " if env["MLC_HOST_OS_TYPE"] == "windows": # have to modify the variable from url to temp_url if it is # going to be used anywhere after this point - url=url.replace("%", "%%") - temp_download_file=env['MLC_DOWNLOAD_FILENAME'].replace( + url = url.replace("%", "%%") + temp_download_file = env['MLC_DOWNLOAD_FILENAME'].replace( "%", "%%") - env['MLC_DOWNLOAD_CMD']=f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer {extra_download_options}" + env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer {extra_download_options}" else: - env['MLC_DOWNLOAD_CMD']=f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['MLC_DOWNLOAD_FILENAME'])}{q} -P --error-on-no-transfer {extra_download_options}" + env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['MLC_DOWNLOAD_FILENAME'])}{q} -P --error-on-no-transfer {extra_download_options}" - filename=env['MLC_DOWNLOAD_FILENAME'] - env['MLC_DOWNLOAD_DOWNLOADED_FILENAME']=filename + filename = env['MLC_DOWNLOAD_FILENAME'] + env['MLC_DOWNLOAD_DOWNLOADED_FILENAME'] = filename - filename=os.path.basename(env['MLC_DOWNLOAD_FILENAME']) - filepath=os.path.join(os.getcwd(), filename) + filename = os.path.basename(env['MLC_DOWNLOAD_FILENAME']) + filepath = os.path.join(os.getcwd(), filename) - env['MLC_DOWNLOAD_DOWNLOADED_PATH']=filepath + env['MLC_DOWNLOAD_DOWNLOADED_PATH'] = filepath # verify checksum if file already present if env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '': - env['MLC_DOWNLOAD_CHECKSUM_CMD']=f"cd {q}{filepath}{q} {xsep} md5sum -c {x_c} {x}{q}{env['MLC_DOWNLOAD_CHECKSUM_FILE']}{q}" + env['MLC_DOWNLOAD_CHECKSUM_CMD'] = f"cd {q}{filepath}{q} {xsep} md5sum -c {x_c} {x}{q}{env['MLC_DOWNLOAD_CHECKSUM_FILE']}{q}" elif env.get('MLC_DOWNLOAD_CHECKSUM', '') != '': if os_info['platform'] == 'windows': - env['MLC_DOWNLOAD_CHECKSUM_CMD']="echo {} {}{} | md5sum {} -c -".format( + env['MLC_DOWNLOAD_CHECKSUM_CMD'] = "echo {} {}{} | md5sum {} -c -".format( env.get('MLC_DOWNLOAD_CHECKSUM'), x, escape_special_chars( env['MLC_DOWNLOAD_FILENAME']), x_c) else: - env['MLC_DOWNLOAD_CHECKSUM_CMD']="echo {} {}{}{}{} | md5sum {} -c -".format( + env['MLC_DOWNLOAD_CHECKSUM_CMD'] = "echo {} {}{}{}{} | md5sum {} -c -".format( env.get('MLC_DOWNLOAD_CHECKSUM'), x, q, env['MLC_DOWNLOAD_FILENAME'], q, x_c) for i in range(1, 5): if env.get('MLC_DOWNLOAD_CHECKSUM' + str(i), '') == '': @@ -300,22 +300,22 @@ def preprocess(i): x_c) # print(env['MLC_DOWNLOAD_CHECKSUM_CMD']) else: - env['MLC_DOWNLOAD_CHECKSUM_CMD']="" + env['MLC_DOWNLOAD_CHECKSUM_CMD'] = "" if not pre_clean: - env['MLC_PRE_DOWNLOAD_CMD']='' + env['MLC_PRE_DOWNLOAD_CMD'] = '' if os_info['platform'] == 'windows' and env.get( 'MLC_DOWNLOAD_CMD', '') != '': - env['MLC_DOWNLOAD_CMD']=escape_special_chars( + env['MLC_DOWNLOAD_CMD'] = escape_special_chars( env['MLC_DOWNLOAD_CMD'], tool) if pre_clean: - env['MLC_PRE_DOWNLOAD_CLEAN_CMD']="del /Q %MLC_DOWNLOAD_FILENAME%" + env['MLC_PRE_DOWNLOAD_CLEAN_CMD'] = "del /Q %MLC_DOWNLOAD_FILENAME%" # Check that if empty CMD, should add "" for x in ['MLC_DOWNLOAD_CMD', 'MLC_DOWNLOAD_CHECKSUM_CMD']: - env[x + '_USED']='YES' if env.get(x, '') != '' else 'NO' + env[x + '_USED'] = 'YES' if env.get(x, '') != '' else 'NO' else: - env['MLC_PRE_DOWNLOAD_CLEAN_CMD']="rm -f {}".format( + env['MLC_PRE_DOWNLOAD_CLEAN_CMD'] = "rm -f {}".format( env['MLC_DOWNLOAD_FILENAME']) return {'return': 0} @@ -323,30 +323,30 @@ def preprocess(i): def postprocess(i): - automation=i['automation'] + automation = i['automation'] - env=i['env'] + env = i['env'] if env.get('MLC_DOWNLOAD_MODE') == "dry": return {'return': 0} - filepath=env['MLC_DOWNLOAD_DOWNLOADED_PATH'] + filepath = env['MLC_DOWNLOAD_DOWNLOADED_PATH'] if not os.path.exists(filepath): return { 'return': 1, 'error': 'Downloaded path {} does not exist. Probably MLC_DOWNLOAD_FILENAME is not set and MLC_DOWNLOAD_URL given is not pointing to a file'.format(filepath)} if env.get('MLC_DOWNLOAD_RENAME_FILE', '') != '': - file_dir=os.path.dirname(filepath) - new_file_name=env['MLC_DOWNLOAD_RENAME_FILE'] - new_file_path=os.path.join(file_dir, new_file_name) + file_dir = os.path.dirname(filepath) + new_file_name = env['MLC_DOWNLOAD_RENAME_FILE'] + new_file_path = os.path.join(file_dir, new_file_name) os.rename(filepath, new_file_path) - filepath=new_file_path + filepath = new_file_path if env.get('MLC_DOWNLOAD_FINAL_ENV_NAME', '') != '': - env[env['MLC_DOWNLOAD_FINAL_ENV_NAME']]=filepath + env[env['MLC_DOWNLOAD_FINAL_ENV_NAME']] = filepath - env['MLC_GET_DEPENDENT_CACHED_PATH']=filepath + env['MLC_GET_DEPENDENT_CACHED_PATH'] = filepath # Since may change directory, check if need to clean some temporal files automation.clean_some_tmp_files({'env': env}) From ccbe74fec954f99fee67180d858058b1462ba985 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 5 May 2025 13:25:05 +0530 Subject: [PATCH 12/13] Update customize.py --- script/download-file/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 4548fe187..a638d083f 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -235,7 +235,7 @@ def preprocess(i): pre_clean = False if not verify_ssl: extra_download_options += " --no-check-certificate " - # rlone fix: check rclone version to add --multi-thread-streams=0 + # rlone fix : check rclone version to add --multi-thread-streams=0 tmp_rclone_version = env.get('MLC_RCLONE_VERSION', '') if tmp_rclone_version == '': logger.warning( From 5363194cd2e908f6334ff7963386282c1f3dcaea Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 6 May 2025 18:04:40 +0530 Subject: [PATCH 13/13] add mit license aggreement from rclone --- script/get-rclone/rclone-custom-install.sh | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/script/get-rclone/rclone-custom-install.sh b/script/get-rclone/rclone-custom-install.sh index 7eaf96f9c..d68e82b2c 100755 --- a/script/get-rclone/rclone-custom-install.sh +++ b/script/get-rclone/rclone-custom-install.sh @@ -1,3 +1,29 @@ +# This script is based on Rclone's original install script: +# https://github.com/rclone/rclone/blob/master/docs/content/install.sh +# +# Rclone is licensed under the MIT License: +# https://github.com/rclone/rclone/blob/master/COPYING +# +# Copyright (C) 2019 by Nick Craig-Wood https://www.craig-wood.com/nick/ + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + #!/usr/bin/env bash # error codes