From 42f3971afdc851424e896c6da5a17ed0a23170e7 Mon Sep 17 00:00:00 2001 From: Damien Regad Date: Sun, 2 Aug 2020 16:05:37 +0200 Subject: Use SourceForge API to set file info The script is now capable of updating the default file to download from the SourceForge files section, based on the uploaded file's extension. Introduces a config file for maintenance scripts, in YAML format, named env.yml. It is used to store the SourceForge API key. --- scripts/.gitignore | 3 ++ scripts/env.yml.sample | 7 +++++ scripts/uploadrelease.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 scripts/env.yml.sample diff --git a/scripts/.gitignore b/scripts/.gitignore index 3e0e62b2..43d23186 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1,2 +1,5 @@ +# ADOdb scripts config file +/env.yml + # Python byte code *.pyc diff --git a/scripts/env.yml.sample b/scripts/env.yml.sample new file mode 100644 index 00000000..fce3840d --- /dev/null +++ b/scripts/env.yml.sample @@ -0,0 +1,7 @@ +# ADOdb maintenance scripts configuration file +# Rename this file to 'env.yml' and update variables below as appropriate. + +# SourceForge API key +# Get it from https://sourceforge.net/auth/preferences/ +# under "Releases API Key" +api_key: diff --git a/scripts/uploadrelease.py b/scripts/uploadrelease.py index 9f6f0aee..e8d673c1 100755 --- a/scripts/uploadrelease.py +++ b/scripts/uploadrelease.py @@ -7,17 +7,24 @@ from distutils.version import LooseVersion import getopt import getpass import glob +import json import os from os import path import re +import requests import subprocess import sys +import yaml # Directories and files to exclude from release tarballs # for debugging, set to a local dir e.g. "localhost:/tmp/sf-adodb/" sf_files = "frs.sourceforge.net:/home/frs/project/adodb/" +# SourceForge Release API base URL +# https://sourceforge.net/p/forge/documentation/Using%20the%20Release%20API/ +sf_api_url = 'https://sourceforge.net/projects/adodb/files/{}/' + # rsync command template rsync_cmd = "rsync -vP --rsh ssh {opt} {src} {usr}@{dst}" @@ -127,6 +134,25 @@ def sourceforge_target_dir(version): return directory +def load_env(): + global api_key + + # Load the config file + env_file = path.join(path.dirname(path.abspath(__file__)), 'env.yml') + try: + stream = file(env_file, 'r') + y = yaml.safe_load(stream) + except IOError: + print("ERROR: Environment file {} not found".format(env_file)) + sys.exit(3) + except yaml.parser.ParserError as e: + print("ERROR: Invalid Environment file") + print(e) + sys.exit(3) + + api_key = y['api_key'] + + def process_command_line(): ''' Retrieve command-line options and set global variables accordingly ''' @@ -186,13 +212,59 @@ def upload_release_files(): ) +def set_sourceforge_file_info(): + print("Updating uploaded files information") + + base_url = sf_api_url.format( + sourceforge_target_dir(get_release_version()) + ) + headers = {'Accept': 'application/json"'} + + # Loop through uploaded files + for file in glob.glob('adodb-*'): + print(" " + file) + + # Determine defaults based on file extension + ext = file.split('.', 3)[3] + if ext == 'zip': + defaults = ['windows'] + elif ext == 'tar.gz': + defaults = ['linux', 'mac', 'bsd', 'solaris', 'others'] + else: + print("WARNING: Unknown extension for file " + file) + continue + + # SourceForge API request + global api_key + url = path.join(base_url, file) + payload = { + 'default': defaults, + 'api_key': api_key + } + req = requests.put(url, headers=headers, params=payload) + + # Print results + if req.status_code == requests.codes.ok: + result = json.loads(req.text)['result'] + print(" Download default for: " + result['x_sf']['default']) + else: + if req.status_code == requests.codes.unauthorized: + err = "access denied" + else: + err = "SourceForge API call failed" + print("ERROR: {} - check API key".format(err)) + break + + def main(): + load_env() process_command_line() # Start upload process print "ADOdb release upload script" upload_release_files() + set_sourceforge_file_info() #end main() -- cgit v1.3 From 8a132503ddb07d6943bb973c97c3e3ae191e854c Mon Sep 17 00:00:00 2001 From: Damien Regad Date: Sun, 2 Aug 2020 16:06:23 +0200 Subject: Code cleanup - Fix typos - PEP8 - remove debug statement - removed reference to no-longer-used global variables --- scripts/uploadrelease.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/uploadrelease.py b/scripts/uploadrelease.py index e8d673c1..e7915bc0 100755 --- a/scripts/uploadrelease.py +++ b/scripts/uploadrelease.py @@ -37,7 +37,7 @@ def usage(): print '''Usage: %s [options] username [release_path] This script will upload the files in the given directory (or the - current one if unspecified) to Sourceforge. + current one if unspecified) to SourceForge. Parameters: release_path Location of the release files to upload @@ -45,12 +45,12 @@ def usage(): Options: -h | --help Show this usage message - -u | --user Sourceforge account (defaults to current user) + -u | --user SourceForge account (defaults to current user) -n | --dry-run Do not upload the files ''' % ( path.basename(__file__) ) -#end usage() +# end usage() def call_rsync(usr, opt, src, dst): @@ -106,7 +106,7 @@ def get_release_version(): def sourceforge_target_dir(version): - ''' Returns the sourceforge target directory, relative to the root + ''' Returns the SourceForge target directory, relative to the root directory (defined in sf_files global variable): basedir/subdir, with basedir: - for ADOdb version 5: adodb-php5-only @@ -156,7 +156,7 @@ def load_env(): def process_command_line(): ''' Retrieve command-line options and set global variables accordingly ''' - global upload_files, upload_doc, dry_run, username, release_path + global dry_run, username, release_path # Get command-line options try: @@ -168,7 +168,6 @@ def process_command_line(): # Default values for flags username = getpass.getuser() - print username dry_run = False for opt, val in opts: @@ -266,7 +265,7 @@ def main(): upload_release_files() set_sourceforge_file_info() -#end main() +# end main() if __name__ == "__main__": main() -- cgit v1.3 From 4d20bfcb49c430fcf8201e66a3ccb7e2650c56a7 Mon Sep 17 00:00:00 2001 From: Damien Regad Date: Sun, 2 Aug 2020 16:36:43 +0200 Subject: Enable dry-run mode for SourceForge API --- scripts/uploadrelease.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/scripts/uploadrelease.py b/scripts/uploadrelease.py index e7915bc0..9ed203da 100755 --- a/scripts/uploadrelease.py +++ b/scripts/uploadrelease.py @@ -179,6 +179,7 @@ def process_command_line(): username = val elif opt in ("-n", "--dry-run"): + print("Dry-run mode - files will not be uploaded or modified") dry_run = True # Mandatory parameters @@ -212,6 +213,8 @@ def upload_release_files(): def set_sourceforge_file_info(): + global api_key, dry_run + print("Updating uploaded files information") base_url = sf_api_url.format( @@ -234,34 +237,38 @@ def set_sourceforge_file_info(): continue # SourceForge API request - global api_key url = path.join(base_url, file) payload = { 'default': defaults, 'api_key': api_key } - req = requests.put(url, headers=headers, params=payload) - - # Print results - if req.status_code == requests.codes.ok: - result = json.loads(req.text)['result'] - print(" Download default for: " + result['x_sf']['default']) + if dry_run: + req = requests.Request('PUT', url, headers=headers, params=payload) + r = req.prepare() + print(" Calling SourceForge Release API: " + r.url) else: - if req.status_code == requests.codes.unauthorized: - err = "access denied" + req = requests.put(url, headers=headers, params=payload) + + # Print results + if req.status_code == requests.codes.ok: + result = json.loads(req.text)['result'] + print(" Download default for: " + result['x_sf']['default']) else: - err = "SourceForge API call failed" - print("ERROR: {} - check API key".format(err)) - break + if req.status_code == requests.codes.unauthorized: + err = "access denied" + else: + err = "SourceForge API call failed" + print("ERROR: {} - check API key".format(err)) + break def main(): - load_env() - process_command_line() - # Start upload process print "ADOdb release upload script" + load_env() + process_command_line() + upload_release_files() set_sourceforge_file_info() -- cgit v1.3 From aa029ba9a0e419a477c3616993e905dff122baac Mon Sep 17 00:00:00 2001 From: Damien Regad Date: Sun, 2 Aug 2020 16:42:13 +0200 Subject: New command-line option to skip SourceForge upload This allows only updating the (previously) uploaded files information via the SourceForge Release API. --- scripts/uploadrelease.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/uploadrelease.py b/scripts/uploadrelease.py index 9ed203da..d6338a52 100755 --- a/scripts/uploadrelease.py +++ b/scripts/uploadrelease.py @@ -29,8 +29,8 @@ sf_api_url = 'https://sourceforge.net/projects/adodb/files/{}/' rsync_cmd = "rsync -vP --rsh ssh {opt} {src} {usr}@{dst}" # Command-line options -options = "hu:n" -long_options = ["help", "user=", "dry-run"] +options = "hu:ns" +long_options = ["help", "user=", "dry-run", "skip-upload"] def usage(): @@ -156,7 +156,7 @@ def load_env(): def process_command_line(): ''' Retrieve command-line options and set global variables accordingly ''' - global dry_run, username, release_path + global dry_run, username, release_path, skip_upload # Get command-line options try: @@ -168,6 +168,7 @@ def process_command_line(): # Default values for flags username = getpass.getuser() + skip_upload = False dry_run = False for opt, val in opts: @@ -178,6 +179,9 @@ def process_command_line(): elif opt in ("-u", "--user"): username = val + elif opt in ("-s", "--skip-upload"): + skip_upload = True + elif opt in ("-n", "--dry-run"): print("Dry-run mode - files will not be uploaded or modified") dry_run = True @@ -269,7 +273,12 @@ def main(): load_env() process_command_line() - upload_release_files() + global skip_upload + if skip_upload: + print("Skipping upload of release files") + else: + upload_release_files() + set_sourceforge_file_info() # end main() -- cgit v1.3 From f9ee36a2cd8bca14024e144eb314a7461e76c8fb Mon Sep 17 00:00:00 2001 From: Damien Regad Date: Sun, 2 Aug 2020 16:47:41 +0200 Subject: List the files to be uploaded --- scripts/uploadrelease.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/uploadrelease.py b/scripts/uploadrelease.py index d6338a52..bdf98e23 100755 --- a/scripts/uploadrelease.py +++ b/scripts/uploadrelease.py @@ -207,6 +207,7 @@ def upload_release_files(): print "Uploading release files..." print " Source:", release_path print " Target: " + target + print " Files: " + ', '.join(glob.glob('*')) print call_rsync( username, @@ -214,6 +215,7 @@ def upload_release_files(): path.join(release_path, "*"), target ) + print def set_sourceforge_file_info(): -- cgit v1.3