From 01b454680562aecc65e4057736f2a1d3d1952692 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Petr=20=C5=A0tetiar?= Date: Thu, 6 Aug 2020 10:13:18 +0200 Subject: [PATCH] collect.py: format with black MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit For automated unified code formatting. Signed-off-by: Petr Å tetiar --- misc/collect.py | 375 ++++++++++++++++++++++++++---------------------- 1 file changed, 201 insertions(+), 174 deletions(-) diff --git a/misc/collect.py b/misc/collect.py index 70e3330..29688e5 100755 --- a/misc/collect.py +++ b/misc/collect.py @@ -10,32 +10,44 @@ import sys import os import re -''' +""" Tool to create overview.json files and update the config.js. -''' +""" parser = argparse.ArgumentParser() -parser.add_argument("--formatted", action="store_true", - help="Output formatted JSON data.") +parser.add_argument( + "--formatted", action="store_true", help="Output formatted JSON data." +) subparsers = parser.add_subparsers(dest="action", required=True) -parser_merge = subparsers.add_parser("merge", - help="Create a grid structure with horizontal and vertical connections.") -parser_merge.add_argument("input_path", nargs="+", - help="Input folder that is traversed for OpenWrt JSON device files.") -parser_merge.add_argument("--download-url", action="store", default="", - help="Link to get the image from. May contain {target}, {version} and {commit}") -#parser_merge.add_argument("--change-prefix", +parser_merge = subparsers.add_parser( + "merge", help="Create a grid structure with horizontal and vertical connections." +) +parser_merge.add_argument( + "input_path", + nargs="+", + help="Input folder that is traversed for OpenWrt JSON device files.", +) +parser_merge.add_argument( + "--download-url", + action="store", + default="", + help="Link to get the image from. May contain {target}, {version} and {commit}", +) +# parser_merge.add_argument("--change-prefix", # help="Change the openwrt- file name prefix.") -parser_scrape = subparsers.add_parser("scrape", - help="Create a grid structure of horizontal, vertical and vertical connections.") -parser_scrape.add_argument("domain", - help="Domain to scrape. E.g. https://downloads.openwrt.org") -parser_scrape.add_argument("selector", - help="Path the config.js file is in.") -parser_scrape.add_argument("--use-wget", action="store_true", - help="Use wget to scrape the site.") +parser_scrape = subparsers.add_parser( + "scrape", + help="Create a grid structure of horizontal, vertical and vertical connections.", +) +parser_scrape.add_argument( + "domain", help="Domain to scrape. E.g. https://downloads.openwrt.org" +) +parser_scrape.add_argument("selector", help="Path the config.js file is in.") +parser_scrape.add_argument( + "--use-wget", action="store_true", help="Use wget to scrape the site." +) args = parser.parse_args() @@ -43,211 +55,226 @@ SUPPORTED_METADATA_VERSION = 1 # accepts {: } def merge_profiles(profiles, download_url): - # json output data - output = {} + # json output data + output = {} - def get_title(title): - if "title" in title: - return title["title"] - else: - return "{} {} {}".format(title.get("vendor", ""), title["model"], title.get("variant", "")).strip() + def get_title(title): + if "title" in title: + return title["title"] + else: + return "{} {} {}".format( + title.get("vendor", ""), title["model"], title.get("variant", "") + ).strip() - def add_profile(id, target, profile, code=None): - images = [] - for image in profile["images"]: - images.append({"name": image["name"], "type": image["type"]}) + def add_profile(id, target, profile, code=None): + images = [] + for image in profile["images"]: + images.append({"name": image["name"], "type": image["type"]}) - if target is None: - target = profile["target"] + if target is None: + target = profile["target"] - #if args.change_prefix: - # change_prefix(images, "openwrt-", args.change_prefix) + # if args.change_prefix: + # change_prefix(images, "openwrt-", args.change_prefix) - for entry in profile["titles"]: - title = get_title(entry) + for entry in profile["titles"]: + title = get_title(entry) - if len(title) == 0: - sys.stderr.write(f"Empty title. Skip title in {path}\n") - continue + if len(title) == 0: + sys.stderr.write(f"Empty title. Skip title in {path}\n") + continue - output["models"][title] = {"id": id, "target": target, "images": images} + output["models"][title] = {"id": id, "target": target, "images": images} - if code is not None: - output["models"][title]["code"] = code + if code is not None: + output["models"][title]["code"] = code - for path, content in profiles.items(): - obj = json.loads(content) + for path, content in profiles.items(): + obj = json.loads(content) - if obj["metadata_version"] != SUPPORTED_METADATA_VERSION: - sys.stderr.write(f"{path} has unsupported metadata version: {obj['metadata_version']} => skip\n") - continue + if obj["metadata_version"] != SUPPORTED_METADATA_VERSION: + sys.stderr.write( + f"{path} has unsupported metadata version: {obj['metadata_version']} => skip\n" + ) + continue - code = obj.get("version_code", obj.get("version_commit")) + code = obj.get("version_code", obj.get("version_commit")) - if not "version_code" in output: - output = { - "version_code": code, - "download_url": download_url, - "models" : {} - } + if not "version_code" in output: + output = {"version_code": code, "download_url": download_url, "models": {}} - # if we have mixed codes/commits, store in device object - if output["version_code"] == code: - code = None; + # if we have mixed codes/commits, store in device object + if output["version_code"] == code: + code = None - try: - if "profiles" in obj: - for id in obj["profiles"]: - add_profile(id, obj.get("target"), obj["profiles"][id], code) - else: - add_profile(obj["id"], obj["target"], obj, code) - except json.decoder.JSONDecodeError as e: - sys.stderr.write(f"Skip {path}\n {e}\n") - except KeyError as e: - sys.stderr.write(f"Abort on {path}\n Missing key {e}\n") - exit(1) + try: + if "profiles" in obj: + for id in obj["profiles"]: + add_profile(id, obj.get("target"), obj["profiles"][id], code) + else: + add_profile(obj["id"], obj["target"], obj, code) + except json.decoder.JSONDecodeError as e: + sys.stderr.write(f"Skip {path}\n {e}\n") + except KeyError as e: + sys.stderr.write(f"Abort on {path}\n Missing key {e}\n") + exit(1) + + return output - return output def update_config(config_path, versions): - content = "" - with open(config_path, "r") as file: - content = file.read() + content = "" + with open(config_path, "r") as file: + content = file.read() - content = re.sub("versions:[\\s]*{[^}]*}", f"versions: {versions}" , content) - with open(config_path, "w+") as file: - # save updated config - file.write(content) + content = re.sub("versions:[\\s]*{[^}]*}", f"versions: {versions}", content) + with open(config_path, "w+") as file: + # save updated config + file.write(content) -''' + +""" Scrape profiles.json using links like https://downloads.openwrt.org/releases/19.07.3/targets/?json Merge into overview.json files. Update config.json. -''' +""" + + def scrape(url, selector_path): - config_path = f"{selector_path}/config.js" - data_path = f"{selector_path}/data" - versions = {} + config_path = f"{selector_path}/config.js" + data_path = f"{selector_path}/data" + versions = {} + + def handle_release(target): + profiles = {} + with urllib.request.urlopen(f"{target}/?json") as file: + array = json.loads(file.read().decode("utf-8")) + for profile in filter(lambda x: x.endswith("/profiles.json"), array): + # print(profile) + with urllib.request.urlopen(f"{target}/{profile}") as file: + profiles[f"{target}/{profile}"] = file.read() + return profiles + + if not os.path.isfile(config_path): + print(f"file not found: {config_path}") + exit(1) - def handle_release(target): - profiles = {} - with urllib.request.urlopen(f"{target}/?json") as file: - array = json.loads(file.read().decode("utf-8")) - for profile in filter(lambda x: x.endswith("/profiles.json"), array): - #print(profile) - with urllib.request.urlopen(f"{target}/{profile}") as file: - profiles[f"{target}/{profile}"] = file.read() - return profiles - - if not os.path.isfile(config_path): - print(f"file not found: {config_path}") - exit(1) - - # fetch release URLs - with urllib.request.urlopen(url) as infile: - for path in re.findall(r"href=[\"']?([^'\" >]+)", str(infile.read())): - if not path.startswith("/") and path.endswith("targets/"): - release = path.strip("/").split("/")[-2] - download_url = f"{url}/{path}/{{target}}" - - profiles = handle_release(f"{url}/{path}") - output = merge_profiles(profiles, download_url) - if len(output) > 0: - Path(f"{data_path}/{release}").mkdir(parents=True, exist_ok=True) - # write overview.json - with open(f"{data_path}/{release}/overview.json", "w") as outfile: - if args.formatted: - json.dump(output, outfile, indent=" ", sort_keys=True) - else: - json.dump(output, outfile, sort_keys=True) + # fetch release URLs + with urllib.request.urlopen(url) as infile: + for path in re.findall(r"href=[\"']?([^'\" >]+)", str(infile.read())): + if not path.startswith("/") and path.endswith("targets/"): + release = path.strip("/").split("/")[-2] + download_url = f"{url}/{path}/{{target}}" + + profiles = handle_release(f"{url}/{path}") + output = merge_profiles(profiles, download_url) + if len(output) > 0: + Path(f"{data_path}/{release}").mkdir(parents=True, exist_ok=True) + # write overview.json + with open(f"{data_path}/{release}/overview.json", "w") as outfile: + if args.formatted: + json.dump(output, outfile, indent=" ", sort_keys=True) + else: + json.dump(output, outfile, sort_keys=True) + + versions[release.upper()] = f"data/{release}/overview.json" - versions[release.upper()] = f"data/{release}/overview.json" + update_config(config_path, versions) - update_config(config_path, versions) -''' +""" Scrape profiles.json using wget (slower but more generic). Merge into overview.json files. Update config.json. -''' +""" + + def scrape_wget(url, selector_path): - config_path = f"{selector_path}/config.js" - data_path = f"{selector_path}/data" - versions = {} + config_path = f"{selector_path}/config.js" + data_path = f"{selector_path}/data" + versions = {} - with tempfile.TemporaryDirectory() as tmp_dir: - # download all profiles.json files - os.system(f"wget -c -r -P {tmp_dir} -A 'profiles.json' --reject-regex 'kmods|packages' --no-parent {url}") + with tempfile.TemporaryDirectory() as tmp_dir: + # download all profiles.json files + os.system( + f"wget -c -r -P {tmp_dir} -A 'profiles.json' --reject-regex 'kmods|packages' --no-parent {url}" + ) - # delete empty folders - os.system(f"find {tmp_dir}/* -type d -empty -delete") + # delete empty folders + os.system(f"find {tmp_dir}/* -type d -empty -delete") - # create overview.json files - for path in glob.glob(f"{tmp_dir}/*/snapshots") + glob.glob(f"{tmp_dir}/*/releases/*"): - release = os.path.basename(path) - base = path[len(tmp_dir)+1:] + # create overview.json files + for path in glob.glob(f"{tmp_dir}/*/snapshots") + glob.glob( + f"{tmp_dir}/*/releases/*" + ): + release = os.path.basename(path) + base = path[len(tmp_dir) + 1 :] - versions[release.upper()] = f"data/{release}/overview.json" - os.system(f"mkdir -p {selector_path}/data/{release}/") + versions[release.upper()] = f"data/{release}/overview.json" + os.system(f"mkdir -p {selector_path}/data/{release}/") - profiles = {} - for ppath in Path(path).rglob('profiles.json'): - with open(ppath, "r") as file: - profiles[ppath] = file.read() + profiles = {} + for ppath in Path(path).rglob("profiles.json"): + with open(ppath, "r") as file: + profiles[ppath] = file.read() - output = merge_profiles(profiles, f"https://{base}/targets/{{target}}") - Path(f"{data_path}/{release}").mkdir(parents=True, exist_ok=True) + output = merge_profiles(profiles, f"https://{base}/targets/{{target}}") + Path(f"{data_path}/{release}").mkdir(parents=True, exist_ok=True) - # write overview.json - with open(f"{data_path}/{release}/overview.json", "w") as outfile: - if args.formatted: - json.dump(output, outfile, indent=" ", sort_keys=True) - else: - json.dump(output, outfile, sort_keys=True) + # write overview.json + with open(f"{data_path}/{release}/overview.json", "w") as outfile: + if args.formatted: + json.dump(output, outfile, indent=" ", sort_keys=True) + else: + json.dump(output, outfile, sort_keys=True) - update_config(config_path, versions) + update_config(config_path, versions) -''' + +""" def change_prefix(images, old_prefix, new_prefix): for image in images: if image["name"].startswith(old_prefix): image["name"] = new_prefix + image["name"][len(old_prefix):] -''' +""" -''' +""" Find and merge json files for a single release. -''' +""" + + def merge(input_paths): - # OpenWrt JSON device files - profiles = {} - - def add_path(path): - #paths.append(path) - with open(path, "r") as file: - profiles[path] = file.read() - - for path in input_paths: - if os.path.isdir(path): - for filepath in Path(path).rglob("*.json"): - add_path(filepath) - else: - if not path.endswith(".json"): - sys.stderr.write(f"Folder does not exists: {path}\n") - exit(1) - add_path(path) + # OpenWrt JSON device files + profiles = {} + + def add_path(path): + # paths.append(path) + with open(path, "r") as file: + profiles[path] = file.read() - output = merge_profiles(profiles, args.download_url) + for path in input_paths: + if os.path.isdir(path): + for filepath in Path(path).rglob("*.json"): + add_path(filepath) + else: + if not path.endswith(".json"): + sys.stderr.write(f"Folder does not exists: {path}\n") + exit(1) + add_path(path) + + output = merge_profiles(profiles, args.download_url) + + if args.formatted: + json.dump(output, sys.stdout, indent=" ", sort_keys=True) + else: + json.dump(output, sys.stdout, sort_keys=True) - if args.formatted: - json.dump(output, sys.stdout, indent=" ", sort_keys=True) - else: - json.dump(output, sys.stdout, sort_keys=True) if args.action == "merge": - merge(args.input_path) + merge(args.input_path) if args.action == "scrape": - if args.use_wget: - scrape_wget(args.domain, args.selector) - else: - scrape(args.domain, args.selector) + if args.use_wget: + scrape_wget(args.domain, args.selector) + else: + scrape(args.domain, args.selector) -- 2.30.2