Rewrite some parts related to handling the collection of bootloaders and kexts

Scripts/datasets/kext_data.py

@@ -268,10 +268,6 @@ kexts = [
         github_repo = {
             "owner": "SongXiaoXi",
             "repo": "AppleIGC"
-        },
-        download_info = {
-            "id": 138279923, 
-            "url": "https://github.com/SongXiaoXi/AppleIGC/releases/download/v1.5/AppleIGC.kext.zip"
         }
     ),
     KextInfo(
@@ -281,10 +277,6 @@ kexts = [
         github_repo = {
             "owner": "Mieze",
             "repo": "AtherosE2200Ethernet"
-        },
-        download_info = {
-            "id": 9746382, 
-            "url": "https://github.com/Mieze/AtherosE2200Ethernet/releases/download/2.2.2/AtherosE2200Ethernet-V2.2.2.zip"
         }
     ),
     KextInfo(
@@ -303,10 +295,6 @@ kexts = [
         github_repo = {
             "owner": "Mieze",
             "repo": "LucyRTL8125Ethernet"
-        },
-        download_info = {
-            "id": 159470181, 
-            "url": "https://github.com/Mieze/LucyRTL8125Ethernet/releases/download/v1.2.0d5/LucyRTL8125Ethernet-V1.2.0d5.zip"
         }
     ),
     KextInfo(
@@ -351,10 +339,6 @@ kexts = [
         github_repo = {
             "owner": "RattletraPM",
             "repo": "GUX-RyzenXHCIFix"
-        },
-        download_info = {
-            "id": 120325166, 
-            "url": "https://github.com/RattletraPM/GUX-RyzenXHCIFix/releases/download/v1.3.0b1-ryzenxhcifix/GenericUSBXHCI.kext.zip"
         }
     ),
     KextInfo(
@@ -378,10 +362,6 @@ kexts = [
         github_repo = {
             "owner": "blankmac",
             "repo": "AlpsHID"
-        },
-        download_info = {
-            "id": 69228327, 
-            "url": "https://github.com/blankmac/AlpsHID/releases/download/v1.2/AlpsHID1.2_release.zip"
         }
     ),
     KextInfo(
@@ -400,10 +380,6 @@ kexts = [
         github_repo = {
             "owner": "VoodooSMBus",
             "repo": "VoodooRMI"
-        },
-        download_info = {
-            "id": 13190749, 
-            "url": "https://github.com/VoodooSMBus/VoodooRMI/releases/download/1.3.5/VoodooRMI-1.3.5-Release.zip"
         }
     ),
     KextInfo(
@@ -462,10 +438,6 @@ kexts = [
         github_repo = {
             "owner": "hieplpvip",
             "repo": "AsusSMC"
-        },
-        download_info = {
-            "id": 41898282, 
-            "url": "https://github.com/hieplpvip/AsusSMC/releases/download/1.4.1/AsusSMC-1.4.1-RELEASE.zip"
         }
     ),
     KextInfo(
@@ -475,10 +447,6 @@ kexts = [
         github_repo = {
             "owner": "Xiashangning",
             "repo": "BigSurface"
-        },
-        download_info = {
-            "id": 18528518, 
-            "url": "https://github.com/Xiashangning/BigSurface/releases/download/v6.5/BigSurface.zip"
         }
     ),
     KextInfo(
@@ -509,10 +477,6 @@ kexts = [
         github_repo = {
             "owner": "0xFireWolf",
             "repo": "RealtekCardReader"
-        },
-        download_info = {
-            "id": 10460478, 
-            "url": "https://github.com/0xFireWolf/RealtekCardReader/releases/download/v0.9.7/RealtekCardReader_0.9.7_006a845_RELEASE.zip"
         }
     ),
     KextInfo(
@@ -525,10 +489,6 @@ kexts = [
         github_repo = {
             "owner": "0xFireWolf",
             "repo": "RealtekCardReaderFriend"
-        },
-        download_info = {
-            "id": 10460478, 
-            "url": "https://github.com/0xFireWolf/RealtekCardReaderFriend/releases/download/v1.0.4/RealtekCardReaderFriend_1.0.4_e1e3301_RELEASE.zip"
         }
     ), 
     KextInfo(
@@ -566,10 +526,6 @@ kexts = [
         github_repo = {
             "owner": "b00t0x",
             "repo": "CpuTopologyRebuild"
-        },
-        download_info = {
-            "id": 13190749, 
-            "url": "https://github.com/b00t0x/CpuTopologyRebuild/releases/download/1.1.0/CpuTopologyRebuild-1.1.0-RELEASE.zip"
         }
     ),
     KextInfo(

Scripts/gathering_files.py

@@ -1,3 +1,4 @@
+from Scripts.datasets.kext_data import kexts
 from Scripts import github
 from Scripts import resource_fetcher
 from Scripts import utils
@@ -11,218 +12,55 @@ class gatheringFiles:
     def __init__(self):
         self.utils = utils.Utils()
         self.github = github.Github()
-        self.fetcher = resource_fetcher.ResourceFetcher(self.github.headers)
+        self.fetcher = resource_fetcher.ResourceFetcher()
         self.dortania_builds_url = "https://raw.githubusercontent.com/dortania/build-repo/builds/latest.json"
         self.amd_vanilla_patches_url = "https://raw.githubusercontent.com/AMD-OSX/AMD_Vanilla/beta/patches.plist"
         self.temporary_dir = tempfile.mkdtemp()
         self.ock_files_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "OCK_Files")
+        self.bootloader_kexts_data_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "bootloader_kexts_data.json")
         self.download_history_file = os.path.join(self.ock_files_dir, "history.json")
-        self.builds = [
-            "AirportBrcmFixup",
-            "BrcmPatchRAM",
-            "BrightnessKeys",
-            "CryptexFixup",
-            "ECEnabler",
-            "HibernationFixup",
-            "IntelBluetoothFirmware",
-            "IntelMausi", 
-            "Lilu", 
-            "NVMeFix", 
-            "RTCMemoryFixup", 
-            "RealtekRTL8111", 
-            "RestrictEvents", 
-            "VirtualSMC", 
-            "VoodooI2C", 
-            "VoodooInput", 
-            "VoodooPS2", 
-            "VoodooSMBus", 
-            "WhateverGreen"
-        ]
-        self.releases = [
-            {
-                "owner": "blankmac",
-                "repo": "AlpsHID"
-            },
-            {
-                "owner": "SongXiaoXi",
-                "repo": "AppleIGC"
-            },
-            {
-                "owner": "hieplpvip",
-                "repo": "AsusSMC"
-            },
-            {
-                "owner": "Xiashangning",
-                "repo": "BigSurface"
-            },
-            {
-                "owner": "b00t0x",
-                "repo": "CpuTopologyRebuild"
-            },
-            {
-                "owner": "0xFireWolf",
-                "repo": "RealtekCardReader"
-            },
-            {
-                "owner": "0xFireWolf",
-                "repo": "RealtekCardReaderFriend"
-            },
-            {
-                "owner": "VoodooSMBus",
-                "repo": "VoodooRMI"
-            },
-            {
-                "owner": "OpenIntelWireless",
-                "repo": "itlwm"
-            },
-            {
-                "owner": "ChefKissInc",
-                "repo": "NootRX"
-            },
-            {
-                "owner": "ChefKissInc",
-                "repo": "NootedRed"
-            },
-            {
-                "owner": "ChefKissInc",
-                "repo": "ForgedInvariant"
-            },
-            {
-                "owner": "wjz304",
-                "repo": "OpenCore_Patch_Build"
-            }
-        ][-5:]
-        self.actions = []
         
     def get_bootloader_kexts_data(self):
-        results = [
-            {
-                "product_name": "AlpsHID", 
-                "id": 69228327, 
-                "url": "https://github.com/blankmac/AlpsHID/releases/download/v1.2/AlpsHID1.2_release.zip"
-            },
-            {
-                "product_name": "AMFIPass", 
-                "id": 926491527, 
-                "url": "https://github.com/dortania/OpenCore-Legacy-Patcher/raw/main/payloads/Kexts/Acidanthera/AMFIPass-v1.4.1-RELEASE.zip"
-            },
-            {
-                "product_name": "AppleALC", 
-                "id": 223994507, 
-                "url": "https://github.com/lzhoang2801/lzhoang2801.github.io/raw/main/public/extra-files/AppleALC-1.9.2-RELEASE.zip"
-            },
-            {
-                "product_name": "AppleIGB", 
-                "id": 736194363, 
-                "url": "https://github.com/lzhoang2801/lzhoang2801.github.io/raw/main/public/extra-files/AppleIGB-v5.11.4.zip"
-            },
-            {
-                "product_name": "AppleIGC", 
-                "id": 138279923, 
-                "url": "https://github.com/SongXiaoXi/AppleIGC/releases/download/v1.5/AppleIGC.kext.zip"
-            },
-            {
-                "product_name": "AppleMCEReporterDisabler", 
-                "id": 738162736, 
-                "url": "https://github.com/acidanthera/bugtracker/files/3703498/AppleMCEReporterDisabler.kext.zip"
-            },
-            {
-                "product_name": "AsusSMC", 
-                "id": 41898282, 
-                "url": "https://github.com/hieplpvip/AsusSMC/releases/download/1.4.1/AsusSMC-1.4.1-RELEASE.zip"
-            },
-            {
-                "product_name": "AtherosE2200Ethernet", 
-                "id": 9746382, 
-                "url": "https://github.com/Mieze/AtherosE2200Ethernet/releases/download/2.2.2/AtherosE2200Ethernet-V2.2.2.zip"
-            },
-            {
-                "product_name": "BigSurface", 
-                "id": 18528518, 
-                "url": "https://github.com/Xiashangning/BigSurface/releases/download/v6.5/BigSurface.zip"
-            },
-            {
-                "product_name": "CtlnaAHCIPort", 
-                "id": 10460478, 
-                "url": "https://github.com/lzhoang2801/lzhoang2801.github.io/raw/main/public/extra-files/CtlnaAHCIPort-v3.4.1.zip"
-            },
-            {
-                "product_name": "CpuTopologyRebuild", 
-                "id": 13190749, 
-                "url": "https://github.com/b00t0x/CpuTopologyRebuild/releases/download/1.1.0/CpuTopologyRebuild-1.1.0-RELEASE.zip"
-            },
-            {
-                "product_name": "IO80211FamilyLegacy", 
-                "id": 817294638, 
-                "url": "https://github.com/dortania/OpenCore-Legacy-Patcher/raw/main/payloads/Kexts/Wifi/IO80211FamilyLegacy-v1.0.0.zip"
-            },
-            {
-                "product_name": "IOSkywalkFamily", 
-                "id": 926584761, 
-                "url": "https://github.com/dortania/OpenCore-Legacy-Patcher/raw/main/payloads/Kexts/Wifi/IOSkywalkFamily-v1.2.0.zip"
-            },
-            {
-                "product_name": "LucyRTL8125Ethernet", 
-                "id": 159470181, 
-                "url": "https://github.com/Mieze/LucyRTL8125Ethernet/releases/download/v1.2.0d5/LucyRTL8125Ethernet-V1.2.0d5.zip"
-            },
-            {
-                "product_name": "NullEthernet", 
-                "id": 182736492, 
-                "url": "https://bitbucket.org/RehabMan/os-x-null-ethernet/downloads/RehabMan-NullEthernet-2016-1220.zip"
-            },
-            {
-                "product_name": "VoodooRMI", 
-                "id": 13190749, 
-                "url": "https://github.com/VoodooSMBus/VoodooRMI/releases/download/1.3.5/VoodooRMI-1.3.5-Release.zip"
-            },
-            {
-                "product_name": "RealtekRTL8100", 
-                "id": 10460478, 
-                "url": "https://github.com/lzhoang2801/lzhoang2801.github.io/raw/main/public/extra-files/RealtekRTL8100-v2.0.1.zip"
-            },
-            {
-                "product_name": "RealtekCardReader", 
-                "id": 10460478, 
-                "url": "https://github.com/0xFireWolf/RealtekCardReader/releases/download/v0.9.7/RealtekCardReader_0.9.7_006a845_RELEASE.zip"
-            },
-            {
-                "product_name": "RealtekCardReaderFriend", 
-                "id": 10460478, 
-                "url": "https://github.com/0xFireWolf/RealtekCardReaderFriend/releases/download/v1.0.4/RealtekCardReaderFriend_1.0.4_e1e3301_RELEASE.zip"
-            },
-            {
-                "product_name": "GenericUSBXHCI", 
-                "id": 120325166, 
-                "url": "https://github.com/RattletraPM/GUX-RyzenXHCIFix/releases/download/v1.3.0b1-ryzenxhcifix/GenericUSBXHCI.kext.zip"
-            },
-            {
-                "product_name": "XHCI-unsupported", 
-                "id": 185465401, 
-                "url": "https://github.com/daliansky/OS-X-USB-Inject-All/releases/download/v0.8.0/XHCI-unsupported.kext.zip"
-            }
-        ]
+        last_updated_str = (self.utils.read_file(self.bootloader_kexts_data_path) or {}).get("last_updated", "2024-07-25T12:00:00")
+        last_updated = datetime.fromisoformat(last_updated_str)
+
+        current_time = datetime.now()
+        if current_time - last_updated < timedelta(minutes=15):
+            return
+        download_urls = []
 
         dortania_builds_data = self.fetcher.fetch_and_parse_content(self.dortania_builds_url, "json")
+        seen_repos = set()
 
-        for product_name, product_data in dortania_builds_data.items():
-            if product_name in self.builds:
-                results.append({
-                    "product_name": product_name, 
-                    "id": product_data["versions"][0]["release"]["id"], 
-                    "url": product_data["versions"][0]["links"]["release"]
-                })
+        for kext in kexts:
+            if kext.download_info:
+                download_urls.append({"product_name": kext.name, **kext.download_info})
+            elif kext.github_repo and kext.github_repo.get("repo") not in seen_repos:
+                name = kext.github_repo.get("repo")
+                seen_repos.add(name)
+                if name in dortania_builds_data:
+                    download_urls.append({
+                        "product_name": name,
+                        "id": dortania_builds_data[name]["versions"][0]["release"]["id"], 
+                        "url": dortania_builds_data[name]["versions"][0]["links"]["release"]
+                    })
+                else:
+                    download_urls.extend(self.github.get_latest_release(kext.github_repo.get("owner"), kext.github_repo.get("repo")))
 
-        for product in self.releases:
-            results.extend(self.github.get_latest_release(product["owner"], product["repo"]))
-        # [self.github.get_latest_artifact_id(product["owner"], product["repo"]) for product in self.actions]
-        
-        sorted_results = sorted(results, key=lambda x:x["product_name"])
+        download_urls.extend(self.github.get_latest_release("wjz304", "OpenCore_Patch_Build"))
+
+        sorted_data = sorted(download_urls, key=lambda x:x["product_name"])
+
+        self.utils.create_folder(self.ock_files_dir)
+        self.utils.write_file(self.bootloader_kexts_data_path, {
+            "download_urls": sorted_data,
+            "last_updated": current_time.isoformat()
+        })
     
-        return sorted_results
+        return sorted_data
     
-    def product_index_in_history(self, product_name, versions):
-        for index, item in enumerate(versions):
+    def product_index_in_history(self, product_name, download_history):
+        for index, item in enumerate(download_history):
             if product_name in item["product_name"]:
                 return index
         return None
@@ -263,27 +101,13 @@ class gatheringFiles:
         return True
 
     def gathering_bootloader_kexts(self):
-        download_history = self.utils.read_file(self.download_history_file) or {
-            "versions": [],
-            "last_updated": "2024-07-25T12:00:00"
-        }
-
-        if not download_history.get("versions"):
-            download_history["versions"] = []
-        last_updated = datetime.fromisoformat(download_history.get("last_updated", "2024-07-25T12:00:00"))
-
-        current_time = datetime.now()
-        if current_time - last_updated < timedelta(minutes=10):
-            return
+        download_history = self.utils.read_file(self.download_history_file) or []
         
-        ock_data = self.get_bootloader_kexts_data()
-
         self.utils.create_folder(self.temporary_dir)
 
-        for product_data in ock_data:
-        
-            product_index = self.product_index_in_history(product_data.get("product_name"), download_history.get("versions"))
-            if not product_index is None and product_data.get("id") == download_history.get("versions")[product_index].get("id"):
+        for product_data in (self.utils.read_file(self.bootloader_kexts_data_path) or {}).get("download_urls", []):
+            product_index = self.product_index_in_history(product_data.get("product_name"), download_history)
+            if not product_index is None and product_data.get("id") == download_history[product_index].get("id"):
                 continue
 
             asset_dir = os.path.join(self.ock_files_dir, product_data.get("product_name"))
@@ -295,20 +119,15 @@ class gatheringFiles:
 
             if self.move_bootloader_kexts_to_product_directory(product_data.get("product_name")):
                 if product_index is None:
-                    download_history["versions"].append({
+                    download_history.append({
                         "product_name": product_data.get("product_name"),
                         "id": product_data.get("id")
                     })
                 else:
-                    download_history["versions"][product_index]["id"] = product_data.get("id")
+                    download_history[product_index]["id"] = product_data.get("id")
                 
                 self.utils.write_file(self.download_history_file, download_history)
 
-        current_time = datetime.now().isoformat()
-        download_history["last_updated"] = current_time
-
-        self.utils.write_file(self.download_history_file, download_history)
-
         shutil.rmtree(self.temporary_dir, ignore_errors=True)
     
     def get_amd_kernel_patches(self):

Scripts/smbios.py

@@ -21,11 +21,10 @@ class SMBIOS:
             download_history = self.utils.read_file(self.g.download_history_file)
 
             if download_history:
-                product_index = self.g.product_index_in_history("OpenCore", download_history["versions"])
+                product_index = self.g.product_index_in_history("OpenCore", download_history)
                 
                 if product_index:
-                    download_history["versions"].pop(product_index)
-                    download_history["last_updated"] = "2024-07-25T12:00:00"
+                    download_history.pop(product_index)
                     self.utils.write_file(self.g.download_history_file, download_history)
 
             raise Exception("{} not found. Please reopen the program to download it".format(macserial_name))